From 1acdd5c0f58b119f6aed3712322474de959a9ea2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 11 Jan 2023 14:59:45 +0100 Subject: [PATCH 001/183] Implement Trainer & TrainingArguments w. tests Note: This commit involves three deprecations: the SetFitTrainer class (and DistilledSetFitTrainer), additional arguments to Trainer.train and the keep_body_frozen argument to Trainer.unfreeze. The first and last of these are 'graceful', i.e. old code will still work, but the Trainer.train changes are breaking in some situations. For example, e.g. um_epochs can no longer be passed to Trainer.train. The new 'deprecated' test files are identical to the old test files. The goal here is to test whether old behaviour is still possible. For the most part it is, with exception of using Trainer.train with extra arguments. As a result, I skipped two tests in test_deprecated_trainer.py. Also note that docstrings have yet to be updated! --- src/setfit/__init__.py | 4 +- src/setfit/modeling.py | 80 ++-- src/setfit/trainer.py | 311 +++++++-------- src/setfit/trainer_distillation.py | 276 ++++++-------- src/setfit/training_args.py | 68 ++++ tests/test_deprecated_trainer.py | 359 ++++++++++++++++++ tests/test_deprecated_trainer_distillation.py | 102 +++++ tests/test_trainer.py | 96 ++--- tests/test_trainer_distillation.py | 25 +- tests/test_training_args.py | 15 + 10 files changed, 901 insertions(+), 435 deletions(-) create mode 100644 src/setfit/training_args.py create mode 100644 tests/test_deprecated_trainer.py create mode 100644 tests/test_deprecated_trainer_distillation.py create mode 100644 tests/test_training_args.py diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index 9c9665fc..37db149b 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -2,5 +2,5 @@ from .data import add_templated_examples, sample_dataset from .modeling import SetFitHead, SetFitModel -from .trainer import SetFitTrainer -from .trainer_distillation import DistillationSetFitTrainer +from .trainer import SetFitTrainer, Trainer +from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 5abcf91d..0fe5abcf 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -1,7 +1,9 @@ import os from dataclasses import dataclass from pathlib import Path +import time from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +import warnings # Google Colab runs on Python 3.7, so we need this to be compatible @@ -14,14 +16,14 @@ import numpy as np import requests import torch -import torch.nn as nn +from torch import nn from huggingface_hub import PyTorchModelHubMixin, hf_hub_download from sentence_transformers import InputExample, SentenceTransformer, models from sklearn.linear_model import LogisticRegression from sklearn.multiclass import OneVsRestClassifier from sklearn.multioutput import ClassifierChain, MultiOutputClassifier from torch.utils.data import DataLoader -from tqdm.auto import tqdm +from tqdm.auto import trange, tqdm from . import logging from .data import SetFitDataset @@ -208,7 +210,7 @@ def predict(self, x_test: torch.Tensor) -> torch.Tensor: return out - def get_loss_fn(self): + def get_loss_fn(self) -> nn.Module: return torch.nn.CrossEntropyLoss() @property @@ -232,9 +234,9 @@ def get_config_dict(self) -> Dict[str, Optional[Union[int, float, bool]]]: @staticmethod def _init_weight(module): if isinstance(module, nn.Linear): - torch.nn.init.xavier_uniform_(module.weight) + nn.init.xavier_uniform_(module.weight) if module.bias is not None: - torch.nn.init.constant_(module.bias, 1e-2) + nn.init.constant_(module.bias, 1e-2) def __repr__(self): return "SetFitHead({})".format(self.get_config_dict()) @@ -270,25 +272,29 @@ def fit( self, x_train: List[str], y_train: List[int], - num_epochs: int, - batch_size: Optional[int] = None, - learning_rate: Optional[float] = None, - body_learning_rate: Optional[float] = None, + classifier_num_epochs: int, + classifier_batch_size: Optional[int] = None, + classifier_learning_rate: Optional[Tuple[float, float]] = (None, None), l2_weight: Optional[float] = None, max_length: Optional[int] = None, - show_progress_bar: Optional[bool] = None, + show_progress_bar: bool = True, + end_to_end: bool = False, + **kwargs ) -> None: if self.has_differentiable_head: # train with pyTorch device = self.model_body.device self.model_body.train() self.model_head.train() + if not end_to_end: + self.freeze("body") - dataloader = self._prepare_dataloader(x_train, y_train, batch_size, max_length) + dataloader = self._prepare_dataloader(x_train, y_train, classifier_batch_size, max_length) criterion = self.model_head.get_loss_fn() - optimizer = self._prepare_optimizer(learning_rate, body_learning_rate, l2_weight) + embedding_learning_rate, classifier_learning_rate = classifier_learning_rate + optimizer = self._prepare_optimizer(classifier_learning_rate, embedding_learning_rate, l2_weight) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5) - for epoch_idx in tqdm(range(num_epochs), desc="Epoch", disable=not show_progress_bar): - for batch in dataloader: + for epoch_idx in trange(classifier_num_epochs, desc="Epoch", disable=not show_progress_bar): + for batch in tqdm(dataloader, desc="Iteration", disable=not show_progress_bar, leave=False): features, labels = batch optimizer.zero_grad() @@ -298,15 +304,18 @@ def fit( outputs = self.model_body(features) if self.normalize_embeddings: - outputs = torch.nn.functional.normalize(outputs, p=2, dim=1) + outputs = nn.functional.normalize(outputs, p=2, dim=1) outputs = self.model_head(outputs) logits = outputs["logits"] - loss = criterion(logits, labels) + loss: torch.Tensor = criterion(logits, labels) loss.backward() optimizer.step() scheduler.step() + + if not end_to_end: + self.unfreeze("body") else: # train with sklearn embeddings = self.model_body.encode(x_train, normalize_embeddings=self.normalize_embeddings) self.model_head.fit(embeddings, y_train) @@ -349,16 +358,16 @@ def _prepare_dataloader( def _prepare_optimizer( self, - learning_rate: float, - body_learning_rate: Optional[float], + classifier_learning_rate: float, + embedding_learning_rate: Optional[float], l2_weight: float, ) -> torch.optim.Optimizer: - body_learning_rate = body_learning_rate or learning_rate + embedding_learning_rate = embedding_learning_rate or classifier_learning_rate l2_weight = l2_weight or self.l2_weight optimizer = torch.optim.AdamW( [ - {"params": self.model_body.parameters(), "lr": body_learning_rate, "weight_decay": l2_weight}, - {"params": self.model_head.parameters(), "lr": learning_rate, "weight_decay": l2_weight}, + {"params": self.model_body.parameters(), "lr": embedding_learning_rate, "weight_decay": l2_weight}, + {"params": self.model_head.parameters(), "lr": classifier_learning_rate, "weight_decay": l2_weight}, ], ) @@ -368,25 +377,30 @@ def freeze(self, component: Optional[Literal["body", "head"]] = None) -> None: if component is None or component == "body": self._freeze_or_not(self.model_body, to_freeze=True) - if component is None or component == "head": + if (component is None or component == "head") and self.has_differentiable_head: self._freeze_or_not(self.model_head, to_freeze=True) - def unfreeze(self, component: Optional[Literal["body", "head"]] = None) -> None: + def unfreeze(self, component: Optional[Literal["body", "head"]] = None, keep_body_frozen: Optional[bool] = None) -> None: + if keep_body_frozen is not None: + warnings.warn("`keep_body_frozen` is deprecated. Please either pass \"head\", \"body\" or no arguments to unfreeze both.") + if component is None or component == "body": self._freeze_or_not(self.model_body, to_freeze=False) - if component is None or component == "head": + if (component is None or component == "head") and self.has_differentiable_head: self._freeze_or_not(self.model_head, to_freeze=False) - def _freeze_or_not(self, model: torch.nn.Module, to_freeze: bool) -> None: + def _freeze_or_not(self, model: nn.Module, to_freeze: bool) -> None: for param in model.parameters(): param.requires_grad = not to_freeze - def predict(self, x_test: List[str], as_numpy: bool = False) -> Union[torch.Tensor, "ndarray"]: - embeddings = self.model_body.encode( - x_test, normalize_embeddings=self.normalize_embeddings, convert_to_tensor=self.has_differentiable_head + def encode(self, inputs: List[str]) -> Union[torch.Tensor, "ndarray"]: + return self.model_body.encode( + inputs, normalize_embeddings=self.normalize_embeddings, convert_to_tensor=self.has_differentiable_head ) + def predict(self, inputs: List[str], as_numpy: bool = False) -> Union[torch.Tensor, "ndarray"]: + embeddings = self.encode(inputs) outputs = self.model_head.predict(embeddings) if as_numpy and self.has_differentiable_head: @@ -396,11 +410,8 @@ def predict(self, x_test: List[str], as_numpy: bool = False) -> Union[torch.Tens return outputs - def predict_proba(self, x_test: List[str], as_numpy: bool = False) -> Union[torch.Tensor, "ndarray"]: - embeddings = self.model_body.encode( - x_test, normalize_embeddings=self.normalize_embeddings, convert_to_tensor=self.has_differentiable_head - ) - + def predict_proba(self, inputs: List[str], as_numpy: bool = False) -> Union[torch.Tensor, "ndarray"]: + embeddings = self.encode(inputs) outputs = self.model_head.predict_proba(embeddings) if as_numpy and self.has_differentiable_head: @@ -419,6 +430,9 @@ def to(self, device: Union[str, torch.device]) -> "SetFitModel": Returns: SetFitModel: Returns the original model, but now on the desired device. """ + # Note that we must also set _target_device, or any SentenceTransformer.fit() call will reset + # the body location + self.model_body._target_device = device if isinstance(device, torch.device) else torch.device(device) self.model_body = self.model_body.to(device) if self.has_differentiable_head: diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 776c0f1d..f05c3e97 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -1,5 +1,6 @@ import math -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +import warnings import evaluate import numpy as np @@ -9,6 +10,8 @@ from torch.utils.data import DataLoader from transformers.trainer_utils import HPSearchBackend, default_compute_objective, number_of_arguments, set_seed +from setfit.training_args import TrainingArguments + from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna from .modeling import SupConLoss, sentence_pairs_generation, sentence_pairs_generation_multilabel @@ -25,7 +28,7 @@ logger = logging.get_logger(__name__) -class SetFitTrainer: +class Trainer: """Trainer to train a SetFit model. Args: @@ -78,44 +81,21 @@ class SetFitTrainer: def __init__( self, model: Optional["SetFitModel"] = None, + args: Optional[TrainingArguments] = None, train_dataset: Optional["Dataset"] = None, eval_dataset: Optional["Dataset"] = None, model_init: Optional[Callable[[], "SetFitModel"]] = None, metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", loss_class=losses.CosineSimilarityLoss, - num_iterations: int = 20, - num_epochs: int = 1, - learning_rate: float = 2e-5, - batch_size: int = 16, - seed: int = 42, column_mapping: Optional[Dict[str, str]] = None, - use_amp: bool = False, - warmup_proportion: float = 0.1, - distance_metric: Callable = BatchHardTripletLossDistanceFunction.cosine_distance, - margin: float = 0.25, - samples_per_label: int = 2, ): - if (warmup_proportion < 0.0) or (warmup_proportion > 1.0): - raise ValueError( - f"warmup_proportion must be greater than or equal to 0.0 and less than or equal to 1.0! But it was: {warmup_proportion}" - ) - + self.args = args self.train_dataset = train_dataset self.eval_dataset = eval_dataset self.model_init = model_init self.metric = metric self.loss_class = loss_class - self.num_iterations = num_iterations - self.num_epochs = num_epochs - self.learning_rate = learning_rate - self.batch_size = batch_size - self.seed = seed self.column_mapping = column_mapping - self.use_amp = use_amp - self.warmup_proportion = warmup_proportion - self.distance_metric = distance_metric - self.margin = margin - self.samples_per_label = samples_per_label if model is None: if model_init is not None: @@ -127,6 +107,10 @@ def __init__( raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument, but not both") self.model = model + # Adopt Trainer.(un)freeze from SetFitModel.(un)freeze + self.freeze = self.model.freeze + self.unfreeze = self.model.unfreeze + self.hp_search_backend = None self._freeze = True # If True, will train the body only; otherwise, train the body and head @@ -226,81 +210,18 @@ def call_model_init(self, params: Optional[Dict[str, Any]] = None): return model - def freeze(self): - """ - Freeze SetFitModel's differentiable head. - Note: call this function only when using the differentiable head. - """ - if not self.model.has_differentiable_head: - raise ValueError("Please use the differentiable head in `SetFitModel` when calling this function.") - - self._freeze = True # Currently use self._freeze as a switch - self.model.freeze("head") - - def unfreeze(self, keep_body_frozen: bool = False): - """ - Unfreeze SetFitModel's differentiable head. - Note: call this function only when using the differentiable head. - - Args: - keep_body_frozen (`bool`, *optional*, defaults to `False`): - Whether to freeze the body when unfreeze the head. - """ - if not self.model.has_differentiable_head: - raise ValueError("Please use the differentiable head in `SetFitModel` when calling this function.") - - self._freeze = False # Currently use self._freeze as a switch - self.model.unfreeze("head") - if keep_body_frozen: - self.model.freeze("body") - else: # ensure to unfreeze the body - self.model.unfreeze("body") - def train( - self, - num_epochs: Optional[int] = None, - batch_size: Optional[int] = None, - learning_rate: Optional[float] = None, - body_learning_rate: Optional[float] = None, - l2_weight: Optional[float] = None, - max_length: Optional[int] = None, - trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, - show_progress_bar: bool = True, + self, args: Optional[TrainingArguments] = None, trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None ): - """ - Main training entry point. + args = args or self.args or TrainingArguments() - Args: - num_epochs (`int`, *optional*): - Temporary change the number of epochs to train the Sentence Transformer body/head for. - If ignore, will use the value given in initialization. - batch_size (`int`, *optional*): - Temporary change the batch size to use for contrastive training or logistic regression. - If ignore, will use the value given in initialization. - learning_rate (`float`, *optional*): - Temporary change the learning rate to use for contrastive training or SetFitModel's head in logistic regression. - If ignore, will use the value given in initialization. - body_learning_rate (`float`, *optional*): - Temporary change the learning rate to use for SetFitModel's body in logistic regression only. - If ignore, will be the same as `learning_rate`. - l2_weight (`float`, *optional*): - Temporary change the weight of L2 regularization for SetFitModel's differentiable head in logistic regression. - max_length (int, *optional*, defaults to `None`): - The maximum number of tokens for one data sample. Currently only for training the differentiable head. - If `None`, will use the maximum number of tokens the model body can accept. - If `max_length` is greater than the maximum number of acceptable tokens the model body can accept, it will be set to the maximum number of acceptable tokens. - trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): - The trial run or the hyperparameter dictionary for hyperparameter search. - show_progress_bar (`bool`, *optional*, defaults to `True`): - Whether to show a bar that indicates training progress. - """ - set_seed(self.seed) # Seed must be set before instantiating the model when using model_init. + set_seed(args.seed) # Seed must be set before instantiating the model when using model_init. if trial: # Trial and model initialization self._hp_search_setup(trial) # sets trainer parameters and initializes model if self.train_dataset is None: - raise ValueError("Training requires a `train_dataset` given to the `SetFitTrainer` initialization.") + raise ValueError(f"Training requires a `train_dataset` given to the `{self.__class__.__name__}` initialization.") self._validate_column_mapping(self.train_dataset) train_dataset = self.train_dataset @@ -308,93 +229,88 @@ def train( logger.info("Applying column mapping to training dataset") train_dataset = self._apply_column_mapping(self.train_dataset, self.column_mapping) - x_train = train_dataset["text"] - y_train = train_dataset["label"] + x_train: List[str] = train_dataset["text"] + y_train: List[int] = train_dataset["label"] if self.loss_class is None: logger.warning("No `loss_class` detected! Using `CosineSimilarityLoss` as the default.") self.loss_class = losses.CosineSimilarityLoss - num_epochs = num_epochs or self.num_epochs - batch_size = batch_size or self.batch_size - learning_rate = learning_rate or self.learning_rate - - if not self.model.has_differentiable_head or self._freeze: - # sentence-transformers adaptation - if self.loss_class in [ - losses.BatchAllTripletLoss, - losses.BatchHardTripletLoss, - losses.BatchSemiHardTripletLoss, - losses.BatchHardSoftMarginTripletLoss, - SupConLoss, - ]: - train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] - train_data_sampler = SentenceLabelDataset(train_examples, samples_per_label=self.samples_per_label) - - batch_size = min(batch_size, len(train_data_sampler)) - train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) - - if self.loss_class is losses.BatchHardSoftMarginTripletLoss: - train_loss = self.loss_class( - model=self.model.model_body, - distance_metric=self.distance_metric, + self.train_embeddings(x_train, y_train, args) + self.train_classifier(x_train, y_train, args) + + def train_embeddings(self, x_train: List[str], y_train: List[int], args: Optional[TrainingArguments] = None): + args = args or self.args or TrainingArguments() + + # sentence-transformers adaptation + if self.loss_class in [ + losses.BatchAllTripletLoss, + losses.BatchHardTripletLoss, + losses.BatchSemiHardTripletLoss, + losses.BatchHardSoftMarginTripletLoss, + SupConLoss, + ]: + train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] + train_data_sampler = SentenceLabelDataset(train_examples, samples_per_label=args.samples_per_label) + + batch_size = min(args.embedding_batch_size, len(train_data_sampler)) + train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) + + if self.loss_class is losses.BatchHardSoftMarginTripletLoss: + train_loss = self.loss_class( + model=self.model.model_body, + distance_metric=args.distance_metric, + ) + elif self.loss_class is SupConLoss: + train_loss = self.loss_class(model=self.model.model_body) + else: + train_loss = self.loss_class( + model=self.model.model_body, + distance_metric=args.distance_metric, + margin=args.margin, + ) + + train_steps = len(train_dataloader) * args.embedding_num_epochs + else: + train_examples = [] + + for _ in range(args.num_iterations): + if self.model.multi_target_strategy is not None: + train_examples = sentence_pairs_generation_multilabel( + np.array(x_train), np.array(y_train), train_examples ) - elif self.loss_class is SupConLoss: - train_loss = self.loss_class(model=self.model.model_body) else: - train_loss = self.loss_class( - model=self.model.model_body, - distance_metric=self.distance_metric, - margin=self.margin, - ) + train_examples = sentence_pairs_generation(np.array(x_train), np.array(y_train), train_examples) + + batch_size = args.embedding_batch_size + train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) + train_loss = self.loss_class(self.model.model_body) + train_steps = len(train_dataloader) * args.embedding_num_epochs + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_examples)}") + logger.info(f" Num epochs = {args.embedding_num_epochs}") + logger.info(f" Total optimization steps = {train_steps}") + logger.info(f" Total train batch size = {batch_size}") + + warmup_steps = math.ceil(train_steps * args.warmup_proportion) + self.model.model_body.fit( + train_objectives=[(train_dataloader, train_loss)], + epochs=args.embedding_num_epochs, + steps_per_epoch=train_steps, + optimizer_params={"lr": args.embedding_learning_rate}, + warmup_steps=warmup_steps, + show_progress_bar=args.show_progress_bar, + use_amp=args.use_amp, + ) - train_steps = len(train_dataloader) * self.num_epochs - else: - train_examples = [] - - for _ in range(self.num_iterations): - if self.model.multi_target_strategy is not None: - train_examples = sentence_pairs_generation_multilabel( - np.array(x_train), np.array(y_train), train_examples - ) - else: - train_examples = sentence_pairs_generation( - np.array(x_train), np.array(y_train), train_examples - ) - - train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) - train_loss = self.loss_class(self.model.model_body) - train_steps = len(train_dataloader) * num_epochs - - logger.info("***** Running training *****") - logger.info(f" Num examples = {len(train_examples)}") - logger.info(f" Num epochs = {num_epochs}") - logger.info(f" Total optimization steps = {train_steps}") - logger.info(f" Total train batch size = {batch_size}") - - warmup_steps = math.ceil(train_steps * self.warmup_proportion) - self.model.model_body.fit( - train_objectives=[(train_dataloader, train_loss)], - epochs=num_epochs, - steps_per_epoch=train_steps, - optimizer_params={"lr": learning_rate}, - warmup_steps=warmup_steps, - show_progress_bar=show_progress_bar, - use_amp=self.use_amp, - ) + def train_classifier(self, x_train: List[str], y_train: List[int], args: Optional[TrainingArguments] = None): + args = args or self.args or TrainingArguments() - if not self.model.has_differentiable_head or not self._freeze: - # Train the final classifier - self.model.fit( - x_train, - y_train, - num_epochs=num_epochs, - batch_size=batch_size, - learning_rate=learning_rate, - body_learning_rate=body_learning_rate, - l2_weight=l2_weight, - max_length=max_length, - show_progress_bar=True, - ) + self.model.fit( + x_train, + y_train, + **args.to_dict(), + ) def evaluate(self): """ @@ -533,3 +449,52 @@ def push_to_hub( config, skip_lfs_files, ) + + +class SetFitTrainer(Trainer): + def __init__( + self, + model: Optional["SetFitModel"] = None, + train_dataset: Optional["Dataset"] = None, + eval_dataset: Optional["Dataset"] = None, + model_init: Optional[Callable[[], "SetFitModel"]] = None, + metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", + loss_class=losses.CosineSimilarityLoss, + num_iterations: int = 20, + num_epochs: int = 1, + learning_rate: float = 2e-5, + batch_size: int = 16, + seed: int = 42, + column_mapping: Optional[Dict[str, str]] = None, + use_amp: bool = False, + warmup_proportion: float = 0.1, + distance_metric: Callable = BatchHardTripletLossDistanceFunction.cosine_distance, + margin: float = 0.25, + samples_per_label: int = 2, + ): + warnings.warn( + "`SetFitTrainer` has been deprecated. Please use `from setfit import Trainer` instead.", DeprecationWarning + ) + args = TrainingArguments( + num_iterations=num_iterations, + num_epochs=num_epochs, + classifier_learning_rate=learning_rate, + embedding_learning_rate=learning_rate, + batch_size=batch_size, + seed=seed, + use_amp=use_amp, + warmup_proportion=warmup_proportion, + distance_metric=distance_metric, + margin=margin, + samples_per_label=samples_per_label, + ) + super().__init__( + model=model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + model_init=model_init, + metric=metric, + loss_class=loss_class, + column_mapping=column_mapping, + ) diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 2546f7ea..9bf53888 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -1,5 +1,5 @@ import math -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import numpy as np import torch @@ -9,8 +9,11 @@ from torch.utils.data import DataLoader from transformers.trainer_utils import set_seed -from . import SetFitTrainer, logging +from setfit.training_args import TrainingArguments + +from . import logging from .modeling import SupConLoss, sentence_pairs_generation_cos_sim +from .trainer import Trainer if TYPE_CHECKING: @@ -23,7 +26,7 @@ logger = logging.get_logger(__name__) -class DistillationSetFitTrainer(SetFitTrainer): +class DistillationTrainer(Trainer): """Trainer to compress a SetFit model with knowledge distillation. Args: @@ -67,177 +70,140 @@ def __init__( self, teacher_model: "SetFitModel", student_model: Optional["SetFitModel"] = None, + args: TrainingArguments = None, train_dataset: Optional["Dataset"] = None, eval_dataset: Optional["Dataset"] = None, model_init: Optional[Callable[[], "SetFitModel"]] = None, metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", loss_class: torch.nn.Module = losses.CosineSimilarityLoss, - num_iterations: int = 20, - num_epochs: int = 1, - learning_rate: float = 2e-5, - batch_size: int = 16, - seed: int = 42, column_mapping: Optional[Dict[str, str]] = None, - use_amp: bool = False, - warmup_proportion: float = 0.1, ) -> None: - super(DistillationSetFitTrainer, self).__init__( + super().__init__( model=student_model, + args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, model_init=model_init, metric=metric, loss_class=loss_class, - num_iterations=num_iterations, - num_epochs=num_epochs, - learning_rate=learning_rate, - batch_size=batch_size, - seed=seed, column_mapping=column_mapping, - use_amp=use_amp, - warmup_proportion=warmup_proportion, ) self.teacher_model = teacher_model self.student_model = self.model - def train( + def train_embeddings( self, - num_epochs: Optional[int] = None, - batch_size: Optional[int] = None, - learning_rate: Optional[float] = None, - body_learning_rate: Optional[float] = None, - l2_weight: Optional[float] = None, - trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, - show_progress_bar: bool = True, + x_train: List[str], + y_train: List[int], + args: Optional[TrainingArguments] = None, ): - """ - Main training entry point. - - Args: - num_epochs (`int`, *optional*): - Temporary change the number of epochs to train the Sentence Transformer body/head for. - If ignore, will use the value given in initialization. - batch_size (`int`, *optional*): - Temporary change the batch size to use for contrastive training or logistic regression. - If ignore, will use the value given in initialization. - learning_rate (`float`, *optional*): - Temporary change the learning rate to use for contrastive training or SetFitModel's head in logistic regression. - If ignore, will use the value given in initialization. - body_learning_rate (`float`, *optional*): - Temporary change the learning rate to use for SetFitModel's body in logistic regression only. - If ignore, will be the same as `learning_rate`. - l2_weight (`float`, *optional*): - Temporary change the weight of L2 regularization for SetFitModel's differentiable head in logistic regression. - trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): - The trial run or the hyperparameter dictionary for hyperparameter search. - show_progress_bar (`bool`, *optional*, defaults to `True`): - Whether to show a bar that indicates training progress. - """ - set_seed(self.seed) # Seed must be set before instantiating the model when using model_init. - - if trial: # Trial and model initialization - self._hp_search_setup(trial) # sets trainer parameters and initializes model - - if self.train_dataset is None: - raise ValueError( - "Training requires a `train_dataset` given to the `DistillationSetFitTrainer` initialization." - ) - - self._validate_column_mapping(self.train_dataset) - train_dataset = self.train_dataset - if self.column_mapping is not None: - logger.info("Applying column mapping to training dataset") - train_dataset = self._apply_column_mapping(self.train_dataset, self.column_mapping) - - x_train = train_dataset["text"] - y_train = train_dataset["label"] - if self.loss_class is None: - logger.warning("No `loss_class` detected! Using `CosineSimilarityLoss` as the default.") - self.loss_class = losses.CosineSimilarityLoss - - num_epochs = num_epochs or self.num_epochs - batch_size = batch_size or self.batch_size - learning_rate = learning_rate or self.learning_rate - - if not self.student_model.has_differentiable_head or self._freeze: - # sentence-transformers adaptation - if self.loss_class in [ - losses.BatchAllTripletLoss, - losses.BatchHardTripletLoss, - losses.BatchSemiHardTripletLoss, - losses.BatchHardSoftMarginTripletLoss, - SupConLoss, - ]: - train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] - train_data_sampler = SentenceLabelDataset(train_examples) - - batch_size = min(batch_size, len(train_data_sampler)) - train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) - - if self.loss_class is losses.BatchHardSoftMarginTripletLoss: - train_loss = self.loss_class( - model=self.student_model, - distance_metric=BatchHardTripletLossDistanceFunction.cosine_distance, - ) - elif self.loss_class is SupConLoss: - train_loss = self.loss_class(model=self.student_model) - else: - - train_loss = self.loss_class( - model=self.student_model, - distance_metric=BatchHardTripletLossDistanceFunction.cosine_distance, - margin=0.25, - ) - - train_steps = len(train_dataloader) * self.num_epochs + args = args or self.args or TrainingArguments() + + # sentence-transformers adaptation + if self.loss_class in [ + losses.BatchAllTripletLoss, + losses.BatchHardTripletLoss, + losses.BatchSemiHardTripletLoss, + losses.BatchHardSoftMarginTripletLoss, + SupConLoss, + ]: + train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] + train_data_sampler = SentenceLabelDataset(train_examples) + + batch_size = min(args.embedding_batch_size, len(train_data_sampler)) + train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) + + if self.loss_class is losses.BatchHardSoftMarginTripletLoss: + train_loss = self.loss_class( + model=self.student_model.model_body, + distance_metric=args.distance_metric, + ) + elif self.loss_class is SupConLoss: + train_loss = self.loss_class(model=self.student_model) else: - train_examples = [] - - # **************** student training **************** - x_train_embd_student = self.teacher_model.model_body.encode(x_train) - y_train = self.teacher_model.model_head.predict(x_train_embd_student) - - cos_sim_matrix = util.cos_sim(x_train_embd_student, x_train_embd_student) - - train_examples = [] - for _ in range(self.num_iterations): - train_examples = sentence_pairs_generation_cos_sim( - np.array(x_train), train_examples, cos_sim_matrix - ) - - # **************** student training END **************** - - train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) - train_loss = self.loss_class(self.student_model.model_body) - train_steps = len(train_dataloader) * num_epochs - - logger.info("***** Running training *****") - logger.info(f" Num examples = {len(train_examples)}") - logger.info(f" Num epochs = {num_epochs}") - logger.info(f" Total optimization steps = {train_steps}") - logger.info(f" Total train batch size = {batch_size}") - - warmup_steps = math.ceil(train_steps * self.warmup_proportion) - self.student_model.model_body.fit( - train_objectives=[(train_dataloader, train_loss)], - epochs=num_epochs, - steps_per_epoch=train_steps, - optimizer_params={"lr": learning_rate}, - warmup_steps=warmup_steps, - show_progress_bar=show_progress_bar, - use_amp=self.use_amp, - ) - - if not self.student_model.has_differentiable_head or not self._freeze: - # Train the final classifier - self.student_model.fit( - x_train, - y_train, - num_epochs=num_epochs, - batch_size=batch_size, - learning_rate=learning_rate, - body_learning_rate=body_learning_rate, - l2_weight=l2_weight, - show_progress_bar=show_progress_bar, - ) + train_loss = self.loss_class( + model=self.student_model.model_body, + distance_metric=args.distance_metric, + margin=args.margin, + ) + + train_steps = len(train_dataloader) * args.embedding_num_epochs + else: + train_examples = [] + + # **************** student training ********************* + # Only this snippet differs from Trainer.train_embeddings + x_train_embd_student = self.teacher_model.model_body.encode(x_train) + y_train = self.teacher_model.model_head.predict(x_train_embd_student) + + cos_sim_matrix = util.cos_sim(x_train_embd_student, x_train_embd_student) + + train_examples = [] + for _ in range(args.num_iterations): + train_examples = sentence_pairs_generation_cos_sim(np.array(x_train), train_examples, cos_sim_matrix) + # **************** student training END ***************** + + batch_size = args.embedding_batch_size + train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) + train_loss = self.loss_class(self.student_model.model_body) + train_steps = len(train_dataloader) * args.embedding_num_epochs + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_examples)}") + logger.info(f" Num epochs = {args.embedding_num_epochs}") + logger.info(f" Total optimization steps = {train_steps}") + logger.info(f" Total train batch size = {batch_size}") + + warmup_steps = math.ceil(train_steps * args.warmup_proportion) + self.student_model.model_body.fit( + train_objectives=[(train_dataloader, train_loss)], + epochs=args.embedding_num_epochs, + steps_per_epoch=train_steps, + optimizer_params={"lr": args.embedding_learning_rate}, + warmup_steps=warmup_steps, + show_progress_bar=args.show_progress_bar, + use_amp=args.use_amp, + ) + + +class DistillationSetFitTrainer(DistillationTrainer): + def __init__( + self, + teacher_model: "SetFitModel", + student_model: Optional["SetFitModel"] = None, + train_dataset: Optional["Dataset"] = None, + eval_dataset: Optional["Dataset"] = None, + model_init: Optional[Callable[[], "SetFitModel"]] = None, + metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", + loss_class: torch.nn.Module = losses.CosineSimilarityLoss, + num_iterations: int = 20, + num_epochs: int = 1, + learning_rate: float = 2e-5, + batch_size: int = 16, + seed: int = 42, + column_mapping: Optional[Dict[str, str]] = None, + use_amp: bool = False, + warmup_proportion: float = 0.1, + ): + args = TrainingArguments( + num_iterations=num_iterations, + num_epochs=num_epochs, + embedding_learning_rate=learning_rate, + classifier_learning_rate=learning_rate, + batch_size=batch_size, + seed=seed, + use_amp=use_amp, + warmup_proportion=warmup_proportion, + ) + super().__init__( + teacher_model=teacher_model, + student_model=student_model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + model_init=model_init, + metric=metric, + loss_class=loss_class, + column_mapping=column_mapping, + ) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py new file mode 100644 index 00000000..5a796d7a --- /dev/null +++ b/src/setfit/training_args.py @@ -0,0 +1,68 @@ +from copy import copy +from dataclasses import dataclass, fields, field +from typing import Callable, Tuple, Union +from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction + + +@dataclass +class TrainingArguments: + + # batch_size is only used to conveniently set `embedding_batch_size` and `classifier_batch_size` + # which are used in practice + batch_size: Union[int, Tuple[int, int]] = field(default=(16, 2), repr=False) + embedding_batch_size: int = None + classifier_batch_size: int = None + + # num_epochs is only used to conveniently set `embedding_num_epochs` and `classifier_num_epochs` + # which are used in practice + num_epochs: Union[int, Tuple[int, int]] = field(default=(1, 16), repr=False) + embedding_num_epochs: int = None + classifier_num_epochs: int = None + + num_iterations: int = 20 + + embedding_learning_rate: float = 2e-5 + classifier_learning_rate: Union[float, Tuple[float, float]] = (1e-5, 1e-2) + + seed: int = 42 + use_amp: bool = False + warmup_proportion: float = 0.1 + distance_metric: Callable = BatchHardTripletLossDistanceFunction.cosine_distance + margin: float = 0.25 + samples_per_label: int = 2 + show_progress_bar: bool = True + + l2_weight: float = None + max_length: int = None + + end_to_end: bool = False + + def __post_init__(self): + if isinstance(self.batch_size, int): + self.batch_size = (self.batch_size, self.batch_size) + if self.embedding_batch_size is None: + self.embedding_batch_size = self.batch_size[0] + if self.classifier_batch_size is None: + self.classifier_batch_size = self.batch_size[1] + + if isinstance(self.num_epochs, int): + self.num_epochs = (self.num_epochs, self.num_epochs) + if self.embedding_num_epochs is None: + self.embedding_num_epochs = self.num_epochs[0] + if self.classifier_num_epochs is None: + self.classifier_num_epochs = self.num_epochs[1] + + if isinstance(self.classifier_learning_rate, float): + self.classifier_learning_rate = (self.embedding_learning_rate, self.classifier_learning_rate) + + if self.warmup_proportion < 0.0 or self.warmup_proportion > 1.0: + raise ValueError( + f"warmup_proportion must be greater than or equal to 0.0 and less than or equal to 1.0! But it was: {self.warmup_proportion}" + ) + + def to_dict(self): + # filter out fields that are defined as field(init=False) + return {field.name: getattr(self, field.name) for field in fields(self) if field.init} + + def copy(self): + return copy(self) \ No newline at end of file diff --git a/tests/test_deprecated_trainer.py b/tests/test_deprecated_trainer.py new file mode 100644 index 00000000..467b77e0 --- /dev/null +++ b/tests/test_deprecated_trainer.py @@ -0,0 +1,359 @@ +from unittest import TestCase + +import evaluate +import pytest +from datasets import Dataset +from sentence_transformers import losses +from transformers.testing_utils import require_optuna +from transformers.utils.hp_naming import TrialShortNamer + +from setfit import logging +from setfit.modeling import SetFitModel, SupConLoss +from setfit.trainer import SetFitTrainer +from setfit.utils import BestRun + + +logging.set_verbosity_warning() +logging.enable_propagation() + + +class SetFitTrainerTest(TestCase): + def setUp(self): + self.model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + self.num_iterations = 1 + + def test_trainer_works_with_model_init(self): + def get_model(): + model_name = "sentence-transformers/paraphrase-albert-small-v2" + return SetFitModel.from_pretrained(model_name) + + dataset = Dataset.from_dict( + {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} + ) + trainer = SetFitTrainer( + model_init=get_model, + train_dataset=dataset, + eval_dataset=dataset, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + trainer.train() + metrics = trainer.evaluate() + self.assertEqual(metrics["accuracy"], 1.0) + + def test_trainer_works_with_column_mapping(self): + dataset = Dataset.from_dict( + {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} + ) + trainer = SetFitTrainer( + model=self.model, + train_dataset=dataset, + eval_dataset=dataset, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + trainer.train() + metrics = trainer.evaluate() + self.assertEqual(metrics["accuracy"], 1.0) + + def test_trainer_works_with_default_columns(self): + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) + trainer = SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) + trainer.train() + metrics = trainer.evaluate() + self.assertEqual(metrics["accuracy"], 1.0) + + def test_trainer_raises_error_with_missing_label(self): + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) + trainer = SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) + with pytest.raises(ValueError): + trainer.train() + + def test_trainer_raises_error_with_missing_text(self): + dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) + trainer = SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) + with pytest.raises(ValueError): + trainer.train() + + def test_column_mapping_with_missing_text(self): + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) + trainer = SetFitTrainer( + model=self.model, + train_dataset=dataset, + eval_dataset=dataset, + num_iterations=self.num_iterations, + column_mapping={"label_new": "label"}, + ) + with pytest.raises(ValueError): + trainer._validate_column_mapping(trainer.train_dataset) + + def test_column_mapping_multilabel(self): + dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[0, 1], [1, 2], [2, 0]]}) + + trainer = SetFitTrainer( + model=self.model, + train_dataset=dataset, + eval_dataset=dataset, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + + trainer._validate_column_mapping(trainer.train_dataset) + formatted_dataset = trainer._apply_column_mapping(trainer.train_dataset, trainer.column_mapping) + + assert formatted_dataset.column_names == ["text", "label"] + + assert formatted_dataset[0]["text"] == "a" + assert formatted_dataset[0]["label"] == [0, 1] + + assert formatted_dataset[1]["text"] == "b" + + def test_trainer_support_callable_as_metric(self): + dataset = Dataset.from_dict( + {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} + ) + + f1_metric = evaluate.load("f1") + accuracy_metric = evaluate.load("accuracy") + + def compute_metrics(y_pred, y_test): + return { + "f1": f1_metric.compute(predictions=y_pred, references=y_test, average="micro")["f1"], + "accuracy": accuracy_metric.compute(predictions=y_pred, references=y_test)["accuracy"], + } + + trainer = SetFitTrainer( + model=self.model, + train_dataset=dataset, + eval_dataset=dataset, + metric=compute_metrics, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + + trainer.train() + metrics = trainer.evaluate() + + self.assertEqual( + { + "f1": 1.0, + "accuracy": 1.0, + }, + metrics, + ) + + def test_raise_when_metric_value_is_invalid(self): + dataset = Dataset.from_dict( + {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} + ) + + trainer = SetFitTrainer( + model=self.model, + train_dataset=dataset, + eval_dataset=dataset, + metric="this-metric-does-not-exist", # invalid metric value + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + + trainer.train() + + with self.assertRaises(FileNotFoundError): + trainer.evaluate() + + def test_trainer_raises_error_with_wrong_warmup_proportion(self): + # warmup_proportion must not be > 1.0 + with pytest.raises(ValueError): + SetFitTrainer(warmup_proportion=1.1) + + # warmup_proportion must not be < 0.0 + with pytest.raises(ValueError): + SetFitTrainer(warmup_proportion=-0.1) + + +class SetFitTrainerDifferentiableHeadTest(TestCase): + def setUp(self): + self.dataset = Dataset.from_dict( + {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} + ) + self.model = SetFitModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", + use_differentiable_head=True, + head_params={"out_features": 3}, + ) + self.num_iterations = 1 + + @pytest.mark.skip(reason="The `trainer.train` argument removals were a hard deprecation, so this test would throw an error.") + def test_trainer_max_length_exceeds_max_acceptable_length(self): + trainer = SetFitTrainer( + model=self.model, + train_dataset=self.dataset, + eval_dataset=self.dataset, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + trainer.unfreeze(keep_body_frozen=True) + with self.assertLogs(level=logging.WARNING) as cm: + max_length = 4096 + max_acceptable_length = self.model.model_body.get_max_seq_length() + trainer.train( + num_epochs=1, + batch_size=3, + learning_rate=1e-2, + l2_weight=0.0, + max_length=max_length, + ) + self.assertEqual( + cm.output, + [ + ( + f"WARNING:setfit.modeling:The specified `max_length`: {max_length} is greater than the maximum length " + f"of the current model body: {max_acceptable_length}. Using {max_acceptable_length} instead." + ) + ], + ) + + @pytest.mark.skip(reason="The `trainer.train` argument removals were a hard deprecation, so this test would throw an error.") + def test_trainer_max_length_is_smaller_than_max_acceptable_length(self): + trainer = SetFitTrainer( + model=self.model, + train_dataset=self.dataset, + eval_dataset=self.dataset, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + trainer.unfreeze(keep_body_frozen=True) + + # An alternative way of `assertNoLogs`, which is new in Python 3.10 + try: + with self.assertLogs(level=logging.WARNING) as cm: + max_length = 32 + trainer.train( + num_epochs=1, + batch_size=3, + learning_rate=1e-2, + l2_weight=0.0, + max_length=max_length, + ) + self.assertEqual(cm.output, []) + except AssertionError as e: + if e.args[0] != "no logs of level WARNING or higher triggered on root": + raise AssertionError(e) + + +class SetFitTrainerMultilabelTest(TestCase): + def setUp(self): + self.model = SetFitModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", multi_target_strategy="one-vs-rest" + ) + self.num_iterations = 1 + + def test_trainer_multilabel_support_callable_as_metric(self): + dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[1, 0, 0], [0, 1, 0], [0, 0, 1]]}) + + multilabel_f1_metric = evaluate.load("f1", "multilabel") + multilabel_accuracy_metric = evaluate.load("accuracy", "multilabel") + + def compute_metrics(y_pred, y_test): + return { + "f1": multilabel_f1_metric.compute(predictions=y_pred, references=y_test, average="micro")["f1"], + "accuracy": multilabel_accuracy_metric.compute(predictions=y_pred, references=y_test)["accuracy"], + } + + trainer = SetFitTrainer( + model=self.model, + train_dataset=dataset, + eval_dataset=dataset, + metric=compute_metrics, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + + trainer.train() + metrics = trainer.evaluate() + + self.assertEqual( + { + "f1": 1.0, + "accuracy": 1.0, + }, + metrics, + ) + + +@require_optuna +class TrainerHyperParameterOptunaIntegrationTest(TestCase): + def setUp(self): + self.dataset = Dataset.from_dict( + {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} + ) + self.num_iterations = 1 + + def test_hyperparameter_search(self): + class MyTrialShortNamer(TrialShortNamer): + DEFAULTS = {"max_iter": 100, "solver": "liblinear"} + + def hp_space(trial): + return { + "learning_rate": trial.suggest_float("learning_rate", 1e-6, 1e-4, log=True), + "batch_size": trial.suggest_categorical("batch_size", [4, 8, 16, 32, 64]), + "max_iter": trial.suggest_int("max_iter", 50, 300), + "solver": trial.suggest_categorical("solver", ["newton-cg", "lbfgs", "liblinear"]), + } + + def model_init(params): + params = params or {} + max_iter = params.get("max_iter", 100) + solver = params.get("solver", "liblinear") + params = { + "head_params": { + "max_iter": max_iter, + "solver": solver, + } + } + return SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", **params) + + def hp_name(trial): + return MyTrialShortNamer.shortname(trial.params) + + trainer = SetFitTrainer( + train_dataset=self.dataset, + eval_dataset=self.dataset, + num_iterations=self.num_iterations, + model_init=model_init, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + result = trainer.hyperparameter_search(direction="minimize", hp_space=hp_space, hp_name=hp_name, n_trials=4) + assert isinstance(result, BestRun) + assert result.hyperparameters.keys() == {"learning_rate", "batch_size", "max_iter", "solver"} + + +# regression test for https://github.com/huggingface/setfit/issues/153 +@pytest.mark.parametrize( + "loss_class", + [ + losses.BatchAllTripletLoss, + losses.BatchHardTripletLoss, + losses.BatchSemiHardTripletLoss, + losses.BatchHardSoftMarginTripletLoss, + SupConLoss, + ], +) +def test_trainer_works_with_non_default_loss_class(loss_class): + dataset = Dataset.from_dict({"text": ["a 1", "b 1", "c 1", "a 2", "b 2", "c 2"], "label": [0, 1, 2, 0, 1, 2]}) + model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + trainer = SetFitTrainer( + model=model, + train_dataset=dataset, + eval_dataset=dataset, + num_iterations=1, + loss_class=loss_class, + ) + trainer.train() + # no asserts here because this is a regression test - we only test if an exception is raised diff --git a/tests/test_deprecated_trainer_distillation.py b/tests/test_deprecated_trainer_distillation.py new file mode 100644 index 00000000..4257a42e --- /dev/null +++ b/tests/test_deprecated_trainer_distillation.py @@ -0,0 +1,102 @@ +from unittest import TestCase + +import pytest +from datasets import Dataset +from sentence_transformers.losses import CosineSimilarityLoss + +from setfit import DistillationSetFitTrainer, SetFitTrainer +from setfit.modeling import SetFitModel + + +class DistillationSetFitTrainerTest(TestCase): + def setUp(self): + self.teacher_model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + self.student_model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-MiniLM-L3-v2") + self.num_iterations = 1 + + def test_trainer_works_with_default_columns(self): + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) + # train a teacher model + teacher_trainer = SetFitTrainer( + model=self.teacher_model, + train_dataset=dataset, + eval_dataset=dataset, + loss_class=CosineSimilarityLoss, + metric="accuracy", + ) + # Teacher Train and evaluate + teacher_trainer.train() + metrics = teacher_trainer.evaluate() + teacher_model = teacher_trainer.model + + student_trainer = DistillationSetFitTrainer( + teacher_model=teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + loss_class=CosineSimilarityLoss, + metric="accuracy", + ) + + # Student Train and evaluate + student_trainer.train() + metrics = student_trainer.evaluate() + print("Student results: ", metrics) + self.assertEqual(metrics["accuracy"], 1.0) + + def test_trainer_raises_error_with_missing_label(self): + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) + trainer = DistillationSetFitTrainer( + teacher_model=self.teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + num_iterations=self.num_iterations, + ) + with pytest.raises(ValueError): + trainer.train() + + def test_trainer_raises_error_with_missing_text(self): + dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) + trainer = DistillationSetFitTrainer( + teacher_model=self.teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + num_iterations=self.num_iterations, + ) + with pytest.raises(ValueError): + trainer.train() + + def test_column_mapping_with_missing_text(self): + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) + trainer = DistillationSetFitTrainer( + teacher_model=self.teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + num_iterations=self.num_iterations, + column_mapping={"label_new": "label"}, + ) + with pytest.raises(ValueError): + trainer._validate_column_mapping(trainer.train_dataset) + + def test_column_mapping_multilabel(self): + dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[0, 1], [1, 2], [2, 0]]}) + + trainer = DistillationSetFitTrainer( + teacher_model=self.teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + + trainer._validate_column_mapping(trainer.train_dataset) + formatted_dataset = trainer._apply_column_mapping(trainer.train_dataset, trainer.column_mapping) + + assert formatted_dataset.column_names == ["text", "label"] + assert formatted_dataset[0]["text"] == "a" + assert formatted_dataset[0]["label"] == [0, 1] + assert formatted_dataset[1]["text"] == "b" diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 439242cd..c5a53ba2 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -9,7 +9,8 @@ from setfit import logging from setfit.modeling import SetFitModel, SupConLoss -from setfit.trainer import SetFitTrainer +from setfit.trainer import Trainer +from setfit.training_args import TrainingArguments from setfit.utils import BestRun @@ -20,7 +21,7 @@ class SetFitTrainerTest(TestCase): def setUp(self): self.model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") - self.num_iterations = 1 + self.args = TrainingArguments(num_iterations=1) def test_trainer_works_with_model_init(self): def get_model(): @@ -30,11 +31,11 @@ def get_model(): dataset = Dataset.from_dict( {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} ) - trainer = SetFitTrainer( + trainer = Trainer( model_init=get_model, + args=self.args, train_dataset=dataset, eval_dataset=dataset, - num_iterations=self.num_iterations, column_mapping={"text_new": "text", "label_new": "label"}, ) trainer.train() @@ -45,11 +46,11 @@ def test_trainer_works_with_column_mapping(self): dataset = Dataset.from_dict( {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} ) - trainer = SetFitTrainer( + trainer = Trainer( model=self.model, + args=self.args, train_dataset=dataset, eval_dataset=dataset, - num_iterations=self.num_iterations, column_mapping={"text_new": "text", "label_new": "label"}, ) trainer.train() @@ -58,36 +59,30 @@ def test_trainer_works_with_column_mapping(self): def test_trainer_works_with_default_columns(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) - trainer = SetFitTrainer( - model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations - ) + trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) trainer.train() metrics = trainer.evaluate() self.assertEqual(metrics["accuracy"], 1.0) def test_trainer_raises_error_with_missing_label(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = SetFitTrainer( - model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations - ) + trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) with pytest.raises(ValueError): trainer.train() def test_trainer_raises_error_with_missing_text(self): dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) - trainer = SetFitTrainer( - model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations - ) + trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) with pytest.raises(ValueError): trainer.train() def test_column_mapping_with_missing_text(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = SetFitTrainer( + trainer = Trainer( model=self.model, + args=self.args, train_dataset=dataset, eval_dataset=dataset, - num_iterations=self.num_iterations, column_mapping={"label_new": "label"}, ) with pytest.raises(ValueError): @@ -96,11 +91,11 @@ def test_column_mapping_with_missing_text(self): def test_column_mapping_multilabel(self): dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[0, 1], [1, 2], [2, 0]]}) - trainer = SetFitTrainer( + trainer = Trainer( model=self.model, + args=self.args, train_dataset=dataset, eval_dataset=dataset, - num_iterations=self.num_iterations, column_mapping={"text_new": "text", "label_new": "label"}, ) @@ -128,12 +123,12 @@ def compute_metrics(y_pred, y_test): "accuracy": accuracy_metric.compute(predictions=y_pred, references=y_test)["accuracy"], } - trainer = SetFitTrainer( + trainer = Trainer( model=self.model, + args=self.args, train_dataset=dataset, eval_dataset=dataset, metric=compute_metrics, - num_iterations=self.num_iterations, column_mapping={"text_new": "text", "label_new": "label"}, ) @@ -153,12 +148,12 @@ def test_raise_when_metric_value_is_invalid(self): {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} ) - trainer = SetFitTrainer( + trainer = Trainer( model=self.model, + args=self.args, train_dataset=dataset, eval_dataset=dataset, metric="this-metric-does-not-exist", # invalid metric value - num_iterations=self.num_iterations, column_mapping={"text_new": "text", "label_new": "label"}, ) @@ -167,15 +162,6 @@ def test_raise_when_metric_value_is_invalid(self): with self.assertRaises(FileNotFoundError): trainer.evaluate() - def test_trainer_raises_error_with_wrong_warmup_proportion(self): - # warmup_proportion must not be > 1.0 - with pytest.raises(ValueError): - SetFitTrainer(warmup_proportion=1.1) - - # warmup_proportion must not be < 0.0 - with pytest.raises(ValueError): - SetFitTrainer(warmup_proportion=-0.1) - class SetFitTrainerDifferentiableHeadTest(TestCase): def setUp(self): @@ -187,27 +173,22 @@ def setUp(self): use_differentiable_head=True, head_params={"out_features": 3}, ) - self.num_iterations = 1 + self.args = TrainingArguments(num_iterations=1) def test_trainer_max_length_exceeds_max_acceptable_length(self): - trainer = SetFitTrainer( + trainer = Trainer( model=self.model, + args=self.args, train_dataset=self.dataset, eval_dataset=self.dataset, - num_iterations=self.num_iterations, column_mapping={"text_new": "text", "label_new": "label"}, ) trainer.unfreeze(keep_body_frozen=True) with self.assertLogs(level=logging.WARNING) as cm: max_length = 4096 max_acceptable_length = self.model.model_body.get_max_seq_length() - trainer.train( - num_epochs=1, - batch_size=3, - learning_rate=1e-2, - l2_weight=0.0, - max_length=max_length, - ) + args = TrainingArguments(num_iterations=1, max_length=max_length) + trainer.train(args) self.assertEqual( cm.output, [ @@ -219,26 +200,20 @@ def test_trainer_max_length_exceeds_max_acceptable_length(self): ) def test_trainer_max_length_is_smaller_than_max_acceptable_length(self): - trainer = SetFitTrainer( + trainer = Trainer( model=self.model, + args=self.args, train_dataset=self.dataset, eval_dataset=self.dataset, - num_iterations=self.num_iterations, column_mapping={"text_new": "text", "label_new": "label"}, ) - trainer.unfreeze(keep_body_frozen=True) # An alternative way of `assertNoLogs`, which is new in Python 3.10 try: with self.assertLogs(level=logging.WARNING) as cm: max_length = 32 - trainer.train( - num_epochs=1, - batch_size=3, - learning_rate=1e-2, - l2_weight=0.0, - max_length=max_length, - ) + args = TrainingArguments(num_iterations=1, max_length=max_length) + trainer.train(args) self.assertEqual(cm.output, []) except AssertionError as e: if e.args[0] != "no logs of level WARNING or higher triggered on root": @@ -250,7 +225,7 @@ def setUp(self): self.model = SetFitModel.from_pretrained( "sentence-transformers/paraphrase-albert-small-v2", multi_target_strategy="one-vs-rest" ) - self.num_iterations = 1 + self.args = TrainingArguments(num_iterations=1) def test_trainer_multilabel_support_callable_as_metric(self): dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[1, 0, 0], [0, 1, 0], [0, 0, 1]]}) @@ -264,12 +239,12 @@ def compute_metrics(y_pred, y_test): "accuracy": multilabel_accuracy_metric.compute(predictions=y_pred, references=y_test)["accuracy"], } - trainer = SetFitTrainer( + trainer = Trainer( model=self.model, + args=self.args, train_dataset=dataset, eval_dataset=dataset, metric=compute_metrics, - num_iterations=self.num_iterations, column_mapping={"text_new": "text", "label_new": "label"}, ) @@ -291,7 +266,7 @@ def setUp(self): self.dataset = Dataset.from_dict( {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} ) - self.num_iterations = 1 + self.args = TrainingArguments(num_iterations=1) def test_hyperparameter_search(self): class MyTrialShortNamer(TrialShortNamer): @@ -320,10 +295,10 @@ def model_init(params): def hp_name(trial): return MyTrialShortNamer.shortname(trial.params) - trainer = SetFitTrainer( + trainer = Trainer( + args=self.args, train_dataset=self.dataset, eval_dataset=self.dataset, - num_iterations=self.num_iterations, model_init=model_init, column_mapping={"text_new": "text", "label_new": "label"}, ) @@ -346,11 +321,12 @@ def hp_name(trial): def test_trainer_works_with_non_default_loss_class(loss_class): dataset = Dataset.from_dict({"text": ["a 1", "b 1", "c 1", "a 2", "b 2", "c 2"], "label": [0, 1, 2, 0, 1, 2]}) model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") - trainer = SetFitTrainer( + args = TrainingArguments(num_iterations=1) + trainer = Trainer( model=model, + args=args, train_dataset=dataset, eval_dataset=dataset, - num_iterations=1, loss_class=loss_class, ) trainer.train() diff --git a/tests/test_trainer_distillation.py b/tests/test_trainer_distillation.py index 4257a42e..df2bad4c 100644 --- a/tests/test_trainer_distillation.py +++ b/tests/test_trainer_distillation.py @@ -4,20 +4,21 @@ from datasets import Dataset from sentence_transformers.losses import CosineSimilarityLoss -from setfit import DistillationSetFitTrainer, SetFitTrainer +from setfit import DistillationTrainer, Trainer from setfit.modeling import SetFitModel +from setfit.training_args import TrainingArguments class DistillationSetFitTrainerTest(TestCase): def setUp(self): self.teacher_model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") self.student_model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-MiniLM-L3-v2") - self.num_iterations = 1 + self.args = TrainingArguments(num_iterations=1) def test_trainer_works_with_default_columns(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) # train a teacher model - teacher_trainer = SetFitTrainer( + teacher_trainer = Trainer( model=self.teacher_model, train_dataset=dataset, eval_dataset=dataset, @@ -29,7 +30,7 @@ def test_trainer_works_with_default_columns(self): metrics = teacher_trainer.evaluate() teacher_model = teacher_trainer.model - student_trainer = DistillationSetFitTrainer( + student_trainer = DistillationTrainer( teacher_model=teacher_model, train_dataset=dataset, student_model=self.student_model, @@ -46,36 +47,36 @@ def test_trainer_works_with_default_columns(self): def test_trainer_raises_error_with_missing_label(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = DistillationSetFitTrainer( + trainer = DistillationTrainer( teacher_model=self.teacher_model, train_dataset=dataset, student_model=self.student_model, eval_dataset=dataset, - num_iterations=self.num_iterations, + args=self.args, ) with pytest.raises(ValueError): trainer.train() def test_trainer_raises_error_with_missing_text(self): dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) - trainer = DistillationSetFitTrainer( + trainer = DistillationTrainer( teacher_model=self.teacher_model, train_dataset=dataset, student_model=self.student_model, eval_dataset=dataset, - num_iterations=self.num_iterations, + args=self.args, ) with pytest.raises(ValueError): trainer.train() def test_column_mapping_with_missing_text(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = DistillationSetFitTrainer( + trainer = DistillationTrainer( teacher_model=self.teacher_model, train_dataset=dataset, student_model=self.student_model, eval_dataset=dataset, - num_iterations=self.num_iterations, + args=self.args, column_mapping={"label_new": "label"}, ) with pytest.raises(ValueError): @@ -84,12 +85,12 @@ def test_column_mapping_with_missing_text(self): def test_column_mapping_multilabel(self): dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[0, 1], [1, 2], [2, 0]]}) - trainer = DistillationSetFitTrainer( + trainer = DistillationTrainer( teacher_model=self.teacher_model, train_dataset=dataset, student_model=self.student_model, eval_dataset=dataset, - num_iterations=self.num_iterations, + args=self.args, column_mapping={"text_new": "text", "label_new": "label"}, ) diff --git a/tests/test_training_args.py b/tests/test_training_args.py new file mode 100644 index 00000000..5575aee6 --- /dev/null +++ b/tests/test_training_args.py @@ -0,0 +1,15 @@ +from unittest import TestCase + +import pytest + +from setfit.training_args import TrainingArguments + +class TestTrainingArguments(TestCase): + def test_training_args_raises_error_with_wrong_warmup_proportion(self): + # warmup_proportion must not be > 1.0 + with pytest.raises(ValueError): + TrainingArguments(warmup_proportion=1.1) + + # warmup_proportion must not be < 0.0 + with pytest.raises(ValueError): + TrainingArguments(warmup_proportion=-0.1) \ No newline at end of file From 89f4435d565effb2b86975ee6a2e5b4dfd47c4c9 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 11 Jan 2023 16:26:22 +0100 Subject: [PATCH 002/183] Readded support for hyperparameter tuning --- src/setfit/trainer.py | 19 ++++++------------- src/setfit/training_args.py | 21 ++++++++++++++++++--- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index f05c3e97..aa124a67 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -162,18 +162,11 @@ def apply_hyperparameters(self, params: Dict[str, Any], final_model: bool = Fals params (`Dict[str, Any]`): The parameters, usually from `BestRun.hyperparameters` final_model (`bool`, *optional*, defaults to `False`): If `True`, replace the `model_init()` function with a fixed model based on the parameters. """ - for key, value in params.items(): - if hasattr(self, key): - old_attr = getattr(self, key, None) - # Casting value to the proper type - if old_attr is not None: - value = type(old_attr)(value) - setattr(self, key, value) - elif number_of_arguments(self.model_init) == 0: # we do not warn if model_init could be using it - logger.warning( - f"Trying to set {key!r} in the hyperparameter search but there is no corresponding field in " - "`SetFitTrainer`, and `model_init` does not take any arguments." - ) + + if self.args: + self.args = self.args.update(params, ignore_extra=True) + else: + self.args = TrainingArguments.from_dict(params, ignore_extra=True) self.model = self.model_init(params) if final_model: @@ -397,7 +390,7 @@ def hyperparameter_search( if backend is None: backend = default_hp_search_backend() if backend is None: - raise RuntimeError("optuna should be installed. " "To install optuna run `pip install optuna`. ") + raise RuntimeError("optuna should be installed. To install optuna run `pip install optuna`. ") backend = HPSearchBackend(backend) if backend == HPSearchBackend.OPTUNA and not is_optuna_available(): raise RuntimeError("You picked the optuna backend, but it is not installed. Use `pip install optuna`.") diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 5a796d7a..2a0f6bfd 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -1,6 +1,9 @@ +from __future__ import annotations + from copy import copy from dataclasses import dataclass, fields, field -from typing import Callable, Tuple, Union +import inspect +from typing import Any, Callable, Dict, Tuple, Union from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction @@ -64,5 +67,17 @@ def to_dict(self): # filter out fields that are defined as field(init=False) return {field.name: getattr(self, field.name) for field in fields(self) if field.init} - def copy(self): - return copy(self) \ No newline at end of file + @classmethod + def from_dict(cls, arguments: Dict[str, Any], ignore_extra: bool = False) -> TrainingArguments: + if ignore_extra: + return cls(**{ + key: value for key, value in arguments.items() + if key in inspect.signature(cls).parameters + }) + return cls(**arguments) + + def copy(self) -> TrainingArguments: + return copy(self) + + def update(self, arguments: Dict[str, Any], ignore_extra: bool = False) -> TrainingArguments: + return TrainingArguments.from_dict({**self.to_dict(), **arguments}, ignore_extra=ignore_extra) From 5f2a6b3c4170f38b59c8b32ef54f8208a693d279 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 11 Jan 2023 16:28:33 +0100 Subject: [PATCH 003/183] Remove unused imports and reformat --- src/setfit/modeling.py | 17 ++++++++++------- src/setfit/trainer.py | 6 ++++-- src/setfit/trainer_distillation.py | 5 +---- src/setfit/training_args.py | 10 ++++------ tests/test_deprecated_trainer.py | 8 ++++++-- tests/test_training_args.py | 3 ++- 6 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 0fe5abcf..c1b673db 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -1,9 +1,8 @@ import os +import warnings from dataclasses import dataclass from pathlib import Path -import time from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union -import warnings # Google Colab runs on Python 3.7, so we need this to be compatible @@ -16,14 +15,14 @@ import numpy as np import requests import torch -from torch import nn from huggingface_hub import PyTorchModelHubMixin, hf_hub_download from sentence_transformers import InputExample, SentenceTransformer, models from sklearn.linear_model import LogisticRegression from sklearn.multiclass import OneVsRestClassifier from sklearn.multioutput import ClassifierChain, MultiOutputClassifier +from torch import nn from torch.utils.data import DataLoader -from tqdm.auto import trange, tqdm +from tqdm.auto import tqdm, trange from . import logging from .data import SetFitDataset @@ -279,7 +278,7 @@ def fit( max_length: Optional[int] = None, show_progress_bar: bool = True, end_to_end: bool = False, - **kwargs + **kwargs, ) -> None: if self.has_differentiable_head: # train with pyTorch device = self.model_body.device @@ -380,9 +379,13 @@ def freeze(self, component: Optional[Literal["body", "head"]] = None) -> None: if (component is None or component == "head") and self.has_differentiable_head: self._freeze_or_not(self.model_head, to_freeze=True) - def unfreeze(self, component: Optional[Literal["body", "head"]] = None, keep_body_frozen: Optional[bool] = None) -> None: + def unfreeze( + self, component: Optional[Literal["body", "head"]] = None, keep_body_frozen: Optional[bool] = None + ) -> None: if keep_body_frozen is not None: - warnings.warn("`keep_body_frozen` is deprecated. Please either pass \"head\", \"body\" or no arguments to unfreeze both.") + warnings.warn( + '`keep_body_frozen` is deprecated. Please either pass "head", "body" or no arguments to unfreeze both.' + ) if component is None or component == "body": self._freeze_or_not(self.model_body, to_freeze=False) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index aa124a67..aa45c3e7 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -1,6 +1,6 @@ import math -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import warnings +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import evaluate import numpy as np @@ -214,7 +214,9 @@ def train( self._hp_search_setup(trial) # sets trainer parameters and initializes model if self.train_dataset is None: - raise ValueError(f"Training requires a `train_dataset` given to the `{self.__class__.__name__}` initialization.") + raise ValueError( + f"Training requires a `train_dataset` given to the `{self.__class__.__name__}` initialization." + ) self._validate_column_mapping(self.train_dataset) train_dataset = self.train_dataset diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 9bf53888..373d037d 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -1,13 +1,11 @@ import math -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union import numpy as np import torch from sentence_transformers import InputExample, losses, util from sentence_transformers.datasets import SentenceLabelDataset -from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction from torch.utils.data import DataLoader -from transformers.trainer_utils import set_seed from setfit.training_args import TrainingArguments @@ -17,7 +15,6 @@ if TYPE_CHECKING: - import optuna from datasets import Dataset from .modeling import SetFitModel diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 2a0f6bfd..7757072c 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -1,9 +1,10 @@ from __future__ import annotations -from copy import copy -from dataclasses import dataclass, fields, field import inspect +from copy import copy +from dataclasses import dataclass, field, fields from typing import Any, Callable, Dict, Tuple, Union + from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction @@ -70,10 +71,7 @@ def to_dict(self): @classmethod def from_dict(cls, arguments: Dict[str, Any], ignore_extra: bool = False) -> TrainingArguments: if ignore_extra: - return cls(**{ - key: value for key, value in arguments.items() - if key in inspect.signature(cls).parameters - }) + return cls(**{key: value for key, value in arguments.items() if key in inspect.signature(cls).parameters}) return cls(**arguments) def copy(self) -> TrainingArguments: diff --git a/tests/test_deprecated_trainer.py b/tests/test_deprecated_trainer.py index 467b77e0..e05fb775 100644 --- a/tests/test_deprecated_trainer.py +++ b/tests/test_deprecated_trainer.py @@ -189,7 +189,9 @@ def setUp(self): ) self.num_iterations = 1 - @pytest.mark.skip(reason="The `trainer.train` argument removals were a hard deprecation, so this test would throw an error.") + @pytest.mark.skip( + reason="The `trainer.train` argument removals were a hard deprecation, so this test would throw an error." + ) def test_trainer_max_length_exceeds_max_acceptable_length(self): trainer = SetFitTrainer( model=self.model, @@ -219,7 +221,9 @@ def test_trainer_max_length_exceeds_max_acceptable_length(self): ], ) - @pytest.mark.skip(reason="The `trainer.train` argument removals were a hard deprecation, so this test would throw an error.") + @pytest.mark.skip( + reason="The `trainer.train` argument removals were a hard deprecation, so this test would throw an error." + ) def test_trainer_max_length_is_smaller_than_max_acceptable_length(self): trainer = SetFitTrainer( model=self.model, diff --git a/tests/test_training_args.py b/tests/test_training_args.py index 5575aee6..61261bb8 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -4,6 +4,7 @@ from setfit.training_args import TrainingArguments + class TestTrainingArguments(TestCase): def test_training_args_raises_error_with_wrong_warmup_proportion(self): # warmup_proportion must not be > 1.0 @@ -12,4 +13,4 @@ def test_training_args_raises_error_with_wrong_warmup_proportion(self): # warmup_proportion must not be < 0.0 with pytest.raises(ValueError): - TrainingArguments(warmup_proportion=-0.1) \ No newline at end of file + TrainingArguments(warmup_proportion=-0.1) From 622f33bbbda5b2919330ca38a2b1c04723b2f2f6 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 11 Jan 2023 16:48:34 +0100 Subject: [PATCH 004/183] Preserve desired behaviour despite deprecation of keep_body_frozen parameter --- src/setfit/modeling.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index c1b673db..d83072de 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -386,6 +386,10 @@ def unfreeze( warnings.warn( '`keep_body_frozen` is deprecated. Please either pass "head", "body" or no arguments to unfreeze both.' ) + # If the body must stay frozen, only unfreeze the head. Eventually, this entire if-branch + # can be removed. + if keep_body_frozen and not component: + component = "head" if component is None or component == "body": self._freeze_or_not(self.model_body, to_freeze=False) From ff591543398d2799eb527cf2f2b91dbf2bcdbb9a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 11 Jan 2023 17:51:01 +0100 Subject: [PATCH 005/183] Ensure that DeprecationWarnings are displayed --- src/setfit/__init__.py | 6 ++++++ src/setfit/modeling.py | 4 +++- src/setfit/trainer.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index 37db149b..ce9f19fc 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -1,6 +1,12 @@ __version__ = "0.6.0.dev0" +import warnings + from .data import add_templated_examples, sample_dataset from .modeling import SetFitHead, SetFitModel from .trainer import SetFitTrainer, Trainer from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer + + +# Ensure that DeprecationWarnings are always shown +warnings.filterwarnings("default", category=DeprecationWarning) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index d83072de..b672392a 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -384,7 +384,9 @@ def unfreeze( ) -> None: if keep_body_frozen is not None: warnings.warn( - '`keep_body_frozen` is deprecated. Please either pass "head", "body" or no arguments to unfreeze both.' + '`keep_body_frozen` is deprecated. Please either pass "head", "body" or no arguments to unfreeze both.', + DeprecationWarning, + stacklevel=2, ) # If the body must stay frozen, only unfreeze the head. Eventually, this entire if-branch # can be removed. diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index aa45c3e7..ce40873b 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -468,7 +468,7 @@ def __init__( samples_per_label: int = 2, ): warnings.warn( - "`SetFitTrainer` has been deprecated. Please use `from setfit import Trainer` instead.", DeprecationWarning + "`SetFitTrainer` has been deprecated. Please use `Trainer` instead.", DeprecationWarning, stacklevel=2 ) args = TrainingArguments( num_iterations=num_iterations, From 3b4ef5812cc5626ff0cc72e49fe809c24c2ace19 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 11 Jan 2023 17:57:15 +0100 Subject: [PATCH 006/183] Set Trainer.freeze and Trainer.unfreeze methods normally --- src/setfit/trainer.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index ce40873b..02572d91 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -2,6 +2,13 @@ import warnings from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union + +# Google Colab runs on Python 3.7, so we need this to be compatible +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal + import evaluate import numpy as np from sentence_transformers import InputExample, losses @@ -107,10 +114,6 @@ def __init__( raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument, but not both") self.model = model - # Adopt Trainer.(un)freeze from SetFitModel.(un)freeze - self.freeze = self.model.freeze - self.unfreeze = self.model.unfreeze - self.hp_search_backend = None self._freeze = True # If True, will train the body only; otherwise, train the body and head @@ -203,6 +206,14 @@ def call_model_init(self, params: Optional[Dict[str, Any]] = None): return model + def freeze(self, component: Optional[Literal["body", "head"]] = None) -> None: + return self.model.freeze(component) + + def unfreeze( + self, component: Optional[Literal["body", "head"]] = None, keep_body_frozen: Optional[bool] = None + ) -> None: + return self.model.unfreeze(component, keep_body_frozen=keep_body_frozen) + def train( self, args: Optional[TrainingArguments] = None, trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None ): From fd68274ade7066a5317ba47dd8f3d1b892dfd117 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 11 Jan 2023 18:33:59 +0100 Subject: [PATCH 007/183] Add TrainingArgument tests for num_epochs, batch_sizes, lr --- tests/test_training_args.py | 75 +++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/test_training_args.py b/tests/test_training_args.py index 61261bb8..941703b5 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -14,3 +14,78 @@ def test_training_args_raises_error_with_wrong_warmup_proportion(self): # warmup_proportion must not be < 0.0 with pytest.raises(ValueError): TrainingArguments(warmup_proportion=-0.1) + + def test_training_args_batch_sizes(self): + batch_size_A = 12 + batch_size_B = 4 + batch_size_C = 6 + + args = TrainingArguments(batch_size=batch_size_A) + assert args.batch_size == (batch_size_A, batch_size_A) + assert args.embedding_batch_size == batch_size_A + assert args.classifier_batch_size == batch_size_A + + args = TrainingArguments(batch_size=(batch_size_A, batch_size_B)) + assert args.batch_size == (batch_size_A, batch_size_B) + assert args.embedding_batch_size == batch_size_A + assert args.classifier_batch_size == batch_size_B + + args = TrainingArguments(batch_size=(batch_size_A, batch_size_B), embedding_batch_size=batch_size_C) + assert args.batch_size == (batch_size_A, batch_size_B) + assert args.embedding_batch_size == batch_size_C + assert args.classifier_batch_size == batch_size_B + + args = TrainingArguments(batch_size=batch_size_A, embedding_batch_size=batch_size_C) + assert args.batch_size == (batch_size_A, batch_size_A) + assert args.embedding_batch_size == batch_size_C + assert args.classifier_batch_size == batch_size_A + + def test_training_args_num_epochs(self): + num_epochs_A = 12 + num_epochs_B = 4 + num_epochs_C = 6 + + args = TrainingArguments(num_epochs=num_epochs_A) + assert args.num_epochs == (num_epochs_A, num_epochs_A) + assert args.embedding_num_epochs == num_epochs_A + assert args.classifier_num_epochs == num_epochs_A + + args = TrainingArguments(num_epochs=(num_epochs_A, num_epochs_B)) + assert args.num_epochs == (num_epochs_A, num_epochs_B) + assert args.embedding_num_epochs == num_epochs_A + assert args.classifier_num_epochs == num_epochs_B + + args = TrainingArguments(num_epochs=(num_epochs_A, num_epochs_B), embedding_num_epochs=num_epochs_C) + assert args.num_epochs == (num_epochs_A, num_epochs_B) + assert args.embedding_num_epochs == num_epochs_C + assert args.classifier_num_epochs == num_epochs_B + + args = TrainingArguments(num_epochs=num_epochs_A, embedding_num_epochs=num_epochs_C) + assert args.num_epochs == (num_epochs_A, num_epochs_A) + assert args.embedding_num_epochs == num_epochs_C + assert args.classifier_num_epochs == num_epochs_A + + def test_training_args_learning_rates(self): + learning_rate_A = 1e-2 + learning_rate_B = 1e-3 + learning_rate_C = 1e-4 + + base = TrainingArguments() + + args = TrainingArguments(classifier_learning_rate=learning_rate_A) + assert args.classifier_learning_rate == (base.embedding_learning_rate, learning_rate_A) + assert args.embedding_learning_rate == base.embedding_learning_rate + + args = TrainingArguments(classifier_learning_rate=learning_rate_A, embedding_learning_rate=learning_rate_B) + assert args.classifier_learning_rate == (learning_rate_B, learning_rate_A) + assert args.embedding_learning_rate == learning_rate_B + + args = TrainingArguments( + classifier_learning_rate=(learning_rate_C, learning_rate_A), embedding_learning_rate=learning_rate_B + ) + assert args.classifier_learning_rate == (learning_rate_C, learning_rate_A) + assert args.embedding_learning_rate == learning_rate_B + + args = TrainingArguments(classifier_learning_rate=(learning_rate_C, learning_rate_A)) + assert args.classifier_learning_rate == (learning_rate_C, learning_rate_A) + assert args.embedding_learning_rate == base.embedding_learning_rate From 14602ea2773f77b82243624ed1bca5e0772519e7 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 11 Jan 2023 19:03:50 +0100 Subject: [PATCH 008/183] Convert trainer.train arguments into a softer deprecation --- src/setfit/trainer.py | 14 +++++++++++++- tests/test_deprecated_trainer.py | 8 ++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 02572d91..e367b7b3 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -215,8 +215,20 @@ def unfreeze( return self.model.unfreeze(component, keep_body_frozen=keep_body_frozen) def train( - self, args: Optional[TrainingArguments] = None, trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None + self, + args: Optional[TrainingArguments] = None, + trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, + **kwargs, ): + if kwargs: + warnings.warn( + f"`{self.__class__.__name__}.train` does not accept keyword arguments anymore. " + f"Please provide training arguments via a `TrainingArguments` instance to the `{self.__class__.__name__}` " + f"initialisation or the `{self.__class__.__name__}.train` method.", + DeprecationWarning, + stacklevel=2, + ) + args = args or self.args or TrainingArguments() set_seed(args.seed) # Seed must be set before instantiating the model when using model_init. diff --git a/tests/test_deprecated_trainer.py b/tests/test_deprecated_trainer.py index e05fb775..b3b8de18 100644 --- a/tests/test_deprecated_trainer.py +++ b/tests/test_deprecated_trainer.py @@ -189,9 +189,7 @@ def setUp(self): ) self.num_iterations = 1 - @pytest.mark.skip( - reason="The `trainer.train` argument removals were a hard deprecation, so this test would throw an error." - ) + @pytest.mark.skip(reason="The `trainer.train` arguments are now ignored, causing this test to fail.") def test_trainer_max_length_exceeds_max_acceptable_length(self): trainer = SetFitTrainer( model=self.model, @@ -221,9 +219,7 @@ def test_trainer_max_length_exceeds_max_acceptable_length(self): ], ) - @pytest.mark.skip( - reason="The `trainer.train` argument removals were a hard deprecation, so this test would throw an error." - ) + @pytest.mark.skip(reason="The `trainer.train` arguments are now ignored, causing this test to fail.") def test_trainer_max_length_is_smaller_than_max_acceptable_length(self): trainer = SetFitTrainer( model=self.model, From 9fc55a699be8fcb8c6c7b4471337fe5072bc0df4 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 23 Jan 2023 12:45:49 +0100 Subject: [PATCH 009/183] Use body/head_learning_rate instead of classifier/embedding_learning_rate The reasoning is that with body_learning_rate, the tuple is for (training embedding phase, training classifier phase), which matches the tuples that you should give to num_epochs and batch_size. --- src/setfit/modeling.py | 20 +++++++------ src/setfit/trainer.py | 6 ++-- src/setfit/trainer_distillation.py | 6 ++-- src/setfit/training_args.py | 20 ++++++++++--- tests/test_training_args.py | 46 ++++++++++++++++++++++-------- 5 files changed, 68 insertions(+), 30 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index b89f8aed..daf22d6b 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -283,7 +283,8 @@ def fit( y_train: Union[List[int], List[List[int]]], classifier_num_epochs: int, classifier_batch_size: Optional[int] = None, - classifier_learning_rate: Optional[Tuple[float, float]] = (None, None), + body_classifier_learning_rate: Optional[float] = None, + head_learning_rate: Optional[float] = None, l2_weight: Optional[float] = None, max_length: Optional[int] = None, show_progress_bar: bool = True, @@ -299,8 +300,7 @@ def fit( dataloader = self._prepare_dataloader(x_train, y_train, classifier_batch_size, max_length) criterion = self.model_head.get_loss_fn() - embedding_learning_rate, classifier_learning_rate = classifier_learning_rate - optimizer = self._prepare_optimizer(classifier_learning_rate, embedding_learning_rate, l2_weight) + optimizer = self._prepare_optimizer(head_learning_rate, body_classifier_learning_rate, l2_weight) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5) for epoch_idx in trange(classifier_num_epochs, desc="Epoch", disable=not show_progress_bar): for batch in tqdm(dataloader, desc="Iteration", disable=not show_progress_bar, leave=False): @@ -367,16 +367,20 @@ def _prepare_dataloader( def _prepare_optimizer( self, - classifier_learning_rate: float, - embedding_learning_rate: Optional[float], + head_learning_rate: float, + body_classifier_learning_rate: Optional[float], l2_weight: float, ) -> torch.optim.Optimizer: - embedding_learning_rate = embedding_learning_rate or classifier_learning_rate + body_classifier_learning_rate = body_classifier_learning_rate or head_learning_rate l2_weight = l2_weight or self.l2_weight optimizer = torch.optim.AdamW( [ - {"params": self.model_body.parameters(), "lr": embedding_learning_rate, "weight_decay": l2_weight}, - {"params": self.model_head.parameters(), "lr": classifier_learning_rate, "weight_decay": l2_weight}, + { + "params": self.model_body.parameters(), + "lr": body_classifier_learning_rate, + "weight_decay": l2_weight, + }, + {"params": self.model_head.parameters(), "lr": head_learning_rate, "weight_decay": l2_weight}, ], ) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index c4fe9b1d..dc5303ac 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -312,7 +312,7 @@ def train_embeddings(self, x_train: List[str], y_train: List[int], args: Optiona self.model.model_body.fit( train_objectives=[(train_dataloader, train_loss)], epochs=args.embedding_num_epochs, - optimizer_params={"lr": args.embedding_learning_rate}, + optimizer_params={"lr": args.body_embedding_learning_rate}, warmup_steps=warmup_steps, show_progress_bar=args.show_progress_bar, use_amp=args.use_amp, @@ -493,8 +493,8 @@ def __init__( args = TrainingArguments( num_iterations=num_iterations, num_epochs=num_epochs, - classifier_learning_rate=learning_rate, - embedding_learning_rate=learning_rate, + body_learning_rate=learning_rate, + head_learning_rate=learning_rate, batch_size=batch_size, seed=seed, use_amp=use_amp, diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 373d037d..6f16c296 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -157,7 +157,7 @@ def train_embeddings( train_objectives=[(train_dataloader, train_loss)], epochs=args.embedding_num_epochs, steps_per_epoch=train_steps, - optimizer_params={"lr": args.embedding_learning_rate}, + optimizer_params={"lr": args.body_embedding_learning_rate}, warmup_steps=warmup_steps, show_progress_bar=args.show_progress_bar, use_amp=args.use_amp, @@ -186,8 +186,8 @@ def __init__( args = TrainingArguments( num_iterations=num_iterations, num_epochs=num_epochs, - embedding_learning_rate=learning_rate, - classifier_learning_rate=learning_rate, + body_learning_rate=learning_rate, + head_learning_rate=learning_rate, batch_size=batch_size, seed=seed, use_amp=use_amp, diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 7757072c..1d4ebcad 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -25,8 +25,12 @@ class TrainingArguments: num_iterations: int = 20 - embedding_learning_rate: float = 2e-5 - classifier_learning_rate: Union[float, Tuple[float, float]] = (1e-5, 1e-2) + # As with batch_size and num_epochs, the first value in the tuple is the learning rate + # for the embeddings step, while the second value is the learning rate for the classifier step. + body_learning_rate: Union[float, Tuple[float, float]] = field(default=(2e-5, 1e-5), repr=False) + body_embedding_learning_rate: float = None + body_classifier_learning_rate: float = None + head_learning_rate: float = 1e-2 seed: int = 42 use_amp: bool = False @@ -42,6 +46,7 @@ class TrainingArguments: end_to_end: bool = False def __post_init__(self): + # Set `self.embedding_batch_size` and `self.classifier_batch_size` using values from `self.batch_size` if isinstance(self.batch_size, int): self.batch_size = (self.batch_size, self.batch_size) if self.embedding_batch_size is None: @@ -49,6 +54,7 @@ def __post_init__(self): if self.classifier_batch_size is None: self.classifier_batch_size = self.batch_size[1] + # Set `self.embedding_num_epochs` and `self.classifier_num_epochs` using values from `self.num_epochs` if isinstance(self.num_epochs, int): self.num_epochs = (self.num_epochs, self.num_epochs) if self.embedding_num_epochs is None: @@ -56,8 +62,14 @@ def __post_init__(self): if self.classifier_num_epochs is None: self.classifier_num_epochs = self.num_epochs[1] - if isinstance(self.classifier_learning_rate, float): - self.classifier_learning_rate = (self.embedding_learning_rate, self.classifier_learning_rate) + # Set `self.body_embedding_learning_rate` and `self.body_classifier_learning_rate` using + # values from `self.body_learning_rate` + if isinstance(self.body_learning_rate, float): + self.body_learning_rate = (self.body_learning_rate, self.body_learning_rate) + if self.body_embedding_learning_rate is None: + self.body_embedding_learning_rate = self.body_learning_rate[0] + if self.body_classifier_learning_rate is None: + self.body_classifier_learning_rate = self.body_learning_rate[1] if self.warmup_proportion < 0.0 or self.warmup_proportion > 1.0: raise ValueError( diff --git a/tests/test_training_args.py b/tests/test_training_args.py index 941703b5..5ad6a850 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -72,20 +72,42 @@ def test_training_args_learning_rates(self): base = TrainingArguments() - args = TrainingArguments(classifier_learning_rate=learning_rate_A) - assert args.classifier_learning_rate == (base.embedding_learning_rate, learning_rate_A) - assert args.embedding_learning_rate == base.embedding_learning_rate + args = TrainingArguments(body_learning_rate=learning_rate_A) + assert args.body_learning_rate == (learning_rate_A, learning_rate_A) + assert args.body_embedding_learning_rate == learning_rate_A + assert args.body_classifier_learning_rate == learning_rate_A + assert args.head_learning_rate == base.head_learning_rate + + args = TrainingArguments(body_learning_rate=(learning_rate_A, learning_rate_B)) + assert args.body_learning_rate == (learning_rate_A, learning_rate_B) + assert args.body_embedding_learning_rate == learning_rate_A + assert args.body_classifier_learning_rate == learning_rate_B + assert args.head_learning_rate == base.head_learning_rate - args = TrainingArguments(classifier_learning_rate=learning_rate_A, embedding_learning_rate=learning_rate_B) - assert args.classifier_learning_rate == (learning_rate_B, learning_rate_A) - assert args.embedding_learning_rate == learning_rate_B + args = TrainingArguments( + body_learning_rate=(learning_rate_A, learning_rate_B), head_learning_rate=learning_rate_C + ) + assert args.body_learning_rate == (learning_rate_A, learning_rate_B) + assert args.body_embedding_learning_rate == learning_rate_A + assert args.body_classifier_learning_rate == learning_rate_B + assert args.head_learning_rate == learning_rate_C args = TrainingArguments( - classifier_learning_rate=(learning_rate_C, learning_rate_A), embedding_learning_rate=learning_rate_B + body_learning_rate=learning_rate_A, + body_embedding_learning_rate=learning_rate_B, + head_learning_rate=learning_rate_C, ) - assert args.classifier_learning_rate == (learning_rate_C, learning_rate_A) - assert args.embedding_learning_rate == learning_rate_B + # Perhaps not ideal, but body_learning_rate is never used directly: + assert args.body_learning_rate == (learning_rate_A, learning_rate_A) + assert args.body_embedding_learning_rate == learning_rate_B + assert args.body_classifier_learning_rate == learning_rate_A + assert args.head_learning_rate == learning_rate_C - args = TrainingArguments(classifier_learning_rate=(learning_rate_C, learning_rate_A)) - assert args.classifier_learning_rate == (learning_rate_C, learning_rate_A) - assert args.embedding_learning_rate == base.embedding_learning_rate + args = TrainingArguments( + body_classifier_learning_rate=learning_rate_A, + body_embedding_learning_rate=learning_rate_B, + head_learning_rate=learning_rate_C, + ) + assert args.body_embedding_learning_rate == learning_rate_B + assert args.body_classifier_learning_rate == learning_rate_A + assert args.head_learning_rate == learning_rate_C From dee70b151e46932049739acf59dd9a80ec45a85e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 14:38:45 +0100 Subject: [PATCH 010/183] Reformat according to the newest black version --- scripts/setfit/distillation_baseline.py | 1 - scripts/setfit/run_fewshot_distillation.py | 1 - scripts/setfit/run_fewshot_multilingual.py | 1 - src/setfit/data.py | 1 - src/setfit/logging.py | 4 ---- src/setfit/modeling.py | 1 - src/setfit/trainer.py | 1 - src/setfit/training_args.py | 1 - 8 files changed, 11 deletions(-) diff --git a/scripts/setfit/distillation_baseline.py b/scripts/setfit/distillation_baseline.py index 98be6bef..75b84b91 100644 --- a/scripts/setfit/distillation_baseline.py +++ b/scripts/setfit/distillation_baseline.py @@ -56,7 +56,6 @@ def compute_metrics_for_regression(self, eval_pred): # ------------------------ Student training ----------------------# # ----------------------------------------------------------------# def standard_model_distillation(self, train_raw_student, x_test, y_test, num_classes): - device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") value2hot = {} diff --git a/scripts/setfit/run_fewshot_distillation.py b/scripts/setfit/run_fewshot_distillation.py index 7d1dabae..ef213b50 100644 --- a/scripts/setfit/run_fewshot_distillation.py +++ b/scripts/setfit/run_fewshot_distillation.py @@ -240,7 +240,6 @@ def train(self): self.trained_teacher_model = teacher_trainer.model if self.mode == self.SETFIT_STUDENT: - # student train data = teacher train data + unlabeled data student_train_dataset = concatenate_datasets([self.teacher_train_dataset, fewshot_ds[name]]) diff --git a/scripts/setfit/run_fewshot_multilingual.py b/scripts/setfit/run_fewshot_multilingual.py index 4ffc5577..1b80fbb5 100644 --- a/scripts/setfit/run_fewshot_multilingual.py +++ b/scripts/setfit/run_fewshot_multilingual.py @@ -108,7 +108,6 @@ def eval_setfit(train_data, test_data, model, loss_class, num_epochs, metric): losses.BatchHardSoftMarginTripletLoss, SupConLoss, ]: - train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] train_data_sampler = SentenceLabelDataset(train_examples) diff --git a/src/setfit/data.py b/src/setfit/data.py index 8e99a4de..ee35a428 100644 --- a/src/setfit/data.py +++ b/src/setfit/data.py @@ -265,7 +265,6 @@ def __getitem__(self, idx: int) -> Tuple[TokenizerOutput, Union[int, List[int]]] return feature, label def collate_fn(self, batch): - features = {input_name: [] for input_name in self.tokenizer.model_input_names} labels = [] diff --git a/src/setfit/logging.py b/src/setfit/logging.py index 91aa793e..13368b07 100644 --- a/src/setfit/logging.py +++ b/src/setfit/logging.py @@ -68,17 +68,14 @@ def _get_default_logging_level(): def _get_library_name() -> str: - return __name__.split(".")[0] def _get_library_root_logger() -> logging.Logger: - return logging.getLogger(_get_library_name()) def _configure_library_root_logger() -> None: - global _default_handler with _lock: @@ -96,7 +93,6 @@ def _configure_library_root_logger() -> None: def _reset_library_root_logger() -> None: - global _default_handler with _lock: diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index c093b945..3bae0e51 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -717,7 +717,6 @@ def sentence_pairs_generation_multilabel(sentences, labels, pairs): if len(np.where(labels.dot(labels[first_idx, :].T) == 0)[0]) == 0: continue else: - for _label in sample_labels: second_idx = np.random.choice(np.where(labels[:, _label] == 1)[0]) positive_sentence = sentences[second_idx] diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index dc5303ac..a34f5615 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -450,7 +450,6 @@ def push_to_hub( config: Optional[dict] = None, skip_lfs_files: bool = False, ): - return self.model.push_to_hub( repo_path_or_name, repo_url, diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 1d4ebcad..333a66ea 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -10,7 +10,6 @@ @dataclass class TrainingArguments: - # batch_size is only used to conveniently set `embedding_batch_size` and `classifier_batch_size` # which are used in practice batch_size: Union[int, Tuple[int, int]] = field(default=(16, 2), repr=False) From abbbb03098d24756c1e541296d52fdd7f630d0f5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 15:48:49 +0100 Subject: [PATCH 011/183] Remove "classifier" from var names in SetFitHead --- src/setfit/modeling.py | 19 +++++++++---------- src/setfit/trainer.py | 9 ++++++++- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 3bae0e51..f12eb6f9 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -281,15 +281,14 @@ def fit( self, x_train: List[str], y_train: Union[List[int], List[List[int]]], - classifier_num_epochs: int, - classifier_batch_size: Optional[int] = None, - body_classifier_learning_rate: Optional[float] = None, + num_epochs: int, + batch_size: Optional[int] = None, + body_learning_rate: Optional[float] = None, head_learning_rate: Optional[float] = None, l2_weight: Optional[float] = None, max_length: Optional[int] = None, show_progress_bar: bool = True, end_to_end: bool = False, - **kwargs, ) -> None: if self.has_differentiable_head: # train with pyTorch device = self.model_body.device @@ -298,11 +297,11 @@ def fit( if not end_to_end: self.freeze("body") - dataloader = self._prepare_dataloader(x_train, y_train, classifier_batch_size, max_length) + dataloader = self._prepare_dataloader(x_train, y_train, batch_size, max_length) criterion = self.model_head.get_loss_fn() - optimizer = self._prepare_optimizer(head_learning_rate, body_classifier_learning_rate, l2_weight) + optimizer = self._prepare_optimizer(head_learning_rate, body_learning_rate, l2_weight) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5) - for epoch_idx in trange(classifier_num_epochs, desc="Epoch", disable=not show_progress_bar): + for epoch_idx in trange(num_epochs, desc="Epoch", disable=not show_progress_bar): for batch in tqdm(dataloader, desc="Iteration", disable=not show_progress_bar, leave=False): features, labels = batch optimizer.zero_grad() @@ -372,16 +371,16 @@ def _prepare_dataloader( def _prepare_optimizer( self, head_learning_rate: float, - body_classifier_learning_rate: Optional[float], + body_learning_rate: Optional[float], l2_weight: float, ) -> torch.optim.Optimizer: - body_classifier_learning_rate = body_classifier_learning_rate or head_learning_rate + body_learning_rate = body_learning_rate or head_learning_rate l2_weight = l2_weight or self.l2_weight optimizer = torch.optim.AdamW( [ { "params": self.model_body.parameters(), - "lr": body_classifier_learning_rate, + "lr": body_learning_rate, "weight_decay": l2_weight, }, {"params": self.model_head.parameters(), "lr": head_learning_rate, "weight_decay": l2_weight}, diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index a34f5615..e034a776 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -324,7 +324,14 @@ def train_classifier(self, x_train: List[str], y_train: List[int], args: Optiona self.model.fit( x_train, y_train, - **args.to_dict(), + num_epochs=args.classifier_num_epochs, + batch_size=args.classifier_batch_size, + body_learning_rate=args.body_classifier_learning_rate, + head_learning_rate=args.head_learning_rate, + l2_weight=args.l2_weight, + max_length=args.max_length, + show_progress_bar=args.show_progress_bar, + end_to_end=args.end_to_end, ) def evaluate(self): From 12d326e01a2f965c138da0f94b2a569743bea0ea Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 16:01:07 +0100 Subject: [PATCH 012/183] Update DeprecationWarnings to include timeline Also add DeprecationWarning for DistillationSetFitTrainer --- src/setfit/__init__.py | 3 ++- src/setfit/modeling.py | 3 ++- src/setfit/trainer.py | 5 ++++- src/setfit/trainer_distillation.py | 7 +++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index ce9f19fc..03b96bd9 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -8,5 +8,6 @@ from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer -# Ensure that DeprecationWarnings are always shown +# Ensure that DeprecationWarnings are shown by default, as recommended by +# https://docs.python.org/3/library/warnings.html#overriding-the-default-filter warnings.filterwarnings("default", category=DeprecationWarning) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index f12eb6f9..44c6d367 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -401,7 +401,8 @@ def unfreeze( ) -> None: if keep_body_frozen is not None: warnings.warn( - '`keep_body_frozen` is deprecated. Please either pass "head", "body" or no arguments to unfreeze both.', + "`keep_body_frozen` is deprecated and will be removed in v2.0.0 of SetFit. " + 'Please either pass "head", "body" or no arguments to unfreeze both.', DeprecationWarning, stacklevel=2, ) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index e034a776..bd007594 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -494,7 +494,10 @@ def __init__( samples_per_label: int = 2, ): warnings.warn( - "`SetFitTrainer` has been deprecated. Please use `Trainer` instead.", DeprecationWarning, stacklevel=2 + "`SetFitTrainer` has been deprecated and will be removed in v2.0.0 of SetFit. " + " Please use `Trainer` instead.", + DeprecationWarning, + stacklevel=2, ) args = TrainingArguments( num_iterations=num_iterations, diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index dedf616f..42864e58 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -1,4 +1,5 @@ import math +import warnings from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union import numpy as np @@ -180,6 +181,12 @@ def __init__( use_amp: bool = False, warmup_proportion: float = 0.1, ): + warnings.warn( + "`DistillationSetFitTrainer` has been deprecated and will be removed in v2.0.0 of SetFit. " + "Please use `DistillationTrainer` instead.", + DeprecationWarning, + stacklevel=2, + ) args = TrainingArguments( num_iterations=num_iterations, num_epochs=num_epochs, From fc246cc0e4b43a7881e42a0b9641d78a1a42b0a4 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 16:21:43 +0100 Subject: [PATCH 013/183] Convert training_argument imports to relative imports --- src/setfit/trainer.py | 3 +-- src/setfit/trainer_distillation.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index bd007594..7a31b5ef 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -17,11 +17,10 @@ from torch.utils.data import DataLoader from transformers.trainer_utils import HPSearchBackend, default_compute_objective, number_of_arguments, set_seed -from setfit.training_args import TrainingArguments - from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna from .modeling import SupConLoss, sentence_pairs_generation, sentence_pairs_generation_multilabel +from .training_args import TrainingArguments from .utils import BestRun, default_hp_space_optuna diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 42864e58..c54f3957 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -8,11 +8,10 @@ from sentence_transformers.datasets import SentenceLabelDataset from torch.utils.data import DataLoader -from setfit.training_args import TrainingArguments - from . import logging from .modeling import SupConLoss, sentence_pairs_generation_cos_sim from .trainer import Trainer +from .training_args import TrainingArguments if TYPE_CHECKING: From 57aa54f5191661756bf56507b0ac6f77b847f81b Mon Sep 17 00:00:00 2001 From: Tom Aarsen <37621491+tomaarsen@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:24:56 +0100 Subject: [PATCH 014/183] Make conditional explicit Co-authored-by: lewtun --- src/setfit/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 7a31b5ef..599e4c4c 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -165,7 +165,7 @@ def apply_hyperparameters(self, params: Dict[str, Any], final_model: bool = Fals final_model (`bool`, *optional*, defaults to `False`): If `True`, replace the `model_init()` function with a fixed model based on the parameters. """ - if self.args: + if self.args is not None: self.args = self.args.update(params, ignore_extra=True) else: self.args = TrainingArguments.from_dict(params, ignore_extra=True) From 7ebdf9302fab7a30deae9c934a1e7242b9844c2a Mon Sep 17 00:00:00 2001 From: Tom Aarsen <37621491+tomaarsen@users.noreply.github.com> Date: Mon, 6 Feb 2023 16:25:13 +0100 Subject: [PATCH 015/183] Make conditional explicit Co-authored-by: lewtun --- src/setfit/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 599e4c4c..44506edd 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -219,7 +219,7 @@ def train( trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, **kwargs, ): - if kwargs: + if kwargs is not None: warnings.warn( f"`{self.__class__.__name__}.train` does not accept keyword arguments anymore. " f"Please provide training arguments via a `TrainingArguments` instance to the `{self.__class__.__name__}` " From 46952933c431cdb1642f952af902e7521d725fc1 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 16:32:22 +0100 Subject: [PATCH 016/183] Use assertEqual rather than assert --- tests/test_training_args.py | 86 ++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/tests/test_training_args.py b/tests/test_training_args.py index 5ad6a850..24250c54 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -21,24 +21,24 @@ def test_training_args_batch_sizes(self): batch_size_C = 6 args = TrainingArguments(batch_size=batch_size_A) - assert args.batch_size == (batch_size_A, batch_size_A) - assert args.embedding_batch_size == batch_size_A - assert args.classifier_batch_size == batch_size_A + self.assertEqual(args.batch_size, (batch_size_A, batch_size_A)) + self.assertEqual(args.embedding_batch_size, batch_size_A) + self.assertEqual(args.classifier_batch_size, batch_size_A) args = TrainingArguments(batch_size=(batch_size_A, batch_size_B)) - assert args.batch_size == (batch_size_A, batch_size_B) - assert args.embedding_batch_size == batch_size_A - assert args.classifier_batch_size == batch_size_B + self.assertEqual(args.batch_size, (batch_size_A, batch_size_B)) + self.assertEqual(args.embedding_batch_size, batch_size_A) + self.assertEqual(args.classifier_batch_size, batch_size_B) args = TrainingArguments(batch_size=(batch_size_A, batch_size_B), embedding_batch_size=batch_size_C) - assert args.batch_size == (batch_size_A, batch_size_B) - assert args.embedding_batch_size == batch_size_C - assert args.classifier_batch_size == batch_size_B + self.assertEqual(args.batch_size, (batch_size_A, batch_size_B)) + self.assertEqual(args.embedding_batch_size, batch_size_C) + self.assertEqual(args.classifier_batch_size, batch_size_B) args = TrainingArguments(batch_size=batch_size_A, embedding_batch_size=batch_size_C) - assert args.batch_size == (batch_size_A, batch_size_A) - assert args.embedding_batch_size == batch_size_C - assert args.classifier_batch_size == batch_size_A + self.assertEqual(args.batch_size, (batch_size_A, batch_size_A)) + self.assertEqual(args.embedding_batch_size, batch_size_C) + self.assertEqual(args.classifier_batch_size, batch_size_A) def test_training_args_num_epochs(self): num_epochs_A = 12 @@ -46,24 +46,24 @@ def test_training_args_num_epochs(self): num_epochs_C = 6 args = TrainingArguments(num_epochs=num_epochs_A) - assert args.num_epochs == (num_epochs_A, num_epochs_A) - assert args.embedding_num_epochs == num_epochs_A - assert args.classifier_num_epochs == num_epochs_A + self.assertEqual(args.num_epochs, (num_epochs_A, num_epochs_A)) + self.assertEqual(args.embedding_num_epochs, num_epochs_A) + self.assertEqual(args.classifier_num_epochs, num_epochs_A) args = TrainingArguments(num_epochs=(num_epochs_A, num_epochs_B)) - assert args.num_epochs == (num_epochs_A, num_epochs_B) - assert args.embedding_num_epochs == num_epochs_A - assert args.classifier_num_epochs == num_epochs_B + self.assertEqual(args.num_epochs, (num_epochs_A, num_epochs_B)) + self.assertEqual(args.embedding_num_epochs, num_epochs_A) + self.assertEqual(args.classifier_num_epochs, num_epochs_B) args = TrainingArguments(num_epochs=(num_epochs_A, num_epochs_B), embedding_num_epochs=num_epochs_C) - assert args.num_epochs == (num_epochs_A, num_epochs_B) - assert args.embedding_num_epochs == num_epochs_C - assert args.classifier_num_epochs == num_epochs_B + self.assertEqual(args.num_epochs, (num_epochs_A, num_epochs_B)) + self.assertEqual(args.embedding_num_epochs, num_epochs_C) + self.assertEqual(args.classifier_num_epochs, num_epochs_B) args = TrainingArguments(num_epochs=num_epochs_A, embedding_num_epochs=num_epochs_C) - assert args.num_epochs == (num_epochs_A, num_epochs_A) - assert args.embedding_num_epochs == num_epochs_C - assert args.classifier_num_epochs == num_epochs_A + self.assertEqual(args.num_epochs, (num_epochs_A, num_epochs_A)) + self.assertEqual(args.embedding_num_epochs, num_epochs_C) + self.assertEqual(args.classifier_num_epochs, num_epochs_A) def test_training_args_learning_rates(self): learning_rate_A = 1e-2 @@ -73,24 +73,24 @@ def test_training_args_learning_rates(self): base = TrainingArguments() args = TrainingArguments(body_learning_rate=learning_rate_A) - assert args.body_learning_rate == (learning_rate_A, learning_rate_A) - assert args.body_embedding_learning_rate == learning_rate_A - assert args.body_classifier_learning_rate == learning_rate_A - assert args.head_learning_rate == base.head_learning_rate + self.assertEqual(args.body_learning_rate, (learning_rate_A, learning_rate_A)) + self.assertEqual(args.body_embedding_learning_rate, learning_rate_A) + self.assertEqual(args.body_classifier_learning_rate, learning_rate_A) + self.assertEqual(args.head_learning_rate, base.head_learning_rate) args = TrainingArguments(body_learning_rate=(learning_rate_A, learning_rate_B)) - assert args.body_learning_rate == (learning_rate_A, learning_rate_B) - assert args.body_embedding_learning_rate == learning_rate_A - assert args.body_classifier_learning_rate == learning_rate_B - assert args.head_learning_rate == base.head_learning_rate + self.assertEqual(args.body_learning_rate, (learning_rate_A, learning_rate_B)) + self.assertEqual(args.body_embedding_learning_rate, learning_rate_A) + self.assertEqual(args.body_classifier_learning_rate, learning_rate_B) + self.assertEqual(args.head_learning_rate, base.head_learning_rate) args = TrainingArguments( body_learning_rate=(learning_rate_A, learning_rate_B), head_learning_rate=learning_rate_C ) - assert args.body_learning_rate == (learning_rate_A, learning_rate_B) - assert args.body_embedding_learning_rate == learning_rate_A - assert args.body_classifier_learning_rate == learning_rate_B - assert args.head_learning_rate == learning_rate_C + self.assertEqual(args.body_learning_rate, (learning_rate_A, learning_rate_B)) + self.assertEqual(args.body_embedding_learning_rate, learning_rate_A) + self.assertEqual(args.body_classifier_learning_rate, learning_rate_B) + self.assertEqual(args.head_learning_rate, learning_rate_C) args = TrainingArguments( body_learning_rate=learning_rate_A, @@ -98,16 +98,16 @@ def test_training_args_learning_rates(self): head_learning_rate=learning_rate_C, ) # Perhaps not ideal, but body_learning_rate is never used directly: - assert args.body_learning_rate == (learning_rate_A, learning_rate_A) - assert args.body_embedding_learning_rate == learning_rate_B - assert args.body_classifier_learning_rate == learning_rate_A - assert args.head_learning_rate == learning_rate_C + self.assertEqual(args.body_learning_rate, (learning_rate_A, learning_rate_A)) + self.assertEqual(args.body_embedding_learning_rate, learning_rate_B) + self.assertEqual(args.body_classifier_learning_rate, learning_rate_A) + self.assertEqual(args.head_learning_rate, learning_rate_C) args = TrainingArguments( body_classifier_learning_rate=learning_rate_A, body_embedding_learning_rate=learning_rate_B, head_learning_rate=learning_rate_C, ) - assert args.body_embedding_learning_rate == learning_rate_B - assert args.body_classifier_learning_rate == learning_rate_A - assert args.head_learning_rate == learning_rate_C + self.assertEqual(args.body_embedding_learning_rate, learning_rate_B) + self.assertEqual(args.body_classifier_learning_rate, learning_rate_A) + self.assertEqual(args.head_learning_rate, learning_rate_C) From 4c6d0fdd9338524110df5a0b24f2e137517d51f0 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 16:33:25 +0100 Subject: [PATCH 017/183] Remove training_arguments from test func names --- tests/test_training_args.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_training_args.py b/tests/test_training_args.py index 24250c54..a573e10e 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -6,7 +6,7 @@ class TestTrainingArguments(TestCase): - def test_training_args_raises_error_with_wrong_warmup_proportion(self): + def test_raises_error_with_wrong_warmup_proportion(self): # warmup_proportion must not be > 1.0 with pytest.raises(ValueError): TrainingArguments(warmup_proportion=1.1) @@ -15,7 +15,7 @@ def test_training_args_raises_error_with_wrong_warmup_proportion(self): with pytest.raises(ValueError): TrainingArguments(warmup_proportion=-0.1) - def test_training_args_batch_sizes(self): + def test_batch_sizes(self): batch_size_A = 12 batch_size_B = 4 batch_size_C = 6 @@ -40,7 +40,7 @@ def test_training_args_batch_sizes(self): self.assertEqual(args.embedding_batch_size, batch_size_C) self.assertEqual(args.classifier_batch_size, batch_size_A) - def test_training_args_num_epochs(self): + def test_num_epochs(self): num_epochs_A = 12 num_epochs_B = 4 num_epochs_C = 6 @@ -65,7 +65,7 @@ def test_training_args_num_epochs(self): self.assertEqual(args.embedding_num_epochs, num_epochs_C) self.assertEqual(args.classifier_num_epochs, num_epochs_A) - def test_training_args_learning_rates(self): + def test_learning_rates(self): learning_rate_A = 1e-2 learning_rate_B = 1e-3 learning_rate_C = 1e-4 From 5937ec25b010171122ce074f5a9cb031f36de7b9 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 17:47:12 +0100 Subject: [PATCH 018/183] Replace loss_class on Trainer with loss on TrainArgs --- src/setfit/trainer.py | 21 ++++++++------------- src/setfit/trainer_distillation.py | 18 ++++++++---------- src/setfit/training_args.py | 12 +++++++----- tests/test_trainer.py | 3 +-- tests/test_trainer_distillation.py | 3 --- 5 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 44506edd..f31b2aba 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -92,7 +92,6 @@ def __init__( eval_dataset: Optional["Dataset"] = None, model_init: Optional[Callable[[], "SetFitModel"]] = None, metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", - loss_class=losses.CosineSimilarityLoss, column_mapping: Optional[Dict[str, str]] = None, ): self.args = args @@ -100,7 +99,6 @@ def __init__( self.eval_dataset = eval_dataset self.model_init = model_init self.metric = metric - self.loss_class = loss_class self.column_mapping = column_mapping if model is None: @@ -248,9 +246,6 @@ def train( x_train: List[str] = train_dataset["text"] y_train: List[int] = train_dataset["label"] - if self.loss_class is None: - logger.warning("No `loss_class` detected! Using `CosineSimilarityLoss` as the default.") - self.loss_class = losses.CosineSimilarityLoss self.train_embeddings(x_train, y_train, args) self.train_classifier(x_train, y_train, args) @@ -259,7 +254,7 @@ def train_embeddings(self, x_train: List[str], y_train: List[int], args: Optiona args = args or self.args or TrainingArguments() # sentence-transformers adaptation - if self.loss_class in [ + if args.loss in [ losses.BatchAllTripletLoss, losses.BatchHardTripletLoss, losses.BatchSemiHardTripletLoss, @@ -272,15 +267,15 @@ def train_embeddings(self, x_train: List[str], y_train: List[int], args: Optiona batch_size = min(args.embedding_batch_size, len(train_data_sampler)) train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) - if self.loss_class is losses.BatchHardSoftMarginTripletLoss: - train_loss = self.loss_class( + if args.loss is losses.BatchHardSoftMarginTripletLoss: + train_loss = args.loss( model=self.model.model_body, distance_metric=args.distance_metric, ) - elif self.loss_class is SupConLoss: - train_loss = self.loss_class(model=self.model.model_body) + elif args.loss is SupConLoss: + train_loss = args.loss(model=self.model.model_body) else: - train_loss = self.loss_class( + train_loss = args.loss( model=self.model.model_body, distance_metric=args.distance_metric, margin=args.margin, @@ -298,7 +293,7 @@ def train_embeddings(self, x_train: List[str], y_train: List[int], args: Optiona batch_size = args.embedding_batch_size train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) - train_loss = self.loss_class(self.model.model_body) + train_loss = args.loss(self.model.model_body) total_train_steps = len(train_dataloader) * args.embedding_num_epochs logger.info("***** Running training *****") @@ -510,6 +505,7 @@ def __init__( distance_metric=distance_metric, margin=margin, samples_per_label=samples_per_label, + loss=loss_class, ) super().__init__( model=model, @@ -518,6 +514,5 @@ def __init__( eval_dataset=eval_dataset, model_init=model_init, metric=metric, - loss_class=loss_class, column_mapping=column_mapping, ) diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index c54f3957..0a2cbae9 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -72,7 +72,6 @@ def __init__( eval_dataset: Optional["Dataset"] = None, model_init: Optional[Callable[[], "SetFitModel"]] = None, metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", - loss_class: torch.nn.Module = losses.CosineSimilarityLoss, column_mapping: Optional[Dict[str, str]] = None, ) -> None: super().__init__( @@ -82,7 +81,6 @@ def __init__( eval_dataset=eval_dataset, model_init=model_init, metric=metric, - loss_class=loss_class, column_mapping=column_mapping, ) @@ -98,7 +96,7 @@ def train_embeddings( args = args or self.args or TrainingArguments() # sentence-transformers adaptation - if self.loss_class in [ + if args.loss in [ losses.BatchAllTripletLoss, losses.BatchHardTripletLoss, losses.BatchSemiHardTripletLoss, @@ -111,15 +109,15 @@ def train_embeddings( batch_size = min(args.embedding_batch_size, len(train_data_sampler)) train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) - if self.loss_class is losses.BatchHardSoftMarginTripletLoss: - train_loss = self.loss_class( + if args.loss is losses.BatchHardSoftMarginTripletLoss: + train_loss = args.loss( model=self.student_model.model_body, distance_metric=args.distance_metric, ) - elif self.loss_class is SupConLoss: - train_loss = self.loss_class(model=self.student_model) + elif args.loss is SupConLoss: + train_loss = args.loss(model=self.student_model) else: - train_loss = self.loss_class( + train_loss = args.loss( model=self.student_model.model_body, distance_metric=args.distance_metric, margin=args.margin, @@ -141,7 +139,7 @@ def train_embeddings( batch_size = args.embedding_batch_size train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) - train_loss = self.loss_class(self.student_model.model_body) + train_loss = args.loss(self.student_model.model_body) total_train_steps = len(train_dataloader) * args.embedding_num_epochs logger.info("***** Running training *****") @@ -195,6 +193,7 @@ def __init__( seed=seed, use_amp=use_amp, warmup_proportion=warmup_proportion, + loss=loss_class, ) super().__init__( teacher_model=teacher_model, @@ -204,6 +203,5 @@ def __init__( eval_dataset=eval_dataset, model_init=model_init, metric=metric, - loss_class=loss_class, column_mapping=column_mapping, ) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 333a66ea..8e357f56 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -3,9 +3,9 @@ import inspect from copy import copy from dataclasses import dataclass, field, fields -from typing import Any, Callable, Dict, Tuple, Union +from typing import Any, Callable, Dict, Optional, Tuple, Union -from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction +from sentence_transformers import losses @dataclass @@ -34,16 +34,18 @@ class TrainingArguments: seed: int = 42 use_amp: bool = False warmup_proportion: float = 0.1 - distance_metric: Callable = BatchHardTripletLossDistanceFunction.cosine_distance + distance_metric: Callable = losses.BatchHardTripletLossDistanceFunction.cosine_distance margin: float = 0.25 samples_per_label: int = 2 show_progress_bar: bool = True - l2_weight: float = None - max_length: int = None + l2_weight: Optional[float] = None + max_length: Optional[int] = None end_to_end: bool = False + loss: Callable = losses.CosineSimilarityLoss + def __post_init__(self): # Set `self.embedding_batch_size` and `self.classifier_batch_size` using values from `self.batch_size` if isinstance(self.batch_size, int): diff --git a/tests/test_trainer.py b/tests/test_trainer.py index b40ec5f1..b274d862 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -364,13 +364,12 @@ def hp_name(trial): def test_trainer_works_with_non_default_loss_class(loss_class): dataset = Dataset.from_dict({"text": ["a 1", "b 1", "c 1", "a 2", "b 2", "c 2"], "label": [0, 1, 2, 0, 1, 2]}) model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") - args = TrainingArguments(num_iterations=1) + args = TrainingArguments(num_iterations=1, loss=loss_class) trainer = Trainer( model=model, args=args, train_dataset=dataset, eval_dataset=dataset, - loss_class=loss_class, ) trainer.train() # no asserts here because this is a regression test - we only test if an exception is raised diff --git a/tests/test_trainer_distillation.py b/tests/test_trainer_distillation.py index df2bad4c..2216b1ad 100644 --- a/tests/test_trainer_distillation.py +++ b/tests/test_trainer_distillation.py @@ -2,7 +2,6 @@ import pytest from datasets import Dataset -from sentence_transformers.losses import CosineSimilarityLoss from setfit import DistillationTrainer, Trainer from setfit.modeling import SetFitModel @@ -22,7 +21,6 @@ def test_trainer_works_with_default_columns(self): model=self.teacher_model, train_dataset=dataset, eval_dataset=dataset, - loss_class=CosineSimilarityLoss, metric="accuracy", ) # Teacher Train and evaluate @@ -35,7 +33,6 @@ def test_trainer_works_with_default_columns(self): train_dataset=dataset, student_model=self.student_model, eval_dataset=dataset, - loss_class=CosineSimilarityLoss, metric="accuracy", ) From f1e3de974d0bbee814d3ae6e597976187b3a4baf Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 17:48:02 +0100 Subject: [PATCH 019/183] Removed dead class argument --- src/setfit/trainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index f31b2aba..8d71b1ae 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -112,7 +112,6 @@ def __init__( self.model = model self.hp_search_backend = None - self._freeze = True # If True, will train the body only; otherwise, train the body and head def _validate_column_mapping(self, dataset: "Dataset") -> None: """ From 6051095ddb7127657d297237322c114bdc69c474 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Feb 2023 18:02:23 +0100 Subject: [PATCH 020/183] Move SupConLoss to losses.py Also remove unused SetFitBaseModel and SKLearnWrapper --- src/setfit/losses.py | 100 ++++++++++++++++++++++ src/setfit/modeling.py | 133 ----------------------------- src/setfit/trainer.py | 3 +- src/setfit/trainer_distillation.py | 3 +- src/setfit/utils.py | 2 +- tests/test_deprecated_trainer.py | 3 +- tests/test_trainer.py | 3 +- 7 files changed, 109 insertions(+), 138 deletions(-) create mode 100644 src/setfit/losses.py diff --git a/src/setfit/losses.py b/src/setfit/losses.py new file mode 100644 index 00000000..369c8451 --- /dev/null +++ b/src/setfit/losses.py @@ -0,0 +1,100 @@ +import torch +from torch import nn + + +class SupConLoss(nn.Module): + """Supervised Contrastive Learning: https://arxiv.org/pdf/2004.11362.pdf. + + It also supports the unsupervised contrastive loss in SimCLR. + """ + + def __init__(self, model, temperature=0.07, contrast_mode="all", base_temperature=0.07): + super(SupConLoss, self).__init__() + self.model = model + self.temperature = temperature + self.contrast_mode = contrast_mode + self.base_temperature = base_temperature + + def forward(self, sentence_features, labels=None, mask=None): + """Computes loss for model. + + If both `labels` and `mask` are None, it degenerates to SimCLR unsupervised loss: + https://arxiv.org/pdf/2002.05709.pdf + + Args: + features: hidden vector of shape [bsz, n_views, ...]. + labels: ground truth of shape [bsz]. + mask: contrastive mask of shape [bsz, bsz], mask_{i,j}=1 if sample j + has the same class as sample i. Can be asymmetric. + + Returns: + A loss scalar. + """ + features = self.model(sentence_features[0])["sentence_embedding"] + + # Normalize embeddings + features = torch.nn.functional.normalize(features, p=2, dim=1) + + # Add n_views dimension + features = torch.unsqueeze(features, 1) + + device = features.device + + if len(features.shape) < 3: + raise ValueError("`features` needs to be [bsz, n_views, ...]," "at least 3 dimensions are required") + if len(features.shape) > 3: + features = features.view(features.shape[0], features.shape[1], -1) + + batch_size = features.shape[0] + if labels is not None and mask is not None: + raise ValueError("Cannot define both `labels` and `mask`") + elif labels is None and mask is None: + mask = torch.eye(batch_size, dtype=torch.float32).to(device) + elif labels is not None: + labels = labels.contiguous().view(-1, 1) + if labels.shape[0] != batch_size: + raise ValueError("Num of labels does not match num of features") + mask = torch.eq(labels, labels.T).float().to(device) + else: + mask = mask.float().to(device) + + contrast_count = features.shape[1] + contrast_feature = torch.cat(torch.unbind(features, dim=1), dim=0) + if self.contrast_mode == "one": + anchor_feature = features[:, 0] + anchor_count = 1 + elif self.contrast_mode == "all": + anchor_feature = contrast_feature + anchor_count = contrast_count + else: + raise ValueError("Unknown mode: {}".format(self.contrast_mode)) + + # Compute logits + anchor_dot_contrast = torch.div(torch.matmul(anchor_feature, contrast_feature.T), self.temperature) + # For numerical stability + logits_max, _ = torch.max(anchor_dot_contrast, dim=1, keepdim=True) + logits = anchor_dot_contrast - logits_max.detach() + + # Tile mask + mask = mask.repeat(anchor_count, contrast_count) + # Mask-out self-contrast cases + logits_mask = torch.scatter( + torch.ones_like(mask), + 1, + torch.arange(batch_size * anchor_count).view(-1, 1).to(device), + 0, + ) + mask = mask * logits_mask + + # Compute log_prob + exp_logits = torch.exp(logits) * logits_mask + log_prob = logits - torch.log(exp_logits.sum(1, keepdim=True)) + + # Compute mean of log-likelihood over positive + mean_log_prob_pos = (mask * log_prob).sum(1) / mask.sum(1) + + # Loss + loss = -(self.temperature / self.base_temperature) * mean_log_prob_pos + loss = loss.view(anchor_count, batch_size).mean() + + return loss diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 44c6d367..9fee15d5 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -90,15 +90,6 @@ """ -class SetFitBaseModel: - def __init__(self, model, max_seq_length: int, add_normalization_layer: bool) -> None: - self.model = SentenceTransformer(model) - self.model.max_seq_length = max_seq_length - - if add_normalization_layer: - self.model._modules["2"] = models.Normalize() - - class SetFitHead(models.Dense): """ A SetFit head that supports multi-class classification for end-to-end training. @@ -586,104 +577,6 @@ def _from_pretrained( ) -class SupConLoss(nn.Module): - """Supervised Contrastive Learning: https://arxiv.org/pdf/2004.11362.pdf. - - It also supports the unsupervised contrastive loss in SimCLR. - """ - - def __init__(self, model, temperature=0.07, contrast_mode="all", base_temperature=0.07): - super(SupConLoss, self).__init__() - self.model = model - self.temperature = temperature - self.contrast_mode = contrast_mode - self.base_temperature = base_temperature - - def forward(self, sentence_features, labels=None, mask=None): - """Computes loss for model. - - If both `labels` and `mask` are None, it degenerates to SimCLR unsupervised loss: - https://arxiv.org/pdf/2002.05709.pdf - - Args: - features: hidden vector of shape [bsz, n_views, ...]. - labels: ground truth of shape [bsz]. - mask: contrastive mask of shape [bsz, bsz], mask_{i,j}=1 if sample j - has the same class as sample i. Can be asymmetric. - - Returns: - A loss scalar. - """ - features = self.model(sentence_features[0])["sentence_embedding"] - - # Normalize embeddings - features = torch.nn.functional.normalize(features, p=2, dim=1) - - # Add n_views dimension - features = torch.unsqueeze(features, 1) - - device = features.device - - if len(features.shape) < 3: - raise ValueError("`features` needs to be [bsz, n_views, ...]," "at least 3 dimensions are required") - if len(features.shape) > 3: - features = features.view(features.shape[0], features.shape[1], -1) - - batch_size = features.shape[0] - if labels is not None and mask is not None: - raise ValueError("Cannot define both `labels` and `mask`") - elif labels is None and mask is None: - mask = torch.eye(batch_size, dtype=torch.float32).to(device) - elif labels is not None: - labels = labels.contiguous().view(-1, 1) - if labels.shape[0] != batch_size: - raise ValueError("Num of labels does not match num of features") - mask = torch.eq(labels, labels.T).float().to(device) - else: - mask = mask.float().to(device) - - contrast_count = features.shape[1] - contrast_feature = torch.cat(torch.unbind(features, dim=1), dim=0) - if self.contrast_mode == "one": - anchor_feature = features[:, 0] - anchor_count = 1 - elif self.contrast_mode == "all": - anchor_feature = contrast_feature - anchor_count = contrast_count - else: - raise ValueError("Unknown mode: {}".format(self.contrast_mode)) - - # Compute logits - anchor_dot_contrast = torch.div(torch.matmul(anchor_feature, contrast_feature.T), self.temperature) - # For numerical stability - logits_max, _ = torch.max(anchor_dot_contrast, dim=1, keepdim=True) - logits = anchor_dot_contrast - logits_max.detach() - - # Tile mask - mask = mask.repeat(anchor_count, contrast_count) - # Mask-out self-contrast cases - logits_mask = torch.scatter( - torch.ones_like(mask), - 1, - torch.arange(batch_size * anchor_count).view(-1, 1).to(device), - 0, - ) - mask = mask * logits_mask - - # Compute log_prob - exp_logits = torch.exp(logits) * logits_mask - log_prob = logits - torch.log(exp_logits.sum(1, keepdim=True)) - - # Compute mean of log-likelihood over positive - mean_log_prob_pos = (mask * log_prob).sum(1) / mask.sum(1) - - # Loss - loss = -(self.temperature / self.base_temperature) * mean_log_prob_pos - loss = loss.view(anchor_count, batch_size).mean() - - return loss - - def sentence_pairs_generation(sentences, labels, pairs): # Initialize two empty lists to hold the (sentence, sentence) pairs and # labels to indicate if a pair is positive or negative @@ -754,29 +647,3 @@ def sentence_pairs_generation_cos_sim(sentences, pairs, cos_sim_matrix): pairs.append(InputExample(texts=[current_sentence, paired_sentence], label=cos_sim)) return pairs - - -class SKLearnWrapper: - def __init__(self, st_model=None, clf=None): - self.st_model = st_model - self.clf = clf - - def fit(self, x_train, y_train): - embeddings = self.st_model.encode(x_train) - self.clf.fit(embeddings, y_train) - - def predict(self, x_test): - embeddings = self.st_model.encode(x_test) - return self.clf.predict(embeddings) - - def predict_proba(self, x_test): - embeddings = self.st_model.encode(x_test) - return self.clf.predict_proba(embeddings) - - def save(self, path): - self.st_model.save(path=path) - joblib.dump(self.clf, f"{path}/setfit_head.pkl") - - def load(self, path): - self.st_model = SentenceTransformer(model_name_or_path=path) - self.clf = joblib.load(f"{path}/setfit_head.pkl") diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 8d71b1ae..57cd88b4 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -19,7 +19,8 @@ from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna -from .modeling import SupConLoss, sentence_pairs_generation, sentence_pairs_generation_multilabel +from .losses import SupConLoss +from .modeling import sentence_pairs_generation, sentence_pairs_generation_multilabel from .training_args import TrainingArguments from .utils import BestRun, default_hp_space_optuna diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 0a2cbae9..a299741a 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -9,7 +9,8 @@ from torch.utils.data import DataLoader from . import logging -from .modeling import SupConLoss, sentence_pairs_generation_cos_sim +from .losses import SupConLoss +from .modeling import sentence_pairs_generation_cos_sim from .trainer import Trainer from .training_args import TrainingArguments diff --git a/src/setfit/utils.py b/src/setfit/utils.py index 409edb05..4620ca61 100644 --- a/src/setfit/utils.py +++ b/src/setfit/utils.py @@ -7,7 +7,7 @@ from sentence_transformers import losses from .data import create_fewshot_splits, create_fewshot_splits_multilabel -from .modeling import SupConLoss +from .losses import SupConLoss SEC_TO_NS_SCALE = 1000000000 diff --git a/tests/test_deprecated_trainer.py b/tests/test_deprecated_trainer.py index d4708f93..2a19163f 100644 --- a/tests/test_deprecated_trainer.py +++ b/tests/test_deprecated_trainer.py @@ -8,7 +8,8 @@ from transformers.utils.hp_naming import TrialShortNamer from setfit import logging -from setfit.modeling import SetFitModel, SupConLoss +from setfit.losses import SupConLoss +from setfit.modeling import SetFitModel from setfit.trainer import SetFitTrainer from setfit.utils import BestRun diff --git a/tests/test_trainer.py b/tests/test_trainer.py index b274d862..ed66286d 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -8,7 +8,8 @@ from transformers.utils.hp_naming import TrialShortNamer from setfit import logging -from setfit.modeling import SetFitModel, SupConLoss +from setfit.losses import SupConLoss +from setfit.modeling import SetFitModel from setfit.trainer import Trainer from setfit.training_args import TrainingArguments from setfit.utils import BestRun From bddd46a776883996da6fc80d38a4b1272882acdf Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 7 Feb 2023 08:06:34 +0100 Subject: [PATCH 021/183] Add deprecation to Trainer.(un)freeze --- src/setfit/trainer.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 57cd88b4..f12a47ab 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -204,11 +204,23 @@ def call_model_init(self, params: Optional[Dict[str, Any]] = None): return model def freeze(self, component: Optional[Literal["body", "head"]] = None) -> None: + warnings.warn( + f"`{self.__class__.__name__}.freeze` is deprecated and will be removed in v2.0.0 of SetFit. " + "Please use `SetFitModel.freeze` directly instead.", + DeprecationWarning, + stacklevel=2, + ) return self.model.freeze(component) def unfreeze( self, component: Optional[Literal["body", "head"]] = None, keep_body_frozen: Optional[bool] = None ) -> None: + warnings.warn( + f"`{self.__class__.__name__}.unfreeze` is deprecated and will be removed in v2.0.0 of SetFit. " + "Please use `SetFitModel.unfreeze` directly instead.", + DeprecationWarning, + stacklevel=2, + ) return self.model.unfreeze(component, keep_body_frozen=keep_body_frozen) def train( From fa8a077164842e6eb8cecec751c861f770c0f224 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 7 Feb 2023 08:18:35 +0100 Subject: [PATCH 022/183] Prevent warning from always triggering --- src/setfit/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index f12a47ab..e12da914 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -229,7 +229,7 @@ def train( trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, **kwargs, ): - if kwargs is not None: + if len(kwargs): warnings.warn( f"`{self.__class__.__name__}.train` does not accept keyword arguments anymore. " f"Please provide training arguments via a `TrainingArguments` instance to the `{self.__class__.__name__}` " From 85a3684d05f928c92ed0c102776e37cf4ae39a35 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 7 Feb 2023 08:18:47 +0100 Subject: [PATCH 023/183] Export TrainingArguments in __init__ --- src/setfit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index 03b96bd9..017d7ff8 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -6,7 +6,7 @@ from .modeling import SetFitHead, SetFitModel from .trainer import SetFitTrainer, Trainer from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer - +from .training_args import TrainingArguments # Ensure that DeprecationWarnings are shown by default, as recommended by # https://docs.python.org/3/library/warnings.html#overriding-the-default-filter From ca625a22598e275e7ec7d70f9b91417e2f4fccf3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 7 Feb 2023 10:28:21 +0100 Subject: [PATCH 024/183] Update & add important missing docstrings --- src/setfit/__init__.py | 1 + src/setfit/modeling.py | 111 ++++++++++++++++++++++++++--- src/setfit/trainer.py | 105 +++++++++++++++++---------- src/setfit/trainer_distillation.py | 43 +++++------ src/setfit/training_args.py | 77 +++++++++++++++++--- 5 files changed, 256 insertions(+), 81 deletions(-) diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index 017d7ff8..cb9af3c4 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -8,6 +8,7 @@ from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer from .training_args import TrainingArguments + # Ensure that DeprecationWarnings are shown by default, as recommended by # https://docs.python.org/3/library/warnings.html#overriding-the-default-filter warnings.filterwarnings("default", category=DeprecationWarning) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 9fee15d5..43c26f9e 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -2,7 +2,7 @@ import warnings from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union # Google Colab runs on Python 3.7, so we need this to be compatible @@ -28,10 +28,6 @@ from .data import SetFitDataset -if TYPE_CHECKING: - from numpy import ndarray - - logging.set_verbosity_info() logger = logging.get_logger(__name__) @@ -276,11 +272,31 @@ def fit( batch_size: Optional[int] = None, body_learning_rate: Optional[float] = None, head_learning_rate: Optional[float] = None, + end_to_end: bool = False, l2_weight: Optional[float] = None, max_length: Optional[int] = None, show_progress_bar: bool = True, - end_to_end: bool = False, ) -> None: + """Train the classifier head, only used if a differentiable PyTorch head is used. + + Args: + x_train (`List[str]`): A list of training sentences. + y_train (`Union[List[int], List[List[int]]]`): A list of labels corresponding to the training sentences. + num_epochs (`int`): The number of epochs to train for. + batch_size (`int`, *optional*): The batch size to use. + body_learning_rate (`float`, *optional*): The learning rate for the `SentenceTransformer` body + in the `AdamW` optimizer. Disregarded if `end_to_end=False`. + head_learning_rate (`float`, *optional*): The learning rate for the differentiable torch head + in the `AdamW` optimizer. + end_to_end (`bool`, defaults to `False`): If True, train the entire model end-to-end. + Otherwise, freeze the `SentenceTransformer` body and only train the head. + l2_weight (`float`, *optional*): The l2 weight for both the model body and head + in the `AdamW` optimizer. + max_length (`int`, *optional*): The maximum token length a tokenizer can generate. If not provided, + the maximum length for the `SentenceTransformer` body is used. + show_progress_bar (`bool`, defaults to `True`): Whether to display a progress bar for the training + epochs and iterations. + """ if self.has_differentiable_head: # train with pyTorch device = self.model_body.device self.model_body.train() @@ -381,6 +397,12 @@ def _prepare_optimizer( return optimizer def freeze(self, component: Optional[Literal["body", "head"]] = None) -> None: + """Freeze the model body and/or the head, preventing further training on that component until unfrozen. + + Args: + component (`Literal["body", "head"]`, *optional*): Either "body" or "head" to freeze that component. + If no component is provided, freeze both. Defaults to None. + """ if component is None or component == "body": self._freeze_or_not(self.model_body, to_freeze=True) @@ -390,6 +412,13 @@ def freeze(self, component: Optional[Literal["body", "head"]] = None) -> None: def unfreeze( self, component: Optional[Literal["body", "head"]] = None, keep_body_frozen: Optional[bool] = None ) -> None: + """Unfreeze the model body and/or the head, allowing further training on that component. + + Args: + component (`Literal["body", "head"]`, *optional*): Either "body" or "head" to unfreeze that component. + If no component is provided, unfreeze both. Defaults to None. + keep_body_frozen (`bool`, *optional*): Deprecated argument, use `component` instead. + """ if keep_body_frozen is not None: warnings.warn( "`keep_body_frozen` is deprecated and will be removed in v2.0.0 of SetFit. " @@ -409,15 +438,40 @@ def unfreeze( self._freeze_or_not(self.model_head, to_freeze=False) def _freeze_or_not(self, model: nn.Module, to_freeze: bool) -> None: + """Set `requires_grad=not to_freeze` for all parameters in `model`""" for param in model.parameters(): param.requires_grad = not to_freeze - def encode(self, inputs: List[str]) -> Union[torch.Tensor, "ndarray"]: + def encode(self, inputs: List[str]) -> Union[torch.Tensor, np.ndarray]: + """Convert input sentences to embeddings using the `SentenceTransformer` body. + + Args: + inputs (`List[str]`): The input sentences to embed. + + Returns: + Union[torch.Tensor, np.ndarray]: A matrix with shape [INPUT_LENGTH, EMBEDDING_SIZE], as a + torch Tensor if this model has a differentiable Torch head, or otherwise as a numpy array. + """ return self.model_body.encode( inputs, normalize_embeddings=self.normalize_embeddings, convert_to_tensor=self.has_differentiable_head ) - def predict(self, inputs: List[str], as_numpy: bool = False) -> Union[torch.Tensor, "ndarray"]: + def predict(self, inputs: List[str], as_numpy: bool = False) -> Union[torch.Tensor, np.ndarray]: + """Predict the various classes. + + Args: + inputs (`List[str]`): The input sentences to predict classes for. + as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. + + Example: + >>> model = SetFitModel.from_pretrained(...) + >>> model.predict(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) + tensor([0, 1, 1], dtype=torch.int32) + + Returns: + `Union[torch.Tensor, np.ndarray]`: A vector with equal length to the inputs, denoting + to which class each input is predicted to belong. + """ embeddings = self.encode(inputs) outputs = self.model_head.predict(embeddings) @@ -428,7 +482,24 @@ def predict(self, inputs: List[str], as_numpy: bool = False) -> Union[torch.Tens return outputs - def predict_proba(self, inputs: List[str], as_numpy: bool = False) -> Union[torch.Tensor, "ndarray"]: + def predict_proba(self, inputs: List[str], as_numpy: bool = False) -> Union[torch.Tensor, np.ndarray]: + """Predict the probabilities of the various classes. + + Args: + inputs (`List[str]`): The input sentences to predict class probabilities for. + as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. + + Example: + >>> model = SetFitModel.from_pretrained(...) + >>> model.predict_proba(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) + tensor([[0.9367, 0.0633], + [0.0627, 0.9373], + [0.0890, 0.9110]], dtype=torch.float64) + + Returns: + `Union[torch.Tensor, np.ndarray]`: A matrix with shape [INPUT_LENGTH, NUM_CLASSES] denoting + probabilities of predicting an input as a class. + """ embeddings = self.encode(inputs) outputs = self.model_head.predict_proba(embeddings) @@ -445,6 +516,12 @@ def to(self, device: Union[str, torch.device]) -> "SetFitModel": Args: device (Union[str, torch.device]): The identifier of the device to move the model to. + Example: + + >>> model = SetFitModel.from_pretrained(...) + >>> model.to("cpu") + >>> model(["cats are cute", "dogs are loyal"]) + Returns: SetFitModel: Returns the original model, but now on the desired device. """ @@ -472,7 +549,21 @@ def create_model_card(self, path: str, model_name: Optional[str] = "SetFit Model with open(os.path.join(path, "README.md"), "w", encoding="utf-8") as f: f.write(model_card_content) - def __call__(self, inputs): + def __call__(self, inputs: List[str]) -> torch.Tensor: + """Predict the various classes. + + Args: + inputs (`List[str]`): The input sentences to predict classes for. + + Example: + >>> model = SetFitModel.from_pretrained(...) + >>> model(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) + tensor([0, 1, 1], dtype=torch.int32) + + Returns: + `torch.Tensor`: A vector with equal length to the inputs, denoting to which class each + input is predicted to belong. + """ return self.predict(inputs) def _save_pretrained(self, save_directory: str) -> None: diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index e12da914..17b18f41 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -41,48 +41,24 @@ class Trainer: Args: model (`SetFitModel`, *optional*): The model to train. If not provided, a `model_init` must be passed. + args (`TrainingArguments`, *optional*): + The training arguments to use. train_dataset (`Dataset`): The training dataset. eval_dataset (`Dataset`, *optional*): The evaluation dataset. model_init (`Callable[[], SetFitModel]`, *optional*): - A function that instantiates the model to be used. If provided, each call to [`~SetFitTrainer.train`] will start - from a new instance of the model as given by this function when a `trial` is passed. + A function that instantiates the model to be used. If provided, each call to + [`~SetFitTrainer.train`] will start from a new instance of the model as given by this + function when a `trial` is passed. metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): - The metric to use for evaluation. If a string is provided, we treat it as the metric name and load it with default settings. + The metric to use for evaluation. If a string is provided, we treat it as the metric + name and load it with default settings. If a callable is provided, it must take two arguments (`y_pred`, `y_test`). - loss_class (`nn.Module`, *optional*, defaults to `CosineSimilarityLoss`): - The loss function to use for contrastive training. - num_iterations (`int`, *optional*, defaults to `20`): - The number of iterations to generate sentence pairs for. - This argument is ignored if triplet loss is used. - It is only used in conjunction with `CosineSimilarityLoss`. - num_epochs (`int`, *optional*, defaults to `1`): - The number of epochs to train the Sentence Transformer body for. - learning_rate (`float`, *optional*, defaults to `2e-5`): - The learning rate to use for contrastive training. - batch_size (`int`, *optional*, defaults to `16`): - The batch size to use for contrastive training. - seed (`int`, *optional*, defaults to 42): - Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use the - [`~SetTrainer.model_init`] function to instantiate the model if it has some randomly initialized parameters. column_mapping (`Dict[str, str]`, *optional*): - A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: {"text_column_name": "text", "label_column_name: "label"}. - use_amp (`bool`, *optional*, defaults to `False`): - Use Automatic Mixed Precision (AMP). Only for Pytorch >= 1.6.0 - warmup_proportion (`float`, *optional*, defaults to `0.1`): - Proportion of the warmup in the total training steps. - Must be greater than or equal to 0.0 and less than or equal to 1.0. - distance_metric (`Callable`, defaults to `BatchHardTripletLossDistanceFunction.cosine_distance`): - Function that returns a distance between two embeddings. - It is set for the triplet loss and - is ignored for `CosineSimilarityLoss` and `SupConLoss`. - margin (`float`, defaults to `0.25`): Margin for the triplet loss. - Negative samples should be at least margin further apart from the anchor than the positive. - This is ignored for `CosineSimilarityLoss`, `BatchHardSoftMarginTripletLoss` and `SupConLoss`. - samples_per_label (`int`, defaults to `2`): Number of consecutive, random and unique samples drawn per label. - This is only relevant for triplet loss and ignored for `CosineSimilarityLoss`. - Batch size should be a multiple of samples_per_label. + A mapping from the column names in the dataset to the column names expected by the model. + The expected format is a dictionary with the following format: + `{"text_column_name": "text", "label_column_name: "label"}`. """ def __init__( @@ -106,10 +82,10 @@ def __init__( if model_init is not None: model = self.call_model_init() else: - raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument") + raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument.") else: if model_init is not None: - raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument, but not both") + raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument, but not both.") self.model = model self.hp_search_backend = None @@ -204,6 +180,14 @@ def call_model_init(self, params: Optional[Dict[str, Any]] = None): return model def freeze(self, component: Optional[Literal["body", "head"]] = None) -> None: + """Freeze the model body and/or the head, preventing further training on that component until unfrozen. + + This method is deprecated, use `SetFitModel.freeze` instead. + + Args: + component (`Literal["body", "head"]`, *optional*): Either "body" or "head" to freeze that component. + If no component is provided, freeze both. Defaults to None. + """ warnings.warn( f"`{self.__class__.__name__}.freeze` is deprecated and will be removed in v2.0.0 of SetFit. " "Please use `SetFitModel.freeze` directly instead.", @@ -215,6 +199,15 @@ def freeze(self, component: Optional[Literal["body", "head"]] = None) -> None: def unfreeze( self, component: Optional[Literal["body", "head"]] = None, keep_body_frozen: Optional[bool] = None ) -> None: + """Unfreeze the model body and/or the head, allowing further training on that component. + + This method is deprecated, use `SetFitModel.unfreeze` instead. + + Args: + component (`Literal["body", "head"]`, *optional*): Either "body" or "head" to unfreeze that component. + If no component is provided, unfreeze both. Defaults to None. + keep_body_frozen (`bool`, *optional*): Deprecated argument, use `component` instead. + """ warnings.warn( f"`{self.__class__.__name__}.unfreeze` is deprecated and will be removed in v2.0.0 of SetFit. " "Please use `SetFitModel.unfreeze` directly instead.", @@ -229,6 +222,15 @@ def train( trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, **kwargs, ): + """ + Main training entry point. + + Args: + args (`TrainingArguments`, *optional*): + Temporarily change the training arguments for this training call. + trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): + The trial run or the hyperparameter dictionary for hyperparameter search. + """ if len(kwargs): warnings.warn( f"`{self.__class__.__name__}.train` does not accept keyword arguments anymore. " @@ -262,7 +264,18 @@ def train( self.train_embeddings(x_train, y_train, args) self.train_classifier(x_train, y_train, args) - def train_embeddings(self, x_train: List[str], y_train: List[int], args: Optional[TrainingArguments] = None): + def train_embeddings( + self, x_train: List[str], y_train: Union[List[int], List[List[int]]], args: Optional[TrainingArguments] = None + ): + """ + Method to perform the embedding phase: finetuning the `SentenceTransformer` body. + + Args: + x_train (`List[str]`): A list of training sentences. + y_train (`Union[List[int], List[List[int]]]`): A list of labels corresponding to the training sentences. + args (`TrainingArguments`, *optional*): + Temporarily change the training arguments for this training call. + """ args = args or self.args or TrainingArguments() # sentence-transformers adaptation @@ -324,7 +337,18 @@ def train_embeddings(self, x_train: List[str], y_train: List[int], args: Optiona use_amp=args.use_amp, ) - def train_classifier(self, x_train: List[str], y_train: List[int], args: Optional[TrainingArguments] = None): + def train_classifier( + self, x_train: List[str], y_train: Union[List[int], List[List[int]]], args: Optional[TrainingArguments] = None + ): + """ + Method to perform the classifier phase: fitting a classifier head. + + Args: + x_train (`List[str]`): A list of training sentences. + y_train (`Union[List[int], List[List[int]]]`): A list of labels corresponding to the training sentences. + args (`TrainingArguments`, *optional*): + Temporarily change the training arguments for this training call. + """ args = args or self.args or TrainingArguments() self.model.fit( @@ -479,6 +503,11 @@ def push_to_hub( class SetFitTrainer(Trainer): + """ + `SetFitTrainer` has been deprecated and will be removed in v2.0.0 of SetFit. + Please use `Trainer` instead. + """ + def __init__( self, model: Optional["SetFitModel"] = None, diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index a299741a..588578c9 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -30,38 +30,26 @@ class DistillationTrainer(Trainer): Args: teacher_model (`SetFitModel`): The teacher model to mimic. + student_model (`SetFitModel`, *optional*): + The model to train. If not provided, a `model_init` must be passed. + args (`TrainingArguments`, *optional*): + The training arguments to use. train_dataset (`Dataset`): The training dataset. - student_model (`SetFitModel`): - The student model to train. If not provided, a `model_init` must be passed. eval_dataset (`Dataset`, *optional*): The evaluation dataset. model_init (`Callable[[], SetFitModel]`, *optional*): - A function that instantiates the model to be used. If provided, each call to [`~DistillationSetFitTrainer.train`] will start - from a new instance of the model as given by this function when a `trial` is passed. + A function that instantiates the model to be used. If provided, each call to + [`~SetFitTrainer.train`] will start from a new instance of the model as given by this + function when a `trial` is passed. metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): - The metric to use for evaluation. If a string is provided, we treat it as the metric name and load it with default settings. + The metric to use for evaluation. If a string is provided, we treat it as the metric + name and load it with default settings. If a callable is provided, it must take two arguments (`y_pred`, `y_test`). - loss_class (`nn.Module`, *optional*, defaults to `CosineSimilarityLoss`): - The loss function to use for contrastive training. - num_iterations (`int`, *optional*, defaults to `20`): - The number of iterations to generate sentence pairs for. - num_epochs (`int`, *optional*, defaults to `1`): - The number of epochs to train the Sentence Transformer body for. - learning_rate (`float`, *optional*, defaults to `2e-5`): - The learning rate to use for contrastive training. - batch_size (`int`, *optional*, defaults to `16`): - The batch size to use for contrastive training. - seed (`int`, *optional*, defaults to 42): - Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use the - [`~SetTrainer.model_init`] function to instantiate the model if it has some randomly initialized parameters. column_mapping (`Dict[str, str]`, *optional*): - A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: {"text_column_name": "text", "label_column_name: "label"}. - use_amp (`bool`, *optional*, defaults to `False`): - Use Automatic Mixed Precision (AMP). Only for Pytorch >= 1.6.0 - warmup_proportion (`float`, *optional*, defaults to `0.1`): - Proportion of the warmup in the total training steps. - Must be greater than or equal to 0.0 and less than or equal to 1.0. + A mapping from the column names in the dataset to the column names expected by the model. + The expected format is a dictionary with the following format: + `{"text_column_name": "text", "label_column_name: "label"}`. """ def __init__( @@ -91,7 +79,7 @@ def __init__( def train_embeddings( self, x_train: List[str], - y_train: List[int], + y_train: Union[List[int], List[List[int]]], args: Optional[TrainingArguments] = None, ): args = args or self.args or TrainingArguments() @@ -161,6 +149,11 @@ def train_embeddings( class DistillationSetFitTrainer(DistillationTrainer): + """ + `DistillationSetFitTrainer` has been deprecated and will be removed in v2.0.0 of SetFit. + Please use `DistillationTrainer` instead. + """ + def __init__( self, teacher_model: "SetFitModel", diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 8e357f56..3d9a371f 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -10,6 +10,65 @@ @dataclass class TrainingArguments: + """ + TrainingArguments is the subset of the arguments which relate to the training loop itself. + + Parameters: + batch_size (`Union[int, Tuple[int, int]]`, defaults to `(16, 2)`): + Set the batch sizes for the embedding and classifier training phases respectively, + or set both if an integer is provided. + Note that the batch size for the classifier is only used with a differentiable PyTorch head. + num_epochs (`Union[int, Tuple[int, int]]`, defaults to `(1, 16)`): + Set the number of epochs the embedding and classifier training phases respectively, + or set both if an integer is provided. + Note that the number of epochs for the classifier is only used with a differentiable PyTorch head. + num_iterations (`int`, defaults to `20`): + The number of iterations to generate sentence pairs for. + This argument is ignored if triplet loss is used. + It is only used in conjunction with `CosineSimilarityLoss`. + body_learning_rate (`Union[float, Tuple[float, float]]`, defaults to `(2e-5, 1e-5)`): + Set the learning rate for the `SentenceTransformer` body for the embedding and classifier + training phases respectively, or set both if a float is provided. + Note that the body learning rate for the classifier is only used with a differentiable PyTorch + head *and* if `end_to_end=True`. + head_learning_rate (`float`, defaults to `1e-2`): + Set the learning rate for the head for the classifier training phase. + loss (`nn.Module`, defaults to `CosineSimilarityLoss`): + The loss function to use for contrastive training of the embedding training phase. + distance_metric (`Callable`, defaults to `BatchHardTripletLossDistanceFunction.cosine_distance`): + Function that returns a distance between two embeddings. + It is set for the triplet loss and ignored for `CosineSimilarityLoss` and `SupConLoss`. + margin (`float`, defaults to `0.25`): + Margin for the triplet loss. + Negative samples should be at least margin further apart from the anchor than the positive. + It is ignored for `CosineSimilarityLoss`, `BatchHardSoftMarginTripletLoss` and `SupConLoss`. + end_to_end (`bool`, defaults to `False`): + If True, train the entire model end-to-end during the classifier training phase. + Otherwise, freeze the `SentenceTransformer` body and only train the head. + Only used with a differentiable PyTorch head. + use_amp (`bool`, defaults to `False`): + Whether to use Automatic Mixed Precision (AMP) during the embedding training phase. + Only for Pytorch >= 1.6.0 + warmup_proportion (`float`, defaults to `0.1`): + Proportion of the warmup in the total training steps. + Must be greater than or equal to 0.0 and less than or equal to 1.0. + l2_weight (`float`, *optional*): + Optional l2 weight for both the model body and head, passed to the `AdamW` optimizer in the + classifier training phase if a differentiable PyTorch head is used. + max_length (`int`, *optional*): + The maximum token length a tokenizer can generate. If not provided, the maximum length for + the `SentenceTransformer` body is used. + samples_per_label (`int`, defaults to `2`): Number of consecutive, random and unique samples drawn per label. + This is only relevant for triplet loss and ignored for `CosineSimilarityLoss`. + Batch size should be a multiple of samples_per_label. + show_progress_bar (`bool`, defaults to `True`): + Whether to display a progress bar for the training epochs and iterations. + seed (`int`, defaults to `42`): + Random seed that will be set at the beginning of training. To ensure reproducibility across + runs, use the [`~SetTrainer.model_init`] function to instantiate the model if it has some + randomly initialized parameters. + """ + # batch_size is only used to conveniently set `embedding_batch_size` and `classifier_batch_size` # which are used in practice batch_size: Union[int, Tuple[int, int]] = field(default=(16, 2), repr=False) @@ -31,20 +90,22 @@ class TrainingArguments: body_classifier_learning_rate: float = None head_learning_rate: float = 1e-2 - seed: int = 42 - use_amp: bool = False - warmup_proportion: float = 0.1 + # Loss-related arguments + loss: Callable = losses.CosineSimilarityLoss distance_metric: Callable = losses.BatchHardTripletLossDistanceFunction.cosine_distance margin: float = 0.25 - samples_per_label: int = 2 - show_progress_bar: bool = True + end_to_end: bool = field(default=False) + + use_amp: bool = False + warmup_proportion: float = 0.1 l2_weight: Optional[float] = None max_length: Optional[int] = None + samples_per_label: int = 2 - end_to_end: bool = False - - loss: Callable = losses.CosineSimilarityLoss + # Arguments that do not affect performance + show_progress_bar: bool = True + seed: int = 42 def __post_init__(self): # Set `self.embedding_batch_size` and `self.classifier_batch_size` using values from `self.batch_size` From 68e9094c730b71eaa7725b0e9da466f089caaf3e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 8 Feb 2023 11:26:23 +0100 Subject: [PATCH 025/183] Use standard dataclass initialization for SetFitModel --- src/setfit/modeling.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 43c26f9e..79256557 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -242,22 +242,11 @@ def __repr__(self): class SetFitModel(PyTorchModelHubMixin): """A SetFit model with integration to the Hugging Face Hub.""" - def __init__( - self, - model_body: Optional[SentenceTransformer] = None, - model_head: Optional[Union[SetFitHead, LogisticRegression]] = None, - multi_target_strategy: Optional[str] = None, - l2_weight: float = 1e-2, - normalize_embeddings: bool = False, - ) -> None: - super(SetFitModel, self).__init__() - self.model_body = model_body - self.model_head = model_head - - self.multi_target_strategy = multi_target_strategy - self.l2_weight = l2_weight - - self.normalize_embeddings = normalize_embeddings + model_body: Optional[SentenceTransformer] = (None,) + model_head: Optional[Union[SetFitHead, LogisticRegression]] = None + multi_target_strategy: Optional[str] = None + l2_weight: float = 1e-2 + normalize_embeddings: bool = False @property def has_differentiable_head(self) -> bool: From 19a6fc8c3a2728b3a034d36d66110d2961776107 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 15 Feb 2023 12:05:55 +0100 Subject: [PATCH 026/183] Merge branch 'main' of https://github.com/huggingface/setfit into refactor_v2 --- scripts/setfit/run_zeroshot.py | 2 +- setup.py | 13 ++------ src/setfit/__init__.py | 2 +- src/setfit/data.py | 24 +++++++++----- src/setfit/modeling.py | 10 +++--- src/setfit/trainer.py | 27 ++++++++++++--- tests/test_deprecated_trainer.py | 57 ++++++++++++++++++++++++++++---- tests/test_trainer.py | 53 +++++++++++++++++++++++++---- 8 files changed, 145 insertions(+), 43 deletions(-) diff --git a/scripts/setfit/run_zeroshot.py b/scripts/setfit/run_zeroshot.py index 231fe33b..cf65f0f6 100644 --- a/scripts/setfit/run_zeroshot.py +++ b/scripts/setfit/run_zeroshot.py @@ -92,7 +92,7 @@ def main(): metric = DEV_DATASET_TO_METRIC.get(args.eval_dataset, TEST_DATASET_TO_METRIC.get(args.eval_dataset, "accuracy")) - if args.reference_dataset is None: + if args.reference_dataset is None and args.candidate_labels is None: args.reference_dataset = args.eval_dataset train_data = get_templated_dataset( diff --git a/setup.py b/setup.py index de78d342..ab703e04 100644 --- a/setup.py +++ b/setup.py @@ -10,26 +10,17 @@ MAINTAINER_EMAIL = "lewis@huggingface.co" INTEGRATIONS_REQUIRE = ["optuna"] - REQUIRED_PKGS = ["datasets>=2.3.0", "sentence-transformers>=2.2.1", "evaluate>=0.3.0"] - QUALITY_REQUIRE = ["black", "flake8", "isort", "tabulate"] - ONNX_REQUIRE = ["onnxruntime", "onnx", "skl2onnx"] - OPENVINO_REQUIRE = ["hummingbird-ml", "openvino>=2022.3"] - TESTS_REQUIRE = ["pytest", "pytest-cov"] + ONNX_REQUIRE + OPENVINO_REQUIRE - -COMPAT_TESTS_REQUIRE = [requirement.replace(">=", "==") for requirement in REQUIRED_PKGS] + TESTS_REQUIRE - EXTRAS_REQUIRE = { "optuna": INTEGRATIONS_REQUIRE, "quality": QUALITY_REQUIRE, "tests": TESTS_REQUIRE, "onnx": ONNX_REQUIRE, "openvino": ONNX_REQUIRE + OPENVINO_REQUIRE, - "compat_tests": COMPAT_TESTS_REQUIRE, } @@ -38,11 +29,11 @@ def combine_requirements(base_keys): EXTRAS_REQUIRE["dev"] = combine_requirements([k for k in EXTRAS_REQUIRE]) - +EXTRAS_REQUIRE["compat_tests"] = [requirement.replace(">=", "==") for requirement in REQUIRED_PKGS] + TESTS_REQUIRE setup( name="setfit", - version="0.6.0.dev0", + version="0.7.0.dev0", description="Efficient few-shot learning with Sentence Transformers", long_description=README_TEXT, long_description_content_type="text/markdown", diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index b4d87dcd..0c79e2b0 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.6.0.dev0" +__version__ = "0.7.0.dev0" import warnings diff --git a/src/setfit/data.py b/src/setfit/data.py index 966d249d..ce809bbd 100644 --- a/src/setfit/data.py +++ b/src/setfit/data.py @@ -3,9 +3,14 @@ import pandas as pd import torch -from datasets import Dataset, DatasetDict, concatenate_datasets, load_dataset +from datasets import Dataset, DatasetDict, load_dataset from torch.utils.data import Dataset as TorchDataset +from . import logging + + +logging.set_verbosity_info() +logger = logging.get_logger(__name__) if TYPE_CHECKING: from transformers import PreTrainedTokenizerBase @@ -160,14 +165,15 @@ def create_samples(df: pd.DataFrame, sample_size: int, seed: int) -> pd.DataFram def sample_dataset(dataset: Dataset, label_column: str = "label", num_samples: int = 8, seed: int = 42) -> Dataset: """Samples a Dataset to create an equal number of samples per class (when possible).""" shuffled_dataset = dataset.shuffle(seed=seed) - num_labels = len(dataset.unique(label_column)) - samples = [] - for label in range(num_labels): - data = shuffled_dataset.filter(lambda example: int(example[label_column]) == label) - num_label_samples = min(len(data), num_samples) - samples.append(data.select([i for i in range(num_label_samples)])) - - all_samples = concatenate_datasets(samples) + + df = shuffled_dataset.to_pandas() + df = df.groupby(label_column) + + # sample num_samples, or at least as much as possible + df = df.apply(lambda x: x.sample(min(num_samples, len(x)))) + df = df.reset_index(drop=True) + + all_samples = Dataset.from_pandas(df) return all_samples.shuffle(seed=seed) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 79256557..0d744384 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -662,19 +662,21 @@ def sentence_pairs_generation(sentences, labels, pairs): # labels to indicate if a pair is positive or negative num_classes = np.unique(labels) - idx = [np.where(labels == i)[0] for i in num_classes] + label_to_idx = {x: i for i, x in enumerate(num_classes)} + positive_idxs = [np.where(labels == i)[0] for i in num_classes] + negative_idxs = [np.where(labels != i)[0] for i in num_classes] for first_idx in range(len(sentences)): current_sentence = sentences[first_idx] label = labels[first_idx] - second_idx = np.random.choice(idx[np.where(num_classes == label)[0][0]]) + second_idx = np.random.choice(positive_idxs[label_to_idx[label]]) positive_sentence = sentences[second_idx] # Prepare a positive pair and update the sentences and labels # lists, respectively pairs.append(InputExample(texts=[current_sentence, positive_sentence], label=1.0)) - negative_idx = np.where(labels != label)[0] - negative_sentence = sentences[np.random.choice(negative_idx)] + third_idx = np.random.choice(negative_idxs[label_to_idx[label]]) + negative_sentence = sentences[third_idx] # Prepare a negative pair of sentences and update our lists pairs.append(InputExample(texts=[current_sentence, negative_sentence], label=0.0)) # Return a 2-tuple of our sentence pairs and labels diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 17b18f41..58735588 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -11,10 +11,12 @@ import evaluate import numpy as np +from datasets import DatasetDict from sentence_transformers import InputExample, losses from sentence_transformers.datasets import SentenceLabelDataset from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction from torch.utils.data import DataLoader +from tqdm.auto import trange from transformers.trainer_utils import HPSearchBackend, default_compute_objective, number_of_arguments, set_seed from . import logging @@ -97,9 +99,23 @@ def _validate_column_mapping(self, dataset: "Dataset") -> None: required_columns = {"text", "label"} column_names = set(dataset.column_names) if self.column_mapping is None and not required_columns.issubset(column_names): - raise ValueError( - f"A column mapping must be provided when the dataset does not contain the following columns: {required_columns}" - ) + # Issue #226: load_dataset will automatically assign points to "train" if no split is specified + if column_names == {"train"} and isinstance(dataset, DatasetDict): + raise ValueError( + "SetFit expected a Dataset, but it got a DatasetDict with the split ['train']. " + "Did you mean to select the training split with dataset['train']?" + ) + elif isinstance(dataset, DatasetDict): + raise ValueError( + f"SetFit expected a Dataset, but it got a DatasetDict with the splits {sorted(column_names)}. " + "Did you mean to select one of these splits from the dataset?" + ) + else: + raise ValueError( + f"SetFit expected the dataset to have the columns {sorted(required_columns)}, " + f"but only the columns {sorted(column_names)} were found. " + "Either make sure these columns are present, or specify which columns to use with column_mapping in SetFitTrainer." + ) if self.column_mapping is not None: missing_columns = required_columns.difference(self.column_mapping.values()) if missing_columns: @@ -108,7 +124,8 @@ def _validate_column_mapping(self, dataset: "Dataset") -> None: ) if not set(self.column_mapping.keys()).issubset(column_names): raise ValueError( - f"The following columns are missing from the dataset: {set(self.column_mapping.keys()).difference(column_names)}. Please provide a mapping for all required columns." + f"The column mapping expected the columns {sorted(self.column_mapping.keys())} in the dataset, " + f"but the dataset had the columns {sorted(column_names)}." ) def _apply_column_mapping(self, dataset: "Dataset", column_mapping: Dict[str, str]) -> "Dataset": @@ -308,7 +325,7 @@ def train_embeddings( else: train_examples = [] - for _ in range(args.num_iterations): + for _ in trange(args.num_iterations, desc="Generating Training Pairs", disable=not args.show_progress_bar): if self.model.multi_target_strategy is not None: train_examples = sentence_pairs_generation_multilabel( np.array(x_train), np.array(y_train), train_examples diff --git a/tests/test_deprecated_trainer.py b/tests/test_deprecated_trainer.py index 2a19163f..3bbcf587 100644 --- a/tests/test_deprecated_trainer.py +++ b/tests/test_deprecated_trainer.py @@ -1,8 +1,11 @@ +import pathlib +import re +import tempfile from unittest import TestCase import evaluate import pytest -from datasets import Dataset +from datasets import Dataset, load_dataset from sentence_transformers import losses from transformers.testing_utils import require_optuna from transformers.utils.hp_naming import TrialShortNamer @@ -75,23 +78,65 @@ def test_trainer_raises_error_with_missing_label(self): trainer.train() def test_trainer_raises_error_with_missing_text(self): + """If the required columns are missing from the dataset, the library should throw an error and list the columns found.""" dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) trainer = SetFitTrainer( model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations ) - with pytest.raises(ValueError): - trainer.train() + expected_message = re.escape( + "SetFit expected the dataset to have the columns ['label', 'text'], " + "but only the columns ['extra_column', 'label'] were found. " + "Either make sure these columns are present, or specify which columns to use with column_mapping in SetFitTrainer." + ) + with pytest.raises(ValueError, match=expected_message): + trainer._validate_column_mapping(trainer.train_dataset) - def test_column_mapping_with_missing_text(self): + def test_column_mapping_raises_error_when_mapped_columns_missing(self): + """If the columns specified in the column mapping are missing from the dataset, the library should throw an error and list the columns found.""" dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) trainer = SetFitTrainer( model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations, - column_mapping={"label_new": "label"}, + column_mapping={"text_new": "text", "label_new": "label"}, ) - with pytest.raises(ValueError): + expected_message = re.escape( + "The column mapping expected the columns ['label_new', 'text_new'] in the dataset, " + "but the dataset had the columns ['extra_column', 'text'].", + ) + with pytest.raises(ValueError, match=expected_message): + trainer._validate_column_mapping(trainer.train_dataset) + + def test_trainer_raises_error_when_dataset_not_split(self): + """Verify that an error is raised if we pass an unsplit dataset to the trainer.""" + dataset = Dataset.from_dict({"text": ["a", "b", "c", "d"], "label": [0, 0, 1, 1]}).train_test_split( + test_size=0.5 + ) + trainer = SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) + expected_message = re.escape( + "SetFit expected a Dataset, but it got a DatasetDict with the splits ['test', 'train']. " + "Did you mean to select one of these splits from the dataset?", + ) + with pytest.raises(ValueError, match=expected_message): + trainer._validate_column_mapping(trainer.train_dataset) + + def test_trainer_raises_error_when_dataset_is_dataset_dict_with_train(self): + """Verify that a useful error is raised if we pass an unsplit dataset with only a `train` split to the trainer.""" + with tempfile.TemporaryDirectory() as tmpdirname: + path = pathlib.Path(tmpdirname) / "test_dataset_dict_with_train.csv" + path.write_text("label,text\n1,good\n0,terrible\n") + dataset = load_dataset("csv", data_files=str(path)) + trainer = SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) + expected_message = re.escape( + "SetFit expected a Dataset, but it got a DatasetDict with the split ['train']. " + "Did you mean to select the training split with dataset['train']?", + ) + with pytest.raises(ValueError, match=expected_message): trainer._validate_column_mapping(trainer.train_dataset) def test_column_mapping_multilabel(self): diff --git a/tests/test_trainer.py b/tests/test_trainer.py index ed66286d..a964aaf2 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -1,8 +1,11 @@ +import pathlib +import re +import tempfile from unittest import TestCase import evaluate import pytest -from datasets import Dataset +from datasets import Dataset, load_dataset from sentence_transformers import losses from transformers.testing_utils import require_optuna from transformers.utils.hp_naming import TrialShortNamer @@ -72,21 +75,59 @@ def test_trainer_raises_error_with_missing_label(self): trainer.train() def test_trainer_raises_error_with_missing_text(self): + """If the required columns are missing from the dataset, the library should throw an error and list the columns found.""" dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) - with pytest.raises(ValueError): - trainer.train() + expected_message = re.escape( + "SetFit expected the dataset to have the columns ['label', 'text'], " + "but only the columns ['extra_column', 'label'] were found. " + "Either make sure these columns are present, or specify which columns to use with column_mapping in SetFitTrainer." + ) + with pytest.raises(ValueError, match=expected_message): + trainer._validate_column_mapping(trainer.train_dataset) - def test_column_mapping_with_missing_text(self): + def test_column_mapping_raises_error_when_mapped_columns_missing(self): + """If the columns specified in the column mapping are missing from the dataset, the library should throw an error and list the columns found.""" dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) trainer = Trainer( model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset, - column_mapping={"label_new": "label"}, + column_mapping={"text_new": "text", "label_new": "label"}, ) - with pytest.raises(ValueError): + expected_message = re.escape( + "The column mapping expected the columns ['label_new', 'text_new'] in the dataset, " + "but the dataset had the columns ['extra_column', 'text'].", + ) + with pytest.raises(ValueError, match=expected_message): + trainer._validate_column_mapping(trainer.train_dataset) + + def test_trainer_raises_error_when_dataset_not_split(self): + """Verify that an error is raised if we pass an unsplit dataset to the trainer.""" + dataset = Dataset.from_dict({"text": ["a", "b", "c", "d"], "label": [0, 0, 1, 1]}).train_test_split( + test_size=0.5 + ) + trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) + expected_message = re.escape( + "SetFit expected a Dataset, but it got a DatasetDict with the splits ['test', 'train']. " + "Did you mean to select one of these splits from the dataset?", + ) + with pytest.raises(ValueError, match=expected_message): + trainer._validate_column_mapping(trainer.train_dataset) + + def test_trainer_raises_error_when_dataset_is_dataset_dict_with_train(self): + """Verify that a useful error is raised if we pass an unsplit dataset with only a `train` split to the trainer.""" + with tempfile.TemporaryDirectory() as tmpdirname: + path = pathlib.Path(tmpdirname) / "test_dataset_dict_with_train.csv" + path.write_text("label,text\n1,good\n0,terrible\n") + dataset = load_dataset("csv", data_files=str(path)) + trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) + expected_message = re.escape( + "SetFit expected a Dataset, but it got a DatasetDict with the split ['train']. " + "Did you mean to select the training split with dataset['train']?", + ) + with pytest.raises(ValueError, match=expected_message): trainer._validate_column_mapping(trainer.train_dataset) def test_column_mapping_multilabel(self): From ca87c4266148267520b3122710ec2b49d04adc35 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 16 Feb 2023 13:07:35 +0100 Subject: [PATCH 027/183] Remove duplicate space in DeprecationWarning --- src/setfit/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 58735588..31cdfcce 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -547,7 +547,7 @@ def __init__( ): warnings.warn( "`SetFitTrainer` has been deprecated and will be removed in v2.0.0 of SetFit. " - " Please use `Trainer` instead.", + "Please use `Trainer` instead.", DeprecationWarning, stacklevel=2, ) From cc5282fe30117d3e9c8d10832e711a25131f1b44 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 3 Mar 2023 16:28:58 +0100 Subject: [PATCH 028/183] No longer require labeled data for DistillationTrainer This bug was pointed out by #320 --- src/setfit/trainer.py | 9 +- src/setfit/trainer_distillation.py | 139 +++++++++++------- tests/test_deprecated_trainer_distillation.py | 29 +++- tests/test_trainer_distillation.py | 29 +++- 4 files changed, 137 insertions(+), 69 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 31cdfcce..0fff050c 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -63,6 +63,8 @@ class Trainer: `{"text_column_name": "text", "label_column_name: "label"}`. """ + _REQUIRED_COLUMNS = {"text", "label"} + def __init__( self, model: Optional["SetFitModel"] = None, @@ -96,9 +98,8 @@ def _validate_column_mapping(self, dataset: "Dataset") -> None: """ Validates the provided column mapping against the dataset. """ - required_columns = {"text", "label"} column_names = set(dataset.column_names) - if self.column_mapping is None and not required_columns.issubset(column_names): + if self.column_mapping is None and not self._REQUIRED_COLUMNS.issubset(column_names): # Issue #226: load_dataset will automatically assign points to "train" if no split is specified if column_names == {"train"} and isinstance(dataset, DatasetDict): raise ValueError( @@ -112,12 +113,12 @@ def _validate_column_mapping(self, dataset: "Dataset") -> None: ) else: raise ValueError( - f"SetFit expected the dataset to have the columns {sorted(required_columns)}, " + f"SetFit expected the dataset to have the columns {sorted(self._REQUIRED_COLUMNS)}, " f"but only the columns {sorted(column_names)} were found. " "Either make sure these columns are present, or specify which columns to use with column_mapping in SetFitTrainer." ) if self.column_mapping is not None: - missing_columns = required_columns.difference(self.column_mapping.values()) + missing_columns = self._REQUIRED_COLUMNS.difference(self.column_mapping.values()) if missing_columns: raise ValueError( f"The following columns are missing from the column mapping: {missing_columns}. Please provide a mapping for all required columns." diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 588578c9..41bfa26d 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -1,21 +1,21 @@ import math import warnings -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import numpy as np import torch -from sentence_transformers import InputExample, losses, util -from sentence_transformers.datasets import SentenceLabelDataset +from sentence_transformers import losses, util from torch.utils.data import DataLoader +from transformers.trainer_utils import set_seed from . import logging -from .losses import SupConLoss from .modeling import sentence_pairs_generation_cos_sim from .trainer import Trainer from .training_args import TrainingArguments if TYPE_CHECKING: + import optuna from datasets import Dataset from .modeling import SetFitModel @@ -52,6 +52,8 @@ class DistillationTrainer(Trainer): `{"text_column_name": "text", "label_column_name: "label"}`. """ + _REQUIRED_COLUMNS = {"text"} + def __init__( self, teacher_model: "SetFitModel", @@ -76,59 +78,81 @@ def __init__( self.teacher_model = teacher_model self.student_model = self.model + def train( + self, + args: Optional[TrainingArguments] = None, + trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, + **kwargs, + ) -> None: + """ + Main training entry point. + + Args: + args (`TrainingArguments`, *optional*): + Temporarily change the training arguments for this training call. + trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): + The trial run or the hyperparameter dictionary for hyperparameter search. + """ + if len(kwargs): + warnings.warn( + f"`{self.__class__.__name__}.train` does not accept keyword arguments anymore. " + f"Please provide training arguments via a `TrainingArguments` instance to the `{self.__class__.__name__}` " + f"initialisation or the `{self.__class__.__name__}.train` method.", + DeprecationWarning, + stacklevel=2, + ) + + args = args or self.args or TrainingArguments() + + set_seed(args.seed) # Seed must be set before instantiating the model when using model_init. + + if trial: # Trial and model initialization + self._hp_search_setup(trial) # sets trainer parameters and initializes model + + if self.train_dataset is None: + raise ValueError( + f"Training requires a `train_dataset` given to the `{self.__class__.__name__}` initialization." + ) + + self._validate_column_mapping(self.train_dataset) + train_dataset = self.train_dataset + if self.column_mapping is not None: + logger.info("Applying column mapping to training dataset") + train_dataset = self._apply_column_mapping(self.train_dataset, self.column_mapping) + + x_train: List[str] = train_dataset["text"] + + self.train_embeddings(x_train, args) + self.train_classifier(x_train, args) + def train_embeddings( self, x_train: List[str], - y_train: Union[List[int], List[List[int]]], args: Optional[TrainingArguments] = None, - ): + ) -> None: + """ + Method to perform the embedding phase: finetuning the student its `SentenceTransformer` body. + + Args: + x_train (`List[str]`): A list of training sentences. + args (`TrainingArguments`, *optional*): + Temporarily change the training arguments for this training call. + """ args = args or self.args or TrainingArguments() - # sentence-transformers adaptation - if args.loss in [ - losses.BatchAllTripletLoss, - losses.BatchHardTripletLoss, - losses.BatchSemiHardTripletLoss, - losses.BatchHardSoftMarginTripletLoss, - SupConLoss, - ]: - train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] - train_data_sampler = SentenceLabelDataset(train_examples) - - batch_size = min(args.embedding_batch_size, len(train_data_sampler)) - train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) - - if args.loss is losses.BatchHardSoftMarginTripletLoss: - train_loss = args.loss( - model=self.student_model.model_body, - distance_metric=args.distance_metric, - ) - elif args.loss is SupConLoss: - train_loss = args.loss(model=self.student_model) - else: - train_loss = args.loss( - model=self.student_model.model_body, - distance_metric=args.distance_metric, - margin=args.margin, - ) - else: - train_examples = [] - - # **************** student training ********************* - # Only this snippet differs from Trainer.train_embeddings - x_train_embd_student = self.teacher_model.model_body.encode(x_train) - y_train = self.teacher_model.model_head.predict(x_train_embd_student) - - cos_sim_matrix = util.cos_sim(x_train_embd_student, x_train_embd_student) - - train_examples = [] - for _ in range(args.num_iterations): - train_examples = sentence_pairs_generation_cos_sim(np.array(x_train), train_examples, cos_sim_matrix) - # **************** student training END ***************** - - batch_size = args.embedding_batch_size - train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) - train_loss = args.loss(self.student_model.model_body) + # **************** student training ********************* + x_train_embd_student = self.teacher_model.model_body.encode(x_train) + + cos_sim_matrix = util.cos_sim(x_train_embd_student, x_train_embd_student) + + train_examples = [] + for _ in range(args.num_iterations): + train_examples = sentence_pairs_generation_cos_sim(np.array(x_train), train_examples, cos_sim_matrix) + # **************** student training END ***************** + + batch_size = args.embedding_batch_size + train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) + train_loss = args.loss(self.student_model.model_body) total_train_steps = len(train_dataloader) * args.embedding_num_epochs logger.info("***** Running training *****") @@ -147,6 +171,19 @@ def train_embeddings( use_amp=args.use_amp, ) + def train_classifier(self, x_train: List[str], args: Optional[TrainingArguments] = None) -> None: + """ + Method to perform the classifier phase: fitting the student classifier head. + + Args: + x_train (`List[str]`): A list of training sentences. + args (`TrainingArguments`, *optional*): + Temporarily change the training arguments for this training call. + """ + x_train_embd_student = self.teacher_model.model_body.encode(x_train) + y_train = self.teacher_model.model_head.predict(x_train_embd_student) + return super().train_classifier(x_train, y_train, args) + class DistillationSetFitTrainer(DistillationTrainer): """ diff --git a/tests/test_deprecated_trainer_distillation.py b/tests/test_deprecated_trainer_distillation.py index 4257a42e..5d59c4f5 100644 --- a/tests/test_deprecated_trainer_distillation.py +++ b/tests/test_deprecated_trainer_distillation.py @@ -26,7 +26,6 @@ def test_trainer_works_with_default_columns(self): ) # Teacher Train and evaluate teacher_trainer.train() - metrics = teacher_trainer.evaluate() teacher_model = teacher_trainer.model student_trainer = DistillationSetFitTrainer( @@ -45,16 +44,32 @@ def test_trainer_works_with_default_columns(self): self.assertEqual(metrics["accuracy"], 1.0) def test_trainer_raises_error_with_missing_label(self): - dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = DistillationSetFitTrainer( + labeled_dataset = Dataset.from_dict( + {"text": ["a", "b", "c"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]} + ) + # train a teacher model + teacher_trainer = SetFitTrainer( + model=self.teacher_model, + train_dataset=labeled_dataset, + eval_dataset=labeled_dataset, + metric="accuracy", + num_iterations=self.num_iterations, + ) + # Teacher Train and evaluate + teacher_trainer.train() + + unlabeled_dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) + student_trainer = DistillationSetFitTrainer( teacher_model=self.teacher_model, - train_dataset=dataset, student_model=self.student_model, - eval_dataset=dataset, + train_dataset=unlabeled_dataset, + eval_dataset=labeled_dataset, num_iterations=self.num_iterations, ) - with pytest.raises(ValueError): - trainer.train() + student_trainer.train() + metrics = student_trainer.evaluate() + print("Student results: ", metrics) + self.assertEqual(metrics["accuracy"], 1.0) def test_trainer_raises_error_with_missing_text(self): dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) diff --git a/tests/test_trainer_distillation.py b/tests/test_trainer_distillation.py index 2216b1ad..71b81e0a 100644 --- a/tests/test_trainer_distillation.py +++ b/tests/test_trainer_distillation.py @@ -25,7 +25,6 @@ def test_trainer_works_with_default_columns(self): ) # Teacher Train and evaluate teacher_trainer.train() - metrics = teacher_trainer.evaluate() teacher_model = teacher_trainer.model student_trainer = DistillationTrainer( @@ -43,16 +42,32 @@ def test_trainer_works_with_default_columns(self): self.assertEqual(metrics["accuracy"], 1.0) def test_trainer_raises_error_with_missing_label(self): - dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = DistillationTrainer( + labeled_dataset = Dataset.from_dict( + {"text": ["a", "b", "c"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]} + ) + # train a teacher model + teacher_trainer = Trainer( + model=self.teacher_model, + train_dataset=labeled_dataset, + eval_dataset=labeled_dataset, + metric="accuracy", + args=self.args, + ) + # Teacher Train and evaluate + teacher_trainer.train() + + unlabeled_dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) + student_trainer = DistillationTrainer( teacher_model=self.teacher_model, - train_dataset=dataset, student_model=self.student_model, - eval_dataset=dataset, + train_dataset=unlabeled_dataset, + eval_dataset=labeled_dataset, args=self.args, ) - with pytest.raises(ValueError): - trainer.train() + student_trainer.train() + metrics = student_trainer.evaluate() + print("Student results: ", metrics) + self.assertEqual(metrics["accuracy"], 1.0) def test_trainer_raises_error_with_missing_text(self): dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) From 36cbbfec240f3fa294a78d1b6841f35c95facf70 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Mar 2023 10:51:53 +0100 Subject: [PATCH 029/183] Update docs for v1.0.0 --- README.md | 118 ++++++++++++++++------------------ docs/source/en/quickstart.mdx | 112 +++++++++++++++----------------- 2 files changed, 106 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 31b3d390..71bddb45 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,7 @@ Here is an end-to-end example using a classification head from `scikit-learn`: ```python from datasets import load_dataset -from sentence_transformers.losses import CosineSimilarityLoss - -from setfit import SetFitModel, SetFitTrainer, sample_dataset +from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset # Load a dataset from the Hugging Face Hub @@ -61,17 +59,19 @@ eval_dataset = dataset["validation"] # Load a SetFit model from Hub model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") -# Create trainer -trainer = SetFitTrainer( +args = TrainingArguments( + batch_size=16, + num_iterations=20, # The number of text pairs to generate for contrastive learning + num_epochs=1 # The number of epochs to use for contrastive learning +) + +trainer = Trainer( model=model, + args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, - loss_class=CosineSimilarityLoss, metric="accuracy", - batch_size=16, - num_iterations=20, # The number of text pairs to generate for contrastive learning - num_epochs=1, # The number of epochs to use for contrastive learning - column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer + column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer ) # Train and evaluate @@ -81,7 +81,7 @@ metrics = trainer.evaluate() # Push model to the Hub trainer.push_to_hub("my-awesome-setfit-model") -# Download from Hub and run inference +# Download from Hub model = SetFitModel.from_pretrained("lewtun/my-awesome-setfit-model") # Run inference preds = model(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) @@ -92,9 +92,7 @@ Here is an end-to-end example using `SetFitHead`: ```python from datasets import load_dataset -from sentence_transformers.losses import CosineSimilarityLoss - -from setfit import SetFitModel, SetFitTrainer, sample_dataset +from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset # Load a dataset from the Hugging Face Hub @@ -103,6 +101,7 @@ dataset = load_dataset("sst2") # Simulate the few-shot regime by sampling 8 examples per class train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) eval_dataset = dataset["validation"] +num_classes = 2 # Load a SetFit model from Hub model = SetFitModel.from_pretrained( @@ -111,36 +110,26 @@ model = SetFitModel.from_pretrained( head_params={"out_features": num_classes}, ) -# Create trainer -trainer = SetFitTrainer( +args = TrainingArguments( + body_learning_rate=2e-5, + head_learning_rate=1e-2, + batch_size=16, + num_iterations=20, # The number of text pairs to generate for contrastive learning + num_epochs=(1, 25), # For finetuning the embeddings and training the classifier, respectively + l2_weight=0.0, + end_to_end=False, # Don't train the classifier end-to-end, i.e. only train the head +) + +trainer = Trainer( model=model, train_dataset=train_dataset, eval_dataset=eval_dataset, - loss_class=CosineSimilarityLoss, metric="accuracy", - batch_size=16, - num_iterations=20, # The number of text pairs to generate for contrastive learning - num_epochs=1, # The number of epochs to use for contrastive learning - column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer + column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer ) # Train and evaluate -trainer.freeze() # Freeze the head -trainer.train() # Train only the body - -# Unfreeze the head and freeze the body -> head-only training -trainer.unfreeze(keep_body_frozen=True) -# or -# Unfreeze the head and unfreeze the body -> end-to-end training -trainer.unfreeze(keep_body_frozen=False) - -trainer.train( - num_epochs=25, # The number of epochs to train the head or the whole model (body and head) - batch_size=16, - body_learning_rate=1e-5, # The body's learning rate - learning_rate=1e-2, # The head's learning rate - l2_weight=0.0, # Weight decay on **both** the body and head. If `None`, will use 0.01. -) +trainer.train() metrics = trainer.evaluate() # Push model to the Hub @@ -175,7 +164,7 @@ This will initialise a multilabel classification head from `sklearn` - the follo * `multi-output`: uses a `MultiOutputClassifier` head. * `classifier-chain`: uses a `ClassifierChain` head. -From here, you can instantiate a `SetFitTrainer` using the same example above, and train it as usual. +From here, you can instantiate a `Trainer` using the same example above, and train it as usual. #### Example using the differentiable `SetFitHead`: @@ -196,7 +185,6 @@ model = SetFitModel.from_pretrained( SetFit can also be applied to scenarios where no labels are available. To do so, create a synthetic dataset of training examples: ```python -from datasets import Dataset from setfit import get_templated_dataset candidate_labels = ["negative", "positive"] @@ -206,22 +194,22 @@ train_dataset = get_templated_dataset(candidate_labels=candidate_labels, sample_ This will create examples of the form `"This sentence is {}"`, where the `{}` is filled in with one of the candidate labels. From here you can train a SetFit model as usual: ```python -from setfit import SetFitModel, SetFitTrainer +from setfit import SetFitModel, Trainer model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") -trainer = SetFitTrainer( +trainer = Trainer( model=model, train_dataset=train_dataset ) trainer.train() ``` -We find this approach typically outperforms the [zero-shot pipeline](https://huggingface.co/docs/transformers/v4.24.0/en/main_classes/pipelines#transformers.ZeroShotClassificationPipeline) in 🤗 Transformers (based on MNLI with Bart), while being 5x faster to generate predictions with. +We find this approach typically outperforms the [zero-shot pipeline](https://huggingface.co/docs/transformers/v4.24.0/en/main_classes/pipelines#transformers.ZeroShotClassificationPipeline) in 🤗 Transformers (based on MNLI with BART), while being 5x faster to generate predictions with. ### Running hyperparameter search -`SetFitTrainer` provides a `hyperparameter_search()` method that you can use to find good hyperparameters for your data. To use this feature, first install the `optuna` backend: +`Trainer` provides a `hyperparameter_search()` method that you can use to find good hyperparameters for your data. To use this feature, first install the `optuna` backend: ```bash python -m pip install setfit[optuna] @@ -267,23 +255,23 @@ def hp_space(trial): # Training parameters **Note:** In practice, we found `num_iterations` to be the most important hyperparameter for the contrastive learning process. -The next step is to instantiate a `SetFitTrainer` and call `hyperparameter_search()`: +The next step is to instantiate a `Trainer` and call `hyperparameter_search()`: ```python from datasets import Dataset -from setfit import SetFitTrainer +from setfit import Trainer dataset = Dataset.from_dict( - {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} - ) + {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} +) -trainer = SetFitTrainer( +trainer = Trainer( train_dataset=dataset, eval_dataset=dataset, model_init=model_init, column_mapping={"text_new": "text", "label_new": "label"}, ) -best_run = trainer.hyperparameter_search(direction="maximize", hp_space=hp_space, n_trials=20) +best_run = trainer.hyperparameter_search(direction="maximize", hp_space=hp_space, n_trials=5) ``` Finally, you can apply the hyperparameters you found to the trainer, and lock in the optimal model, before training for @@ -300,9 +288,8 @@ If you have access to unlabeled data, you can use knowledge distillation to comp ```python from datasets import load_dataset -from sentence_transformers.losses import CosineSimilarityLoss - -from setfit import SetFitModel, SetFitTrainer, DistillationSetFitTrainer, sample_dataset +from setfit import SetFitModel, Trainer, DistillationTrainer, sample_dataset +from setfit.training_args import TrainingArguments # Load a dataset from the Hugging Face Hub dataset = load_dataset("ag_news") @@ -320,34 +307,37 @@ teacher_model = SetFitModel.from_pretrained( ) # Create trainer for teacher model -teacher_trainer = SetFitTrainer( +teacher_trainer = Trainer( model=teacher_model, train_dataset=train_dataset_teacher, eval_dataset=eval_dataset, - loss_class=CosineSimilarityLoss, ) # Train teacher model teacher_trainer.train() +teacher_metrics = teacher_trainer.evaluate() # Load small student model student_model = SetFitModel.from_pretrained("paraphrase-MiniLM-L3-v2") +args = TrainingArguments( + batch_size=16, + num_iterations=20, + num_epochs=1 +) + # Create trainer for knowledge distillation -student_trainer = DistillationSetFitTrainer( +student_trainer = DistillationTrainer( teacher_model=teacher_model, - train_dataset=train_dataset_student, student_model=student_model, + args=args, + train_dataset=train_dataset_student, eval_dataset=eval_dataset, - loss_class=CosineSimilarityLoss, - metric="accuracy", - batch_size=16, - num_iterations=20, - num_epochs=1, ) # Train student with knowledge distillation student_trainer.train() +student_metrics = student_trainer.evaluate() ``` @@ -402,7 +392,8 @@ make style && make quality ## Citation -```@misc{https://doi.org/10.48550/arxiv.2209.11055, +``` +@misc{https://doi.org/10.48550/arxiv.2209.11055, doi = {10.48550/ARXIV.2209.11055}, url = {https://arxiv.org/abs/2209.11055}, author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}, @@ -410,5 +401,6 @@ make style && make quality title = {Efficient Few-Shot Learning Without Prompts}, publisher = {arXiv}, year = {2022}, - copyright = {Creative Commons Attribution 4.0 International}} + copyright = {Creative Commons Attribution 4.0 International} +} ``` diff --git a/docs/source/en/quickstart.mdx b/docs/source/en/quickstart.mdx index cc10ba5b..74bd75de 100644 --- a/docs/source/en/quickstart.mdx +++ b/docs/source/en/quickstart.mdx @@ -18,9 +18,7 @@ Here is an end-to-end example using a classification head from `scikit-learn`: ```python from datasets import load_dataset -from sentence_transformers.losses import CosineSimilarityLoss - -from setfit import SetFitModel, SetFitTrainer, sample_dataset +from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset # Load a dataset from the Hugging Face Hub @@ -33,17 +31,19 @@ eval_dataset = dataset["validation"] # Load a SetFit model from Hub model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") -# Create trainer -trainer = SetFitTrainer( +args = TrainingArguments( + batch_size=16, + num_iterations=20, # The number of text pairs to generate for contrastive learning + num_epochs=1 # The number of epochs to use for contrastive learning +) + +trainer = Trainer( model=model, + args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, - loss_class=CosineSimilarityLoss, metric="accuracy", - batch_size=16, - num_iterations=20, # The number of text pairs to generate for contrastive learning - num_epochs=1, # The number of epochs to use for contrastive learning - column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer + column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer ) # Train and evaluate @@ -53,7 +53,7 @@ metrics = trainer.evaluate() # Push model to the Hub trainer.push_to_hub("my-awesome-setfit-model") -# Download from Hub and run inference +# Download from Hub model = SetFitModel.from_pretrained("lewtun/my-awesome-setfit-model") # Run inference preds = model(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) @@ -64,9 +64,7 @@ Here is an end-to-end example using `SetFitHead`: ```python from datasets import load_dataset -from sentence_transformers.losses import CosineSimilarityLoss - -from setfit import SetFitModel, SetFitTrainer, sample_dataset +from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset # Load a dataset from the Hugging Face Hub @@ -75,6 +73,7 @@ dataset = load_dataset("sst2") # Simulate the few-shot regime by sampling 8 examples per class train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) eval_dataset = dataset["validation"] +num_classes = 2 # Load a SetFit model from Hub model = SetFitModel.from_pretrained( @@ -83,36 +82,26 @@ model = SetFitModel.from_pretrained( head_params={"out_features": num_classes}, ) -# Create trainer -trainer = SetFitTrainer( +args = TrainingArguments( + body_learning_rate=2e-5, + head_learning_rate=1e-2, + batch_size=16, + num_iterations=20, # The number of text pairs to generate for contrastive learning + num_epochs=(1, 25), # For finetuning the embeddings and training the classifier, respectively + l2_weight=0.0, + end_to_end=False, # Don't train the classifier end-to-end, i.e. only train the head +) + +trainer = Trainer( model=model, train_dataset=train_dataset, eval_dataset=eval_dataset, - loss_class=CosineSimilarityLoss, metric="accuracy", - batch_size=16, - num_iterations=20, # The number of text pairs to generate for contrastive learning - num_epochs=1, # The number of epochs to use for contrastive learning - column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer + column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer ) # Train and evaluate -trainer.freeze() # Freeze the head -trainer.train() # Train only the body - -# Unfreeze the head and freeze the body -> head-only training -trainer.unfreeze(keep_body_frozen=True) -# or -# Unfreeze the head and unfreeze the body -> end-to-end training -trainer.unfreeze(keep_body_frozen=False) - -trainer.train( - num_epochs=25, # The number of epochs to train the head or the whole model (body and head) - batch_size=16, - body_learning_rate=1e-5, # The body's learning rate - learning_rate=1e-2, # The head's learning rate - l2_weight=0.0, # Weight decay on **both** the body and head. If `None`, will use 0.01. -) +trainer.train() metrics = trainer.evaluate() # Push model to the Hub @@ -147,7 +136,7 @@ This will initialise a multilabel classification head from `sklearn` - the follo * `multi-output`: uses a `MultiOutputClassifier` head. * `classifier-chain`: uses a `ClassifierChain` head. -From here, you can instantiate a `SetFitTrainer` using the same example above, and train it as usual. +From here, you can instantiate a `Trainer` using the same example above, and train it as usual. #### Example using the differentiable `SetFitHead`: @@ -168,7 +157,6 @@ model = SetFitModel.from_pretrained( SetFit can also be applied to scenarios where no labels are available. To do so, create a synthetic dataset of training examples: ```python -from datasets import Dataset from setfit import get_templated_dataset candidate_labels = ["negative", "positive"] @@ -178,22 +166,22 @@ train_dataset = get_templated_dataset(candidate_labels=candidate_labels, sample_ This will create examples of the form `"This sentence is {}"`, where the `{}` is filled in with one of the candidate labels. From here you can train a SetFit model as usual: ```python -from setfit import SetFitModel, SetFitTrainer +from setfit import SetFitModel, Trainer model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") -trainer = SetFitTrainer( +trainer = Trainer( model=model, train_dataset=train_dataset ) trainer.train() ``` -We find this approach typically outperforms the [zero-shot pipeline](https://huggingface.co/docs/transformers/v4.24.0/en/main_classes/pipelines#transformers.ZeroShotClassificationPipeline) in 🤗 Transformers (based on MNLI with Bart), while being 5x faster to generate predictions with. +We find this approach typically outperforms the [zero-shot pipeline](https://huggingface.co/docs/transformers/v4.24.0/en/main_classes/pipelines#transformers.ZeroShotClassificationPipeline) in 🤗 Transformers (based on MNLI with BART), while being 5x faster to generate predictions with. ### Running hyperparameter search -`SetFitTrainer` provides a `hyperparameter_search()` method that you can use to find good hyperparameters for your data. To use this feature, first install the `optuna` backend: +`Trainer` provides a `hyperparameter_search()` method that you can use to find good hyperparameters for your data. To use this feature, first install the `optuna` backend: ```bash python -m pip install setfit[optuna] @@ -239,23 +227,23 @@ def hp_space(trial): # Training parameters **Note:** In practice, we found `num_iterations` to be the most important hyperparameter for the contrastive learning process. -The next step is to instantiate a `SetFitTrainer` and call `hyperparameter_search()`: +The next step is to instantiate a `Trainer` and call `hyperparameter_search()`: ```python from datasets import Dataset -from setfit import SetFitTrainer +from setfit import Trainer dataset = Dataset.from_dict( - {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} - ) + {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} +) -trainer = SetFitTrainer( +trainer = Trainer( train_dataset=dataset, eval_dataset=dataset, model_init=model_init, column_mapping={"text_new": "text", "label_new": "label"}, ) -best_run = trainer.hyperparameter_search(direction="maximize", hp_space=hp_space, n_trials=20) +best_run = trainer.hyperparameter_search(direction="maximize", hp_space=hp_space, n_trials=5) ``` Finally, you can apply the hyperparameters you found to the trainer, and lock in the optimal model, before training for @@ -272,9 +260,8 @@ If you have access to unlabeled data, you can use knowledge distillation to comp ```python from datasets import load_dataset -from sentence_transformers.losses import CosineSimilarityLoss - -from setfit import SetFitModel, SetFitTrainer, DistillationSetFitTrainer, sample_dataset +from setfit import SetFitModel, Trainer, DistillationTrainer, sample_dataset +from setfit.training_args import TrainingArguments # Load a dataset from the Hugging Face Hub dataset = load_dataset("ag_news") @@ -292,32 +279,35 @@ teacher_model = SetFitModel.from_pretrained( ) # Create trainer for teacher model -teacher_trainer = SetFitTrainer( +teacher_trainer = Trainer( model=teacher_model, train_dataset=train_dataset_teacher, eval_dataset=eval_dataset, - loss_class=CosineSimilarityLoss, ) # Train teacher model teacher_trainer.train() +teacher_metrics = teacher_trainer.evaluate() # Load small student model student_model = SetFitModel.from_pretrained("paraphrase-MiniLM-L3-v2") +args = TrainingArguments( + batch_size=16, + num_iterations=20, + num_epochs=1 +) + # Create trainer for knowledge distillation -student_trainer = DistillationSetFitTrainer( +student_trainer = DistillationTrainer( teacher_model=teacher_model, - train_dataset=train_dataset_student, student_model=student_model, + args=args, + train_dataset=train_dataset_student, eval_dataset=eval_dataset, - loss_class=CosineSimilarityLoss, - metric="accuracy", - batch_size=16, - num_iterations=20, - num_epochs=1, ) # Train student with knowledge distillation student_trainer.train() +student_metrics = student_trainer.evaluate() ``` \ No newline at end of file From deb57fffd03b733e049b8d4750a6dec3799c04b9 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Mar 2023 11:04:11 +0100 Subject: [PATCH 030/183] Remove references of SetFitTrainer --- README.md | 2 +- docs/source/en/api/trainer.mdx | 8 ++++---- docs/source/en/quickstart.mdx | 2 +- src/setfit/integrations.py | 4 ++-- src/setfit/trainer.py | 10 +++++----- src/setfit/trainer_distillation.py | 2 +- src/setfit/utils.py | 2 +- tests/exporters/test_onnx.py | 29 ++++++++++++++--------------- tests/test_trainer.py | 10 +++++----- tests/test_trainer_distillation.py | 2 +- 10 files changed, 35 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 71bddb45..93bc9efe 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The examples below provide a quick overview on the various features supported in `setfit` is integrated with the [Hugging Face Hub](https://huggingface.co/) and provides two main classes: * `SetFitModel`: a wrapper that combines a pretrained body from `sentence_transformers` and a classification head from either [`scikit-learn`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) or [`SetFitHead`](https://github.com/huggingface/setfit/blob/main/src/setfit/modeling.py) (a differentiable head built upon `PyTorch` with similar APIs to `sentence_transformers`). -* `SetFitTrainer`: a helper class that wraps the fine-tuning process of SetFit. +* `Trainer`: a helper class that wraps the fine-tuning process of SetFit. Here is an end-to-end example using a classification head from `scikit-learn`: diff --git a/docs/source/en/api/trainer.mdx b/docs/source/en/api/trainer.mdx index a51df833..4b605dc8 100644 --- a/docs/source/en/api/trainer.mdx +++ b/docs/source/en/api/trainer.mdx @@ -1,8 +1,8 @@ -# SetFitTrainer +# Trainer -[[autodoc]] SetFitTrainer +[[autodoc]] Trainer -# DistillationSetFitTrainer +# DistillationTrainer -[[autodoc]] DistillationSetFitTrainer \ No newline at end of file +[[autodoc]] DistillationTrainer \ No newline at end of file diff --git a/docs/source/en/quickstart.mdx b/docs/source/en/quickstart.mdx index 74bd75de..9e46933b 100644 --- a/docs/source/en/quickstart.mdx +++ b/docs/source/en/quickstart.mdx @@ -11,7 +11,7 @@ The examples below provide a quick overview on the various features supported in `setfit` is integrated with the [Hugging Face Hub](https://huggingface.co/) and provides two main classes: * `SetFitModel`: a wrapper that combines a pretrained body from `sentence_transformers` and a classification head from either [`scikit-learn`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) or [`SetFitHead`](https://github.com/huggingface/setfit/blob/main/src/setfit/modeling.py) (a differentiable head built upon `PyTorch` with similar APIs to `sentence_transformers`). -* `SetFitTrainer`: a helper class that wraps the fine-tuning process of SetFit. +* `Trainer`: a helper class that wraps the fine-tuning process of SetFit. Here is an end-to-end example using a classification head from `scikit-learn`: diff --git a/src/setfit/integrations.py b/src/setfit/integrations.py index 94d7161e..44b4858a 100644 --- a/src/setfit/integrations.py +++ b/src/setfit/integrations.py @@ -5,7 +5,7 @@ if TYPE_CHECKING: - from .trainer import SetFitTrainer + from .trainer import Trainer def is_optuna_available(): @@ -17,7 +17,7 @@ def default_hp_search_backend(): return "optuna" -def run_hp_search_optuna(trainer: "SetFitTrainer", n_trials: int, direction: str, **kwargs) -> BestRun: +def run_hp_search_optuna(trainer: "Trainer", n_trials: int, direction: str, **kwargs) -> BestRun: import optuna # Heavily inspired by transformers.integrations.run_hp_search_optuna diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 0fff050c..aa2d08d9 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -51,7 +51,7 @@ class Trainer: The evaluation dataset. model_init (`Callable[[], SetFitModel]`, *optional*): A function that instantiates the model to be used. If provided, each call to - [`~SetFitTrainer.train`] will start from a new instance of the model as given by this + [`~Trainer.train`] will start from a new instance of the model as given by this function when a `trial` is passed. metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): The metric to use for evaluation. If a string is provided, we treat it as the metric @@ -86,10 +86,10 @@ def __init__( if model_init is not None: model = self.call_model_init() else: - raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument.") + raise RuntimeError("`Trainer` requires either a `model` or `model_init` argument.") else: if model_init is not None: - raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument, but not both.") + raise RuntimeError("`Trainer` requires either a `model` or `model_init` argument, but not both.") self.model = model self.hp_search_backend = None @@ -115,7 +115,7 @@ def _validate_column_mapping(self, dataset: "Dataset") -> None: raise ValueError( f"SetFit expected the dataset to have the columns {sorted(self._REQUIRED_COLUMNS)}, " f"but only the columns {sorted(column_names)} were found. " - "Either make sure these columns are present, or specify which columns to use with column_mapping in SetFitTrainer." + "Either make sure these columns are present, or specify which columns to use with column_mapping in Trainer." ) if self.column_mapping is not None: missing_columns = self._REQUIRED_COLUMNS.difference(self.column_mapping.values()) @@ -432,7 +432,7 @@ def hyperparameter_search( - To use this method, you need to have provided a `model_init` when initializing your [`SetFitTrainer`]: we need to + To use this method, you need to have provided a `model_init` when initializing your [`Trainer`]: we need to reinitialize the model at each new run. diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 41bfa26d..f0ecb91e 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -40,7 +40,7 @@ class DistillationTrainer(Trainer): The evaluation dataset. model_init (`Callable[[], SetFitModel]`, *optional*): A function that instantiates the model to be used. If provided, each call to - [`~SetFitTrainer.train`] will start from a new instance of the model as given by this + [`~DistillationTrainer.train`] will start from a new instance of the model as given by this function when a `trial` is passed. metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): The metric to use for evaluation. If a string is provided, we treat it as the metric diff --git a/src/setfit/utils.py b/src/setfit/utils.py index 352f9c03..d75dc7cf 100644 --- a/src/setfit/utils.py +++ b/src/setfit/utils.py @@ -135,7 +135,7 @@ def summary(self) -> None: class BestRun(NamedTuple): """ - The best run found by a hyperparameter search (see [`~SetFitTrainer.hyperparameter_search`]). + The best run found by a hyperparameter search (see [`~Trainer.hyperparameter_search`]). Parameters: run_id (`str`): diff --git a/tests/exporters/test_onnx.py b/tests/exporters/test_onnx.py index 9eb68578..6b49b96b 100644 --- a/tests/exporters/test_onnx.py +++ b/tests/exporters/test_onnx.py @@ -8,7 +8,8 @@ from setfit import SetFitModel from setfit.data import get_templated_dataset from setfit.exporters.onnx import export_onnx -from setfit.trainer import SetFitTrainer +from setfit.trainer import Trainer +from setfit.training_args import TrainingArguments def test_export_onnx_sklearn_head(): @@ -63,25 +64,23 @@ def test_export_onnx_torch_head(out_features): model_path, use_differentiable_head=True, head_params={"out_features": out_features} ) - trainer = SetFitTrainer( + args = TrainingArguments( + num_iterations=15, + num_epochs=(1, 15), + batch_size=16, + body_learning_rate=(2e-5, 1e-5), + head_learning_rate=1e-2, + l2_weight=0.0, + end_to_end=True, + ) + trainer = Trainer( model=model, + args=args, train_dataset=dataset, eval_dataset=dataset, - num_iterations=15, column_mapping={"text": "text", "label": "label"}, ) - # Train and evaluate - trainer.freeze() # Freeze the head - trainer.train() # Train only the body - # Unfreeze the head and unfreeze the body -> end-to-end training - trainer.unfreeze(keep_body_frozen=False) - trainer.train( - num_epochs=15, - batch_size=16, - body_learning_rate=1e-5, - learning_rate=1e-2, - l2_weight=0.0, - ) + trainer.train() # Export the sklearn based model output_path = "model.onnx" diff --git a/tests/test_trainer.py b/tests/test_trainer.py index a964aaf2..33c6e14b 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -22,7 +22,7 @@ logging.enable_propagation() -class SetFitTrainerTest(TestCase): +class TrainerTest(TestCase): def setUp(self): self.model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") self.args = TrainingArguments(num_iterations=1) @@ -81,7 +81,7 @@ def test_trainer_raises_error_with_missing_text(self): expected_message = re.escape( "SetFit expected the dataset to have the columns ['label', 'text'], " "but only the columns ['extra_column', 'label'] were found. " - "Either make sure these columns are present, or specify which columns to use with column_mapping in SetFitTrainer." + "Either make sure these columns are present, or specify which columns to use with column_mapping in Trainer." ) with pytest.raises(ValueError, match=expected_message): trainer._validate_column_mapping(trainer.train_dataset) @@ -205,7 +205,7 @@ def test_raise_when_metric_value_is_invalid(self): trainer.evaluate() -class SetFitTrainerDifferentiableHeadTest(TestCase): +class TrainerDifferentiableHeadTest(TestCase): def setUp(self): self.dataset = Dataset.from_dict( {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} @@ -262,7 +262,7 @@ def test_trainer_max_length_is_smaller_than_max_acceptable_length(self): raise AssertionError(e) -class SetFitTrainerMultilabelTest(TestCase): +class TrainerMultilabelTest(TestCase): def setUp(self): self.model = SetFitModel.from_pretrained( "sentence-transformers/paraphrase-albert-small-v2", multi_target_strategy="one-vs-rest" @@ -302,7 +302,7 @@ def compute_metrics(y_pred, y_test): ) -class SetFitTrainerMultilabelDifferentiableTest(TestCase): +class TrainerMultilabelDifferentiableTest(TestCase): def setUp(self): self.model = SetFitModel.from_pretrained( "sentence-transformers/paraphrase-albert-small-v2", diff --git a/tests/test_trainer_distillation.py b/tests/test_trainer_distillation.py index 71b81e0a..a9d80c47 100644 --- a/tests/test_trainer_distillation.py +++ b/tests/test_trainer_distillation.py @@ -8,7 +8,7 @@ from setfit.training_args import TrainingArguments -class DistillationSetFitTrainerTest(TestCase): +class DistillationTrainerTest(TestCase): def setUp(self): self.teacher_model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") self.student_model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-MiniLM-L3-v2") From 46922d5d72be1ab400733a9ce3f0abf49e5133e3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 6 Mar 2023 11:13:31 +0100 Subject: [PATCH 031/183] Update expected test output --- tests/test_deprecated_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_deprecated_trainer.py b/tests/test_deprecated_trainer.py index 3bbcf587..b8f122d3 100644 --- a/tests/test_deprecated_trainer.py +++ b/tests/test_deprecated_trainer.py @@ -86,7 +86,7 @@ def test_trainer_raises_error_with_missing_text(self): expected_message = re.escape( "SetFit expected the dataset to have the columns ['label', 'text'], " "but only the columns ['extra_column', 'label'] were found. " - "Either make sure these columns are present, or specify which columns to use with column_mapping in SetFitTrainer." + "Either make sure these columns are present, or specify which columns to use with column_mapping in Trainer." ) with pytest.raises(ValueError, match=expected_message): trainer._validate_column_mapping(trainer.train_dataset) From b0f9f582e3782d3aa3c54341fb48c588c29b25dd Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 19 Apr 2023 09:04:34 +0200 Subject: [PATCH 032/183] Remove unused pipeline --- src/setfit/pipeline.py | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 src/setfit/pipeline.py diff --git a/src/setfit/pipeline.py b/src/setfit/pipeline.py deleted file mode 100644 index 51e551ff..00000000 --- a/src/setfit/pipeline.py +++ /dev/null @@ -1,12 +0,0 @@ -from .modeling import SKLearnWrapper - - -class SetFitPipeline: - def __init__(self, model_name_or_path) -> None: - base_model = SKLearnWrapper() - base_model.load(model_name_or_path) - self.model = base_model - - def __call__(self, inputs, *args, **kwargs): - model_outputs = self.model.predict(inputs) - return model_outputs From 339f332e820c4d1f78194abbee4269db19a584d4 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 19 Apr 2023 09:05:35 +0200 Subject: [PATCH 033/183] Execute deprecations --- src/setfit/data.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/setfit/data.py b/src/setfit/data.py index ce809bbd..8128e376 100644 --- a/src/setfit/data.py +++ b/src/setfit/data.py @@ -21,15 +21,6 @@ SAMPLE_SIZES = [2, 4, 8, 16, 32, 64] -def get_augmented_samples(*args, **kwargs) -> None: - warnings.warn( - "`get_augmented_samples` has been deprecated and will be removed in v1.0.0 of SetFit. " - "Please use `get_templated_dataset` instead.", - DeprecationWarning, - stacklevel=2, - ) - - def get_templated_dataset( dataset: Optional[Dataset] = None, candidate_labels: Optional[List[str]] = None, @@ -115,15 +106,6 @@ def get_templated_dataset( return dataset -def add_templated_examples(*args, **kwargs) -> None: - warnings.warn( - "`add_templated_examples` has been deprecated and will be removed in v1.0.0 of SetFit. " - "Please use `get_templated_dataset` instead.", - DeprecationWarning, - stacklevel=2, - ) - - def get_candidate_labels(dataset_name: str, label_names_column: str = "label_text") -> List[str]: dataset = load_dataset(dataset_name, split="train") From 9e0bf78d1c86188e14f17023bbbf4819abce98e9 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 19 Apr 2023 09:16:53 +0200 Subject: [PATCH 034/183] Stop importing now-removed function --- src/setfit/__init__.py | 2 +- src/setfit/data.py | 1 - src/setfit/modeling.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index 02dad09b..c36d630d 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -2,7 +2,7 @@ import warnings -from .data import add_templated_examples, get_templated_dataset, sample_dataset +from .data import get_templated_dataset, sample_dataset from .modeling import SetFitHead, SetFitModel from .trainer import SetFitTrainer, Trainer from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer diff --git a/src/setfit/data.py b/src/setfit/data.py index 8128e376..44507018 100644 --- a/src/setfit/data.py +++ b/src/setfit/data.py @@ -1,4 +1,3 @@ -import warnings from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union import pandas as pd diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index e5f23cf1..1a0b947e 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -1,6 +1,6 @@ import os -import warnings import tempfile +import warnings from dataclasses import dataclass from pathlib import Path from typing import Dict, List, Optional, Tuple, Union From ecabbcfbdc842cbc2b3c7592a933d9fb9efd7bd5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 6 Jul 2023 22:22:57 +0200 Subject: [PATCH 035/183] Initial setup for logging & callbacks --- .gitignore | 4 + src/setfit/sentence_transformer.py | 296 +++++++++++++++++++++++++++++ src/setfit/trainer.py | 179 +++++++++++++---- src/setfit/training_args.py | 116 +++++++++++ 4 files changed, 553 insertions(+), 42 deletions(-) create mode 100644 src/setfit/sentence_transformer.py diff --git a/.gitignore b/.gitignore index a13745c3..6e89ff50 100644 --- a/.gitignore +++ b/.gitignore @@ -149,3 +149,7 @@ scripts/tfew/run_tmux.sh # macOS .DS_Store .vscode/settings.json + +# Common SetFit Trainer logging folders +wandb +runs/ \ No newline at end of file diff --git a/src/setfit/sentence_transformer.py b/src/setfit/sentence_transformer.py new file mode 100644 index 00000000..b24b64a1 --- /dev/null +++ b/src/setfit/sentence_transformer.py @@ -0,0 +1,296 @@ +import time +from typing import Callable, Dict, Iterable, Tuple, Type +from sentence_transformers import SentenceTransformer +from sentence_transformers.evaluation import SentenceEvaluator +from sentence_transformers.util import batch_to_device +from torch.utils.data import DataLoader +from torch import nn +from torch.optim import Optimizer +import torch +from tqdm.autonotebook import trange, tqdm +from transformers.trainer_callback import TrainerState, TrainerControl, CallbackHandler +from transformers.trainer_utils import speed_metrics + +from setfit.training_args import TrainingArguments + + +def log(args: TrainingArguments, callback_handler: CallbackHandler, state: TrainerState, control: TrainerControl, logs: Dict[str, float]) -> None: + """ + Log `logs` on the various objects watching training. + + Subclass and override this method to inject custom behavior. + + Args: + logs (`Dict[str, float]`): + The values to log. + """ + if state.epoch is not None: + logs["epoch"] = round(state.epoch, 2) + + output = {**logs, **{"step": state.global_step}} + state.log_history.append(output) + return callback_handler.on_log(args, state, control, logs) + + +def fit( + model_body: SentenceTransformer, + train_dataloader: DataLoader, + eval_dataloader: DataLoader, + loss_func: nn.Module, + args: TrainingArguments, + callback_handler: CallbackHandler, + state: TrainerState, + control: TrainerControl, + # evaluator: SentenceEvaluator = None, # <- remove + # epochs: int = 1, # <- remove + # steps_per_epoch=None, # <- remove? + scheduler: str = "WarmupLinear", + warmup_steps: int = 10000, + optimizer_class: Type[Optimizer] = torch.optim.AdamW, + optimizer_params: Dict[str, object] = {"lr": 2e-5}, + weight_decay: float = 0.01, + output_path: str = None, + save_best_model: bool = True, + max_grad_norm: float = 1, + use_amp: bool = False, + # callback: Callable[[float, int, int], None] = None, # <- remove + show_progress_bar: bool = True, + checkpoint_path: str = None, # <- remove + checkpoint_save_steps: int = 500, # <- remove + checkpoint_save_total_limit: int = 0, # <- remove +): + """ + Train the model with the given training objective + Each training objective is sampled in turn for one batch. + We sample only as many batches from each objective as there are in the smallest one + to make sure of equal training with each dataset. + + :param train_objectives: Tuples of (DataLoader, LossFunction). Pass more than one for multi-task learning + :param evaluator: An evaluator (sentence_transformers.evaluation) evaluates the model performance during training on held-out dev data. It is used to determine the best model that is saved to disc. + :param epochs: Number of epochs for training + :param steps_per_epoch: Number of training steps per epoch. If set to None (default), one epoch is equal the DataLoader size from train_objectives. + :param scheduler: Learning rate scheduler. Available schedulers: constantlr, warmupconstant, warmuplinear, warmupcosine, warmupcosinewithhardrestarts + :param warmup_steps: Behavior depends on the scheduler. For WarmupLinear (default), the learning rate is increased from o up to the maximal learning rate. After these many training steps, the learning rate is decreased linearly back to zero. + :param optimizer_class: Optimizer + :param optimizer_params: Optimizer parameters + :param weight_decay: Weight decay for model parameters + :param evaluation_steps: If > 0, evaluate the model using evaluator after each number of training steps + :param output_path: Storage path for the model and evaluation files + :param save_best_model: If true, the best model (according to evaluator) is stored at output_path + :param max_grad_norm: Used for gradient normalization. + :param use_amp: Use Automatic Mixed Precision (AMP). Only for Pytorch >= 1.6.0 + :param callback: Callback function that is invoked after each evaluation. + It must accept the following three parameters in this order: + `score`, `epoch`, `steps` + :param show_progress_bar: If True, output a tqdm progress bar + :param checkpoint_path: Folder to save checkpoints during training + :param checkpoint_save_steps: Will save a checkpoint after so many steps + :param checkpoint_save_total_limit: Total number of checkpoints to store + """ + + """ + ##Add info to model card + # info_loss_functions = "\n".join(["- {} with {} training examples".format(str(loss), len(dataloader)) for dataloader, loss in train_objectives]) + info_loss_functions = [] + for dataloader, loss in train_objectives: + info_loss_functions.extend(ModelCardTemplate.get_train_objective_info(dataloader, loss)) + info_loss_functions = "\n\n".join([text for text in info_loss_functions]) + + info_fit_parameters = json.dumps( + { + "evaluator": fullname(evaluator), + "epochs": epochs, + "steps_per_epoch": steps_per_epoch, + "scheduler": scheduler, + "warmup_steps": warmup_steps, + "optimizer_class": str(optimizer_class), + "optimizer_params": optimizer_params, + "weight_decay": weight_decay, + "evaluation_steps": evaluation_steps, + "max_grad_norm": max_grad_norm, + }, + indent=4, + sort_keys=True, + ) + self._model_card_text = None + self._model_card_vars["{TRAINING_SECTION}"] = ModelCardTemplate.__TRAINING_SECTION__.replace( + "{LOSS_FUNCTIONS}", info_loss_functions + ).replace("{FIT_PARAMETERS}", info_fit_parameters) + """ + # TODO: Loading best model + # TODO: Saving/checkpointing + # TODO: args.gradient_accumulation_steps + # TODO: fp16/bf16, etc. + + state.epoch = 0 + start_time = time.time() + # TODO: Add max_steps via args.max_steps here? + state.max_steps = len(train_dataloader) * args.embedding_num_epochs + control = callback_handler.on_train_begin(args, state, control) + + if use_amp: + from torch.cuda.amp import autocast + + scaler = torch.cuda.amp.GradScaler() + + model_body.to(model_body._target_device) + + # Use smart batching + train_dataloader.collate_fn = model_body.smart_batching_collate + if eval_dataloader: + eval_dataloader.collate_fn = model_body.smart_batching_collate + + loss_func.to(model_body._target_device) + + model_body.best_score = -9999999 + + steps_per_epoch = len(train_dataloader) + num_train_steps = int(steps_per_epoch * args.embedding_num_epochs) + + # Prepare optimizers + param_optimizer = list(loss_func.named_parameters()) + + no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], + "weight_decay": weight_decay, + }, + {"params": [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], "weight_decay": 0.0}, + ] + + optimizer = optimizer_class(optimizer_grouped_parameters, **optimizer_params) + scheduler_obj = model_body._get_scheduler( + optimizer, scheduler=scheduler, warmup_steps=warmup_steps, t_total=num_train_steps + ) + + data_iterator = iter(train_dataloader) + + skip_scheduler = False + for epoch in range(args.embedding_num_epochs): + control = callback_handler.on_epoch_begin(args, state, control) + + training_steps = 0 + + loss_func.zero_grad() + loss_func.train() + + for step in range(steps_per_epoch): + control = callback_handler.on_step_begin(args, state, control) + + try: + data = next(data_iterator) + except StopIteration: + data_iterator = iter(train_dataloader) + data = next(data_iterator) + + features, labels = data + labels = labels.to(model_body._target_device) + features = list(map(lambda batch: batch_to_device(batch, model_body._target_device), features)) + + if use_amp: + with autocast(): + loss_value = loss_func(features, labels) + + scale_before_step = scaler.get_scale() + scaler.scale(loss_value).backward() + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(loss_func.parameters(), max_grad_norm) + scaler.step(optimizer) + scaler.update() + + skip_scheduler = scaler.get_scale() != scale_before_step + else: + loss_value = loss_func(features, labels) + loss_value.backward() + torch.nn.utils.clip_grad_norm_(loss_func.parameters(), max_grad_norm) + optimizer.step() + + optimizer.zero_grad() + + if not skip_scheduler: + scheduler_obj.step() + + training_steps += 1 + + state.global_step += 1 + state.epoch = epoch + (step + 1) / steps_per_epoch + control = callback_handler.on_step_end(args, state, control) + + if control.should_log: + learning_rate = scheduler_obj.get_last_lr()[0] + metrics = {"embedding_loss": round(loss_value.item(), 4), "learning_rate": learning_rate} + control = log(args, callback_handler, state, control, metrics) + + if control.should_evaluate: + # self._eval_during_training(evaluator, output_path, save_best_model, epoch, training_steps, callback) + eval_loss = evaluate_with_loss(model_body, eval_dataloader, loss_func, show_progress_bar, use_amp) + learning_rate = scheduler_obj.get_last_lr()[0] + metrics = {"eval_embedding_loss": round(eval_loss, 4), "learning_rate": learning_rate} + control = log(args, callback_handler, state, control, metrics) + control = callback_handler.on_evaluate(args, state, control, metrics) + if state.best_metric is None or eval_loss < state.best_metric: + state.best_metric = eval_loss + + loss_func.zero_grad() + loss_func.train() + + if ( + checkpoint_path is not None + and checkpoint_save_steps is not None + and checkpoint_save_steps > 0 + and state.global_step % checkpoint_save_steps == 0 + ): + model_body._save_checkpoint(checkpoint_path, checkpoint_save_total_limit, state.global_step) + + if control.should_epoch_stop or control.should_training_stop: + break + + control = callback_handler.on_epoch_end(args, state, control) + + if control.should_training_stop: + break + + if output_path is not None: # No evaluator, but output path: save final model version + model_body.save(output_path) + + if checkpoint_path is not None: + model_body._save_checkpoint(checkpoint_path, checkpoint_save_total_limit, state.global_step) + + control = callback_handler.on_train_end(args, state, control) + + num_train_samples = state.max_steps * args.embedding_batch_size # * args.gradient_accumulation_steps + metrics = speed_metrics("train", start_time, num_samples=num_train_samples, num_steps=state.max_steps) + # TODO: This isn't always printed + log(args, callback_handler, state, control, metrics) + + # eval_start_time = time.time() + # num_eval_samples = len(eval_dataloader) # args.max_steps * args.embedding_batch_size # * args.gradient_accumulation_steps + # num_eval_steps = num_eval_samples * args.embedding_num_epochs + # metrics.update(speed_metrics("eval", eval_start_time, num_samples=num_eval_samples, num_steps=num_eval_steps)) + + +def evaluate_with_loss(model_body: SentenceTransformer, eval_dataloader: DataLoader, loss_func: nn.Module, show_progress_bar: bool, use_amp: bool): + model_body.eval() + + if use_amp: + from torch.cuda.amp import autocast + + scaler = torch.cuda.amp.GradScaler() + + losses = [] + for data in tqdm(iter(eval_dataloader), leave=False): + features, labels = data + labels = labels.to(model_body._target_device) + features = list(map(lambda batch: batch_to_device(batch, model_body._target_device), features)) + + if use_amp: + with autocast(): + loss_value = loss_func(features, labels) + + losses.append(scaler.scale(loss_value).item()) + else: + losses.append(loss_func(features, labels).item()) + + model_body.train() + return sum(losses) / len(losses) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 5fb4b37f..32ad05c3 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -2,6 +2,8 @@ import warnings from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +from setfit import sentence_transformer + # Google Colab runs on Python 3.7, so we need this to be compatible try: @@ -18,6 +20,18 @@ from torch.utils.data import DataLoader from tqdm.auto import trange from transformers.trainer_utils import HPSearchBackend, default_compute_objective, number_of_arguments, set_seed +import transformers +from transformers.trainer_callback import ( + CallbackHandler, + DefaultFlowCallback, + PrinterCallback, + ProgressCallback, + TrainerCallback, + TrainerState, + TrainerControl, +) +from transformers.integrations import get_reporting_integration_callbacks, get_available_reporting_integrations +from transformers.utils.import_utils import is_in_notebook from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna @@ -37,6 +51,15 @@ logger = logging.get_logger(__name__) +DEFAULT_CALLBACKS = [DefaultFlowCallback] +DEFAULT_PROGRESS_CALLBACK = ProgressCallback + +if is_in_notebook(): + from transformers.utils.notebook import NotebookProgressCallback + + DEFAULT_PROGRESS_CALLBACK = NotebookProgressCallback + + class Trainer: """Trainer to train a SetFit model. @@ -77,9 +100,10 @@ def __init__( model_init: Optional[Callable[[], "SetFitModel"]] = None, metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", metric_kwargs: Optional[Dict[str, Any]] = None, + callbacks: Optional[List[TrainerCallback]] = None, column_mapping: Optional[Dict[str, str]] = None, ): - self.args = args + self.args = args or TrainingArguments() self.train_dataset = train_dataset self.eval_dataset = eval_dataset self.model_init = model_init @@ -99,6 +123,57 @@ def __init__( self.model = model self.hp_search_backend = None + # Setup the callbacks + default_callbacks = DEFAULT_CALLBACKS + get_reporting_integration_callbacks(self.args.report_to) + callbacks = default_callbacks if callbacks is None else default_callbacks + callbacks + # TODO: Observe optimizer and scheduler by wrapping SentenceTransformer._get_scheduler + self.callback_handler = CallbackHandler( + callbacks, self.model.model_body, self.model.model_body.tokenizer, None, None + ) + self.state = TrainerState() + self.control = TrainerControl() + self.add_callback(DEFAULT_PROGRESS_CALLBACK if self.args.show_progress_bar else PrinterCallback) + + self.control = self.callback_handler.on_init_end(args, self.state, self.control) + + def add_callback(self, callback): + """ + Add a callback to the current list of [`~transformer.TrainerCallback`]. + + Args: + callback (`type` or [`~transformer.TrainerCallback`]): + A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + first case, will instantiate a member of that class. + """ + self.callback_handler.add_callback(callback) + + def pop_callback(self, callback): + """ + Remove a callback from the current list of [`~transformer.TrainerCallback`] and returns it. + + If the callback is not found, returns `None` (and no error is raised). + + Args: + callback (`type` or [`~transformer.TrainerCallback`]): + A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + first case, will pop the first member of that class found in the list of callbacks. + + Returns: + [`~transformer.TrainerCallback`]: The callback removed, if found. + """ + return self.callback_handler.pop_callback(callback) + + def remove_callback(self, callback): + """ + Remove a callback from the current list of [`~transformer.TrainerCallback`]. + + Args: + callback (`type` or [`~transformer.TrainerCallback`]): + A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + first case, will remove the first member of that class found in the list of callbacks. + """ + self.callback_handler.remove_callback(callback) + def _validate_column_mapping(self, dataset: "Dataset") -> None: """ Validates the provided column mapping against the dataset. @@ -275,20 +350,28 @@ def train( f"Training requires a `train_dataset` given to the `{self.__class__.__name__}` initialization." ) - self._validate_column_mapping(self.train_dataset) - train_dataset = self.train_dataset - if self.column_mapping is not None: - logger.info("Applying column mapping to training dataset") - train_dataset = self._apply_column_mapping(self.train_dataset, self.column_mapping) + parameters = [] + for dataset, dataset_name in [(self.train_dataset, "training"), (self.eval_dataset, "evaluation")]: + if dataset is None: + continue - x_train: List[str] = train_dataset["text"] - y_train: List[int] = train_dataset["label"] + self._validate_column_mapping(dataset) + if self.column_mapping is not None: + logger.info(f"Applying column mapping to {dataset_name} dataset") + dataset = self._apply_column_mapping(dataset, self.column_mapping) - self.train_embeddings(x_train, y_train, args) - self.train_classifier(x_train, y_train, args) + parameters.extend([dataset["text"], dataset["label"]]) + + self.train_embeddings(*parameters, args=args) + self.train_classifier(*parameters[:2], args=args) def train_embeddings( - self, x_train: List[str], y_train: Union[List[int], List[List[int]]], args: Optional[TrainingArguments] = None + self, + x_train: List[str], + y_train: Union[List[int], List[List[int]]], + x_eval: List[str], + y_eval: Union[List[int], List[List[int]]], + args: Optional[TrainingArguments] = None, ) -> None: """ Method to perform the embedding phase: finetuning the `SentenceTransformer` body. @@ -301,6 +384,36 @@ def train_embeddings( """ args = args or self.args or TrainingArguments() + train_dataloader, loss_func, batch_size = self.get_dataloader(x_train, y_train, args=args) + if x_eval is not None: + eval_dataloader, _, _ = self.get_dataloader(x_eval, y_eval, args=args) + else: + eval_dataloader = None + + total_train_steps = len(train_dataloader) * args.embedding_num_epochs + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataloader)}") + logger.info(f" Num epochs = {args.embedding_num_epochs}") + logger.info(f" Total optimization steps = {total_train_steps}") + logger.info(f" Total train batch size = {batch_size}") + + warmup_steps = math.ceil(total_train_steps * args.warmup_proportion) + sentence_transformer.fit( + self.model.model_body, + train_dataloader=train_dataloader, + loss_func=loss_func, + eval_dataloader=eval_dataloader, + args=args, + callback_handler=self.callback_handler, + state=self.state, + control=self.control, + optimizer_params={"lr": args.body_embedding_learning_rate}, + warmup_steps=warmup_steps, + show_progress_bar=args.show_progress_bar, + use_amp=args.use_amp, + ) + + def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], args: TrainingArguments): # sentence-transformers adaptation if args.loss in [ losses.BatchAllTripletLoss, @@ -309,56 +422,38 @@ def train_embeddings( losses.BatchHardSoftMarginTripletLoss, SupConLoss, ]: - train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] - train_data_sampler = SentenceLabelDataset(train_examples, samples_per_label=args.samples_per_label) + examples = [InputExample(texts=[text], label=label) for text, label in zip(x, y)] + data_sampler = SentenceLabelDataset(examples, samples_per_label=args.samples_per_label) - batch_size = min(args.embedding_batch_size, len(train_data_sampler)) - train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) + batch_size = min(args.embedding_batch_size, len(data_sampler)) + dataloader = DataLoader(data_sampler, batch_size=batch_size, drop_last=True) if args.loss is losses.BatchHardSoftMarginTripletLoss: - train_loss = args.loss( + loss = args.loss( model=self.model.model_body, distance_metric=args.distance_metric, ) elif args.loss is SupConLoss: - train_loss = args.loss(model=self.model.model_body) + loss = args.loss(model=self.model.model_body) else: - train_loss = args.loss( + loss = args.loss( model=self.model.model_body, distance_metric=args.distance_metric, margin=args.margin, ) else: - train_examples = [] + examples = [] for _ in trange(args.num_iterations, desc="Generating Training Pairs", disable=not args.show_progress_bar): if self.model.multi_target_strategy is not None: - train_examples = sentence_pairs_generation_multilabel( - np.array(x_train), np.array(y_train), train_examples - ) + examples = sentence_pairs_generation_multilabel(np.array(x), np.array(y), examples) else: - train_examples = sentence_pairs_generation(np.array(x_train), np.array(y_train), train_examples) + examples = sentence_pairs_generation(np.array(x), np.array(y), examples) batch_size = args.embedding_batch_size - train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) - train_loss = args.loss(self.model.model_body) - - total_train_steps = len(train_dataloader) * args.embedding_num_epochs - logger.info("***** Running training *****") - logger.info(f" Num examples = {len(train_examples)}") - logger.info(f" Num epochs = {args.embedding_num_epochs}") - logger.info(f" Total optimization steps = {total_train_steps}") - logger.info(f" Total train batch size = {batch_size}") - - warmup_steps = math.ceil(total_train_steps * args.warmup_proportion) - self.model.model_body.fit( - train_objectives=[(train_dataloader, train_loss)], - epochs=args.embedding_num_epochs, - optimizer_params={"lr": args.body_embedding_learning_rate}, - warmup_steps=warmup_steps, - show_progress_bar=args.show_progress_bar, - use_amp=args.use_amp, - ) + dataloader = DataLoader(examples, shuffle=True, batch_size=batch_size) + loss = args.loss(self.model.model_body) + return dataloader, loss, batch_size def train_classifier( self, x_train: List[str], y_train: Union[List[int], List[List[int]]], args: Optional[TrainingArguments] = None diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index ed3cbbdd..2a1aeb3b 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -3,9 +3,15 @@ import inspect from copy import copy from dataclasses import dataclass, field, fields +import json from typing import Any, Callable, Dict, Optional, Tuple, Union from sentence_transformers import losses +import torch +from transformers.integrations import get_available_reporting_integrations +from transformers.training_args import default_logdir +from transformers.utils import is_torch_available +from transformers import IntervalStrategy @dataclass @@ -67,6 +73,53 @@ class TrainingArguments: Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use the [`~SetTrainer.model_init`] function to instantiate the model if it has some randomly initialized parameters. + report_to (`str` or `List[str]`, *optional*, defaults to `"all"`): + The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`, + `"comet_ml"`, `"mlflow"`, `"neptune"`, `"tensorboard"`,`"clearml"` and `"wandb"`. Use `"all"` to report to + all integrations installed, `"none"` for no integrations. + run_name (`str`, *optional*): + A descriptor for the run. Typically used for [wandb](https://www.wandb.com/) and + [mlflow](https://www.mlflow.org/) logging. + logging_dir (`str`, *optional*): + [TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to + *runs/**CURRENT_DATETIME_HOSTNAME***. + logging_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): + The logging strategy to adopt during training. Possible values are: + + - `"no"`: No logging is done during training. + - `"epoch"`: Logging is done at the end of each epoch. + - `"steps"`: Logging is done every `logging_steps`. + + logging_first_step (`bool`, *optional*, defaults to `False`): + Whether to log and evaluate the first `global_step` or not. + logging_steps (`int`, *optional*, defaults to 500): + Number of update steps between two logs if `logging_strategy="steps"`. + evaluation_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`): + The evaluation strategy to adopt during training. Possible values are: + + - `"no"`: No evaluation is done during training. + - `"steps"`: Evaluation is done (and logged) every `eval_steps`. + - `"epoch"`: Evaluation is done at the end of each epoch. + + eval_steps (`int`, *optional*): + Number of update steps between two evaluations if `evaluation_strategy="steps"`. Will default to the same + value as `logging_steps` if not set. + eval_delay (`float`, *optional*): + Number of epochs or steps to wait for before the first evaluation can be performed, depending on the + evaluation_strategy. + + save_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): + The checkpoint save strategy to adopt during training. Possible values are: + + - `"no"`: No save is done during training. + - `"epoch"`: Save is done at the end of each epoch. + - `"steps"`: Save is done every `save_steps`. + save_steps (`int`, *optional*, defaults to 500): + Number of updates steps before two checkpoint saves if `save_strategy="steps"`. + save_total_limit (`int`, *optional*): + If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in + `output_dir`. + """ # batch_size is only used to conveniently set `embedding_batch_size` and `classifier_batch_size` @@ -107,6 +160,26 @@ class TrainingArguments: show_progress_bar: bool = True seed: int = 42 + # Logging & callbacks + report_to: str = "all" + run_name: Optional[str] = None + logging_dir: Optional[str] = None + logging_strategy: str = "steps" + logging_first_step: bool = False + logging_steps: int = 5 + + evaluation_strategy: str = "steps" + eval_steps: Optional[int] = None + eval_delay: int = 0 + + save_strategy: str = "steps" + save_steps: int = 500 + save_total_limit: Optional[int] = None + + load_best_model_at_end: bool = True + metric_for_best_model: str = field(default="embedding_loss", repr=False) + greater_is_better: bool = field(default=False, repr=False) + def __post_init__(self) -> None: # Set `self.embedding_batch_size` and `self.classifier_batch_size` using values from `self.batch_size` if isinstance(self.batch_size, int): @@ -138,6 +211,29 @@ def __post_init__(self) -> None: f"warmup_proportion must be greater than or equal to 0.0 and less than or equal to 1.0! But it was: {self.warmup_proportion}" ) + if self.report_to in (None, "all"): + self.report_to = get_available_reporting_integrations() + + if self.logging_dir is None: + self.logging_dir = default_logdir() + + self.logging_strategy = IntervalStrategy(self.logging_strategy) + self.evaluation_strategy = IntervalStrategy(self.evaluation_strategy) + + # eval_steps has to be defined and non-zero, fallbacks to logging_steps if the latter is non-zero + if self.evaluation_strategy == IntervalStrategy.STEPS and (self.eval_steps is None or self.eval_steps == 0): + if self.logging_steps > 0: + self.eval_steps = self.logging_steps + else: + raise ValueError( + f"evaluation strategy {self.evaluation_strategy} requires either non-zero `eval_steps` or" + " `logging_steps`" + ) + + # logging_steps must be non-zero for logging_strategy that is other than 'no' + if self.logging_strategy == IntervalStrategy.STEPS and self.logging_steps == 0: + raise ValueError(f"logging strategy {self.logging_strategy} requires non-zero --logging_steps") + def to_dict(self) -> Dict[str, Any]: # filter out fields that are defined as field(init=False) return {field.name: getattr(self, field.name) for field in fields(self) if field.init} @@ -153,3 +249,23 @@ def copy(self) -> TrainingArguments: def update(self, arguments: Dict[str, Any], ignore_extra: bool = False) -> TrainingArguments: return TrainingArguments.from_dict({**self.to_dict(), **arguments}, ignore_extra=ignore_extra) + + def to_json_string(self): + """ + Serializes this instance to a JSON string. + """ + # TODO: This needs to be improved + return json.dumps({key: str(value) for key, value in self.to_dict().items()}, indent=2) + + def to_sanitized_dict(self) -> Dict[str, Any]: + """ + Sanitized serialization to use with TensorBoard’s hparams + """ + d = self.to_dict() + d = {**d, **{"train_batch_size": self.embedding_batch_size, "eval_batch_size": self.embedding_batch_size}} + + valid_types = [bool, int, float, str] + if is_torch_available(): + valid_types.append(torch.Tensor) + + return {k: v if type(v) in valid_types else str(v) for k, v in d.items()} From 6e6720b8054b7f92ca48cf47a544b7a84f6cf249 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 6 Jul 2023 23:28:31 +0200 Subject: [PATCH 036/183] Move sentence-transformer training into trainer.py --- src/setfit/sentence_transformer.py | 296 ----------------------------- src/setfit/trainer.py | 239 ++++++++++++++++++++--- src/setfit/training_args.py | 12 +- 3 files changed, 222 insertions(+), 325 deletions(-) delete mode 100644 src/setfit/sentence_transformer.py diff --git a/src/setfit/sentence_transformer.py b/src/setfit/sentence_transformer.py deleted file mode 100644 index b24b64a1..00000000 --- a/src/setfit/sentence_transformer.py +++ /dev/null @@ -1,296 +0,0 @@ -import time -from typing import Callable, Dict, Iterable, Tuple, Type -from sentence_transformers import SentenceTransformer -from sentence_transformers.evaluation import SentenceEvaluator -from sentence_transformers.util import batch_to_device -from torch.utils.data import DataLoader -from torch import nn -from torch.optim import Optimizer -import torch -from tqdm.autonotebook import trange, tqdm -from transformers.trainer_callback import TrainerState, TrainerControl, CallbackHandler -from transformers.trainer_utils import speed_metrics - -from setfit.training_args import TrainingArguments - - -def log(args: TrainingArguments, callback_handler: CallbackHandler, state: TrainerState, control: TrainerControl, logs: Dict[str, float]) -> None: - """ - Log `logs` on the various objects watching training. - - Subclass and override this method to inject custom behavior. - - Args: - logs (`Dict[str, float]`): - The values to log. - """ - if state.epoch is not None: - logs["epoch"] = round(state.epoch, 2) - - output = {**logs, **{"step": state.global_step}} - state.log_history.append(output) - return callback_handler.on_log(args, state, control, logs) - - -def fit( - model_body: SentenceTransformer, - train_dataloader: DataLoader, - eval_dataloader: DataLoader, - loss_func: nn.Module, - args: TrainingArguments, - callback_handler: CallbackHandler, - state: TrainerState, - control: TrainerControl, - # evaluator: SentenceEvaluator = None, # <- remove - # epochs: int = 1, # <- remove - # steps_per_epoch=None, # <- remove? - scheduler: str = "WarmupLinear", - warmup_steps: int = 10000, - optimizer_class: Type[Optimizer] = torch.optim.AdamW, - optimizer_params: Dict[str, object] = {"lr": 2e-5}, - weight_decay: float = 0.01, - output_path: str = None, - save_best_model: bool = True, - max_grad_norm: float = 1, - use_amp: bool = False, - # callback: Callable[[float, int, int], None] = None, # <- remove - show_progress_bar: bool = True, - checkpoint_path: str = None, # <- remove - checkpoint_save_steps: int = 500, # <- remove - checkpoint_save_total_limit: int = 0, # <- remove -): - """ - Train the model with the given training objective - Each training objective is sampled in turn for one batch. - We sample only as many batches from each objective as there are in the smallest one - to make sure of equal training with each dataset. - - :param train_objectives: Tuples of (DataLoader, LossFunction). Pass more than one for multi-task learning - :param evaluator: An evaluator (sentence_transformers.evaluation) evaluates the model performance during training on held-out dev data. It is used to determine the best model that is saved to disc. - :param epochs: Number of epochs for training - :param steps_per_epoch: Number of training steps per epoch. If set to None (default), one epoch is equal the DataLoader size from train_objectives. - :param scheduler: Learning rate scheduler. Available schedulers: constantlr, warmupconstant, warmuplinear, warmupcosine, warmupcosinewithhardrestarts - :param warmup_steps: Behavior depends on the scheduler. For WarmupLinear (default), the learning rate is increased from o up to the maximal learning rate. After these many training steps, the learning rate is decreased linearly back to zero. - :param optimizer_class: Optimizer - :param optimizer_params: Optimizer parameters - :param weight_decay: Weight decay for model parameters - :param evaluation_steps: If > 0, evaluate the model using evaluator after each number of training steps - :param output_path: Storage path for the model and evaluation files - :param save_best_model: If true, the best model (according to evaluator) is stored at output_path - :param max_grad_norm: Used for gradient normalization. - :param use_amp: Use Automatic Mixed Precision (AMP). Only for Pytorch >= 1.6.0 - :param callback: Callback function that is invoked after each evaluation. - It must accept the following three parameters in this order: - `score`, `epoch`, `steps` - :param show_progress_bar: If True, output a tqdm progress bar - :param checkpoint_path: Folder to save checkpoints during training - :param checkpoint_save_steps: Will save a checkpoint after so many steps - :param checkpoint_save_total_limit: Total number of checkpoints to store - """ - - """ - ##Add info to model card - # info_loss_functions = "\n".join(["- {} with {} training examples".format(str(loss), len(dataloader)) for dataloader, loss in train_objectives]) - info_loss_functions = [] - for dataloader, loss in train_objectives: - info_loss_functions.extend(ModelCardTemplate.get_train_objective_info(dataloader, loss)) - info_loss_functions = "\n\n".join([text for text in info_loss_functions]) - - info_fit_parameters = json.dumps( - { - "evaluator": fullname(evaluator), - "epochs": epochs, - "steps_per_epoch": steps_per_epoch, - "scheduler": scheduler, - "warmup_steps": warmup_steps, - "optimizer_class": str(optimizer_class), - "optimizer_params": optimizer_params, - "weight_decay": weight_decay, - "evaluation_steps": evaluation_steps, - "max_grad_norm": max_grad_norm, - }, - indent=4, - sort_keys=True, - ) - self._model_card_text = None - self._model_card_vars["{TRAINING_SECTION}"] = ModelCardTemplate.__TRAINING_SECTION__.replace( - "{LOSS_FUNCTIONS}", info_loss_functions - ).replace("{FIT_PARAMETERS}", info_fit_parameters) - """ - # TODO: Loading best model - # TODO: Saving/checkpointing - # TODO: args.gradient_accumulation_steps - # TODO: fp16/bf16, etc. - - state.epoch = 0 - start_time = time.time() - # TODO: Add max_steps via args.max_steps here? - state.max_steps = len(train_dataloader) * args.embedding_num_epochs - control = callback_handler.on_train_begin(args, state, control) - - if use_amp: - from torch.cuda.amp import autocast - - scaler = torch.cuda.amp.GradScaler() - - model_body.to(model_body._target_device) - - # Use smart batching - train_dataloader.collate_fn = model_body.smart_batching_collate - if eval_dataloader: - eval_dataloader.collate_fn = model_body.smart_batching_collate - - loss_func.to(model_body._target_device) - - model_body.best_score = -9999999 - - steps_per_epoch = len(train_dataloader) - num_train_steps = int(steps_per_epoch * args.embedding_num_epochs) - - # Prepare optimizers - param_optimizer = list(loss_func.named_parameters()) - - no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"] - optimizer_grouped_parameters = [ - { - "params": [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], - "weight_decay": weight_decay, - }, - {"params": [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], "weight_decay": 0.0}, - ] - - optimizer = optimizer_class(optimizer_grouped_parameters, **optimizer_params) - scheduler_obj = model_body._get_scheduler( - optimizer, scheduler=scheduler, warmup_steps=warmup_steps, t_total=num_train_steps - ) - - data_iterator = iter(train_dataloader) - - skip_scheduler = False - for epoch in range(args.embedding_num_epochs): - control = callback_handler.on_epoch_begin(args, state, control) - - training_steps = 0 - - loss_func.zero_grad() - loss_func.train() - - for step in range(steps_per_epoch): - control = callback_handler.on_step_begin(args, state, control) - - try: - data = next(data_iterator) - except StopIteration: - data_iterator = iter(train_dataloader) - data = next(data_iterator) - - features, labels = data - labels = labels.to(model_body._target_device) - features = list(map(lambda batch: batch_to_device(batch, model_body._target_device), features)) - - if use_amp: - with autocast(): - loss_value = loss_func(features, labels) - - scale_before_step = scaler.get_scale() - scaler.scale(loss_value).backward() - scaler.unscale_(optimizer) - torch.nn.utils.clip_grad_norm_(loss_func.parameters(), max_grad_norm) - scaler.step(optimizer) - scaler.update() - - skip_scheduler = scaler.get_scale() != scale_before_step - else: - loss_value = loss_func(features, labels) - loss_value.backward() - torch.nn.utils.clip_grad_norm_(loss_func.parameters(), max_grad_norm) - optimizer.step() - - optimizer.zero_grad() - - if not skip_scheduler: - scheduler_obj.step() - - training_steps += 1 - - state.global_step += 1 - state.epoch = epoch + (step + 1) / steps_per_epoch - control = callback_handler.on_step_end(args, state, control) - - if control.should_log: - learning_rate = scheduler_obj.get_last_lr()[0] - metrics = {"embedding_loss": round(loss_value.item(), 4), "learning_rate": learning_rate} - control = log(args, callback_handler, state, control, metrics) - - if control.should_evaluate: - # self._eval_during_training(evaluator, output_path, save_best_model, epoch, training_steps, callback) - eval_loss = evaluate_with_loss(model_body, eval_dataloader, loss_func, show_progress_bar, use_amp) - learning_rate = scheduler_obj.get_last_lr()[0] - metrics = {"eval_embedding_loss": round(eval_loss, 4), "learning_rate": learning_rate} - control = log(args, callback_handler, state, control, metrics) - control = callback_handler.on_evaluate(args, state, control, metrics) - if state.best_metric is None or eval_loss < state.best_metric: - state.best_metric = eval_loss - - loss_func.zero_grad() - loss_func.train() - - if ( - checkpoint_path is not None - and checkpoint_save_steps is not None - and checkpoint_save_steps > 0 - and state.global_step % checkpoint_save_steps == 0 - ): - model_body._save_checkpoint(checkpoint_path, checkpoint_save_total_limit, state.global_step) - - if control.should_epoch_stop or control.should_training_stop: - break - - control = callback_handler.on_epoch_end(args, state, control) - - if control.should_training_stop: - break - - if output_path is not None: # No evaluator, but output path: save final model version - model_body.save(output_path) - - if checkpoint_path is not None: - model_body._save_checkpoint(checkpoint_path, checkpoint_save_total_limit, state.global_step) - - control = callback_handler.on_train_end(args, state, control) - - num_train_samples = state.max_steps * args.embedding_batch_size # * args.gradient_accumulation_steps - metrics = speed_metrics("train", start_time, num_samples=num_train_samples, num_steps=state.max_steps) - # TODO: This isn't always printed - log(args, callback_handler, state, control, metrics) - - # eval_start_time = time.time() - # num_eval_samples = len(eval_dataloader) # args.max_steps * args.embedding_batch_size # * args.gradient_accumulation_steps - # num_eval_steps = num_eval_samples * args.embedding_num_epochs - # metrics.update(speed_metrics("eval", eval_start_time, num_samples=num_eval_samples, num_steps=num_eval_steps)) - - -def evaluate_with_loss(model_body: SentenceTransformer, eval_dataloader: DataLoader, loss_func: nn.Module, show_progress_bar: bool, use_amp: bool): - model_body.eval() - - if use_amp: - from torch.cuda.amp import autocast - - scaler = torch.cuda.amp.GradScaler() - - losses = [] - for data in tqdm(iter(eval_dataloader), leave=False): - features, labels = data - labels = labels.to(model_body._target_device) - features = list(map(lambda batch: batch_to_device(batch, model_body._target_device), features)) - - if use_amp: - with autocast(): - loss_value = loss_func(features, labels) - - losses.append(scaler.scale(loss_value).item()) - else: - losses.append(loss_func(features, labels).item()) - - model_body.train() - return sum(losses) / len(losses) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 32ad05c3..53b0d763 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -1,38 +1,41 @@ import math +import time import warnings from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union -from setfit import sentence_transformer - - -# Google Colab runs on Python 3.7, so we need this to be compatible -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal - import evaluate import numpy as np +import torch from datasets import DatasetDict -from sentence_transformers import InputExample, losses +from sentence_transformers import InputExample, SentenceTransformer, losses from sentence_transformers.datasets import SentenceLabelDataset from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction +from sentence_transformers.util import batch_to_device +from torch import nn +from torch.cuda.amp import autocast from torch.utils.data import DataLoader -from tqdm.auto import trange -from transformers.trainer_utils import HPSearchBackend, default_compute_objective, number_of_arguments, set_seed -import transformers +from tqdm.autonotebook import tqdm, trange +from transformers.integrations import get_reporting_integration_callbacks from transformers.trainer_callback import ( CallbackHandler, DefaultFlowCallback, PrinterCallback, ProgressCallback, TrainerCallback, - TrainerState, TrainerControl, + TrainerState, +) +from transformers.trainer_utils import ( + HPSearchBackend, + default_compute_objective, + number_of_arguments, + set_seed, + speed_metrics, ) -from transformers.integrations import get_reporting_integration_callbacks, get_available_reporting_integrations from transformers.utils.import_utils import is_in_notebook +from setfit.training_args import TrainingArguments + from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna from .losses import SupConLoss @@ -41,6 +44,13 @@ from .utils import BestRun, default_hp_space_optuna +# Google Colab runs on Python 3.7, so we need this to be compatible +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal + + if TYPE_CHECKING: import optuna from datasets import Dataset @@ -398,19 +408,13 @@ def train_embeddings( logger.info(f" Total train batch size = {batch_size}") warmup_steps = math.ceil(total_train_steps * args.warmup_proportion) - sentence_transformer.fit( + self._train_sentence_transformer( self.model.model_body, train_dataloader=train_dataloader, - loss_func=loss_func, eval_dataloader=eval_dataloader, args=args, - callback_handler=self.callback_handler, - state=self.state, - control=self.control, - optimizer_params={"lr": args.body_embedding_learning_rate}, + loss_func=loss_func, warmup_steps=warmup_steps, - show_progress_bar=args.show_progress_bar, - use_amp=args.use_amp, ) def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], args: TrainingArguments): @@ -455,6 +459,195 @@ def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], arg loss = args.loss(self.model.model_body) return dataloader, loss, batch_size + def log(self, args: TrainingArguments, logs: Dict[str, float]) -> None: + """ + Log `logs` on the various objects watching training. + + Subclass and override this method to inject custom behavior. + + Args: + logs (`Dict[str, float]`): + The values to log. + """ + if self.state.epoch is not None: + logs["epoch"] = round(self.state.epoch, 2) + + output = {**logs, **{"step": self.state.global_step}} + self.state.log_history.append(output) + return self.callback_handler.on_log(args, self.state, self.control, logs) + + def _train_sentence_transformer( + self, + model_body: SentenceTransformer, + train_dataloader: DataLoader, + eval_dataloader: DataLoader, + args: TrainingArguments, + loss_func: nn.Module, + warmup_steps: int = 10000, + ) -> None: + """ + Train the model with the given training objective + Each training objective is sampled in turn for one batch. + We sample only as many batches from each objective as there are in the smallest one + to make sure of equal training with each dataset. + """ + # TODO: Loading best model + # TODO: Saving/checkpointing + # TODO: args.gradient_accumulation_steps + # TODO: fp16/bf16, etc. + + # Hardcoded training arguments + max_grad_norm = 1 + weight_decay = 0.01 + + self.state.epoch = 0 + start_time = time.time() + # TODO: Add max_steps via args.max_steps here? + self.state.max_steps = len(train_dataloader) * args.embedding_num_epochs + self.control = self.callback_handler.on_train_begin(args, self.state, self.control) + + if args.use_amp: + scaler = torch.cuda.amp.GradScaler() + + model_body.to(model_body._target_device) + loss_func.to(model_body._target_device) + + # Use smart batching + train_dataloader.collate_fn = model_body.smart_batching_collate + if eval_dataloader: + eval_dataloader.collate_fn = model_body.smart_batching_collate + + steps_per_epoch = len(train_dataloader) + num_train_steps = int(steps_per_epoch * args.embedding_num_epochs) + + # Prepare optimizers + param_optimizer = list(loss_func.named_parameters()) + + no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], + "weight_decay": weight_decay, + }, + {"params": [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], "weight_decay": 0.0}, + ] + + optimizer = torch.optim.AdamW(optimizer_grouped_parameters, **{"lr": args.body_embedding_learning_rate}) + scheduler_obj = model_body._get_scheduler( + optimizer, scheduler="WarmupLinear", warmup_steps=warmup_steps, t_total=num_train_steps + ) + + data_iterator = iter(train_dataloader) + skip_scheduler = False + for epoch in range(args.embedding_num_epochs): + self.control = self.callback_handler.on_epoch_begin(args, self.state, self.control) + + loss_func.zero_grad() + loss_func.train() + + for step in range(steps_per_epoch): + self.control = self.callback_handler.on_step_begin(args, self.state, self.control) + + try: + data = next(data_iterator) + except StopIteration: + data_iterator = iter(train_dataloader) + data = next(data_iterator) + + features, labels = data + labels = labels.to(model_body._target_device) + features = list(map(lambda batch: batch_to_device(batch, model_body._target_device), features)) + + if args.use_amp: + with autocast(): + loss_value = loss_func(features, labels) + + scale_before_step = scaler.get_scale() + scaler.scale(loss_value).backward() + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(loss_func.parameters(), max_grad_norm) + scaler.step(optimizer) + scaler.update() + + skip_scheduler = scaler.get_scale() != scale_before_step + else: + loss_value = loss_func(features, labels) + loss_value.backward() + torch.nn.utils.clip_grad_norm_(loss_func.parameters(), max_grad_norm) + optimizer.step() + + optimizer.zero_grad() + + if not skip_scheduler: + scheduler_obj.step() + + self.state.global_step += 1 + self.state.epoch = epoch + (step + 1) / steps_per_epoch + self.control = self.callback_handler.on_step_end(args, self.state, self.control) + + if self.control.should_log: + learning_rate = scheduler_obj.get_last_lr()[0] + metrics = {"embedding_loss": round(loss_value.item(), 4), "learning_rate": learning_rate} + self.control = self.log(args, metrics) + + if self.control.should_evaluate: + eval_loss = self._evaluate_with_loss(model_body, eval_dataloader, args, loss_func) + learning_rate = scheduler_obj.get_last_lr()[0] + metrics = {"eval_embedding_loss": round(eval_loss, 4), "learning_rate": learning_rate} + self.control = self.log(args, metrics) + + self.control = self.callback_handler.on_evaluate(args, self.state, self.control, metrics) + if self.state.best_metric is None or eval_loss < self.state.best_metric: + self.state.best_metric = eval_loss + + loss_func.zero_grad() + loss_func.train() + + if self.control.should_epoch_stop or self.control.should_training_stop: + break + + self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) + + if self.control.should_training_stop: + break + + # Ensure logging the speed metrics + num_train_samples = self.state.max_steps * args.embedding_batch_size # * args.gradient_accumulation_steps + metrics = speed_metrics("train", start_time, num_samples=num_train_samples, num_steps=self.state.max_steps) + self.control.should_log = True + self.log(args, metrics) + + self.control = self.callback_handler.on_train_end(args, self.state, self.control) + + def _evaluate_with_loss( + self, + model_body: SentenceTransformer, + eval_dataloader: DataLoader, + args: TrainingArguments, + loss_func: nn.Module, + ) -> float: + model_body.eval() + + if args.use_amp: + scaler = torch.cuda.amp.GradScaler() + + losses = [] + for data in tqdm(iter(eval_dataloader), leave=False, disable=not args.show_progress_bar): + features, labels = data + labels = labels.to(model_body._target_device) + features = list(map(lambda batch: batch_to_device(batch, model_body._target_device), features)) + + if args.use_amp: + with autocast(): + loss_value = loss_func(features, labels) + + losses.append(scaler.scale(loss_value).item()) + else: + losses.append(loss_func(features, labels).item()) + + model_body.train() + return sum(losses) / len(losses) + def train_classifier( self, x_train: List[str], y_train: Union[List[int], List[List[int]]], args: Optional[TrainingArguments] = None ) -> None: diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 2a1aeb3b..9a6d9ee8 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -1,17 +1,17 @@ from __future__ import annotations import inspect +import json from copy import copy from dataclasses import dataclass, field, fields -import json from typing import Any, Callable, Dict, Optional, Tuple, Union -from sentence_transformers import losses import torch +from sentence_transformers import losses +from transformers import IntervalStrategy from transformers.integrations import get_available_reporting_integrations from transformers.training_args import default_logdir from transformers.utils import is_torch_available -from transformers import IntervalStrategy @dataclass @@ -165,7 +165,7 @@ class TrainingArguments: run_name: Optional[str] = None logging_dir: Optional[str] = None logging_strategy: str = "steps" - logging_first_step: bool = False + logging_first_step: bool = True logging_steps: int = 5 evaluation_strategy: str = "steps" @@ -177,8 +177,8 @@ class TrainingArguments: save_total_limit: Optional[int] = None load_best_model_at_end: bool = True - metric_for_best_model: str = field(default="embedding_loss", repr=False) - greater_is_better: bool = field(default=False, repr=False) + metric_for_best_model: str = field(default="embedding_loss", repr=False, init=False) + greater_is_better: bool = field(default=False, repr=False, init=False) def __post_init__(self) -> None: # Set `self.embedding_batch_size` and `self.classifier_batch_size` using values from `self.batch_size` From 826eb538b919b4a4a747eede65979ac6701e1c42 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 28 Jul 2023 16:33:46 +0200 Subject: [PATCH 037/183] Add checkpointing, support EarlyStoppingCallback --- src/setfit/trainer.py | 52 ++++++++++++++++++++++++++------- src/setfit/training_args.py | 57 ++++++++++++++++++++++++++++++++----- 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 53b0d763..ddedc2c3 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -1,4 +1,6 @@ import math +from pathlib import Path +import shutil import time import warnings from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union @@ -34,8 +36,6 @@ ) from transformers.utils.import_utils import is_in_notebook -from setfit.training_args import TrainingArguments - from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna from .losses import SupConLoss @@ -379,8 +379,8 @@ def train_embeddings( self, x_train: List[str], y_train: Union[List[int], List[List[int]]], - x_eval: List[str], - y_eval: Union[List[int], List[List[int]]], + x_eval: List[str] = None, + y_eval: Union[List[int], List[List[int]]] = None, args: Optional[TrainingArguments] = None, ) -> None: """ @@ -480,7 +480,7 @@ def _train_sentence_transformer( self, model_body: SentenceTransformer, train_dataloader: DataLoader, - eval_dataloader: DataLoader, + eval_dataloader: Optional[DataLoader], args: TrainingArguments, loss_func: nn.Module, warmup_steps: int = 10000, @@ -491,10 +491,9 @@ def _train_sentence_transformer( We sample only as many batches from each objective as there are in the smallest one to make sure of equal training with each dataset. """ - # TODO: Loading best model - # TODO: Saving/checkpointing # TODO: args.gradient_accumulation_steps # TODO: fp16/bf16, etc. + # TODO: Safetensors # Hardcoded training arguments max_grad_norm = 1 @@ -590,19 +589,28 @@ def _train_sentence_transformer( metrics = {"embedding_loss": round(loss_value.item(), 4), "learning_rate": learning_rate} self.control = self.log(args, metrics) - if self.control.should_evaluate: + eval_loss = None + if self.control.should_evaluate and eval_dataloader: eval_loss = self._evaluate_with_loss(model_body, eval_dataloader, args, loss_func) learning_rate = scheduler_obj.get_last_lr()[0] metrics = {"eval_embedding_loss": round(eval_loss, 4), "learning_rate": learning_rate} self.control = self.log(args, metrics) self.control = self.callback_handler.on_evaluate(args, self.state, self.control, metrics) - if self.state.best_metric is None or eval_loss < self.state.best_metric: - self.state.best_metric = eval_loss loss_func.zero_grad() loss_func.train() + if self.control.should_save: + checkpoint_dir = self._checkpoint( + self.args.output_dir, args.save_total_limit, self.state.global_step + ) + self.control = self.callback_handler.on_save(self.args, self.state, self.control) + + if eval_loss is not None and (self.state.best_metric is None or eval_loss < self.state.best_metric): + self.state.best_metric = eval_loss + self.state.best_model_checkpoint = checkpoint_dir + if self.control.should_epoch_stop or self.control.should_training_stop: break @@ -611,6 +619,12 @@ def _train_sentence_transformer( if self.control.should_training_stop: break + if self.args.load_best_model_at_end and self.state.best_model_checkpoint: + dir_name = Path(self.state.best_model_checkpoint).name + if dir_name.startswith("step_"): + logger.info(f"Loading best SentenceTransformer model from step {dir_name[5:]}.") + self.model.model_body = SentenceTransformer(self.state.best_model_checkpoint, device=model_body.device) + # Ensure logging the speed metrics num_train_samples = self.state.max_steps * args.embedding_batch_size # * args.gradient_accumulation_steps metrics = speed_metrics("train", start_time, num_samples=num_train_samples, num_steps=self.state.max_steps) @@ -648,6 +662,24 @@ def _evaluate_with_loss( model_body.train() return sum(losses) / len(losses) + def _checkpoint(self, checkpoint_path: str, checkpoint_save_total_limit: int, step: int) -> None: + # Delete old checkpoints + if checkpoint_save_total_limit is not None and checkpoint_save_total_limit > 0: + old_checkpoints = [] + for subdir in Path(checkpoint_path).glob("step_*"): + if subdir.name[5:].isdigit() and ( + self.state.best_model_checkpoint is None or subdir != Path(self.state.best_model_checkpoint) + ): + old_checkpoints.append({"step": int(subdir.name[5:]), "path": str(subdir)}) + + if len(old_checkpoints) > checkpoint_save_total_limit - 1: + old_checkpoints = sorted(old_checkpoints, key=lambda x: x["step"]) + shutil.rmtree(old_checkpoints[0]["path"]) + + checkpoint_file_path = str(Path(checkpoint_path) / f"step_{step}") + self.model.save_pretrained(checkpoint_file_path) + return checkpoint_file_path + def train_classifier( self, x_train: List[str], y_train: Union[List[int], List[List[int]]], args: Optional[TrainingArguments] = None ) -> None: diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 9a6d9ee8..3ba751cb 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -20,6 +20,8 @@ class TrainingArguments: TrainingArguments is the subset of the arguments which relate to the training loop itself. Parameters: + output_dir (`str`, defaults to `"checkpoints"`): + The output directory where the model predictions and checkpoints will be written. batch_size (`Union[int, Tuple[int, int]]`, defaults to `(16, 2)`): Set the batch sizes for the embedding and classifier training phases respectively, or set both if an integer is provided. @@ -116,12 +118,22 @@ class TrainingArguments: - `"steps"`: Save is done every `save_steps`. save_steps (`int`, *optional*, defaults to 500): Number of updates steps before two checkpoint saves if `save_strategy="steps"`. - save_total_limit (`int`, *optional*): + save_total_limit (`int`, *optional*, defaults to `1`): If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in - `output_dir`. + `output_dir`. Note, the best model is always preserved if the `evaluation_strategy` is not `"no"`. + load_best_model_at_end (`bool`, *optional*, defaults to `False`): + Whether or not to load the best model found during training at the end of training. + + + When set to `True`, the parameters `save_strategy` needs to be the same as `evaluation_strategy`, and in + the case it is "steps", `save_steps` must be a round multiple of `eval_steps`. + + """ + output_dir: str = "checkpoints" + # batch_size is only used to conveniently set `embedding_batch_size` and `classifier_batch_size` # which are used in practice batch_size: Union[int, Tuple[int, int]] = field(default=(16, 2), repr=False) @@ -174,11 +186,11 @@ class TrainingArguments: save_strategy: str = "steps" save_steps: int = 500 - save_total_limit: Optional[int] = None + save_total_limit: Optional[int] = 1 - load_best_model_at_end: bool = True - metric_for_best_model: str = field(default="embedding_loss", repr=False, init=False) - greater_is_better: bool = field(default=False, repr=False, init=False) + load_best_model_at_end: bool = False + metric_for_best_model: str = field(default="embedding_loss", repr=False) + greater_is_better: bool = field(default=False, repr=False) def __post_init__(self) -> None: # Set `self.embedding_batch_size` and `self.classifier_batch_size` using values from `self.batch_size` @@ -211,8 +223,12 @@ def __post_init__(self) -> None: f"warmup_proportion must be greater than or equal to 0.0 and less than or equal to 1.0! But it was: {self.warmup_proportion}" ) - if self.report_to in (None, "all"): + if self.report_to in (None, "all", ["all"]): self.report_to = get_available_reporting_integrations() + elif self.report_to in ("none", ["none"]): + self.report_to = [] + elif not isinstance(self.report_to, list): + self.report_to = [self.report_to] if self.logging_dir is None: self.logging_dir = default_logdir() @@ -230,6 +246,33 @@ def __post_init__(self) -> None: " `logging_steps`" ) + # Sanity checks for load_best_model_at_end: we require save and eval strategies to be compatible. + if self.load_best_model_at_end: + if self.evaluation_strategy != self.save_strategy: + raise ValueError( + "`load_best_model_at_end` requires the save and eval strategy to match, but found\n- Evaluation " + f"strategy: {self.evaluation_strategy}\n- Save strategy: {self.save_strategy}" + ) + if self.evaluation_strategy == IntervalStrategy.STEPS and self.save_steps % self.eval_steps != 0: + if self.eval_steps < 1 or self.save_steps < 1: + if not (self.eval_steps < 1 and self.save_steps < 1): + raise ValueError( + "`load_best_model_at_end` requires the saving steps to be a multiple of the evaluation " + "steps, which cannot get guaranteed when mixing ratio and absolute steps for save_steps" + f"{self.save_steps} and eval_steps {self.eval_steps}." + ) + # Work around floating point precision issues + LARGE_MULTIPLIER = 1_000_000 + if (self.save_steps * LARGE_MULTIPLIER) % (self.eval_steps * LARGE_MULTIPLIER) != 0: + raise ValueError( + "`load_best_model_at_end` requires the saving steps to be a multiple of the evaluation " + f"steps, but found {self.save_steps}, which is not a multiple of {self.eval_steps}." + ) + raise ValueError( + "`load_best_model_at_end` requires the saving steps to be a round multiple of the evaluation " + f"steps, but found {self.save_steps}, which is not a round multiple of {self.eval_steps}." + ) + # logging_steps must be non-zero for logging_strategy that is other than 'no' if self.logging_strategy == IntervalStrategy.STEPS and self.logging_steps == 0: raise ValueError(f"logging strategy {self.logging_strategy} requires non-zero --logging_steps") From 1930973c6a862c2914f4162254e4ad7d5ce460c5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Sat, 29 Jul 2023 16:43:29 +0200 Subject: [PATCH 038/183] Run formatting --- tests/test_deprecated_trainer.py | 6 ++---- tests/test_trainer.py | 8 ++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_deprecated_trainer.py b/tests/test_deprecated_trainer.py index 82a2eea8..8e1ce1d5 100644 --- a/tests/test_deprecated_trainer.py +++ b/tests/test_deprecated_trainer.py @@ -5,9 +5,9 @@ import evaluate import pytest +import torch from datasets import Dataset, load_dataset from sentence_transformers import losses -import torch from transformers.testing_utils import require_optuna from transformers.utils.hp_naming import TrialShortNamer @@ -511,9 +511,7 @@ def test_trainer_evaluate_multilabel_f1(): def test_trainer_evaluate_on_cpu() -> None: # This test used to fail if CUDA was available - dataset = Dataset.from_dict( - {"text": ["positive sentence", "negative sentence"], "label": [1, 0]} - ) + dataset = Dataset.from_dict({"text": ["positive sentence", "negative sentence"], "label": [1, 0]}) model = SetFitModel.from_pretrained( "sentence-transformers/paraphrase-albert-small-v2", use_differentiable_head=True ) diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 4cde93e2..792888f8 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -74,9 +74,7 @@ def test_trainer_works_with_alternate_dataset_for_evaluate(self): alternate_dataset = Dataset.from_dict( {"text": ["x", "y", "z"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]} ) - trainer = Trainer( - model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset - ) + trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) trainer.train() metrics = trainer.evaluate(alternate_dataset) self.assertNotEqual(metrics["accuracy"], 1.0) @@ -470,9 +468,7 @@ def test_trainer_evaluate_multilabel_f1(): def test_trainer_evaluate_on_cpu() -> None: # This test used to fail if CUDA was available - dataset = Dataset.from_dict( - {"text": ["positive sentence", "negative sentence"], "label": [1, 0]} - ) + dataset = Dataset.from_dict({"text": ["positive sentence", "negative sentence"], "label": [1, 0]}) model = SetFitModel.from_pretrained( "sentence-transformers/paraphrase-albert-small-v2", use_differentiable_head=True ) From a87cdc0395186b9f39972b66269c92e392bc87f0 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Sat, 29 Jul 2023 22:21:30 +0200 Subject: [PATCH 039/183] Add additional trainer tests --- src/setfit/trainer.py | 2 +- tests/conftest.py | 8 +++++ tests/test_trainer.py | 79 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tests/conftest.py diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index bf9d78fb..fadf5a7b 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -801,7 +801,7 @@ def hyperparameter_search( if backend is None: backend = default_hp_search_backend() if backend is None: - raise RuntimeError("optuna should be installed. To install optuna run `pip install optuna`. ") + raise RuntimeError("optuna should be installed. To install optuna run `pip install optuna`.") backend = HPSearchBackend(backend) if backend == HPSearchBackend.OPTUNA and not is_optuna_available(): raise RuntimeError("You picked the optuna backend, but it is not installed. Use `pip install optuna`.") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..acf5b825 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,8 @@ +import pytest + +from setfit import SetFitModel + + +@pytest.fixture() +def model() -> SetFitModel: + return SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 792888f8..e5e0fa08 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -1,3 +1,4 @@ +import os import pathlib import re import tempfile @@ -8,6 +9,7 @@ import torch from datasets import Dataset, load_dataset from sentence_transformers import losses +from transformers import TrainerCallback from transformers.testing_utils import require_optuna from transformers.utils.hp_naming import TrialShortNamer @@ -428,11 +430,10 @@ def test_trainer_works_with_non_default_loss_class(loss_class): # no asserts here because this is a regression test - we only test if an exception is raised -def test_trainer_evaluate_with_strings(): +def test_trainer_evaluate_with_strings(model: SetFitModel): dataset = Dataset.from_dict( {"text": ["positive sentence", "negative sentence"], "label": ["positive", "negative"]} ) - model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") trainer = Trainer( model=model, args=TrainingArguments(num_iterations=1), @@ -487,3 +488,77 @@ def compute_metric(y_pred, y_test) -> None: ) trainer.train() trainer.evaluate() + + +def test_no_model_no_model_init(): + with pytest.raises(RuntimeError, match="`Trainer` requires either a `model` or `model_init` argument."): + Trainer() + + +def test_model_and_model_init(model: SetFitModel): + def model_init() -> SetFitModel: + return model + + with pytest.raises(RuntimeError, match="`Trainer` requires either a `model` or `model_init` argument."): + Trainer(model=model, model_init=model_init) + + +def test_trainer_callbacks(model: SetFitModel): + trainer = Trainer(model=model) + assert len(trainer.callback_handler.callbacks) == 2 + + class TestCallback(TrainerCallback): + pass + + callback = TestCallback() + trainer.add_callback(callback) + assert len(trainer.callback_handler.callbacks) == 3 + assert trainer.callback_handler.callbacks[-1] == callback + + assert trainer.pop_callback(callback) == callback + trainer.add_callback(callback) + assert trainer.callback_handler.callbacks[-1] == callback + trainer.remove_callback(callback) + assert callback not in trainer.callback_handler.callbacks + + +def test_trainer_warn_freeze(model: SetFitModel): + trainer = Trainer(model) + with pytest.warns( + DeprecationWarning, + match="Trainer.freeze` is deprecated and will be removed in v2.0.0 of SetFit. " + "Please use `SetFitModel.freeze` directly instead.", + ): + trainer.freeze() + + +def test_train_with_kwargs(model: SetFitModel): + train_dataset = Dataset.from_dict({"text": ["positive sentence", "negative sentence"], "label": [1, 0]}) + trainer = Trainer(model, train_dataset=train_dataset) + with pytest.warns(DeprecationWarning, match="`Trainer.train` does not accept keyword arguments anymore."): + trainer.train(num_epochs=5) + + +def test_train_no_dataset(model: SetFitModel): + trainer = Trainer(model) + with pytest.raises(ValueError, match="Training requires a `train_dataset` given to the `Trainer` initialization."): + trainer.train() + + +def test_train_amp_save(model: SetFitModel, tmp_path): + args = TrainingArguments(output_dir=tmp_path, use_amp=True, save_steps=5) + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2]}) + trainer = Trainer(model, args=args, train_dataset=dataset, eval_dataset=dataset) + trainer.train() + assert trainer.evaluate() == {"accuracy": 1.0} + assert os.listdir(tmp_path) == ["step_5"] + + +def test_train_load_best(model: SetFitModel, tmp_path, caplog): + args = TrainingArguments(output_dir=tmp_path, save_steps=5, eval_steps=5, load_best_model_at_end=True) + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2]}) + trainer = Trainer(model, args=args, train_dataset=dataset, eval_dataset=dataset) + with caplog.at_level(logging.INFO): + trainer.train() + + assert any("Load pretrained SentenceTransformer" in text for _, _, text in caplog.record_tuples) From d418759dc17618d2c83f6938c2b790f89b343124 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Sat, 29 Jul 2023 22:35:01 +0200 Subject: [PATCH 040/183] Use isinstance, required by flake8 release from 1hr ago --- src/setfit/exporters/onnx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/exporters/onnx.py b/src/setfit/exporters/onnx.py index 51e4b41f..cd05c464 100644 --- a/src/setfit/exporters/onnx.py +++ b/src/setfit/exporters/onnx.py @@ -153,7 +153,7 @@ def export_sklearn_head_to_onnx(model_head: LogisticRegression, opset: int) -> o # If the datatype of the model is double we need to cast the outputs # from the setfit model to doubles for compatibility inside of ONNX. - if type(dtype) == onnxconverter_common.data_types.DoubleTensorType: + if isinstance(dtype, onnxconverter_common.data_types.DoubleTensorType): sklearn_model = Pipeline([("castdouble", CastTransformer(dtype=np.double)), ("head", model_head)]) else: sklearn_model = model_head From 08892f6753a82b95c165b2a7ca7ee8eb1ff20438 Mon Sep 17 00:00:00 2001 From: danstan5 Date: Thu, 14 Sep 2023 12:04:56 +0100 Subject: [PATCH 041/183] sampler for refactor WIP --- src/setfit/modeling.py | 74 ---- src/setfit/sampler.py | 144 ++++++++ src/setfit/trainer.py | 25 +- src/setfit/trainer_distillation.py | 2 +- src/setfit/trainer_unique_pairs.py | 531 +++++++++++++++++++++++++++++ 5 files changed, 687 insertions(+), 89 deletions(-) create mode 100644 src/setfit/sampler.py create mode 100644 src/setfit/trainer_unique_pairs.py diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 17740a3e..835968f4 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -673,77 +673,3 @@ def _from_pretrained( multi_target_strategy=multi_target_strategy, normalize_embeddings=normalize_embeddings, ) - - -def sentence_pairs_generation(sentences, labels, pairs): - # Initialize two empty lists to hold the (sentence, sentence) pairs and - # labels to indicate if a pair is positive or negative - - num_classes = np.unique(labels) - label_to_idx = {x: i for i, x in enumerate(num_classes)} - positive_idxs = [np.where(labels == i)[0] for i in num_classes] - negative_idxs = [np.where(labels != i)[0] for i in num_classes] - - for first_idx in range(len(sentences)): - current_sentence = sentences[first_idx] - label = labels[first_idx] - second_idx = np.random.choice(positive_idxs[label_to_idx[label]]) - positive_sentence = sentences[second_idx] - # Prepare a positive pair and update the sentences and labels - # lists, respectively - pairs.append(InputExample(texts=[current_sentence, positive_sentence], label=1.0)) - - third_idx = np.random.choice(negative_idxs[label_to_idx[label]]) - negative_sentence = sentences[third_idx] - # Prepare a negative pair of sentences and update our lists - pairs.append(InputExample(texts=[current_sentence, negative_sentence], label=0.0)) - # Return a 2-tuple of our sentence pairs and labels - return pairs - - -def sentence_pairs_generation_multilabel(sentences, labels, pairs): - # Initialize two empty lists to hold the (sentence, sentence) pairs and - # labels to indicate if a pair is positive or negative - for first_idx in range(len(sentences)): - current_sentence = sentences[first_idx] - sample_labels = np.where(labels[first_idx, :] == 1)[0] - if len(np.where(labels.dot(labels[first_idx, :].T) == 0)[0]) == 0: - continue - else: - for _label in sample_labels: - second_idx = np.random.choice(np.where(labels[:, _label] == 1)[0]) - positive_sentence = sentences[second_idx] - # Prepare a positive pair and update the sentences and labels - # lists, respectively - pairs.append(InputExample(texts=[current_sentence, positive_sentence], label=1.0)) - - # Search for sample that don't have a label in common with current - # sentence - negative_idx = np.where(labels.dot(labels[first_idx, :].T) == 0)[0] - negative_sentence = sentences[np.random.choice(negative_idx)] - # Prepare a negative pair of sentences and update our lists - pairs.append(InputExample(texts=[current_sentence, negative_sentence], label=0.0)) - # Return a 2-tuple of our sentence pairs and labels - return pairs - - -def sentence_pairs_generation_cos_sim(sentences, pairs, cos_sim_matrix): - # initialize two empty lists to hold the (sentence, sentence) pairs and - # labels to indicate if a pair is positive or negative - - idx = list(range(len(sentences))) - - for first_idx in range(len(sentences)): - current_sentence = sentences[first_idx] - second_idx = int(np.random.choice([x for x in idx if x != first_idx])) - - cos_sim = float(cos_sim_matrix[first_idx][second_idx]) - paired_sentence = sentences[second_idx] - pairs.append(InputExample(texts=[current_sentence, paired_sentence], label=cos_sim)) - - third_idx = np.random.choice([x for x in idx if x != first_idx]) - cos_sim = float(cos_sim_matrix[first_idx][third_idx]) - paired_sentence = sentences[third_idx] - pairs.append(InputExample(texts=[current_sentence, paired_sentence], label=cos_sim)) - - return pairs diff --git a/src/setfit/sampler.py b/src/setfit/sampler.py new file mode 100644 index 00000000..f4ae97ce --- /dev/null +++ b/src/setfit/sampler.py @@ -0,0 +1,144 @@ +from typing import Generator, Iterable, List, Optional + +import numpy as np +from torch.utils.data import IterableDataset + +from sentence_transformers import InputExample + +from . import logging + + +logging.set_verbosity_info() +logger = logging.get_logger(__name__) + + +def sentence_pairs_generation_cos_sim(sentences, pairs, cos_sim_matrix): + # initialize two empty lists to hold the (sentence, sentence) pairs and + # labels to indicate if a pair is positive or negative + + idx = list(range(len(sentences))) + + for first_idx in range(len(sentences)): + current_sentence = sentences[first_idx] + second_idx = int(np.random.choice([x for x in idx if x != first_idx])) + + cos_sim = float(cos_sim_matrix[first_idx][second_idx]) + paired_sentence = sentences[second_idx] + pairs.append(InputExample(texts=[current_sentence, paired_sentence], label=cos_sim)) + + third_idx = np.random.choice([x for x in idx if x != first_idx]) + cos_sim = float(cos_sim_matrix[first_idx][third_idx]) + paired_sentence = sentences[third_idx] + pairs.append(InputExample(texts=[current_sentence, paired_sentence], label=cos_sim)) + + return pairs + + +def shuffle_combinations(iterable: Iterable, replacement: bool = True) -> Generator: + """Generates shuffled pair combinations for any iterable data provided. + + Args: + iterable: data to generate pair combinations from + replacement: enable to include combinations of same samples, + equivalent to itertools.combinations_with_replacement + + Returns: + Generator of shuffled pairs as a tuple + """ + n = len(iterable) + k = 1 if not replacement else 0 + idxs = np.stack(np.triu_indices(n, k), axis=-1) + for i in np.random.RandomState(seed=42).permutation(len(idxs)): + _idx, idx = idxs[i, :] + yield iterable[_idx], iterable[idx] + + +class ConstrastiveDataset(IterableDataset): + def __init__(self, + examples: InputExample, + multilabel: bool, + num_iterations: Optional[None] = None, + sampling_strategy: str = "oversampling", + ): + """Generates positive and negative text pairs for contrastive learning. + + Args: + examples (InputExample): text and labels in a text transformer dataclass + multilabel: set to process "multilabel" labels array + sampling_strategy: "unique", "oversampling", or "undersampling" + num_iterations: if provided explicitly sets the number of pairs to be generated + where n_pairs = n_iterations * n_sentences * 2 (for pos & neg pairs) + """ + super().__init__() + self.pos_index = 0 + self.neg_index = 0 + self.pos_pairs = [] + self.neg_pairs = [] + self.sentences = np.array([s.texts[0] for s in examples]) + self.labels = np.array([s.label for s in examples]) + self.sentence_labels = list(zip(self.sentences, self.labels)) + + if multilabel: + self.generate_multilabel_pairs() + else: + self.generate_pairs() + + if num_iterations is not None and num_iterations > 0: + self.len_pos_pairs = num_iterations * len(self.sentences) + self.len_neg_pairs = num_iterations * len(self.sentences) + + elif sampling_strategy == "unique": + self.len_pos_pairs = len(self.pos_pairs) + self.len_neg_pairs = len(self.neg_pairs) + + elif sampling_strategy == "undersampling": + self.len_pos_pairs = min(len(self.pos_pairs), len(self.neg_pairs)) + self.len_neg_pairs = min(len(self.pos_pairs), len(self.neg_pairs)) + + elif sampling_strategy == "oversampling": + self.len_pos_pairs = max(len(self.pos_pairs), len(self.neg_pairs)) + self.len_neg_pairs = max(len(self.pos_pairs), len(self.neg_pairs)) + + else: + raise ValueError("Invalid sampling strategy. Must be one of 'unique', 'oversampling', or 'undersampling'.") + + def generate_pairs(self) -> None: + for (_text, _label), (text, label) in shuffle_combinations(self.sentence_labels): + if _label == label: + self.pos_pairs.append(InputExample(texts=[_text, text], label=1.0)) + else: + self.neg_pairs.append(InputExample(texts=[_text, text], label=0.0)) + + def generate_multilabel_pairs(self) -> None: + for (_text, _label), (text, label) in shuffle_combinations(self.sentence_labels): + if any(np.logical_and(_label, label)): + # logical_and checks if labels are both set for each class + self.pos_pairs.append(InputExample(texts=[_text, text], label=1.0)) + else: + self.neg_pairs.append(InputExample(texts=[_text, text], label=0.0)) + + def get_positive_pairs(self) -> List[InputExample]: + pairs = [] + for _ in range(self.len_pos_pairs): + if self.pos_index >= len(self.pos_pairs): + self.pos_index = 0 + pairs.append(self.pos_pairs[self.pos_index]) + self.pos_index += 1 + return pairs + + def get_negative_pairs(self) -> List[InputExample]: + pairs = [] + for _ in range(self.len_neg_pairs): + if self.neg_index >= len(self.neg_pairs): + self.neg_index = 0 + pairs.append(self.neg_pairs[self.neg_index]) + self.neg_index += 1 + return pairs + + def __iter__(self): + for pos_pair, neg_pair in zip(self.get_positive_pairs(), self.get_negative_pairs()): + yield pos_pair + yield neg_pair + + def __len__(self): + return self.len_pos_pairs + self.len_neg_pairs diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index fadf5a7b..5d7c2a17 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -39,7 +39,7 @@ from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna from .losses import SupConLoss -from .modeling import sentence_pairs_generation, sentence_pairs_generation_multilabel +from .sampler import ConstrastiveDataset from .training_args import TrainingArguments from .utils import BestRun, default_hp_space_optuna @@ -417,7 +417,10 @@ def train_embeddings( ) def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], args: TrainingArguments): + # sentence-transformers adaptation + input_data = [InputExample(texts=[text], label=label) for text, label in zip(x, y)] + if args.loss in [ losses.BatchAllTripletLoss, losses.BatchHardTripletLoss, @@ -425,9 +428,7 @@ def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], arg losses.BatchHardSoftMarginTripletLoss, SupConLoss, ]: - examples = [InputExample(texts=[text], label=label) for text, label in zip(x, y)] - data_sampler = SentenceLabelDataset(examples, samples_per_label=args.samples_per_label) - + data_sampler = SentenceLabelDataset(input_data, samples_per_label=args.samples_per_label) batch_size = min(args.embedding_batch_size, len(data_sampler)) dataloader = DataLoader(data_sampler, batch_size=batch_size, drop_last=True) @@ -445,17 +446,13 @@ def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], arg margin=args.margin, ) else: - examples = [] - - for _ in trange(args.num_iterations, desc="Generating Training Pairs", disable=not args.show_progress_bar): - if self.model.multi_target_strategy is not None: - examples = sentence_pairs_generation_multilabel(np.array(x), np.array(y), examples) - else: - examples = sentence_pairs_generation(np.array(x), np.array(y), examples) - - batch_size = args.embedding_batch_size - dataloader = DataLoader(examples, shuffle=True, batch_size=batch_size) + data_sampler = ConstrastiveDataset( + input_data, self.model.multi_target_strategy, args.num_iterations + ) # sets default sampling_strategy="oversampling" + batch_size = min(args.embedding_batch_size, len(data_sampler)) + dataloader = DataLoader(data_sampler, batch_size=batch_size, drop_last=False) # shuffle=True can be dropped in for 'randomising' loss = args.loss(self.model.model_body) + return dataloader, loss, batch_size def log(self, args: TrainingArguments, logs: Dict[str, float]) -> None: diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 5a27f585..faebe43c 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -9,7 +9,7 @@ from transformers.trainer_utils import set_seed from . import logging -from .modeling import sentence_pairs_generation_cos_sim +from .sampler import sentence_pairs_generation_cos_sim from .trainer import Trainer from .training_args import TrainingArguments diff --git a/src/setfit/trainer_unique_pairs.py b/src/setfit/trainer_unique_pairs.py new file mode 100644 index 00000000..461b880d --- /dev/null +++ b/src/setfit/trainer_unique_pairs.py @@ -0,0 +1,531 @@ +import math +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union + +import evaluate +from sentence_transformers import InputExample, losses +from sentence_transformers.datasets import SentenceLabelDataset +from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction +from torch.utils.data import DataLoader +from transformers.trainer_utils import HPSearchBackend, default_compute_objective, number_of_arguments, set_seed + +from . import logging +from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna +from .modeling import SupConLoss +from .sampler import OVERSAMPLE, ConstrastiveDataset +from .utils import BestRun, default_hp_space_optuna + + +if TYPE_CHECKING: + import optuna + from datasets import Dataset + + from .modeling import SetFitModel + +logging.set_verbosity_info() +logger = logging.get_logger(__name__) + + +class SetFitTrainer: + """Trainer to train a SetFit model. + + Args: + model (`SetFitModel`, *optional*): + The model to train. If not provided, a `model_init` must be passed. + train_dataset (`Dataset`): + The training dataset. + eval_dataset (`Dataset`, *optional*): + The evaluation dataset. + model_init (`Callable[[], SetFitModel]`, *optional*): + A function that instantiates the model to be used. If provided, each call to [`~SetFitTrainer.train`] will start + from a new instance of the model as given by this function when a `trial` is passed. + metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): + The metric to use for evaluation. If a string is provided, we treat it as the metric name and load it with default settings. + If a callable is provided, it must take two arguments (`y_pred`, `y_test`). + loss_class (`nn.Module`, *optional*, defaults to `CosineSimilarityLoss`): + The loss function to use for contrastive training. + num_iterations (`int`, *optional*, defaults to `20`): + The number of iterations to generate sentence pairs for. + This argument is ignored if triplet loss is used. + It is only used in conjunction with `CosineSimilarityLoss`. + num_epochs (`int`, *optional*, defaults to `1`): + The number of epochs to train the Sentence Transformer body for. + learning_rate (`float`, *optional*, defaults to `2e-5`): + The learning rate to use for contrastive training. + batch_size (`int`, *optional*, defaults to `16`): + The batch size to use for contrastive training. + seed (`int`, *optional*, defaults to 42): + Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use the + [`~SetTrainer.model_init`] function to instantiate the model if it has some randomly initialized parameters. + column_mapping (`Dict[str, str]`, *optional*): + A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: {"text_column_name": "text", "label_column_name: "label"}. + use_amp (`bool`, *optional*, defaults to `False`): + Use Automatic Mixed Precision (AMP). Only for Pytorch >= 1.6.0 + warmup_proportion (`float`, *optional*, defaults to `0.1`): + Proportion of the warmup in the total training steps. + Must be greater than or equal to 0.0 and less than or equal to 1.0. + distance_metric (`Callable`, defaults to `BatchHardTripletLossDistanceFunction.cosine_distance`): + Function that returns a distance between two embeddings. + It is set for the triplet loss and + is ignored for `CosineSimilarityLoss` and `SupConLoss`. + margin (`float`, defaults to `0.25`): Margin for the triplet loss. + Negative samples should be at least margin further apart from the anchor than the positive. + This is ignored for `CosineSimilarityLoss`, `BatchHardSoftMarginTripletLoss` and `SupConLoss`. + samples_per_label (`int`, defaults to `2`): Number of consecutive, random and unique samples drawn per label. + This is only relevant for triplet loss and ignored for `CosineSimilarityLoss`. + Batch size should be a multiple of samples_per_label. + """ + + def __init__( + self, + model: Optional["SetFitModel"] = None, + train_dataset: Optional["Dataset"] = None, + eval_dataset: Optional["Dataset"] = None, + model_init: Optional[Callable[[], "SetFitModel"]] = None, + metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", + loss_class: Optional[Any] = None, + num_iterations: int = 20, + num_epochs: int = 1, + learning_rate: float = 2e-5, + batch_size: int = 16, + seed: int = 42, + column_mapping: Optional[Dict[str, str]] = None, + use_amp: bool = False, + warmup_proportion: float = 0.1, + distance_metric: Callable = BatchHardTripletLossDistanceFunction.cosine_distance, + margin: float = 0.25, + samples_per_label: int = 2, + sampling_strategy: int = OVERSAMPLE, + ): + if (warmup_proportion < 0.0) or (warmup_proportion > 1.0): + raise ValueError( + f"warmup_proportion must be greater than or equal to 0.0 and less than or equal to 1.0! But it was: {warmup_proportion}" + ) + + self.train_dataset = train_dataset + self.eval_dataset = eval_dataset + self.model_init = model_init + self.metric = metric + self.loss_class = loss_class + self.num_iterations = num_iterations + self.num_epochs = num_epochs + self.learning_rate = learning_rate + self.batch_size = batch_size + self.seed = seed + self.column_mapping = column_mapping + self.use_amp = use_amp + self.warmup_proportion = warmup_proportion + self.distance_metric = distance_metric + self.margin = margin + self.samples_per_label = samples_per_label + self.sampling_strategy = sampling_strategy + + if model is None: + if model_init is not None: + model = self.call_model_init() + else: + raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument") + else: + if model_init is not None: + raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument, but not both") + + self.model = model + self.hp_search_backend = None + self._freeze = True # If True, will train the body only; otherwise, train the body and head + + def _validate_column_mapping(self, dataset: "Dataset") -> None: + """ + Validates the provided column mapping against the dataset. + """ + required_columns = {"text", "label"} + column_names = set(dataset.column_names) + if self.column_mapping is None and not required_columns.issubset(column_names): + raise ValueError( + f"A column mapping must be provided when the dataset does not contain the following columns: {required_columns}" + ) + if self.column_mapping is not None: + missing_columns = required_columns.difference(self.column_mapping.values()) + if missing_columns: + raise ValueError( + f"The following columns are missing from the column mapping: {missing_columns}. Please provide a mapping for all required columns." + ) + if not set(self.column_mapping.keys()).issubset(column_names): + raise ValueError( + f"The following columns are missing from the dataset: {set(self.column_mapping.keys()).difference(column_names)}. Please provide a mapping for all required columns." + ) + + def _apply_column_mapping(self, dataset: "Dataset", column_mapping: Dict[str, str]) -> "Dataset": + """ + Applies the provided column mapping to the dataset, renaming columns accordingly. + Extra features not in the column mapping are prefixed with `"feat_"`. + """ + dataset = dataset.rename_columns( + { + **column_mapping, + **{col: f"feat_{col}" for col in dataset.column_names if col not in column_mapping}, + } + ) + dset_format = dataset.format + dataset = dataset.with_format( + type=dset_format["type"], + columns=dataset.column_names, + output_all_columns=dset_format["output_all_columns"], + **dset_format["format_kwargs"], + ) + return dataset + + def apply_hyperparameters(self, params: Dict[str, Any], final_model: bool = False): + """Applies a dictionary of hyperparameters to both the trainer and the model + + Args: + params (`Dict[str, Any]`): The parameters, usually from `BestRun.hyperparameters` + final_model (`bool`, *optional*, defaults to `False`): If `True`, replace the `model_init()` function with a fixed model based on the parameters. + """ + for key, value in params.items(): + if hasattr(self, key): + old_attr = getattr(self, key, None) + # Casting value to the proper type + if old_attr is not None: + value = type(old_attr)(value) + setattr(self, key, value) + elif number_of_arguments(self.model_init) == 0: # we do not warn if model_init could be using it + logger.warning( + f"Trying to set {key!r} in the hyperparameter search but there is no corresponding field in " + "`SetFitTrainer`, and `model_init` does not take any arguments." + ) + + self.model = self.model_init(params) + if final_model: + self.model_init = None + + def _hp_search_setup(self, trial: Union["optuna.Trial", Dict[str, Any]]): + """HP search setup code""" + + # Heavily inspired by transformers.Trainer._hp_search_setup + if self.hp_search_backend is None or trial is None: + return + + if isinstance(trial, Dict): # For passing a Dict to train() -- mostly unused for now + params = trial + elif self.hp_search_backend == HPSearchBackend.OPTUNA: + params = self.hp_space(trial) + else: + raise ValueError("Invalid trial parameter") + + logger.info(f"Trial: {params}") + self.apply_hyperparameters(params, final_model=False) + + def call_model_init(self, params: Optional[Dict[str, Any]] = None): + model_init_argcount = number_of_arguments(self.model_init) + if model_init_argcount == 0: + model = self.model_init() + elif model_init_argcount == 1: + model = self.model_init(params) + else: + raise RuntimeError("`model_init` should have 0 or 1 argument.") + + if model is None: + raise RuntimeError("`model_init` should not return None.") + + return model + + def freeze(self): + """ + Freeze SetFitModel's differentiable head. + Note: call this function only when using the differentiable head. + """ + if not self.model.has_differentiable_head: + raise ValueError("Please use the differentiable head in `SetFitModel` when calling this function.") + + self._freeze = True # Currently use self._freeze as a switch + self.model.freeze("head") + + def unfreeze(self, keep_body_frozen: bool = False): + """ + Unfreeze SetFitModel's differentiable head. + Note: call this function only when using the differentiable head. + + Args: + keep_body_frozen (`bool`, *optional*, defaults to `False`): + Whether to freeze the body when unfreeze the head. + """ + if not self.model.has_differentiable_head: + raise ValueError("Please use the differentiable head in `SetFitModel` when calling this function.") + + self._freeze = False # Currently use self._freeze as a switch + self.model.unfreeze("head") + if keep_body_frozen: + self.model.freeze("body") + else: # ensure to unfreeze the body + self.model.unfreeze("body") + + def train( + self, + num_epochs: Optional[int] = None, + batch_size: Optional[int] = None, + learning_rate: Optional[float] = None, + body_learning_rate: Optional[float] = None, + l2_weight: Optional[float] = None, + max_length: Optional[int] = None, + trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, + show_progress_bar: bool = True, + ): + """ + Main training entry point. + + Args: + num_epochs (`int`, *optional*): + Temporary change the number of epochs to train the Sentence Transformer body/head for. + If ignore, will use the value given in initialization. + batch_size (`int`, *optional*): + Temporary change the batch size to use for contrastive training or logistic regression. + If ignore, will use the value given in initialization. + learning_rate (`float`, *optional*): + Temporary change the learning rate to use for contrastive training or SetFitModel's head in logistic regression. + If ignore, will use the value given in initialization. + body_learning_rate (`float`, *optional*): + Temporary change the learning rate to use for SetFitModel's body in logistic regression only. + If ignore, will be the same as `learning_rate`. + l2_weight (`float`, *optional*): + Temporary change the weight of L2 regularization for SetFitModel's differentiable head in logistic regression. + max_length (int, *optional*, defaults to `None`): + The maximum number of tokens for one data sample. Currently only for training the differentiable head. + If `None`, will use the maximum number of tokens the model body can accept. + If `max_length` is greater than the maximum number of acceptable tokens the model body can accept, it will be set to the maximum number of acceptable tokens. + trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): + The trial run or the hyperparameter dictionary for hyperparameter search. + show_progress_bar (`bool`, *optional*, defaults to `True`): + Whether to show a bar that indicates training progress. + """ + set_seed(self.seed) # Seed must be set before instantiating the model when using model_init. + + if trial: # Trial and model initialization + self._hp_search_setup(trial) # sets trainer parameters and initializes model + + if self.train_dataset is None: + raise ValueError("Training requires a `train_dataset` given to the `SetFitTrainer` initialization.") + + self._validate_column_mapping(self.train_dataset) + train_dataset = self.train_dataset + if self.column_mapping is not None: + logger.info("Applying column mapping to training dataset") + train_dataset = self._apply_column_mapping(self.train_dataset, self.column_mapping) + + if self.loss_class is None: + logger.warning("No `loss_class` detected! Using `CosineSimilarityLoss` as the default.") + self.loss_class = losses.CosineSimilarityLoss + + multilabel = True if self.model.multi_target_strategy is not None else False + + num_epochs = num_epochs or self.num_epochs + batch_size = batch_size or self.batch_size + learning_rate = learning_rate or self.learning_rate + + # dataset generation + x_train = train_dataset["text"] + y_train = train_dataset["label"] + train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] + + if not self.model.has_differentiable_head or self._freeze: + # sentence-transformers adaptation + if self.loss_class in [ + losses.BatchAllTripletLoss, + losses.BatchHardTripletLoss, + losses.BatchSemiHardTripletLoss, + losses.BatchHardSoftMarginTripletLoss, + SupConLoss, + ]: + train_data_sampler = SentenceLabelDataset(train_examples, samples_per_label=self.samples_per_label) + batch_size = min(batch_size, len(train_data_sampler)) + train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) + else: + train_data_sampler = ConstrastiveDataset(train_examples, multilabel, self.sampling_strategy) + train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=False) + + total_train_steps = len(train_dataloader) * num_epochs + logger.info("***** Running training *****") + logger.info(f" Num examples per epoch = {len(train_data_sampler)}") + logger.info(f" Num epochs = {num_epochs}") + logger.info(f" Total optimization steps = {total_train_steps}") + logger.info(f" Total train batch size = {batch_size}") + + # setup training loss + if self.loss_class in [ + losses.BatchAllTripletLoss, + losses.BatchHardTripletLoss, + losses.BatchSemiHardTripletLoss, + losses.BatchHardSoftMarginTripletLoss, + ]: + train_loss = self.loss_class( + model=self.model.model_body, + distance_metric=self.distance_metric, + margin=self.margin, + ) + elif self.loss_class is losses.BatchHardSoftMarginTripletLoss: + train_loss = self.loss_class( + model=self.model.model_body, + distance_metric=self.distance_metric, + ) + else: + train_loss = self.loss_class(model=self.model.model_body) + + warmup_steps = math.ceil(total_train_steps * self.warmup_proportion) + self.model.model_body.fit( + train_objectives=[(train_dataloader, train_loss)], + epochs=num_epochs, + optimizer_params={"lr": learning_rate}, + warmup_steps=warmup_steps, + show_progress_bar=show_progress_bar, + use_amp=self.use_amp, + ) + + if not self.model.has_differentiable_head or not self._freeze: + # Train the final classifier + self.model.fit( + x_train, + y_train, + num_epochs=num_epochs, + batch_size=batch_size, + learning_rate=learning_rate, + body_learning_rate=body_learning_rate, + l2_weight=l2_weight, + max_length=max_length, + show_progress_bar=True, + ) + + def evaluate(self): + """ + Computes the metrics for a given classifier. + + Returns: + `Dict[str, float]`: The evaluation metrics. + """ + + self._validate_column_mapping(self.eval_dataset) + eval_dataset = self.eval_dataset + + if self.column_mapping is not None: + logger.info("Applying column mapping to evaluation dataset") + eval_dataset = self._apply_column_mapping(self.eval_dataset, self.column_mapping) + + x_test = eval_dataset["text"] + y_test = eval_dataset["label"] + + logger.info("***** Running evaluation *****") + y_pred = self.model.predict(x_test) + + if isinstance(self.metric, str): + metric_config = "multilabel" if self.model.multi_target_strategy is not None else None + metric_fn = evaluate.load(self.metric, config_name=metric_config) + + return metric_fn.compute(predictions=y_pred, references=y_test) + + elif callable(self.metric): + return self.metric(y_pred, y_test) + + else: + raise ValueError("metric must be a string or a callable") + + def hyperparameter_search( + self, + hp_space: Optional[Callable[["optuna.Trial"], Dict[str, float]]] = None, + compute_objective: Optional[Callable[[Dict[str, float]], float]] = None, + n_trials: int = 10, + direction: str = "maximize", + backend: Optional[Union["str", HPSearchBackend]] = None, + hp_name: Optional[Callable[["optuna.Trial"], str]] = None, + **kwargs, + ) -> BestRun: + """ + Launch a hyperparameter search using `optuna`. The optimized quantity is determined + by `compute_objective`, which defaults to a function returning the evaluation loss when no metric is provided, + the sum of all metrics otherwise. + + + + To use this method, you need to have provided a `model_init` when initializing your [`SetFitTrainer`]: we need to + reinitialize the model at each new run. + + + + Args: + hp_space (`Callable[["optuna.Trial"], Dict[str, float]]`, *optional*): + A function that defines the hyperparameter search space. Will default to + [`~trainer_utils.default_hp_space_optuna`]. + compute_objective (`Callable[[Dict[str, float]], float]`, *optional*): + A function computing the objective to minimize or maximize from the metrics returned by the `evaluate` + method. Will default to [`~trainer_utils.default_compute_objective`] which uses the sum of metrics. + n_trials (`int`, *optional*, defaults to 100): + The number of trial runs to test. + direction (`str`, *optional*, defaults to `"maximize"`): + Whether to optimize greater or lower objects. Can be `"minimize"` or `"maximize"`, you should pick + `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or several metrics. + backend (`str` or [`~training_utils.HPSearchBackend`], *optional*): + The backend to use for hyperparameter search. Only optuna is supported for now. + TODO: add support for ray and sigopt. + hp_name (`Callable[["optuna.Trial"], str]]`, *optional*): + A function that defines the trial/run name. Will default to None. + kwargs (`Dict[str, Any]`, *optional*): + Additional keyword arguments passed along to `optuna.create_study`. For more + information see: + + - the documentation of + [optuna.create_study](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.html) + + Returns: + [`trainer_utils.BestRun`]: All the information about the best run. + """ + if backend is None: + backend = default_hp_search_backend() + if backend is None: + raise RuntimeError("optuna should be installed. " "To install optuna run `pip install optuna`. ") + backend = HPSearchBackend(backend) + if backend == HPSearchBackend.OPTUNA and not is_optuna_available(): + raise RuntimeError("You picked the optuna backend, but it is not installed. Use `pip install optuna`.") + elif backend != HPSearchBackend.OPTUNA: + raise RuntimeError("Only optuna backend is supported for hyperparameter search.") + self.hp_search_backend = backend + if self.model_init is None: + raise RuntimeError( + "To use hyperparameter search, you need to pass your model through a model_init function." + ) + + self.hp_space = default_hp_space_optuna if hp_space is None else hp_space + self.hp_name = hp_name + self.compute_objective = default_compute_objective if compute_objective is None else compute_objective + + backend_dict = { + HPSearchBackend.OPTUNA: run_hp_search_optuna, + } + best_run = backend_dict[backend](self, n_trials, direction, **kwargs) + + self.hp_search_backend = None + return best_run + + def push_to_hub( + self, + repo_path_or_name: Optional[str] = None, + repo_url: Optional[str] = None, + commit_message: Optional[str] = "Add SetFit model", + organization: Optional[str] = None, + private: Optional[bool] = None, + api_endpoint: Optional[str] = None, + use_auth_token: Optional[Union[bool, str]] = None, + git_user: Optional[str] = None, + git_email: Optional[str] = None, + config: Optional[dict] = None, + skip_lfs_files: bool = False, + ): + + return self.model.push_to_hub( + repo_path_or_name, + repo_url, + commit_message, + organization, + private, + api_endpoint, + use_auth_token, + git_user, + git_email, + config, + skip_lfs_files, + ) From 173f0845baa6133491bfc89c33328a4dd8c2227c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 17 Oct 2023 21:49:50 +0200 Subject: [PATCH 042/183] Run formatters --- src/setfit/modeling.py | 2 +- src/setfit/sampler.py | 18 +++++++++--------- src/setfit/trainer.py | 12 +++++------- src/setfit/trainer_unique_pairs.py | 3 +-- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 466d47a3..0662d2d3 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -17,7 +17,7 @@ import requests import torch from huggingface_hub import PyTorchModelHubMixin, hf_hub_download -from sentence_transformers import InputExample, SentenceTransformer, models +from sentence_transformers import SentenceTransformer, models from sklearn.linear_model import LogisticRegression from sklearn.multiclass import OneVsRestClassifier from sklearn.multioutput import ClassifierChain, MultiOutputClassifier diff --git a/src/setfit/sampler.py b/src/setfit/sampler.py index f4ae97ce..38e4ea92 100644 --- a/src/setfit/sampler.py +++ b/src/setfit/sampler.py @@ -1,9 +1,8 @@ -from typing import Generator, Iterable, List, Optional +from typing import Generator, Iterable, Iterator, List, Optional import numpy as np -from torch.utils.data import IterableDataset - from sentence_transformers import InputExample +from torch.utils.data import IterableDataset from . import logging @@ -54,12 +53,13 @@ def shuffle_combinations(iterable: Iterable, replacement: bool = True) -> Genera class ConstrastiveDataset(IterableDataset): - def __init__(self, + def __init__( + self, examples: InputExample, multilabel: bool, num_iterations: Optional[None] = None, sampling_strategy: str = "oversampling", - ): + ) -> None: """Generates positive and negative text pairs for contrastive learning. Args: @@ -105,7 +105,7 @@ def __init__(self, def generate_pairs(self) -> None: for (_text, _label), (text, label) in shuffle_combinations(self.sentence_labels): if _label == label: - self.pos_pairs.append(InputExample(texts=[_text, text], label=1.0)) + self.pos_pairs.append(InputExample(texts=[_text, text], label=1.0)) else: self.neg_pairs.append(InputExample(texts=[_text, text], label=0.0)) @@ -113,7 +113,7 @@ def generate_multilabel_pairs(self) -> None: for (_text, _label), (text, label) in shuffle_combinations(self.sentence_labels): if any(np.logical_and(_label, label)): # logical_and checks if labels are both set for each class - self.pos_pairs.append(InputExample(texts=[_text, text], label=1.0)) + self.pos_pairs.append(InputExample(texts=[_text, text], label=1.0)) else: self.neg_pairs.append(InputExample(texts=[_text, text], label=0.0)) @@ -135,10 +135,10 @@ def get_negative_pairs(self) -> List[InputExample]: self.neg_index += 1 return pairs - def __iter__(self): + def __iter__(self) -> Iterator[InputExample]: for pos_pair, neg_pair in zip(self.get_positive_pairs(), self.get_negative_pairs()): yield pos_pair yield neg_pair - def __len__(self): + def __len__(self) -> int: return self.len_pos_pairs + self.len_neg_pairs diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 5d7c2a17..6d2e3130 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import evaluate -import numpy as np import torch from datasets import Dataset, DatasetDict from sentence_transformers import InputExample, SentenceTransformer, losses @@ -16,7 +15,7 @@ from torch import nn from torch.cuda.amp import autocast from torch.utils.data import DataLoader -from tqdm.autonotebook import tqdm, trange +from tqdm.autonotebook import tqdm from transformers.integrations import get_reporting_integration_callbacks from transformers.trainer_callback import ( CallbackHandler, @@ -417,7 +416,6 @@ def train_embeddings( ) def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], args: TrainingArguments): - # sentence-transformers adaptation input_data = [InputExample(texts=[text], label=label) for text, label in zip(x, y)] @@ -446,11 +444,11 @@ def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], arg margin=args.margin, ) else: - data_sampler = ConstrastiveDataset( - input_data, self.model.multi_target_strategy, args.num_iterations - ) # sets default sampling_strategy="oversampling" + # sets default sampling_strategy="oversampling" + data_sampler = ConstrastiveDataset(input_data, self.model.multi_target_strategy, args.num_iterations) batch_size = min(args.embedding_batch_size, len(data_sampler)) - dataloader = DataLoader(data_sampler, batch_size=batch_size, drop_last=False) # shuffle=True can be dropped in for 'randomising' + # shuffle=True can be dropped in for 'randomising' + dataloader = DataLoader(data_sampler, batch_size=batch_size, drop_last=False) loss = args.loss(self.model.model_body) return dataloader, loss, batch_size diff --git a/src/setfit/trainer_unique_pairs.py b/src/setfit/trainer_unique_pairs.py index 461b880d..73004574 100644 --- a/src/setfit/trainer_unique_pairs.py +++ b/src/setfit/trainer_unique_pairs.py @@ -337,7 +337,7 @@ def train( train_data_sampler = SentenceLabelDataset(train_examples, samples_per_label=self.samples_per_label) batch_size = min(batch_size, len(train_data_sampler)) train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) - else: + else: train_data_sampler = ConstrastiveDataset(train_examples, multilabel, self.sampling_strategy) train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=False) @@ -515,7 +515,6 @@ def push_to_hub( config: Optional[dict] = None, skip_lfs_files: bool = False, ): - return self.model.push_to_hub( repo_path_or_name, repo_url, From c23959aec721c5d086404be29730181130bacfbb Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 17 Oct 2023 22:11:33 +0200 Subject: [PATCH 043/183] Remove tests from modeling.py The code here is moved to sampler.py, which will need its own tester file --- tests/test_modeling.py | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/tests/test_modeling.py b/tests/test_modeling.py index c31417d2..a5e279f6 100644 --- a/tests/test_modeling.py +++ b/tests/test_modeling.py @@ -10,42 +10,12 @@ from sklearn.multioutput import ClassifierChain, MultiOutputClassifier from setfit import SetFitHead, SetFitModel -from setfit.modeling import MODEL_HEAD_NAME, sentence_pairs_generation, sentence_pairs_generation_multilabel +from setfit.modeling import MODEL_HEAD_NAME torch_cuda_available = pytest.mark.skipif(not torch.cuda.is_available(), reason="PyTorch must be compiled with CUDA") -def test_sentence_pairs_generation(): - sentences = np.array(["sent 1", "sent 2", "sent 3"]) - labels = np.array(["label 1", "label 2", "label 3"]) - - pairs = [] - n_iterations = 2 - - for _ in range(n_iterations): - pairs = sentence_pairs_generation(sentences, labels, pairs) - - assert len(pairs) == 12 - assert pairs[0].texts == ["sent 1", "sent 1"] - assert pairs[0].label == 1.0 - - -def test_sentence_pairs_generation_multilabel(): - sentences = np.array(["sent 1", "sent 2", "sent 3"]) - labels = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]]) - - pairs = [] - n_iterations = 2 - - for _ in range(n_iterations): - pairs = sentence_pairs_generation_multilabel(sentences, labels, pairs) - - assert len(pairs) == 12 - assert pairs[0].texts == ["sent 1", "sent 1"] - assert pairs[0].label == 1.0 - - def test_setfit_model_body(): model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") From 0fa3870c00e178e750b4b5b322b7d009b7607c1e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 17 Oct 2023 22:12:38 +0200 Subject: [PATCH 044/183] Add missing type hint --- src/setfit/trainer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index fadf5a7b..ca9f3973 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -3,7 +3,7 @@ import time import warnings from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union import evaluate import numpy as np @@ -416,7 +416,9 @@ def train_embeddings( warmup_steps=warmup_steps, ) - def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], args: TrainingArguments): + def get_dataloader( + self, x: List[str], y: Union[List[int], List[List[int]]], args: TrainingArguments + ) -> Tuple[DataLoader, nn.Module, int]: # sentence-transformers adaptation if args.loss in [ losses.BatchAllTripletLoss, From 3969f3833d0d89543b9d83bd7ee6ed2b2794cce0 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 17 Oct 2023 22:15:15 +0200 Subject: [PATCH 045/183] Adjust test to still pass if W&B/Tensorboard are installed --- tests/test_trainer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_trainer.py b/tests/test_trainer.py index e5e0fa08..1a75fefb 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -505,14 +505,16 @@ def model_init() -> SetFitModel: def test_trainer_callbacks(model: SetFitModel): trainer = Trainer(model=model) - assert len(trainer.callback_handler.callbacks) == 2 + assert len(trainer.callback_handler.callbacks) >= 2 + callback_names = {callback.__class__.__name__ for callback in trainer.callback_handler.callbacks} + assert {"DefaultFlowCallback", "ProgressCallback"} <= callback_names class TestCallback(TrainerCallback): pass callback = TestCallback() trainer.add_callback(callback) - assert len(trainer.callback_handler.callbacks) == 3 + assert len(trainer.callback_handler.callbacks) == len(callback_names) + 1 assert trainer.callback_handler.callbacks[-1] == callback assert trainer.pop_callback(callback) == callback From 851f0bb1d7cb298bc06eb3bce4e2b5e2ec7e5058 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 17 Oct 2023 23:06:16 +0200 Subject: [PATCH 046/183] The log/eval/save steps should be saved on the state instead Since transformers v4.32.0, --- src/setfit/trainer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index ca9f3973..a4ec70f2 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -142,7 +142,6 @@ def __init__( self.state = TrainerState() self.control = TrainerControl() self.add_callback(DEFAULT_PROGRESS_CALLBACK if self.args.show_progress_bar else PrinterCallback) - self.control = self.callback_handler.on_init_end(args, self.state, self.control) def add_callback(self, callback): @@ -392,6 +391,10 @@ def train_embeddings( Temporarily change the training arguments for this training call. """ args = args or self.args or TrainingArguments() + # Since transformers v4.32.0, the log/eval/save steps should be saved on the state instead + self.state.logging_steps = args.logging_steps + self.state.eval_steps = args.eval_steps + self.state.save_steps = args.save_steps train_dataloader, loss_func, batch_size = self.get_dataloader(x_train, y_train, args=args) if x_eval is not None: From d37ee09ba5764ca6793521586f3de7267c6f55a2 Mon Sep 17 00:00:00 2001 From: danstan5 Date: Thu, 19 Oct 2023 12:37:46 +0100 Subject: [PATCH 047/183] sampler logic fix "unique" strategy --- src/setfit/sampler.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/setfit/sampler.py b/src/setfit/sampler.py index f4ae97ce..24146274 100644 --- a/src/setfit/sampler.py +++ b/src/setfit/sampler.py @@ -1,3 +1,4 @@ +from itertools import zip_longest from typing import Generator, Iterable, List, Optional import numpy as np @@ -136,9 +137,11 @@ def get_negative_pairs(self) -> List[InputExample]: return pairs def __iter__(self): - for pos_pair, neg_pair in zip(self.get_positive_pairs(), self.get_negative_pairs()): - yield pos_pair - yield neg_pair + for pos_pair, neg_pair in zip_longest(self.get_positive_pairs(), self.get_negative_pairs()): + if pos_pair is not None: + yield pos_pair + if neg_pair is not None: + yield neg_pair def __len__(self): return self.len_pos_pairs + self.len_neg_pairs From 0ef88378bff9bcb5bda19ea03e1fb632ce70c367 Mon Sep 17 00:00:00 2001 From: danstan5 Date: Thu, 19 Oct 2023 12:38:37 +0100 Subject: [PATCH 048/183] add sampler tests (not complete) --- src/setfit/trainer_unique_pairs.py | 531 ----------------------------- tests/test_modeling.py | 30 -- tests/test_sampler.py | 50 +++ 3 files changed, 50 insertions(+), 561 deletions(-) delete mode 100644 src/setfit/trainer_unique_pairs.py create mode 100644 tests/test_sampler.py diff --git a/src/setfit/trainer_unique_pairs.py b/src/setfit/trainer_unique_pairs.py deleted file mode 100644 index 461b880d..00000000 --- a/src/setfit/trainer_unique_pairs.py +++ /dev/null @@ -1,531 +0,0 @@ -import math -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union - -import evaluate -from sentence_transformers import InputExample, losses -from sentence_transformers.datasets import SentenceLabelDataset -from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction -from torch.utils.data import DataLoader -from transformers.trainer_utils import HPSearchBackend, default_compute_objective, number_of_arguments, set_seed - -from . import logging -from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna -from .modeling import SupConLoss -from .sampler import OVERSAMPLE, ConstrastiveDataset -from .utils import BestRun, default_hp_space_optuna - - -if TYPE_CHECKING: - import optuna - from datasets import Dataset - - from .modeling import SetFitModel - -logging.set_verbosity_info() -logger = logging.get_logger(__name__) - - -class SetFitTrainer: - """Trainer to train a SetFit model. - - Args: - model (`SetFitModel`, *optional*): - The model to train. If not provided, a `model_init` must be passed. - train_dataset (`Dataset`): - The training dataset. - eval_dataset (`Dataset`, *optional*): - The evaluation dataset. - model_init (`Callable[[], SetFitModel]`, *optional*): - A function that instantiates the model to be used. If provided, each call to [`~SetFitTrainer.train`] will start - from a new instance of the model as given by this function when a `trial` is passed. - metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): - The metric to use for evaluation. If a string is provided, we treat it as the metric name and load it with default settings. - If a callable is provided, it must take two arguments (`y_pred`, `y_test`). - loss_class (`nn.Module`, *optional*, defaults to `CosineSimilarityLoss`): - The loss function to use for contrastive training. - num_iterations (`int`, *optional*, defaults to `20`): - The number of iterations to generate sentence pairs for. - This argument is ignored if triplet loss is used. - It is only used in conjunction with `CosineSimilarityLoss`. - num_epochs (`int`, *optional*, defaults to `1`): - The number of epochs to train the Sentence Transformer body for. - learning_rate (`float`, *optional*, defaults to `2e-5`): - The learning rate to use for contrastive training. - batch_size (`int`, *optional*, defaults to `16`): - The batch size to use for contrastive training. - seed (`int`, *optional*, defaults to 42): - Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use the - [`~SetTrainer.model_init`] function to instantiate the model if it has some randomly initialized parameters. - column_mapping (`Dict[str, str]`, *optional*): - A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: {"text_column_name": "text", "label_column_name: "label"}. - use_amp (`bool`, *optional*, defaults to `False`): - Use Automatic Mixed Precision (AMP). Only for Pytorch >= 1.6.0 - warmup_proportion (`float`, *optional*, defaults to `0.1`): - Proportion of the warmup in the total training steps. - Must be greater than or equal to 0.0 and less than or equal to 1.0. - distance_metric (`Callable`, defaults to `BatchHardTripletLossDistanceFunction.cosine_distance`): - Function that returns a distance between two embeddings. - It is set for the triplet loss and - is ignored for `CosineSimilarityLoss` and `SupConLoss`. - margin (`float`, defaults to `0.25`): Margin for the triplet loss. - Negative samples should be at least margin further apart from the anchor than the positive. - This is ignored for `CosineSimilarityLoss`, `BatchHardSoftMarginTripletLoss` and `SupConLoss`. - samples_per_label (`int`, defaults to `2`): Number of consecutive, random and unique samples drawn per label. - This is only relevant for triplet loss and ignored for `CosineSimilarityLoss`. - Batch size should be a multiple of samples_per_label. - """ - - def __init__( - self, - model: Optional["SetFitModel"] = None, - train_dataset: Optional["Dataset"] = None, - eval_dataset: Optional["Dataset"] = None, - model_init: Optional[Callable[[], "SetFitModel"]] = None, - metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", - loss_class: Optional[Any] = None, - num_iterations: int = 20, - num_epochs: int = 1, - learning_rate: float = 2e-5, - batch_size: int = 16, - seed: int = 42, - column_mapping: Optional[Dict[str, str]] = None, - use_amp: bool = False, - warmup_proportion: float = 0.1, - distance_metric: Callable = BatchHardTripletLossDistanceFunction.cosine_distance, - margin: float = 0.25, - samples_per_label: int = 2, - sampling_strategy: int = OVERSAMPLE, - ): - if (warmup_proportion < 0.0) or (warmup_proportion > 1.0): - raise ValueError( - f"warmup_proportion must be greater than or equal to 0.0 and less than or equal to 1.0! But it was: {warmup_proportion}" - ) - - self.train_dataset = train_dataset - self.eval_dataset = eval_dataset - self.model_init = model_init - self.metric = metric - self.loss_class = loss_class - self.num_iterations = num_iterations - self.num_epochs = num_epochs - self.learning_rate = learning_rate - self.batch_size = batch_size - self.seed = seed - self.column_mapping = column_mapping - self.use_amp = use_amp - self.warmup_proportion = warmup_proportion - self.distance_metric = distance_metric - self.margin = margin - self.samples_per_label = samples_per_label - self.sampling_strategy = sampling_strategy - - if model is None: - if model_init is not None: - model = self.call_model_init() - else: - raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument") - else: - if model_init is not None: - raise RuntimeError("`SetFitTrainer` requires either a `model` or `model_init` argument, but not both") - - self.model = model - self.hp_search_backend = None - self._freeze = True # If True, will train the body only; otherwise, train the body and head - - def _validate_column_mapping(self, dataset: "Dataset") -> None: - """ - Validates the provided column mapping against the dataset. - """ - required_columns = {"text", "label"} - column_names = set(dataset.column_names) - if self.column_mapping is None and not required_columns.issubset(column_names): - raise ValueError( - f"A column mapping must be provided when the dataset does not contain the following columns: {required_columns}" - ) - if self.column_mapping is not None: - missing_columns = required_columns.difference(self.column_mapping.values()) - if missing_columns: - raise ValueError( - f"The following columns are missing from the column mapping: {missing_columns}. Please provide a mapping for all required columns." - ) - if not set(self.column_mapping.keys()).issubset(column_names): - raise ValueError( - f"The following columns are missing from the dataset: {set(self.column_mapping.keys()).difference(column_names)}. Please provide a mapping for all required columns." - ) - - def _apply_column_mapping(self, dataset: "Dataset", column_mapping: Dict[str, str]) -> "Dataset": - """ - Applies the provided column mapping to the dataset, renaming columns accordingly. - Extra features not in the column mapping are prefixed with `"feat_"`. - """ - dataset = dataset.rename_columns( - { - **column_mapping, - **{col: f"feat_{col}" for col in dataset.column_names if col not in column_mapping}, - } - ) - dset_format = dataset.format - dataset = dataset.with_format( - type=dset_format["type"], - columns=dataset.column_names, - output_all_columns=dset_format["output_all_columns"], - **dset_format["format_kwargs"], - ) - return dataset - - def apply_hyperparameters(self, params: Dict[str, Any], final_model: bool = False): - """Applies a dictionary of hyperparameters to both the trainer and the model - - Args: - params (`Dict[str, Any]`): The parameters, usually from `BestRun.hyperparameters` - final_model (`bool`, *optional*, defaults to `False`): If `True`, replace the `model_init()` function with a fixed model based on the parameters. - """ - for key, value in params.items(): - if hasattr(self, key): - old_attr = getattr(self, key, None) - # Casting value to the proper type - if old_attr is not None: - value = type(old_attr)(value) - setattr(self, key, value) - elif number_of_arguments(self.model_init) == 0: # we do not warn if model_init could be using it - logger.warning( - f"Trying to set {key!r} in the hyperparameter search but there is no corresponding field in " - "`SetFitTrainer`, and `model_init` does not take any arguments." - ) - - self.model = self.model_init(params) - if final_model: - self.model_init = None - - def _hp_search_setup(self, trial: Union["optuna.Trial", Dict[str, Any]]): - """HP search setup code""" - - # Heavily inspired by transformers.Trainer._hp_search_setup - if self.hp_search_backend is None or trial is None: - return - - if isinstance(trial, Dict): # For passing a Dict to train() -- mostly unused for now - params = trial - elif self.hp_search_backend == HPSearchBackend.OPTUNA: - params = self.hp_space(trial) - else: - raise ValueError("Invalid trial parameter") - - logger.info(f"Trial: {params}") - self.apply_hyperparameters(params, final_model=False) - - def call_model_init(self, params: Optional[Dict[str, Any]] = None): - model_init_argcount = number_of_arguments(self.model_init) - if model_init_argcount == 0: - model = self.model_init() - elif model_init_argcount == 1: - model = self.model_init(params) - else: - raise RuntimeError("`model_init` should have 0 or 1 argument.") - - if model is None: - raise RuntimeError("`model_init` should not return None.") - - return model - - def freeze(self): - """ - Freeze SetFitModel's differentiable head. - Note: call this function only when using the differentiable head. - """ - if not self.model.has_differentiable_head: - raise ValueError("Please use the differentiable head in `SetFitModel` when calling this function.") - - self._freeze = True # Currently use self._freeze as a switch - self.model.freeze("head") - - def unfreeze(self, keep_body_frozen: bool = False): - """ - Unfreeze SetFitModel's differentiable head. - Note: call this function only when using the differentiable head. - - Args: - keep_body_frozen (`bool`, *optional*, defaults to `False`): - Whether to freeze the body when unfreeze the head. - """ - if not self.model.has_differentiable_head: - raise ValueError("Please use the differentiable head in `SetFitModel` when calling this function.") - - self._freeze = False # Currently use self._freeze as a switch - self.model.unfreeze("head") - if keep_body_frozen: - self.model.freeze("body") - else: # ensure to unfreeze the body - self.model.unfreeze("body") - - def train( - self, - num_epochs: Optional[int] = None, - batch_size: Optional[int] = None, - learning_rate: Optional[float] = None, - body_learning_rate: Optional[float] = None, - l2_weight: Optional[float] = None, - max_length: Optional[int] = None, - trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, - show_progress_bar: bool = True, - ): - """ - Main training entry point. - - Args: - num_epochs (`int`, *optional*): - Temporary change the number of epochs to train the Sentence Transformer body/head for. - If ignore, will use the value given in initialization. - batch_size (`int`, *optional*): - Temporary change the batch size to use for contrastive training or logistic regression. - If ignore, will use the value given in initialization. - learning_rate (`float`, *optional*): - Temporary change the learning rate to use for contrastive training or SetFitModel's head in logistic regression. - If ignore, will use the value given in initialization. - body_learning_rate (`float`, *optional*): - Temporary change the learning rate to use for SetFitModel's body in logistic regression only. - If ignore, will be the same as `learning_rate`. - l2_weight (`float`, *optional*): - Temporary change the weight of L2 regularization for SetFitModel's differentiable head in logistic regression. - max_length (int, *optional*, defaults to `None`): - The maximum number of tokens for one data sample. Currently only for training the differentiable head. - If `None`, will use the maximum number of tokens the model body can accept. - If `max_length` is greater than the maximum number of acceptable tokens the model body can accept, it will be set to the maximum number of acceptable tokens. - trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): - The trial run or the hyperparameter dictionary for hyperparameter search. - show_progress_bar (`bool`, *optional*, defaults to `True`): - Whether to show a bar that indicates training progress. - """ - set_seed(self.seed) # Seed must be set before instantiating the model when using model_init. - - if trial: # Trial and model initialization - self._hp_search_setup(trial) # sets trainer parameters and initializes model - - if self.train_dataset is None: - raise ValueError("Training requires a `train_dataset` given to the `SetFitTrainer` initialization.") - - self._validate_column_mapping(self.train_dataset) - train_dataset = self.train_dataset - if self.column_mapping is not None: - logger.info("Applying column mapping to training dataset") - train_dataset = self._apply_column_mapping(self.train_dataset, self.column_mapping) - - if self.loss_class is None: - logger.warning("No `loss_class` detected! Using `CosineSimilarityLoss` as the default.") - self.loss_class = losses.CosineSimilarityLoss - - multilabel = True if self.model.multi_target_strategy is not None else False - - num_epochs = num_epochs or self.num_epochs - batch_size = batch_size or self.batch_size - learning_rate = learning_rate or self.learning_rate - - # dataset generation - x_train = train_dataset["text"] - y_train = train_dataset["label"] - train_examples = [InputExample(texts=[text], label=label) for text, label in zip(x_train, y_train)] - - if not self.model.has_differentiable_head or self._freeze: - # sentence-transformers adaptation - if self.loss_class in [ - losses.BatchAllTripletLoss, - losses.BatchHardTripletLoss, - losses.BatchSemiHardTripletLoss, - losses.BatchHardSoftMarginTripletLoss, - SupConLoss, - ]: - train_data_sampler = SentenceLabelDataset(train_examples, samples_per_label=self.samples_per_label) - batch_size = min(batch_size, len(train_data_sampler)) - train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=True) - else: - train_data_sampler = ConstrastiveDataset(train_examples, multilabel, self.sampling_strategy) - train_dataloader = DataLoader(train_data_sampler, batch_size=batch_size, drop_last=False) - - total_train_steps = len(train_dataloader) * num_epochs - logger.info("***** Running training *****") - logger.info(f" Num examples per epoch = {len(train_data_sampler)}") - logger.info(f" Num epochs = {num_epochs}") - logger.info(f" Total optimization steps = {total_train_steps}") - logger.info(f" Total train batch size = {batch_size}") - - # setup training loss - if self.loss_class in [ - losses.BatchAllTripletLoss, - losses.BatchHardTripletLoss, - losses.BatchSemiHardTripletLoss, - losses.BatchHardSoftMarginTripletLoss, - ]: - train_loss = self.loss_class( - model=self.model.model_body, - distance_metric=self.distance_metric, - margin=self.margin, - ) - elif self.loss_class is losses.BatchHardSoftMarginTripletLoss: - train_loss = self.loss_class( - model=self.model.model_body, - distance_metric=self.distance_metric, - ) - else: - train_loss = self.loss_class(model=self.model.model_body) - - warmup_steps = math.ceil(total_train_steps * self.warmup_proportion) - self.model.model_body.fit( - train_objectives=[(train_dataloader, train_loss)], - epochs=num_epochs, - optimizer_params={"lr": learning_rate}, - warmup_steps=warmup_steps, - show_progress_bar=show_progress_bar, - use_amp=self.use_amp, - ) - - if not self.model.has_differentiable_head or not self._freeze: - # Train the final classifier - self.model.fit( - x_train, - y_train, - num_epochs=num_epochs, - batch_size=batch_size, - learning_rate=learning_rate, - body_learning_rate=body_learning_rate, - l2_weight=l2_weight, - max_length=max_length, - show_progress_bar=True, - ) - - def evaluate(self): - """ - Computes the metrics for a given classifier. - - Returns: - `Dict[str, float]`: The evaluation metrics. - """ - - self._validate_column_mapping(self.eval_dataset) - eval_dataset = self.eval_dataset - - if self.column_mapping is not None: - logger.info("Applying column mapping to evaluation dataset") - eval_dataset = self._apply_column_mapping(self.eval_dataset, self.column_mapping) - - x_test = eval_dataset["text"] - y_test = eval_dataset["label"] - - logger.info("***** Running evaluation *****") - y_pred = self.model.predict(x_test) - - if isinstance(self.metric, str): - metric_config = "multilabel" if self.model.multi_target_strategy is not None else None - metric_fn = evaluate.load(self.metric, config_name=metric_config) - - return metric_fn.compute(predictions=y_pred, references=y_test) - - elif callable(self.metric): - return self.metric(y_pred, y_test) - - else: - raise ValueError("metric must be a string or a callable") - - def hyperparameter_search( - self, - hp_space: Optional[Callable[["optuna.Trial"], Dict[str, float]]] = None, - compute_objective: Optional[Callable[[Dict[str, float]], float]] = None, - n_trials: int = 10, - direction: str = "maximize", - backend: Optional[Union["str", HPSearchBackend]] = None, - hp_name: Optional[Callable[["optuna.Trial"], str]] = None, - **kwargs, - ) -> BestRun: - """ - Launch a hyperparameter search using `optuna`. The optimized quantity is determined - by `compute_objective`, which defaults to a function returning the evaluation loss when no metric is provided, - the sum of all metrics otherwise. - - - - To use this method, you need to have provided a `model_init` when initializing your [`SetFitTrainer`]: we need to - reinitialize the model at each new run. - - - - Args: - hp_space (`Callable[["optuna.Trial"], Dict[str, float]]`, *optional*): - A function that defines the hyperparameter search space. Will default to - [`~trainer_utils.default_hp_space_optuna`]. - compute_objective (`Callable[[Dict[str, float]], float]`, *optional*): - A function computing the objective to minimize or maximize from the metrics returned by the `evaluate` - method. Will default to [`~trainer_utils.default_compute_objective`] which uses the sum of metrics. - n_trials (`int`, *optional*, defaults to 100): - The number of trial runs to test. - direction (`str`, *optional*, defaults to `"maximize"`): - Whether to optimize greater or lower objects. Can be `"minimize"` or `"maximize"`, you should pick - `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or several metrics. - backend (`str` or [`~training_utils.HPSearchBackend`], *optional*): - The backend to use for hyperparameter search. Only optuna is supported for now. - TODO: add support for ray and sigopt. - hp_name (`Callable[["optuna.Trial"], str]]`, *optional*): - A function that defines the trial/run name. Will default to None. - kwargs (`Dict[str, Any]`, *optional*): - Additional keyword arguments passed along to `optuna.create_study`. For more - information see: - - - the documentation of - [optuna.create_study](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.html) - - Returns: - [`trainer_utils.BestRun`]: All the information about the best run. - """ - if backend is None: - backend = default_hp_search_backend() - if backend is None: - raise RuntimeError("optuna should be installed. " "To install optuna run `pip install optuna`. ") - backend = HPSearchBackend(backend) - if backend == HPSearchBackend.OPTUNA and not is_optuna_available(): - raise RuntimeError("You picked the optuna backend, but it is not installed. Use `pip install optuna`.") - elif backend != HPSearchBackend.OPTUNA: - raise RuntimeError("Only optuna backend is supported for hyperparameter search.") - self.hp_search_backend = backend - if self.model_init is None: - raise RuntimeError( - "To use hyperparameter search, you need to pass your model through a model_init function." - ) - - self.hp_space = default_hp_space_optuna if hp_space is None else hp_space - self.hp_name = hp_name - self.compute_objective = default_compute_objective if compute_objective is None else compute_objective - - backend_dict = { - HPSearchBackend.OPTUNA: run_hp_search_optuna, - } - best_run = backend_dict[backend](self, n_trials, direction, **kwargs) - - self.hp_search_backend = None - return best_run - - def push_to_hub( - self, - repo_path_or_name: Optional[str] = None, - repo_url: Optional[str] = None, - commit_message: Optional[str] = "Add SetFit model", - organization: Optional[str] = None, - private: Optional[bool] = None, - api_endpoint: Optional[str] = None, - use_auth_token: Optional[Union[bool, str]] = None, - git_user: Optional[str] = None, - git_email: Optional[str] = None, - config: Optional[dict] = None, - skip_lfs_files: bool = False, - ): - - return self.model.push_to_hub( - repo_path_or_name, - repo_url, - commit_message, - organization, - private, - api_endpoint, - use_auth_token, - git_user, - git_email, - config, - skip_lfs_files, - ) diff --git a/tests/test_modeling.py b/tests/test_modeling.py index c31417d2..844ec6e6 100644 --- a/tests/test_modeling.py +++ b/tests/test_modeling.py @@ -16,36 +16,6 @@ torch_cuda_available = pytest.mark.skipif(not torch.cuda.is_available(), reason="PyTorch must be compiled with CUDA") -def test_sentence_pairs_generation(): - sentences = np.array(["sent 1", "sent 2", "sent 3"]) - labels = np.array(["label 1", "label 2", "label 3"]) - - pairs = [] - n_iterations = 2 - - for _ in range(n_iterations): - pairs = sentence_pairs_generation(sentences, labels, pairs) - - assert len(pairs) == 12 - assert pairs[0].texts == ["sent 1", "sent 1"] - assert pairs[0].label == 1.0 - - -def test_sentence_pairs_generation_multilabel(): - sentences = np.array(["sent 1", "sent 2", "sent 3"]) - labels = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]]) - - pairs = [] - n_iterations = 2 - - for _ in range(n_iterations): - pairs = sentence_pairs_generation_multilabel(sentences, labels, pairs) - - assert len(pairs) == 12 - assert pairs[0].texts == ["sent 1", "sent 1"] - assert pairs[0].label == 1.0 - - def test_setfit_model_body(): model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") diff --git a/tests/test_sampler.py b/tests/test_sampler.py new file mode 100644 index 00000000..67e1bf07 --- /dev/null +++ b/tests/test_sampler.py @@ -0,0 +1,50 @@ +import pytest +import numpy as np + +from sentence_transformers import InputExample + +from setfit.sampler import ConstrastiveDataset + + +@pytest.mark.parametrize("sampling_strategy, expected_pos_pairs, expected_neg_pairs", [ + ("unique", 4, 2), + ("undersampling", 2, 2), + ("oversampling", 4, 4) +]) +def test_sentence_pairs_generation(sampling_strategy: str, expected_pos_pairs: int, expected_neg_pairs: int): + sentences = np.array(["sent 1", "sent 2", "sent 3"]) + labels = np.array(["label 1", "label 1", "label 2"]) + + data = [InputExample(texts=[text], label=label) for text, label in zip(sentences, labels)] + multilabel = False + + data_sampler = ConstrastiveDataset(data, multilabel, sampling_strategy=sampling_strategy) + + assert data_sampler.len_pos_pairs == expected_pos_pairs + assert data_sampler.len_neg_pairs == expected_neg_pairs + + pairs = [i for i in data_sampler] + + assert len(pairs) == expected_pos_pairs + expected_neg_pairs + assert pairs[0].texts == ["sent 1", "sent 1"] + assert pairs[0].label == 1.0 + + +@pytest.mark.parametrize("sampling_strategy, expected_pos_pairs, expected_neg_pairs", [ + ("unique", 6, 4), + ("undersampling", 4, 4), + ("oversampling", 6, 6) +]) +def test_sentence_pairs_generation_multilabel(sampling_strategy: str, expected_pos_pairs: int, expected_neg_pairs: int): + sentences = np.array(["sent 1", "sent 2", "sent 3", "sent 4"]) + labels = np.array([[1, 0, 0, 1], [0, 1, 1, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) + + data = [InputExample(texts=[text], label=label) for text, label in zip(sentences, labels)] + multilabel = True + + data_sampler = ConstrastiveDataset(data, multilabel, sampling_strategy=sampling_strategy) + assert data_sampler.len_pos_pairs == expected_pos_pairs + assert data_sampler.len_neg_pairs == expected_neg_pairs + + pairs = [i for i in data_sampler] + assert len(pairs) == expected_pos_pairs + expected_neg_pairs From 131aa267b2570296b4c29e6d92d614eebb6ef03b Mon Sep 17 00:00:00 2001 From: danstan5 Date: Thu, 19 Oct 2023 12:39:03 +0100 Subject: [PATCH 049/183] add sampling_strategy into TrainingArguments --- src/setfit/trainer.py | 8 +++++--- src/setfit/training_args.py | 22 +++++++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 5d7c2a17..a4c841b4 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -447,10 +447,12 @@ def get_dataloader(self, x: List[str], y: Union[List[int], List[List[int]]], arg ) else: data_sampler = ConstrastiveDataset( - input_data, self.model.multi_target_strategy, args.num_iterations - ) # sets default sampling_strategy="oversampling" + input_data, self.model.multi_target_strategy, args.num_iterations, args.sampling_strategy + ) + # shuffle_sampler = True can be dropped in for further 'randomising' + shuffle_sampler = True if args.sampling_strategy == "unique" else False batch_size = min(args.embedding_batch_size, len(data_sampler)) - dataloader = DataLoader(data_sampler, batch_size=batch_size, drop_last=False) # shuffle=True can be dropped in for 'randomising' + dataloader = DataLoader(data_sampler, batch_size=batch_size, shuffle=shuffle_sampler, drop_last=False) loss = args.loss(self.model.model_body) return dataloader, loss, batch_size diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 3ba751cb..53f04818 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -30,10 +30,25 @@ class TrainingArguments: Set the number of epochs the embedding and classifier training phases respectively, or set both if an integer is provided. Note that the number of epochs for the classifier is only used with a differentiable PyTorch head. - num_iterations (`int`, defaults to `20`): - The number of iterations to generate sentence pairs for. + num_iterations (`int`, *optional*): + If not set the `sampling_strategy` will determine the number of sentence pairs to generate. + This argument sets the number of iterations to generate sentence pairs for + and provides compatability with Setfit Date: Thu, 19 Oct 2023 12:54:13 +0100 Subject: [PATCH 050/183] num_iterations removed from TrainingArguments --- src/setfit/trainer.py | 11 +++++++++-- src/setfit/training_args.py | 11 ++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index fd867f21..e744e0fe 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -95,6 +95,11 @@ class Trainer: A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: `{"text_column_name": "text", "label_column_name: "label"}`. + num_iterations (`int`, *optional*): + This argument sets the number of iterations to generate sentence pairs for + and provides compatability with Setfit None: self.args = args or TrainingArguments() self.train_dataset = train_dataset @@ -118,6 +124,7 @@ def __init__( self.metric = metric self.metric_kwargs = metric_kwargs self.column_mapping = column_mapping + self.num_iterations = num_iterations if model is None: if model_init is not None: @@ -450,7 +457,7 @@ def get_dataloader( ) else: data_sampler = ConstrastiveDataset( - input_data, self.model.multi_target_strategy, args.num_iterations, args.sampling_strategy + input_data, self.model.multi_target_strategy, self.num_iterations, args.sampling_strategy ) # shuffle_sampler = True can be dropped in for further 'randomising' shuffle_sampler = True if args.sampling_strategy == "unique" else False @@ -905,7 +912,6 @@ def __init__( stacklevel=2, ) args = TrainingArguments( - num_iterations=num_iterations, num_epochs=num_epochs, body_learning_rate=learning_rate, head_learning_rate=learning_rate, @@ -927,4 +933,5 @@ def __init__( metric=metric, metric_kwargs=metric_kwargs, column_mapping=column_mapping, + num_iterations=num_iterations ) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 53f04818..71526c87 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -30,12 +30,6 @@ class TrainingArguments: Set the number of epochs the embedding and classifier training phases respectively, or set both if an integer is provided. Note that the number of epochs for the classifier is only used with a differentiable PyTorch head. - num_iterations (`int`, *optional*): - If not set the `sampling_strategy` will determine the number of sentence pairs to generate. - This argument sets the number of iterations to generate sentence pairs for - and provides compatability with Setfit Date: Fri, 20 Oct 2023 14:27:32 +0100 Subject: [PATCH 051/183] run_fewshot compatible with Date: Wed, 25 Oct 2023 13:44:21 +0200 Subject: [PATCH 052/183] Run make style --- src/setfit/trainer.py | 4 ++-- src/setfit/training_args.py | 6 +++--- tests/test_sampler.py | 29 ++++++++++++++--------------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index e744e0fe..cdda7adc 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -462,7 +462,7 @@ def get_dataloader( # shuffle_sampler = True can be dropped in for further 'randomising' shuffle_sampler = True if args.sampling_strategy == "unique" else False batch_size = min(args.embedding_batch_size, len(data_sampler)) - dataloader = DataLoader(data_sampler, batch_size=batch_size, shuffle=shuffle_sampler, drop_last=False) + dataloader = DataLoader(data_sampler, batch_size=batch_size, shuffle=shuffle_sampler, drop_last=False) loss = args.loss(self.model.model_body) return dataloader, loss, batch_size @@ -933,5 +933,5 @@ def __init__( metric=metric, metric_kwargs=metric_kwargs, column_mapping=column_mapping, - num_iterations=num_iterations + num_iterations=num_iterations, ) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 71526c87..85edd757 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -33,15 +33,15 @@ class TrainingArguments: sampling_strategy (`str`, defaults to `"oversampling"`): The sampling strategy of how to draw pairs in training. Possible values are: - - `"oversampling"`: Draws even number of positive/ negative sentence pairs until every + - `"oversampling"`: Draws even number of positive/ negative sentence pairs until every sentence pair has been drawn. - `"undersampling"`: Draws the minimum number of positive/ negative sentence pairs until every sentence pair in the minority class has been drawn. - - `"unique"`: Draws every sentence pair combination (likely resulting in unbalanced + - `"unique"`: Draws every sentence pair combination (likely resulting in unbalanced number of positive/ negative sentence pairs). The default is set to `"oversampling"` ensuring all sentence pairs are drawn at least once. - Alternatively setting the num_iterations in the SetFitTrainer class will override this + Alternatively setting the num_iterations in the SetFitTrainer class will override this argument and determine the number of generated sentence pairs. body_learning_rate (`Union[float, Tuple[float, float]]`, defaults to `(2e-5, 1e-5)`): Set the learning rate for the `SentenceTransformer` body for the embedding and classifier diff --git a/tests/test_sampler.py b/tests/test_sampler.py index 67e1bf07..c3207592 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -1,16 +1,14 @@ -import pytest import numpy as np - +import pytest from sentence_transformers import InputExample from setfit.sampler import ConstrastiveDataset -@pytest.mark.parametrize("sampling_strategy, expected_pos_pairs, expected_neg_pairs", [ - ("unique", 4, 2), - ("undersampling", 2, 2), - ("oversampling", 4, 4) -]) +@pytest.mark.parametrize( + "sampling_strategy, expected_pos_pairs, expected_neg_pairs", + [("unique", 4, 2), ("undersampling", 2, 2), ("oversampling", 4, 4)], +) def test_sentence_pairs_generation(sampling_strategy: str, expected_pos_pairs: int, expected_neg_pairs: int): sentences = np.array(["sent 1", "sent 2", "sent 3"]) labels = np.array(["label 1", "label 1", "label 2"]) @@ -22,7 +20,7 @@ def test_sentence_pairs_generation(sampling_strategy: str, expected_pos_pairs: i assert data_sampler.len_pos_pairs == expected_pos_pairs assert data_sampler.len_neg_pairs == expected_neg_pairs - + pairs = [i for i in data_sampler] assert len(pairs) == expected_pos_pairs + expected_neg_pairs @@ -30,12 +28,13 @@ def test_sentence_pairs_generation(sampling_strategy: str, expected_pos_pairs: i assert pairs[0].label == 1.0 -@pytest.mark.parametrize("sampling_strategy, expected_pos_pairs, expected_neg_pairs", [ - ("unique", 6, 4), - ("undersampling", 4, 4), - ("oversampling", 6, 6) -]) -def test_sentence_pairs_generation_multilabel(sampling_strategy: str, expected_pos_pairs: int, expected_neg_pairs: int): +@pytest.mark.parametrize( + "sampling_strategy, expected_pos_pairs, expected_neg_pairs", + [("unique", 6, 4), ("undersampling", 4, 4), ("oversampling", 6, 6)], +) +def test_sentence_pairs_generation_multilabel( + sampling_strategy: str, expected_pos_pairs: int, expected_neg_pairs: int +): sentences = np.array(["sent 1", "sent 2", "sent 3", "sent 4"]) labels = np.array([[1, 0, 0, 1], [0, 1, 1, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) @@ -45,6 +44,6 @@ def test_sentence_pairs_generation_multilabel(sampling_strategy: str, expected_p data_sampler = ConstrastiveDataset(data, multilabel, sampling_strategy=sampling_strategy) assert data_sampler.len_pos_pairs == expected_pos_pairs assert data_sampler.len_neg_pairs == expected_neg_pairs - + pairs = [i for i in data_sampler] assert len(pairs) == expected_pos_pairs + expected_neg_pairs From 978daeed6965146caf03c2092e6325a746c8e5f3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 25 Oct 2023 13:48:15 +0200 Subject: [PATCH 053/183] Use "no" as the default evaluation_strategy This matches the default from 'transformers' --- src/setfit/training_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 85edd757..a118a151 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -189,7 +189,7 @@ class TrainingArguments: logging_first_step: bool = True logging_steps: int = 5 - evaluation_strategy: str = "steps" + evaluation_strategy: str = "no" eval_steps: Optional[int] = None eval_delay: int = 0 From 2802a3f8f225734b0bdb9758472e14510405b0c1 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 25 Oct 2023 14:08:04 +0200 Subject: [PATCH 054/183] Move num_iterations back to TrainingArguments --- src/setfit/trainer.py | 11 ++--------- src/setfit/training_args.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index cdda7adc..55e72fcf 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -95,11 +95,6 @@ class Trainer: A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: `{"text_column_name": "text", "label_column_name: "label"}`. - num_iterations (`int`, *optional*): - This argument sets the number of iterations to generate sentence pairs for - and provides compatability with Setfit None: self.args = args or TrainingArguments() self.train_dataset = train_dataset @@ -124,7 +118,6 @@ def __init__( self.metric = metric self.metric_kwargs = metric_kwargs self.column_mapping = column_mapping - self.num_iterations = num_iterations if model is None: if model_init is not None: @@ -457,7 +450,7 @@ def get_dataloader( ) else: data_sampler = ConstrastiveDataset( - input_data, self.model.multi_target_strategy, self.num_iterations, args.sampling_strategy + input_data, self.model.multi_target_strategy, args.num_iterations, args.sampling_strategy ) # shuffle_sampler = True can be dropped in for further 'randomising' shuffle_sampler = True if args.sampling_strategy == "unique" else False @@ -912,6 +905,7 @@ def __init__( stacklevel=2, ) args = TrainingArguments( + num_iterations=num_iterations, num_epochs=num_epochs, body_learning_rate=learning_rate, head_learning_rate=learning_rate, @@ -933,5 +927,4 @@ def __init__( metric=metric, metric_kwargs=metric_kwargs, column_mapping=column_mapping, - num_iterations=num_iterations, ) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index a118a151..6a13ef80 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -40,9 +40,15 @@ class TrainingArguments: - `"unique"`: Draws every sentence pair combination (likely resulting in unbalanced number of positive/ negative sentence pairs). - The default is set to `"oversampling"` ensuring all sentence pairs are drawn at least once. - Alternatively setting the num_iterations in the SetFitTrainer class will override this - argument and determine the number of generated sentence pairs. + The default is set to `"oversampling"`, ensuring all sentence pairs are drawn at least once. + Alternatively setting `num_iterations` will override this argument and determine the number + of generated sentence pairs. + num_iterations (`int`, *optional*): + If not set the `sampling_strategy` will determine the number of sentence pairs to generate. + This argument sets the number of iterations to generate sentence pairs for + and provides compatability with Setfit Date: Wed, 25 Oct 2023 14:34:54 +0200 Subject: [PATCH 055/183] Fix broken trainer tests due to new default sampling --- tests/test_trainer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 1a75fefb..2c699ea2 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -548,7 +548,7 @@ def test_train_no_dataset(model: SetFitModel): def test_train_amp_save(model: SetFitModel, tmp_path): - args = TrainingArguments(output_dir=tmp_path, use_amp=True, save_steps=5) + args = TrainingArguments(output_dir=tmp_path, use_amp=True, save_steps=5, num_epochs=5) dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2]}) trainer = Trainer(model, args=args, train_dataset=dataset, eval_dataset=dataset) trainer.train() @@ -557,7 +557,14 @@ def test_train_amp_save(model: SetFitModel, tmp_path): def test_train_load_best(model: SetFitModel, tmp_path, caplog): - args = TrainingArguments(output_dir=tmp_path, save_steps=5, eval_steps=5, load_best_model_at_end=True) + args = TrainingArguments( + output_dir=tmp_path, + save_steps=5, + eval_steps=5, + evaluation_strategy="steps", + load_best_model_at_end=True, + num_epochs=5, + ) dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2]}) trainer = Trainer(model, args=args, train_dataset=dataset, eval_dataset=dataset) with caplog.at_level(logging.INFO): From f8b7253348e9f76534b6df2843926ca758031bad Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 25 Oct 2023 16:15:14 +0200 Subject: [PATCH 056/183] Use the Contrastive Dataset for Distillation --- src/setfit/sampler.py | 57 ++++++++------ src/setfit/trainer.py | 20 +++-- src/setfit/trainer_distillation.py | 122 ++++++----------------------- tests/test_sampler.py | 6 +- 4 files changed, 71 insertions(+), 134 deletions(-) diff --git a/src/setfit/sampler.py b/src/setfit/sampler.py index 54d72e46..1bea2e78 100644 --- a/src/setfit/sampler.py +++ b/src/setfit/sampler.py @@ -3,6 +3,7 @@ import numpy as np from sentence_transformers import InputExample +import torch from torch.utils.data import IterableDataset from . import logging @@ -12,28 +13,6 @@ logger = logging.get_logger(__name__) -def sentence_pairs_generation_cos_sim(sentences, pairs, cos_sim_matrix): - # initialize two empty lists to hold the (sentence, sentence) pairs and - # labels to indicate if a pair is positive or negative - - idx = list(range(len(sentences))) - - for first_idx in range(len(sentences)): - current_sentence = sentences[first_idx] - second_idx = int(np.random.choice([x for x in idx if x != first_idx])) - - cos_sim = float(cos_sim_matrix[first_idx][second_idx]) - paired_sentence = sentences[second_idx] - pairs.append(InputExample(texts=[current_sentence, paired_sentence], label=cos_sim)) - - third_idx = np.random.choice([x for x in idx if x != first_idx]) - cos_sim = float(cos_sim_matrix[first_idx][third_idx]) - paired_sentence = sentences[third_idx] - pairs.append(InputExample(texts=[current_sentence, paired_sentence], label=cos_sim)) - - return pairs - - def shuffle_combinations(iterable: Iterable, replacement: bool = True) -> Generator: """Generates shuffled pair combinations for any iterable data provided. @@ -53,10 +32,10 @@ def shuffle_combinations(iterable: Iterable, replacement: bool = True) -> Genera yield iterable[_idx], iterable[idx] -class ConstrastiveDataset(IterableDataset): +class ContrastiveDataset(IterableDataset): def __init__( self, - examples: InputExample, + examples: List[InputExample], multilabel: bool, num_iterations: Optional[None] = None, sampling_strategy: str = "oversampling", @@ -145,3 +124,33 @@ def __iter__(self): def __len__(self) -> int: return self.len_pos_pairs + self.len_neg_pairs + + +class ContrastiveDistillationDataset(ContrastiveDataset): + def __init__( + self, + examples: List[InputExample], + cos_sim_matrix: torch.Tensor, + num_iterations: Optional[None] = None, + sampling_strategy: str = "oversampling", + ) -> None: + self.cos_sim_matrix = cos_sim_matrix + super().__init__( + examples, + multilabel=False, + num_iterations=num_iterations, + sampling_strategy=sampling_strategy, + ) + # Internally we store all pairs in pos_pairs, regardless of sampling strategy. + # After all, without labels, there isn't much of a strategy. + self.sentence_labels = list(enumerate(self.sentences)) + + self.len_neg_pairs = 0 + if num_iterations is not None and num_iterations > 0: + self.len_pos_pairs = num_iterations * len(self.sentences) + else: + self.len_pos_pairs = len(self.pos_pairs) + + def generate_pairs(self) -> None: + for (text_one, id_one), (text_two, id_two) in shuffle_combinations(self.sentence_labels): + self.pos_pairs.append(InputExample(texts=[text_one, text_two], label=self.cos_sim_matrix[id_one][id_two])) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 55e72fcf..606c9c37 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -3,7 +3,7 @@ import time import warnings from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple, Union import evaluate import torch @@ -38,7 +38,7 @@ from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna from .losses import SupConLoss -from .sampler import ConstrastiveDataset +from .sampler import ContrastiveDataset from .training_args import TrainingArguments from .utils import BestRun, default_hp_space_optuna @@ -367,17 +367,21 @@ def train( logger.info(f"Applying column mapping to {dataset_name} dataset") dataset = self._apply_column_mapping(dataset, self.column_mapping) - parameters.extend([dataset["text"], dataset["label"]]) + parameters.extend(self.dataset_to_parameters(dataset)) self.train_embeddings(*parameters, args=args) - self.train_classifier(*parameters[:2], args=args) + training_parameters = parameters[:len(parameters) // 2] if self.eval_dataset else parameters + self.train_classifier(*training_parameters, args=args) + + def dataset_to_parameters(self, dataset: Dataset) -> List[Iterable]: + return [dataset["text"], dataset["label"]] def train_embeddings( self, x_train: List[str], - y_train: Union[List[int], List[List[int]]], - x_eval: List[str] = None, - y_eval: Union[List[int], List[List[int]]] = None, + y_train: Optional[Union[List[int], List[List[int]]]] = None, + x_eval: Optional[List[str]] = None, + y_eval: Optional[Union[List[int], List[List[int]]]] = None, args: Optional[TrainingArguments] = None, ) -> None: """ @@ -449,7 +453,7 @@ def get_dataloader( margin=args.margin, ) else: - data_sampler = ConstrastiveDataset( + data_sampler = ContrastiveDataset( input_data, self.model.multi_target_strategy, args.num_iterations, args.sampling_strategy ) # shuffle_sampler = True can be dropped in for further 'randomising' diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index faebe43c..2bc2d629 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -1,23 +1,19 @@ -import math import warnings -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Union -import numpy as np +from datasets import Dataset import torch -from sentence_transformers import losses, util +from sentence_transformers import losses, util, InputExample +from torch import nn from torch.utils.data import DataLoader -from transformers.trainer_utils import set_seed from . import logging -from .sampler import sentence_pairs_generation_cos_sim +from .sampler import ContrastiveDistillationDataset from .trainer import Trainer from .training_args import TrainingArguments if TYPE_CHECKING: - import optuna - from datasets import Dataset - from .modeling import SetFitModel logging.set_verbosity_info() @@ -78,99 +74,27 @@ def __init__( self.teacher_model = teacher_model self.student_model = self.model - def train( - self, - args: Optional[TrainingArguments] = None, - trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, - **kwargs, - ) -> None: - """ - Main training entry point. - - Args: - args (`TrainingArguments`, *optional*): - Temporarily change the training arguments for this training call. - trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): - The trial run or the hyperparameter dictionary for hyperparameter search. - """ - if len(kwargs): - warnings.warn( - f"`{self.__class__.__name__}.train` does not accept keyword arguments anymore. " - f"Please provide training arguments via a `TrainingArguments` instance to the `{self.__class__.__name__}` " - f"initialisation or the `{self.__class__.__name__}.train` method.", - DeprecationWarning, - stacklevel=2, - ) - - args = args or self.args or TrainingArguments() - - set_seed(args.seed) # Seed must be set before instantiating the model when using model_init. - - if trial: # Trial and model initialization - self._hp_search_setup(trial) # sets trainer parameters and initializes model - - if self.train_dataset is None: - raise ValueError( - f"Training requires a `train_dataset` given to the `{self.__class__.__name__}` initialization." - ) - - self._validate_column_mapping(self.train_dataset) - train_dataset = self.train_dataset - if self.column_mapping is not None: - logger.info("Applying column mapping to training dataset") - train_dataset = self._apply_column_mapping(self.train_dataset, self.column_mapping) + def dataset_to_parameters(self, dataset: Dataset) -> List[Iterable]: + return [dataset["text"]] - x_train: List[str] = train_dataset["text"] - - self.train_embeddings(x_train, args) - self.train_classifier(x_train, args) - - def train_embeddings( - self, - x_train: List[str], - args: Optional[TrainingArguments] = None, - ) -> None: - """ - Method to perform the embedding phase: finetuning the student its `SentenceTransformer` body. - - Args: - x_train (`List[str]`): A list of training sentences. - args (`TrainingArguments`, *optional*): - Temporarily change the training arguments for this training call. - """ - args = args or self.args or TrainingArguments() - - # **************** student training ********************* - x_train_embd_student = self.teacher_model.model_body.encode( - x_train, convert_to_tensor=self.teacher_model.has_differentiable_head + def get_dataloader( + self, x: List[str], y: Optional[Union[List[int], List[List[int]]]], args: TrainingArguments + ) -> Tuple[DataLoader, nn.Module, int]: + x_embd_student = self.teacher_model.model_body.encode( + x, convert_to_tensor=self.teacher_model.has_differentiable_head ) - cos_sim_matrix = util.cos_sim(x_train_embd_student, x_train_embd_student) - - train_examples = [] - for _ in range(args.num_iterations): - train_examples = sentence_pairs_generation_cos_sim(np.array(x_train), train_examples, cos_sim_matrix) - # **************** student training END ***************** - - batch_size = args.embedding_batch_size - train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=batch_size) - train_loss = args.loss(self.student_model.model_body) - - total_train_steps = len(train_dataloader) * args.embedding_num_epochs - logger.info("***** Running training *****") - logger.info(f" Num examples = {len(train_examples)}") - logger.info(f" Num epochs = {args.embedding_num_epochs}") - logger.info(f" Total optimization steps = {total_train_steps}") - logger.info(f" Total train batch size = {batch_size}") - - warmup_steps = math.ceil(total_train_steps * args.warmup_proportion) - self.student_model.model_body.fit( - train_objectives=[(train_dataloader, train_loss)], - epochs=args.embedding_num_epochs, - optimizer_params={"lr": args.body_embedding_learning_rate}, - warmup_steps=warmup_steps, - show_progress_bar=args.show_progress_bar, - use_amp=args.use_amp, + cos_sim_matrix = util.cos_sim(x_embd_student, x_embd_student) + + input_data = [InputExample(texts=[text]) for text in x] + data_sampler = ContrastiveDistillationDataset( + input_data, cos_sim_matrix, args.num_iterations, args.sampling_strategy ) + # shuffle_sampler = True can be dropped in for further 'randomising' + shuffle_sampler = True if args.sampling_strategy == "unique" else False + batch_size = min(args.embedding_batch_size, len(data_sampler)) + dataloader = DataLoader(data_sampler, batch_size=batch_size, shuffle=shuffle_sampler, drop_last=False) + loss = args.loss(self.model.model_body) + return dataloader, loss, batch_size def train_classifier(self, x_train: List[str], args: Optional[TrainingArguments] = None) -> None: """ diff --git a/tests/test_sampler.py b/tests/test_sampler.py index c3207592..d8d37712 100644 --- a/tests/test_sampler.py +++ b/tests/test_sampler.py @@ -2,7 +2,7 @@ import pytest from sentence_transformers import InputExample -from setfit.sampler import ConstrastiveDataset +from setfit.sampler import ContrastiveDataset @pytest.mark.parametrize( @@ -16,7 +16,7 @@ def test_sentence_pairs_generation(sampling_strategy: str, expected_pos_pairs: i data = [InputExample(texts=[text], label=label) for text, label in zip(sentences, labels)] multilabel = False - data_sampler = ConstrastiveDataset(data, multilabel, sampling_strategy=sampling_strategy) + data_sampler = ContrastiveDataset(data, multilabel, sampling_strategy=sampling_strategy) assert data_sampler.len_pos_pairs == expected_pos_pairs assert data_sampler.len_neg_pairs == expected_neg_pairs @@ -41,7 +41,7 @@ def test_sentence_pairs_generation_multilabel( data = [InputExample(texts=[text], label=label) for text, label in zip(sentences, labels)] multilabel = True - data_sampler = ConstrastiveDataset(data, multilabel, sampling_strategy=sampling_strategy) + data_sampler = ContrastiveDataset(data, multilabel, sampling_strategy=sampling_strategy) assert data_sampler.len_pos_pairs == expected_pos_pairs assert data_sampler.len_neg_pairs == expected_neg_pairs From 38e96070cddff9d6428bc506194722743a44f9f4 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 25 Oct 2023 16:15:22 +0200 Subject: [PATCH 057/183] Set the default logging steps at 50 --- src/setfit/training_args.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 6a13ef80..d0be79ec 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -109,7 +109,7 @@ class TrainingArguments: logging_first_step (`bool`, *optional*, defaults to `False`): Whether to log and evaluate the first `global_step` or not. - logging_steps (`int`, *optional*, defaults to 500): + logging_steps (`int`, *optional*, defaults to 50): Number of update steps between two logs if `logging_strategy="steps"`. evaluation_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`): The evaluation strategy to adopt during training. Possible values are: @@ -194,7 +194,7 @@ class TrainingArguments: logging_dir: Optional[str] = None logging_strategy: str = "steps" logging_first_step: bool = True - logging_steps: int = 5 + logging_steps: int = 50 evaluation_strategy: str = "no" eval_steps: Optional[int] = None From 4ead15dc4caa71f1a2d4acea23ac744f5c360ad7 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 25 Oct 2023 17:45:52 +0200 Subject: [PATCH 058/183] Add max_steps argument to TrainingArguments --- src/setfit/trainer.py | 6 ++++-- src/setfit/training_args.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 606c9c37..3bbc1789 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -506,8 +506,10 @@ def _train_sentence_transformer( self.state.epoch = 0 start_time = time.time() - # TODO: Add max_steps via args.max_steps here? - self.state.max_steps = len(train_dataloader) * args.embedding_num_epochs + if args.max_steps: + self.state.max_steps = args.max_steps + else: + self.state.max_steps = len(train_dataloader) * args.embedding_num_epochs self.control = self.callback_handler.on_train_begin(args, self.state, self.control) if args.use_amp: diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index d0be79ec..73fd95f8 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -30,6 +30,9 @@ class TrainingArguments: Set the number of epochs the embedding and classifier training phases respectively, or set both if an integer is provided. Note that the number of epochs for the classifier is only used with a differentiable PyTorch head. + max_steps (`int`, *optional*, defaults to `-1`): + If set to a positive number, the total number of training steps to perform. Overrides `num_epochs`. + The training may stop before reaching the set number of steps when all data is exhausted. sampling_strategy (`str`, defaults to `"oversampling"`): The sampling strategy of how to draw pairs in training. Possible values are: @@ -161,6 +164,8 @@ class TrainingArguments: embedding_num_epochs: int = None classifier_num_epochs: int = None + max_steps: int = -1 + sampling_strategy: str = "oversampling" num_iterations: Optional[int] = None From eb703363f15ac0636ae32d97e1c7bcc9ff8d1d5e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 25 Oct 2023 17:52:08 +0200 Subject: [PATCH 059/183] Change max_steps conditional The old conditional was True with the default -1, not ideal --- src/setfit/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 3bbc1789..0abdd110 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -506,7 +506,7 @@ def _train_sentence_transformer( self.state.epoch = 0 start_time = time.time() - if args.max_steps: + if args.max_steps > 0: self.state.max_steps = args.max_steps else: self.state.max_steps = len(train_dataloader) * args.embedding_num_epochs From 5b39f062d1f3c4b684703af389c88806931b0681 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 11:57:03 +0100 Subject: [PATCH 060/183] Seeds are now correctly applied for reproducibility --- src/setfit/data.py | 2 +- src/setfit/trainer.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/setfit/data.py b/src/setfit/data.py index 7eb36224..2d9cd5f8 100644 --- a/src/setfit/data.py +++ b/src/setfit/data.py @@ -151,7 +151,7 @@ def sample_dataset(dataset: Dataset, label_column: str = "label", num_samples: i df = df.groupby(label_column) # sample num_samples, or at least as much as possible - df = df.apply(lambda x: x.sample(min(num_samples, len(x)))) + df = df.apply(lambda x: x.sample(min(num_samples, len(x)), random_state=seed)) df = df.reset_index(drop=True) all_samples = Dataset.from_pandas(df, features=dataset.features) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 0abdd110..9b1b2333 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -119,6 +119,9 @@ def __init__( self.metric_kwargs = metric_kwargs self.column_mapping = column_mapping + # Seed must be set before instantiating the model when using model_init. + set_seed(12) + if model is None: if model_init is not None: model = self.call_model_init() @@ -347,7 +350,8 @@ def train( args = args or self.args or TrainingArguments() - set_seed(args.seed) # Seed must be set before instantiating the model when using model_init. + # Seed must be set before instantiating the model when using model_init. + set_seed(args.seed) if trial: # Trial and model initialization self._hp_search_setup(trial) # sets trainer parameters and initializes model From d8177db12c0383050a433741aee99ec079f2888b Mon Sep 17 00:00:00 2001 From: MosheWasserb <51365978+MosheWasserb@users.noreply.github.com> Date: Thu, 9 Nov 2023 13:23:43 +0200 Subject: [PATCH 061/183] Add files via upload --- ..._small_SST_2_setfit_Optimum_TensorRT.ipynb | 12652 ++++++++++++++++ 1 file changed, 12652 insertions(+) create mode 100644 notebooks/bge_small_SST_2_setfit_Optimum_TensorRT.ipynb diff --git a/notebooks/bge_small_SST_2_setfit_Optimum_TensorRT.ipynb b/notebooks/bge_small_SST_2_setfit_Optimum_TensorRT.ipynb new file mode 100644 index 00000000..12722a26 --- /dev/null +++ b/notebooks/bge_small_SST_2_setfit_Optimum_TensorRT.ipynb @@ -0,0 +1,12652 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7a40382d-295b-45e5-b694-52331d61e657", + "metadata": { + "id": "7a40382d-295b-45e5-b694-52331d61e657" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "id": "76571396-8f54-40ed-9e81-6c7531e6eaee", + "metadata": { + "id": "76571396-8f54-40ed-9e81-6c7531e6eaee" + }, + "source": [ + "# Efficently run SetFit Models with Optimum" + ] + }, + { + "cell_type": "markdown", + "id": "24fd5853-812f-45a4-8a7b-0a0c9a60d0a2", + "metadata": { + "id": "24fd5853-812f-45a4-8a7b-0a0c9a60d0a2" + }, + "source": [ + "[SetFit](https://github.com/huggingface/setfit) is a technique for few-shot text classification that uses contrastive learning to fine-tune Sentence Transformers in domains where little to no labeled data is available. It achieves comparable performance to existing state-of-the-art methods based on large language models, yet requires no prompts and is efficient to train (typically a few seconds on a GPU to minutes on a CPU).\n", + "\n", + "In this notebook you'll learn how to further compress SetFit models for faster inference & deployment on GPU using Optimum Onnx." + ] + }, + { + "cell_type": "markdown", + "id": "a3b30b35-7875-498f-a771-068132f4084f", + "metadata": { + "tags": [], + "id": "a3b30b35-7875-498f-a771-068132f4084f" + }, + "source": [ + "## 1. Setup development environment" + ] + }, + { + "cell_type": "markdown", + "id": "dc40c7af-1f4f-4324-847c-dc9b7797b60c", + "metadata": { + "id": "dc40c7af-1f4f-4324-847c-dc9b7797b60c" + }, + "source": [ + "Our first step is to install SetFit. Running the following cell will install all the required packages for us." + ] + }, + { + "cell_type": "code", + "source": [ + "!pip install setfit" + ], + "metadata": { + "id": "Cu9et-iSaU0i", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "3c1bd043-edf4-443d-9a44-5dc480656943" + }, + "id": "Cu9et-iSaU0i", + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting setfit\n", + " Downloading setfit-0.7.0-py3-none-any.whl (45 kB)\n", + "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/45.9 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.9/45.9 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting datasets>=2.3.0 (from setfit)\n", + " Downloading datasets-2.14.6-py3-none-any.whl (493 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m493.7/493.7 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting sentence-transformers>=2.2.1 (from setfit)\n", + " Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Collecting evaluate>=0.3.0 (from setfit)\n", + " Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (1.23.5)\n", + "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (9.0.0)\n", + "Collecting dill<0.3.8,>=0.3.0 (from datasets>=2.3.0->setfit)\n", + " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (1.5.3)\n", + "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (4.66.1)\n", + "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (3.4.1)\n", + "Collecting multiprocess (from datasets>=2.3.0->setfit)\n", + " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (2023.6.0)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (3.8.6)\n", + "Collecting huggingface-hub<1.0.0,>=0.14.0 (from datasets>=2.3.0->setfit)\n", + " Downloading huggingface_hub-0.19.0-py3-none-any.whl (311 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m311.2/311.2 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (23.2)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (6.0.1)\n", + "Collecting responses<0.19 (from evaluate>=0.3.0->setfit)\n", + " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", + "Collecting transformers<5.0.0,>=4.6.0 (from sentence-transformers>=2.2.1->setfit)\n", + " Downloading transformers-4.35.0-py3-none-any.whl (7.9 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.9/7.9 MB\u001b[0m \u001b[31m47.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (2.1.0+cu118)\n", + "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (0.16.0+cu118)\n", + "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (1.2.2)\n", + "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (1.11.3)\n", + "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (3.8.1)\n", + "Collecting sentencepiece (from sentence-transformers>=2.2.1->setfit)\n", + " Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m74.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (23.1.0)\n", + "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (3.3.2)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (6.0.4)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (4.0.3)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (1.9.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (1.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (1.3.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets>=2.3.0->setfit) (3.13.1)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets>=2.3.0->setfit) (4.5.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.3.0->setfit) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.3.0->setfit) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.3.0->setfit) (2023.7.22)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (3.1.2)\n", + "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (2.1.0)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.1->setfit) (2023.6.3)\n", + "Collecting tokenizers<0.15,>=0.14 (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.1->setfit)\n", + " Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m98.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.1->setfit)\n", + " Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m81.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers>=2.2.1->setfit) (8.1.7)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers>=2.2.1->setfit) (1.3.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.3.0->setfit) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.3.0->setfit) (2023.3.post1)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers>=2.2.1->setfit) (3.2.0)\n", + "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence-transformers>=2.2.1->setfit) (9.4.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets>=2.3.0->setfit) (1.16.0)\n", + "Collecting huggingface-hub<1.0.0,>=0.14.0 (from datasets>=2.3.0->setfit)\n", + " Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m36.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (2.1.3)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (1.3.0)\n", + "Building wheels for collected packages: sentence-transformers\n", + " Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125923 sha256=162630274e9f88976ff1d7790162c5e378a11580d7977416bcefdcbacb07ac7a\n", + " Stored in directory: /root/.cache/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n", + "Successfully built sentence-transformers\n", + "Installing collected packages: sentencepiece, safetensors, dill, responses, multiprocess, huggingface-hub, tokenizers, transformers, datasets, sentence-transformers, evaluate, setfit\n", + "Successfully installed datasets-2.14.6 dill-0.3.7 evaluate-0.4.1 huggingface-hub-0.17.3 multiprocess-0.70.15 responses-0.18.0 safetensors-0.4.0 sentence-transformers-2.2.2 sentencepiece-0.1.99 setfit-0.7.0 tokenizers-0.14.1 transformers-4.35.0\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "id": "1832e3f2-09e7-44b5-a438-87ea658d49ea", + "metadata": { + "id": "1832e3f2-09e7-44b5-a438-87ea658d49ea" + }, + "source": [ + "While we're at it, let's turn off some of the warnings from the 🤗 Datasets library and the tokenizers:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b67e17b0-743a-47ba-9d55-9baea453a0d8", + "metadata": { + "id": "b67e17b0-743a-47ba-9d55-9baea453a0d8", + "outputId": "4a68c21f-eb4e-4652-a1d0-bcf1eb8b9682", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "env: TOKENIZERS_PARALLELISM=false\n" + ] + } + ], + "source": [ + "import datasets\n", + "\n", + "datasets.logging.set_verbosity_error()\n", + "\n", + "%env TOKENIZERS_PARALLELISM=false" + ] + }, + { + "cell_type": "markdown", + "id": "c442bab8-3ae1-4d1d-9ebe-723d7a829bae", + "metadata": { + "id": "c442bab8-3ae1-4d1d-9ebe-723d7a829bae" + }, + "source": [ + "To be able to share your model with the community, there are a few more steps to follow.\n", + "\n", + "First, you have to store your authentication token from the Hugging Face Hub (sign up here if you haven't already!). To do so, execute the following cell and input an **access token with write permissions** associated with your account:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "004da15f-6c49-4e7a-b1f6-b285e22eb6cb", + "metadata": { + "id": "004da15f-6c49-4e7a-b1f6-b285e22eb6cb", + "outputId": "3a5ea685-01e8-495b-a7e3-dc0ce8f87072", + "colab": { + "base_uri": "https://localhost:8080/", + "referenced_widgets": [ + "c22db575e86b45229ab97eb5f5193fe1", + "1da311f97a57441988a622d5b019fa5b", + "458a3ee5ff8d44fca9ca6f4888d79174", + "09d3ebe126e2484d8cc50653f7d8f8fb", + "f867a0c844b547e8bd1d294245c4fa13", + "1993399b93224ec6865eaa01b6a73d8b", + "b35a241e895c41ca8be0f334615df0a3", + "99ef6333ea314434a1dd3ef05fa0e42b", + "ac1617fabea24f8a912e7cb6bb9abca4", + "108c04b373e243218924ab3b101e8908", + "16019f898e814155ae1c27ab2475565a", + "d1df847b4fbe4ab992c64d5d2262cbcf", + "eeb36e1cf46b46328725481ce4b9a23b", + "58c9a6969edf4b5ab636c0fdcb9b5b5b", + "abe2bd79c6574e5aafa865bd18f14221", + "317f736683394e5cb974117fd8f5e7fe", + "a34b80886d56489cb82c36ba1415841c" + ] + } + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "VBox(children=(HTML(value='
:30: MatplotlibDeprecationWarning: The legendHandles attribute was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use legend_handles instead.\n", + " for handle in legend.legendHandles:\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/0ElEQVR4nO3deXgV5f3//9fJShKykIRsNIQQIOyyytYilWhAioCogLQsQUBMRVSkxYpKwVKou/YL0tKwaBDpR1RsBQkaFkVEBKKCASISEAItkhwCSchy//7wx6lHAibkhJMMz8d1zXUxM/fc8z6ZxPNy5p4ZmzHGCAAAwKI83F0AAABAbSLsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAAS3Nr2Dlz5oymTZumuLg4+fn5qXfv3tqxY4dj/bhx42Sz2ZymAQMGuLFiAABQ33i5c+d33323vvjiC61YsUIxMTF65ZVXlJSUpL1796pJkyaSpAEDBigtLc2xja+vr7vKBQAA9ZDNXS8CLSoqUmBgoN566y0NGjTIsbxr164aOHCg5s6dq3Hjxik/P19vvvmmO0oEAAAW4LYzO2VlZSovL1eDBg2clvv5+Wnr1q2O+czMTEVERKhRo0a68cYbNXfuXIWFhV2y35KSEpWUlDjmKyoq9N133yksLEw2m831HwQAALicMUZnzpxRTEyMPDxqOOrGuFGvXr3MDTfcYL799ltTVlZmVqxYYTw8PEyrVq2MMcasXLnSvPXWWyYrK8usWbPGtGnTxnTv3t2UlZVdss/HH3/cSGJiYmJiYmKywHTkyJEa5w23XcaSpJycHKWkpGjz5s3y9PRUly5d1KpVK+3cuVP79u27qP3XX3+thIQEZWRkqH///pX2+eMzOwUFBWratKmOHDmioKCgWvssAADAdex2u2JjY5Wfn6/g4OAa9eXWAcoJCQnatGmTzp49K7vdrujoaI0YMULNmzevtH3z5s0VHh6ugwcPXjLs+Pr6VjqIOSgoiLADAEA944ohKHXiOTsBAQGKjo7W6dOntX79eg0ZMqTSdkePHtWpU6cUHR19lSsEAAD1lVvP7Kxfv17GGCUmJurgwYN6+OGH1bp1a40fP16FhYWaPXu2hg8frqioKOXk5GjGjBlq0aKFkpOT3Vk2AACoR9x6ZqegoECpqalq3bq1xowZo5///Odav369vL295enpqaysLN16661q1aqVJkyYoK5du2rLli08awcAAFSZWwcoXw12u13BwcEqKChgzA4ASysvL1dpaam7ywCq5MKJjUtx5fe3Wy9jAQBqzhijvLw85efnu7sUoFpCQkIUFRVV68/BI+wAQD13IehERETI39+fB6iizjPG6Ny5czp58qQk1fqNR4QdAKjHysvLHUHnck+XB+oaPz8/SdLJkycVERFx2UtaNVUnbj0HAFyZC2N0/P393VwJUH0Xfm9re6wZYQcALIBLV6iPrtbvLWEHAABYGmEHAOAW/fr107Rp09xdxlWzdOlShYSEOOafeOIJderUyW311IZmzZrpueeec3cZFyHsAABQB33zzTey2WyOKSwsTDfffLN27dpVpe2bNWvmtP2Pp3HjxtXuB6hDuBsLAIA6LCMjQ+3atdPRo0c1depUDRw4UF999ZXTWaLK7NixQ+Xl5ZKkjz76SMOHD1d2drbjAX0X7oaqqtLSUnl7e1/RZ3A3zuwAABxOnz2vQ/89q9Nnz1+V/ZWVlem3v/2tgoODFR4erlmzZumHD/Y/fvy4Bg0aJD8/P8XHxys9Pf2iSyX5+fm6++671bhxYwUFBenGG2/Unj17LrvfzMxMXX/99QoICFBISIj69Omjw4cPS/rf5aV//OMfatq0qRo2bKh7771X5eXlWrBggaKiohQREaEnn3zSqc9nnnlGHTp0UEBAgGJjY3XvvfeqsLCwxj+jsLAwRUVFqVu3bnrqqad04sQJbd++XX/84x/Vvn37i9p36tRJs2bNUuPGjRUVFaWoqCiFhoZKkiIiIhzL0tPTlZCQIB8fHyUmJmrFihVO/dhsNi1cuFC33nqrAgICHJ937dq16t69uxo0aKDw8HANGzbMabtz584pJSVFgYGBatq0qRYvXlzjn0FNEXYAACouLdc/dx7R/HVf6dkN2Zq/7iv9c+cRFZeW1+p+ly1bJi8vL33yySd6/vnn9cwzz+jvf/+7Y/2YMWN07NgxZWZm6v/+7/+0ePFix4PoLrjjjjt08uRJvfvuu9q5c6e6dOmi/v3767vvvqt0n2VlZRo6dKhuuOEGZWVladu2bZo0aZLTnUE5OTl69913tW7dOq1cuVJLlizRoEGDdPToUW3atEnz58/Xo48+qu3btzu28fDw0AsvvKAvv/xSy5Yt0/vvv68ZM2a49Od14WzM+fPnlZKSon379mnHjh2O9bt27VJWVpbGjx9/2X7WrFmj+++/Xw899JC++OILTZ48WePHj9cHH3zg1O6JJ57QsGHD9PnnnyslJUX/+te/NGzYMN1yyy3atWuXNm7cqOuvv95pm6efflrdunXTrl27dO+992rKlCnKzs520U/gChmLKygoMJJMQUGBu0sBAJcrKioye/fuNUVFRTXqZ/WnuWbS8h1m5v9lmXn/3mtm/l+WmbR8h1n9aa6LKr3YDTfcYNq0aWMqKiocy373u9+ZNm3aGGOM2bdvn5FkduzY4Vh/4MABI8k8++yzxhhjtmzZYoKCgkxxcbFT3wkJCebll1+udL+nTp0ykkxmZmal6x9//HHj7+9v7Ha7Y1lycrJp1qyZKS8vdyxLTEw08+bNu+TnW716tQkLC3PMp6WlmeDgYKf9XHfddZfc/tChQ0aS2bVrlzHGmNOnT5thw4aZhg0bmry8PGOMMQMHDjRTpkxxbHPfffeZfv36XdTXBx98YCSZ06dPG2OM6d27t5k4caJTmzvuuMPccsstjnlJZtq0aU5tevXqZUaPHn3JmuPi4syvf/1rx3xFRYWJiIgwCxcurLT95X5/Xfn9zZkdALjGnT57Xp9+c1phAb5qHOgrXy9PNQ70VViAr3Z+c7pWL2n17NnT6YxKr169dODAAZWXlys7O1teXl7q0qWLY32LFi3UqFEjx/yePXtUWFiosLAwNWzY0DEdOnRIOTk5ys3NdVr+pz/9SaGhoRo3bpySk5M1ePBgPf/88zp+/LhTXc2aNVNgYKBjPjIyUm3btpWHh4fTsh+eZcrIyFD//v3VpEkTBQYG6je/+Y1OnTqlc+fO1ehn1Lt3bzVs2FCNGjXSnj17tGrVKkVGRkqSJk6cqJUrV6q4uFjnz59Xenq6UlJSfrLPffv2qU+fPk7L+vTpo3379jkt69atm9P87t271b9//8v23bFjR8e/bTaboqKiLjobd7UxQBkArnH5RaU6d75MMSHOA1aD/Lx0LL9I+UWlahTg46bqLq+wsFDR0dHKzMy8aF1ISIhCQkK0e/dux7ILY1fS0tI0depUrVu3TqtWrdKjjz6qDRs2qGfPnpJ00UBcm81W6bKKigpJ39859atf/UpTpkzRk08+qdDQUG3dulUTJkzQ+fPna/SE61WrVqlt27YKCwu7aFDy4MGD5evrqzVr1sjHx0elpaW6/fbbr3hfPxYQEOA0X5VBzZf7ObkLYQcArnEhft7y9/GSvahMjQP/934ie1GZAny8FOJXe3fg/HDMiyR9/PHHatmypTw9PZWYmKiysjLt2rVLXbt2lSQdPHhQp0+fdrTv0qWL8vLy5OXlpWbNmlW6jxYtWlS6vHPnzurcubNmzpypXr16KT093RF2qmvnzp2qqKjQ008/7Tj78/rrr19RXz8WGxurhISEStd5eXlp7NixSktLk4+Pj0aOHFmlQNKmTRt9+OGHGjt2rGPZhx9+qLZt2152u44dO2rjxo0/OSaoriHsAMA1rlGAj7o1a6QNe09I+v6Mjr2oTKfOluimtpG1elYnNzdXDz74oCZPnqzPPvtML774op5++mlJUuvWrZWUlKRJkyZp4cKF8vb21kMPPSQ/Pz/Hpa+kpCT16tVLQ4cO1YIFC9SqVSsdO3bMMZD2x5dhJOnQoUNavHixbr31VsXExCg7O1sHDhzQmDFjrvhztGjRQqWlpXrxxRc1ePBgffjhh1q0aNEV91cdd999t9q0aSPp+8BSFQ8//LDuvPNOde7cWUlJSVq7dq3eeOMNZWRkXHa7xx9/XP3791dCQoJGjhypsrIy/fvf/9bvfve7Gn+O2sSYHQCAftUxRje1jZQxRsfyi2SM0U1tI/WrjjG1ut8xY8aoqKhI119/vVJTU3X//fdr0qRJjvXLly9XZGSk+vbtq2HDhmnixIkKDAxUgwYNJH1/ieTf//63+vbtq/Hjx6tVq1YaOXKkDh8+7BjX8mP+/v766quvNHz4cLVq1UqTJk1SamqqJk+efMWf47rrrtMzzzyj+fPnq3379nr11Vc1b968K+6vOlq2bKnevXurdevW6tGjR5W2GTp0qJ5//nk99dRTateunV5++WWlpaWpX79+l92uX79+Wr16td5++2116tRJN954oz755BMXfIraZTPmBw80sCC73a7g4GAVFBQ4HqQEAFZRXFysQ4cOKT4+3hEAauL02fPKLypViJ93nRync/ToUcXGxjoGA0Myxqhly5a699579eCDD7q7nGq53O+vK7+/uYwFAHBoFOBTp0LO+++/r8LCQnXo0EHHjx/XjBkz1KxZM/Xt29fdpdUJ//nPf/Taa68pLy+v3o2juZoIOwCAOqu0tFSPPPKIvv76awUGBqp379569dVX6+1rC1wtIiJC4eHhWrx4sdMt+XBG2AEA1FnJyclKTk52dxl1lsVHorgMA5QBAIClEXYAwAL4P3zUR1fr95awAwD12IWxKzV9JQHgDhd+b2t7DBZjdgCgHvP09FRISIjj3UP+/v5O75oC6iJjjM6dO6eTJ08qJCREnp6eP71RDRB2AKCei4qKkiS3v2wRqK6QkBDH729tIuwAQD1ns9kUHR2tiIgIlZaWurscoEq8vb1r/YzOBYQdALAIT0/Pq/blAdQnDFAGAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBcM2qqODlmcC1gIcKArhmlJZXKDvvjHYePq3c787qfFmFfLw81DQ0QF3jGikxKlDenvw/IGA1hB0AlmeM0e4j+Xpvb56OfFekigqjwAbe8vSwqbC4XNu/PqUd33yn2FA/3dw2Sp2bNnJ3yQBciLADwNKMMdp68L9a89m3Kq2oUJMQPzXwdn6lQlRwAxWXluvo6SKt2HZYhSVl+kXLxm6qGICrcb4WgKXtPpKvNZ99Ky9Pm5qHN7wo6FzQwNtTzcMbysvTpjW7vtXuI/lXt1AAtYawA8CySssr9N7ePJVWVCg62K9K20QH+32/3Zd5Ki2vqOUKAVwNhB0AlpWdd0ZHvitSTBWDzgUxwX7K/e6csvPO1FJlAK4mwg4Ay9p5+LQqKswlL11dSgNvT1VUGO08fLqWKgNwNRF2AFhW7ndnFdjA+4q2bdjAS7nfnXNxRQDcgbADwJIqKozOl1XI08N2Rdt7eth0vqycBw8CFkDYAWBJHh42+Xh5qPwKw0p5hZGPl6c8rjAsAag7CDsALKtpaIDOFJde0baFxWVqGurv4ooAuINbw86ZM2c0bdo0xcXFyc/PT71799aOHTsc640xeuyxxxQdHS0/Pz8lJSXpwIEDbqwYQH3SNa6RPDxsKi4tr9Z2xaXl8vCwqWscT1IGrMCtYefuu+/Whg0btGLFCn3++ee6+eablZSUpG+//VaStGDBAr3wwgtatGiRtm/froCAACUnJ6u4uNidZQOoJxKjAhUb6qdjBUXV2u5YQZGahvorMSqwlioDcDXZjDFuGX1XVFSkwMBAvfXWWxo0aJBjedeuXTVw4EDNmTNHMTExeuihhzR9+nRJUkFBgSIjI7V06VKNHDmySvux2+0KDg5WQUGBgoKCauWzAKi7duWe1opth+XlaavSgwWPFxSprMJoTK9m6hQbUvsFAqiUK7+/3XZmp6ysTOXl5WrQoIHTcj8/P23dulWHDh1SXl6ekpKSHOuCg4PVo0cPbdu27ZL9lpSUyG63O00Arl2dYkM0rEsTlZUbff3fwkte0iouLdfX/y1UWYXRsM5NCDqAhbjtRaCBgYHq1auX5syZozZt2igyMlIrV67Utm3b1KJFC+Xl5UmSIiMjnbaLjIx0rKvMvHnzNHv27FqtHUD9YbPZ9IuWjRXYwFvvfZmn3O/OqaLCqGEDL3l62FReYVRYXCYPD5uahvrr5nZRBB3AYtz61vMVK1YoJSVFTZo0kaenp7p06aJRo0Zp586dV9znzJkz9eCDDzrm7Xa7YmNjXVEugHqsU2yI2sUEKTvvjHYePq3c787pfFm5/H281C4mWF3jGikxKlDentykCliNW8NOQkKCNm3apLNnz8putys6OlojRoxQ8+bNFRUVJUk6ceKEoqOjHducOHFCnTp1umSfvr6+8vX1re3SAdRD3p4eat8kWO2bBEv6/sGDPEcHsL468b8wAQEBio6O1unTp7V+/XoNGTJE8fHxioqK0saNGx3t7Ha7tm/frl69ermxWgBWQdABrg1uPbOzfv16GWOUmJiogwcP6uGHH1br1q01fvx42Ww2TZs2TXPnzlXLli0VHx+vWbNmKSYmRkOHDnVn2QAAoB5xa9gpKCjQzJkzdfToUYWGhmr48OF68skn5e39/Yv7ZsyYobNnz2rSpEnKz8/Xz3/+c61bt+6iO7gAAAAuxW3P2blaeM4OAAD1jyWeswMAAHA1EHYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAICluTXslJeXa9asWYqPj5efn58SEhI0Z84cGWMcbcaNGyebzeY0DRgwwI1VAwCA+sTLnTufP3++Fi5cqGXLlqldu3b69NNPNX78eAUHB2vq1KmOdgMGDFBaWppj3tfX1x3lAgCAesitYeejjz7SkCFDNGjQIElSs2bNtHLlSn3yySdO7Xx9fRUVFeWOEgEAQD3n1stYvXv31saNG7V//35J0p49e7R161YNHDjQqV1mZqYiIiKUmJioKVOm6NSpU+4oFwAA1ENuPbPz+9//Xna7Xa1bt5anp6fKy8v15JNPavTo0Y42AwYM0G233ab4+Hjl5OTokUce0cCBA7Vt2zZ5enpe1GdJSYlKSkoc83a7/ap8FgAAUDe5Ney8/vrrevXVV5Wenq527dpp9+7dmjZtmmJiYjR27FhJ0siRIx3tO3TooI4dOyohIUGZmZnq37//RX3OmzdPs2fPvmqfAQAA1G0288Nbn66y2NhY/f73v1dqaqpj2dy5c/XKK6/oq6++uuR2jRs31ty5czV58uSL1lV2Zic2NlYFBQUKCgpy7QcAAAC1wm63Kzg42CXf3249s3Pu3Dl5eDgPG/L09FRFRcUltzl69KhOnTql6OjoStf7+vpytxYAAHBwa9gZPHiwnnzySTVt2lTt2rXTrl279MwzzyglJUWSVFhYqNmzZ2v48OGKiopSTk6OZsyYoRYtWig5OdmdpQMAgHrCrZexzpw5o1mzZmnNmjU6efKkYmJiNGrUKD322GPy8fFRUVGRhg4dql27dik/P18xMTG6+eabNWfOHEVGRlZpH648DQYAAK4OV35/uzXsXA2EHQAA6h9Xfn/zbiwAAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBpXtVpXFFRoU2bNmnLli06fPiwzp07p8aNG6tz585KSkpSbGxsbdUJAABwRap0ZqeoqEhz585VbGysbrnlFr377rvKz8+Xp6enDh48qMcff1zx8fG65ZZb9PHHH9d2zQAAAFVWpTM7rVq1Uq9evfS3v/1NN910k7y9vS9qc/jwYaWnp2vkyJH6wx/+oIkTJ7q8WAAAgOqyGWPMTzXat2+f2rRpU6UOS0tLlZubq4SEhBoX5wp2u13BwcEqKChQUFCQu8sBAABV4Mrv7ypdxqpq0JEkb2/vOhN0AAAAqjVA+YfKysr08ssvKzMzU+Xl5erTp49SU1PVoEEDV9YHAABQI1ccdqZOnar9+/frtttuU2lpqZYvX65PP/1UK1eudGV9AAAANVLlsLNmzRoNGzbMMf/ee+8pOztbnp6ekqTk5GT17NnT9RUCAADUQJUfKviPf/xDQ4cO1bFjxyRJXbp00T333KN169Zp7dq1mjFjhrp3715rhQIAAFyJKoedtWvXatSoUerXr59efPFFLV68WEFBQfrDH/6gWbNmKTY2Vunp6bVZKwAAQLVV6dbzH8rPz9eMGTO0Z88eLVq0SJ07d66t2lyCW88BAKh/rvqt5z8UEhKixYsX6y9/+YvGjBmjhx9+WMXFxTUqAgAAoLZUOezk5ubqzjvvVIcOHTR69Gi1bNlSO3fulL+/v6677jq9++67tVknAADAFanyZax+/fopKipK48aN0/r165WTk6O3335b0vdPWJ48ebKioqL0+uuv12rB1cVlLAAA6h9Xfn9X+dbzTz/9VHv27FFCQoKSk5MVHx/vWNemTRtt3rxZixcvrlExAAAArlblsNO1a1c99thjGjt2rDIyMtShQ4eL2kyaNMmlxQEAANRUlcfsLF++XCUlJXrggQf07bff6uWXX67NugAAAFyiymd24uLi9M9//rM2awEAAHC5Kp3ZOXv2bLU6rW57AACA2lKlsNOiRQv9+c9/1vHjxy/ZxhijDRs2aODAgXrhhRdcViAAAEBNVOkyVmZmph555BE98cQTuu6669StWzfFxMSoQYMGOn36tPbu3att27bJy8tLM2fO1OTJk2u7bgAAgCqp1usicnNztXr1am3ZskWHDx9WUVGRwsPD1blzZyUnJ2vgwIGOt6DXFTxnBwCA+seV39/VfjdWfUPYAQCg/nHru7EAAADqE8IOAACwNMIOAACwNMIOAACwNMIOAACwtGqHnWbNmumPf/yjcnNza7zz8vJyzZo1S/Hx8fLz81NCQoLmzJmjH94gZozRY489pujoaPn5+SkpKUkHDhyo8b4BAMC1odphZ9q0aXrjjTfUvHlz3XTTTXrttddUUlJyRTufP3++Fi5cqJdeekn79u3T/PnztWDBAr344ouONgsWLNALL7ygRYsWafv27QoICFBycrKKi4uvaJ8AAODacsXP2fnss8+0dOlSrVy5UuXl5brrrruUkpKiLl26VLmPX/3qV4qMjNSSJUscy4YPHy4/Pz+98sorMsYoJiZGDz30kKZPny5JKigoUGRkpJYuXaqRI0f+5D54zg4AAPVPnXjOTpcuXfTCCy/o2LFjevzxx/X3v/9d3bt3V6dOnfSPf/xDVclQvXv31saNG7V//35J0p49e7R161YNHDhQknTo0CHl5eUpKSnJsU1wcLB69Oihbdu2VdpnSUmJ7Ha70wQAAK5dVXo3VmVKS0u1Zs0apaWlacOGDerZs6cmTJigo0eP6pFHHlFGRobS09Mv28fvf/972e12tW7dWp6eniovL9eTTz6p0aNHS5Ly8vIkSZGRkU7bRUZGOtb92Lx58zR79uwr/VgAAMBiqh12PvvsM6WlpWnlypXy8PDQmDFj9Oyzz6p169aONsOGDVP37t1/sq/XX39dr776qtLT09WuXTvt3r1b06ZNU0xMjMaOHVvd0iRJM2fO1IMPPuiYt9vtio2NvaK+AABA/VftsNO9e3fddNNNWrhwoYYOHSpvb++L2sTHx1dpPM3DDz+s3//+9462HTp00OHDhzVv3jyNHTtWUVFRkqQTJ04oOjrasd2JEyfUqVOnSvv09fWVr69vdT8WAACwqGqHna+//lpxcXGXbRMQEKC0tLSf7OvcuXPy8HAeNuTp6amKigpJ34emqKgobdy40RFu7Ha7tm/frilTplS3dAAAcA2qdtg5efKk8vLy1KNHD6fl27dvl6enp7p161blvgYPHqwnn3xSTZs2Vbt27bRr1y4988wzSklJkSTZbDZNmzZNc+fOVcuWLRUfH69Zs2YpJiZGQ4cOrW7pAADgGlTtu7FSU1N15MiRi5Z/++23Sk1NrVZfL774om6//Xbde++9atOmjaZPn67Jkydrzpw5jjYzZszQfffdp0mTJql79+4qLCzUunXr1KBBg+qWDgAArkHVfs5Ow4YNlZWVpebNmzstP3TokDp27KgzZ864tMCa4jk7AADUP259zo6vr69OnDhx0fLjx4/Ly+uK72QHAACoFdUOOzfffLNmzpypgoICx7L8/Hw98sgjuummm1xaHAAAQE1V+1TMU089pb59+youLk6dO3eWJO3evVuRkZFasWKFywsEAACoiWqHnSZNmigrK0uvvvqq9uzZIz8/P40fP16jRo2q9Jk7AAAA7nRFg2wCAgI0adIkV9cCAADgclc8onjv3r3Kzc3V+fPnnZbfeuutNS4KAADAVa7oCcrDhg3T559/LpvN5ni7uc1mkySVl5e7tkIAAIAaqPbdWPfff7/i4+N18uRJ+fv768svv9TmzZvVrVs3ZWZm1kKJAAAAV67aZ3a2bdum999/X+Hh4fLw8JCHh4d+/vOfa968eZo6dap27dpVG3UCAABckWqf2SkvL1dgYKAkKTw8XMeOHZMkxcXFKTs727XVAQAA1FC1z+y0b99ee/bsUXx8vHr06KEFCxbIx8dHixcvvugVEgAAAO5W7bDz6KOP6uzZs5KkP/7xj/rVr36lX/ziFwoLC9OqVatcXiAAAEBNVPtFoJX57rvv1KhRI8cdWXUJLwIFAKD+cduLQEtLS+Xl5aUvvvjCaXloaGidDDoAAADVCjve3t5q2rQpz9IBAAD1RrXvxvrDH/6gRx55RN99911t1AMAAOBS1R6g/NJLL+ngwYOKiYlRXFycAgICnNZ/9tlnLisOAACgpqoddoYOHVoLZQAAANQOl9yNVZdxNxYAAPWP2+7GAgAAqG+qfRnLw8PjsreZc6cWAACoS6oddtasWeM0X1paql27dmnZsmWaPXu2ywoDAABwBZeN2UlPT9eqVav01ltvuaI7l2HMDgAA9U+dHLPTs2dPbdy40VXdAQAAuIRLwk5RUZFeeOEFNWnSxBXdAQAAuEy1x+z8+IWfxhidOXNG/v7+euWVV1xaHAAAQE1VO+w8++yzTmHHw8NDjRs3Vo8ePdSoUSOXFgcAAFBT1Q4748aNq4UyAAAAake1x+ykpaVp9erVFy1fvXq1li1b5pKiAAAAXKXaYWfevHkKDw+/aHlERIT+9Kc/uaQoAAAAV6l22MnNzVV8fPxFy+Pi4pSbm+uSogAAAFyl2mEnIiJCWVlZFy3fs2ePwsLCXFIUAACAq1Q77IwaNUpTp07VBx98oPLycpWXl+v999/X/fffr5EjR9ZGjQAAAFes2ndjzZkzR99884369+8vL6/vN6+oqNCYMWMYswMAAOqcK3431oEDB7R79275+fmpQ4cOiouLc3VtLsG7sQAAqH9c+f1d7TM7F7Rs2VItW7as0c4BAABqW7XH7AwfPlzz58+/aPmCBQt0xx13uKQoAAAAV6l22Nm8ebNuueWWi5YPHDhQmzdvdklRAAAArlLtsFNYWCgfH5+Llnt7e8tut7ukKAAAAFepdtjp0KGDVq1addHy1157TW3btnVJUQAAAK5S7QHKs2bN0m233aacnBzdeOONkqSNGzdq5cqVlb4zCwAAwJ2qHXYGDx6sN998U3/605/0z3/+U35+furYsaMyMjJ0ww031EaNAAAAV+yKn7NTmS+++ELt27d3VXcuwXN2AACof1z5/V3tMTs/dubMGS1evFjXX3+9rrvuupp2BwAA4FJXHHY2b96sMWPGKDo6Wk899ZRuvPFGffzxx66sDQAAoMaqNWYnLy9PS5cu1ZIlS2S323XnnXeqpKREb775JndiAQCAOqnKZ3YGDx6sxMREZWVl6bnnntOxY8f04osv1mZtAAAANVblsPPuu+9qwoQJmj17tgYNGiRPT88a77xZs2ay2WwXTampqZKkfv36XbTunnvuqfF+AQDAtaPKYWfr1q06c+aMunbtqh49euill17Sf//73xrtfMeOHTp+/Lhj2rBhgyQ5vWNr4sSJTm0WLFhQo30CAIBrS5XDTs+ePfW3v/1Nx48f1+TJk/Xaa68pJiZGFRUV2rBhg86cOVPtnTdu3FhRUVGO6Z133lFCQoLT83r8/f2d2nD7OAAAqI5q340VEBCglJQUbd26VZ9//rkeeugh/fnPf1ZERIRuvfXWKy7k/PnzeuWVV5SSkiKbzeZY/uqrryo8PFzt27fXzJkzde7cucv2U1JSIrvd7jQBAIBrV42es5OYmKgFCxbo6NGjWrlyZY0KefPNN5Wfn69x48Y5lt1111165ZVX9MEHH2jmzJlasWKFfv3rX1+2n3nz5ik4ONgxxcbG1qguAABQv7n0Cco1kZycLB8fH61du/aSbd5//331799fBw8eVEJCQqVtSkpKVFJS4pi32+2KjY3lCcoAANQjrnyCcrXfjVUbDh8+rIyMDL3xxhuXbdejRw9JumzY8fX1la+vr8trBAAA9VONXxfhCmlpaYqIiNCgQYMu22737t2SpOjo6KtQFQAAsAK3n9mpqKhQWlqaxo4dKy+v/5WTk5Oj9PR03XLLLQoLC1NWVpYeeOAB9e3bVx07dnRjxQAAoD5xe9jJyMhQbm6uUlJSnJb7+PgoIyNDzz33nM6ePavY2FgNHz5cjz76qJsqBQAA9VGdGaBcW1w5wAkAAFwdrvz+rhNjdgAAAGoLYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFiaW8NOs2bNZLPZLppSU1MlScXFxUpNTVVYWJgaNmyo4cOH68SJE+4sGQAA1DNuDTs7duzQ8ePHHdOGDRskSXfccYck6YEHHtDatWu1evVqbdq0SceOHdNtt93mzpIBAEA9YzPGGHcXccG0adP0zjvv6MCBA7Lb7WrcuLHS09N1++23S5K++uortWnTRtu2bVPPnj2r1KfdbldwcLAKCgoUFBRUm+UDAAAXceX3d50Zs3P+/Hm98sorSklJkc1m086dO1VaWqqkpCRHm9atW6tp06batm3bJfspKSmR3W53mgAAwLWrzoSdN998U/n5+Ro3bpwkKS8vTz4+PgoJCXFqFxkZqby8vEv2M2/ePAUHBzum2NjYWqwaAADUdXUm7CxZskQDBw5UTExMjfqZOXOmCgoKHNORI0dcVCEAAKiPvNxdgCQdPnxYGRkZeuONNxzLoqKidP78eeXn5zud3Tlx4oSioqIu2Zevr698fX1rs1wAAFCP1IkzO2lpaYqIiNCgQYMcy7p27Spvb29t3LjRsSw7O1u5ubnq1auXO8oEAAD1kNvP7FRUVCgtLU1jx46Vl9f/ygkODtaECRP04IMPKjQ0VEFBQbrvvvvUq1evKt+JBQAA4Pawk5GRodzcXKWkpFy07tlnn5WHh4eGDx+ukpISJScn6//9v//nhioBAEB9Vaees1MbeM4OAAD1jyWfswMAAFAbCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDS3B52vv32W/36179WWFiY/Pz81KFDB3366aeO9ePGjZPNZnOaBgwY4MaKAQBAfeLlzp2fPn1affr00S9/+Uu9++67aty4sQ4cOKBGjRo5tRswYIDS0tIc876+vle7VAAAUE+5NezMnz9fsbGxTkEmPj7+ona+vr6Kioq6mqUBAACLcOtlrLffflvdunXTHXfcoYiICHXu3Fl/+9vfLmqXmZmpiIgIJSYmasqUKTp16tQl+ywpKZHdbneaAADAtcutYefrr7/WwoUL1bJlS61fv15TpkzR1KlTtWzZMkebAQMGaPny5dq4caPmz5+vTZs2aeDAgSovL6+0z3nz5ik4ONgxxcbGXq2PAwAA6iCbMca4a+c+Pj7q1q2bPvroI8eyqVOnaseOHdq2bVul23z99ddKSEhQRkaG+vfvf9H6kpISlZSUOObtdrtiY2NVUFCgoKAg138IAADgcna7XcHBwS75/nbrmZ3o6Gi1bdvWaVmbNm2Um5t7yW2aN2+u8PBwHTx4sNL1vr6+CgoKcpoAAMC1y61hp0+fPsrOznZatn//fsXFxV1ym6NHj+rUqVOKjo6u7fIAAIAFuDXsPPDAA/r444/1pz/9SQcPHlR6eroWL16s1NRUSVJhYaEefvhhffzxx/rmm2+0ceNGDRkyRC1atFBycrI7SwcAAPWEW8NO9+7dtWbNGq1cuVLt27fXnDlz9Nxzz2n06NGSJE9PT2VlZenWW29Vq1atNGHCBHXt2lVbtmzhWTsAAKBK3DpA+Wpw5QAnAABwdVhmgDIAAEBtI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABL83J3AbXNGCNJstvtbq4EAABU1YXv7Qvf4zVh+bBz5swZSVJsbKybKwEAANV16tQpBQcH16gPm3FFZKrDKioqdOzYMQUGBspms7m7nHrNbrcrNjZWR44cUVBQkLvLwQ9wbOoujk3dxvGpuwoKCtS0aVOdPn1aISEhNerL8md2PDw89LOf/czdZVhKUFAQ/1Goozg2dRfHpm7j+NRdHh41H17MAGUAAGBphB0AAGBphB1Uma+vrx5//HH5+vq6uxT8CMem7uLY1G0cn7rLlcfG8gOUAQDAtY0zOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIO7isJ554QjabzWlq3bq1u8u6Zm3evFmDBw9WTEyMbDab3nzzTaf1xhg99thjio6Olp+fn5KSknTgwAH3FHuN+aljM27cuIv+lgYMGOCeYq8x8+bNU/fu3RUYGKiIiAgNHTpU2dnZTm2Ki4uVmpqqsLAwNWzYUMOHD9eJEyfcVPG1oyrHpl+/fhf97dxzzz3V2g9hBz+pXbt2On78uGPaunWru0u6Zp09e1bXXXed/vrXv1a6fsGCBXrhhRe0aNEibd++XQEBAUpOTlZxcfFVrvTa81PHRpIGDBjg9Le0cuXKq1jhtWvTpk1KTU3Vxx9/rA0bNqi0tFQ333yzzp4962jzwAMPaO3atVq9erU2bdqkY8eO6bbbbnNj1deGqhwbSZo4caLT386CBQuqtyMDXMbjjz9urrvuOneXgUpIMmvWrHHMV1RUmKioKPOXv/zFsSw/P9/4+vqalStXuqHCa9ePj40xxowdO9YMGTLELfXA2cmTJ40ks2nTJmPM938n3t7eZvXq1Y42+/btM5LMtm3b3FXmNenHx8YYY2644QZz//3316hfzuzgJx04cEAxMTFq3ry5Ro8erdzcXHeXhEocOnRIeXl5SkpKciwLDg5Wjx49tG3bNjdWhgsyMzMVERGhxMRETZkyRadOnXJ3SdekgoICSVJoaKgkaefOnSotLXX622ndurWaNm3K385V9uNjc8Grr76q8PBwtW/fXjNnztS5c+eq1a/lXwSKmunRo4eWLl2qxMREHT9+XLNnz9YvfvELffHFFwoMDHR3efiBvLw8SVJkZKTT8sjISMc6uM+AAQN02223KT4+Xjk5OXrkkUc0cOBAbdu2TZ6enu4u75pRUVGhadOmqU+fPmrfvr2k7/92fHx8LnqzNn87V1dlx0aS7rrrLsXFxSkmJkZZWVn63e9+p+zsbL3xxhtV7puwg8saOHCg498dO3ZUjx49FBcXp9dff10TJkxwY2VA/TJy5EjHvzt06KCOHTsqISFBmZmZ6t+/vxsru7akpqbqiy++YOxhHXSpYzNp0iTHvzt06KDo6Gj1799fOTk5SkhIqFLfXMZCtYSEhKhVq1Y6ePCgu0vBj0RFRUnSRXeQnDhxwrEOdUfz5s0VHh7O39JV9Nvf/lbvvPOOPvjgA/3sZz9zLI+KitL58+eVn5/v1J6/navnUsemMj169JCkav3tEHZQLYWFhcrJyVF0dLS7S8GPxMfHKyoqShs3bnQss9vt2r59u3r16uXGylCZo0eP6tSpU/wtXQXGGP32t7/VmjVr9P777ys+Pt5pfdeuXeXt7e30t5Odna3c3Fz+dmrZTx2byuzevVuSqvW3w2UsXNb06dM1ePBgxcXF6dixY3r88cfl6empUaNGubu0a1JhYaHT/80cOnRIu3fvVmhoqJo2bapp06Zp7ty5atmypeLj4zVr1izFxMRo6NCh7iv6GnG5YxMaGqrZs2dr+PDhioqKUk5OjmbMmKEWLVooOTnZjVVfG1JTU5Wenq633npLgYGBjnE4wcHB8vPzU3BwsCZMmKAHH3xQoaGhCgoK0n333adevXqpZ8+ebq7e2n7q2OTk5Cg9PV233HKLwsLClJWVpQceeEB9+/ZVx44dq76jGt3LBcsbMWKEiY6ONj4+PqZJkyZmxIgR5uDBg+4u65r1wQcfGEkXTWPHjjXGfH/7+axZs0xkZKTx9fU1/fv3N9nZ2e4t+hpxuWNz7tw5c/PNN5vGjRsbb29vExcXZyZOnGjy8vLcXfY1obLjIsmkpaU52hQVFZl7773XNGrUyPj7+5thw4aZ48ePu6/oa8RPHZvc3FzTt29fExoaanx9fU2LFi3Mww8/bAoKCqq1H9v/vzMAAABLYswOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOgDrlm2++kc1mczwS3orOnz+vFi1a6KOPPqq1fSxatEiDBw+utf6B+oSwA9QD27Ztk6enpwYNGuTuUuqkcePG1atXYixatEjx8fHq3bt3re0jJSVFn332mbZs2VJr+wDqC8IOUA8sWbJE9913nzZv3qxjx47V6r6MMSorK6vVfVzLjDF66aWXNGHChFrdj4+Pj+666y698MILtbofoD4g7AB1XGFhoVatWqUpU6Zo0KBBWrp0qWPdXXfdpREjRji1Ly0tVXh4uJYvXy5Jqqio0Lx58xQfHy8/Pz9dd911+uc//+lon5mZKZvNpnfffVddu3aVr6+vtm7dqpycHA0ZMkSRkZFq2LChunfvroyMDKd9HT9+XIMGDZKfn5/i4+OVnp6uZs2a6bnnnnO0yc/P1913363GjRsrKChIN954o/bs2VPlz19eXq4JEyY46k9MTNTzzz/vWP/EE09o2bJleuutt2Sz2WSz2ZSZmSlJOnLkiO68806FhIQoNDRUQ4YM0TfffOPY9sIZoaeeekrR0dEKCwtTamqqSktLHW1KSkr0u9/9TrGxsfL19VWLFi20ZMkSGWPUokULPfXUU0717t69WzabzemloD+0c+dO5eTkOJ2lu3Dp7vXXX9cvfvEL+fn5qXv37tq/f7927Nihbt26qWHDhho4cKD+85//OB2766+/XgEBAQoJCVGfPn10+PBhx/rBgwfr7bffVlFRUZV/3oAlufaVXgBcbcmSJaZbt27GGGPWrl1rEhISTEVFhTHGmHfeecf4+fmZM2fOONqvXbvW+Pn5GbvdbowxZu7cuaZ169Zm3bp1Jicnx6SlpRlfX1+TmZlpjPnfCyw7duxo3nvvPXPw4EFz6tQps3v3brNo0SLz+eefm/3795tHH33UNGjQwBw+fNixr6SkJNOpUyfz8ccfm507d5obbrjB+Pn5mWeffdapzeDBg82OHTvM/v37zUMPPWTCwsLMqVOnKv28hw4dMpLMrl27jDHGnD9/3jz22GNmx44d5uuvvzavvPKK8ff3N6tWrTLGGHPmzBlz5513mgEDBpjjx4+b48ePm5KSEnP+/HnTpk0bk5KSYrKysszevXvNXXfdZRITE01JSYkxxpixY8eaoKAgc88995h9+/aZtWvXGn9/f7N48WJHPXfeeaeJjY01b7zxhsnJyTEZGRnmtddeM8YY8+STT5q2bds61T916lTTt2/fSx7PZ555xrRu3brSz3zhOO3du9f07NnTdO3a1fTr189s3brVfPbZZ6ZFixbmnnvuMcYYU1paaoKDg8306dPNwYMHzd69e83SpUudjs/Zs2eNh4eH+eCDDy5ZD3AtIOwAdVzv3r3Nc889Z4z5/gsuPDzc8eV1YX758uWO9qNGjTIjRowwxhhTXFxs/P39zUcffeTU54QJE8yoUaOMMf8LO2+++eZP1tKuXTvz4osvGmOM2bdvn5FkduzY4Vh/4MABI8kRdrZs2WKCgoJMcXGxUz8JCQnm5ZdfrnQfPw47lUlNTTXDhw93zI8dO9YMGTLEqc2KFStMYmKiIxgaY0xJSYnx8/Mz69evd2wXFxdnysrKHG3uuOMOx88vOzvbSDIbNmyotI5vv/3WeHp6mu3btxtjvg9m4eHhZunSpZes/f777zc33nhjpZ/573//u2PZypUrjSSzceNGx7J58+aZxMREY4wxp06dMpIcofVSGjVqdNl6gGsBl7GAOiw7O1uffPKJRo0aJUny8vLSiBEjtGTJEsf8nXfeqVdffVWSdPbsWb311lsaPXq0JOngwYM6d+6cbrrpJjVs2NAxLV++XDk5OU776tatm9N8YWGhpk+frjZt2igkJEQNGzbUvn37lJub66jNy8tLXbp0cWzTokULNWrUyDG/Z88eFRYWKiwszGn/hw4dumj/l/PXv/5VXbt2VePGjdWwYUMtXrzYUcel7NmzRwcPHlRgYKBjv6GhoSouLnbad7t27eTp6emYj46O1smTJyV9f0nK09NTN9xwQ6X7iImJ0aBBg/SPf/xDkrR27VqVlJTojjvuuGRdRUVFatCgQaXrOnbs6Ph3ZGSkJKlDhw5Oyy7UFhoaqnHjxik5OVmDBw/W888/r+PHj1/Up5+fn86dO3fJeoBrgZe7CwBwaUuWLFFZWZliYmIcy4wx8vX11UsvvaTg4GCNHj1aN9xwg06ePKkNGzbIz89PAwYMkPR9YJGkf/3rX2rSpIlT376+vk7zAQEBTvPTp0/Xhg0b9NRTT6lFixby8/PT7bffrvPnz1e5/sLCQkVHRzvG0PxQSEhIlfp47bXXNH36dD399NPq1auXAgMD9Ze//EXbt2//yX137drVEQR/qHHjxo5/e3t7O62z2WyqqKiQ9H1Q+Cl33323fvOb3+jZZ59VWlqaRowYIX9//0u2Dw8P1+eff17puh/WYrPZKl12oTZJSktL09SpU7Vu3TqtWrVKjz76qDZs2KCePXs62nz33XdOnxe4FhF2gDqqrKxMy5cv19NPP62bb77Zad3QoUO1cuVK3XPPPerdu7diY2O1atUqvfvuu7rjjjscX5Bt27aVr6+vcnNzL3l24lI+/PBDjRs3TsOGDZP0fXj44eDexMRElZWVadeuXeratauk788knT592tGmS5cuysvLk5eXl5o1a3YFP4Xv6+jdu7fuvfdex7IfnxXy8fFReXm507IuXbpo1apVioiIUFBQ0BXtu0OHDqqoqNCmTZuUlJRUaZtbbrlFAQEBWrhwodatW6fNmzdfts/OnTtr4cKFMsY4Ak1NdO7cWZ07d9bMmTPVq1cvpaenO8JOTk6OiouL1blz5xrvB6jPuIwF1FHvvPOOTp8+rQkTJqh9+/ZO0/Dhwx2XsqTv78patGiRNmzY4LiEJUmBgYGaPn26HnjgAS1btkw5OTn67LPP9OKLL2rZsmWX3X/Lli31xhtvaPfu3dqzZ4/uuusup7MKrVu3VlJSkiZNmqRPPvlEu3bt0qRJk+Tn5+f4Ek9KSlKvXr00dOhQvffee/rmm2/00Ucf6Q9/+IM+/fTTKv0cWrZsqU8//VTr16/X/v37NWvWLO3YscOpTbNmzZSVlaXs7Gz997//VWlpqUaPHq3w8HANGTJEW7Zs0aFDh5SZmampU6fq6NGjVdp3s2bNNHbsWKWkpOjNN9909PH666872nh6emrcuHGaOXOmWrZsqV69el22z1/+8pcqLCzUl19+WaUaLuXQoUOaOXOmtm3bpsOHD+u9997TgQMH1KZNG0ebLVu2qHnz5kpISKjRvoD6jrAD1FFLlixRUlKSgoODL1o3fPhwffrpp8rKypIkjR49Wnv37lWTJk3Up08fp7Zz5szRrFmzNG/ePLVp00YDBgzQv/71L8XHx192/88884waNWqk3r17a/DgwUpOTnYanyNJy5cvV2RkpPr27athw4Zp4sSJCgwMdIxJsdls+ve//62+fftq/PjxatWqlUaOHKnDhw87xqT8lMmTJ+u2227TiBEj1KNHD506dcrpLI8kTZw4UYmJierWrZsaN26sDz/8UP7+/tq8ebOaNm2q2267TW3atNGECRNUXFxcrTM9Cxcu1O233657771XrVu31sSJE3X27FmnNhMmTND58+c1fvz4n+wvLCxMw4YNq/TyWnX4+/vrq6++0vDhw9WqVStNmjRJqampmjx5sqPNypUrNXHixBrtB7ACmzHGuLsIANZw9OhRxcbGKiMjQ/3793d3OVfNli1b1L9/fx05cqRKIS4rK0s33XSTcnJy1LBhw1qp6csvv9SNN96o/fv3VxqYgWsJYQfAFXv//fdVWFioDh066Pjx45oxY4a+/fZb7d+//6KBv1ZUUlKi//znPxo7dqyioqKqdbZm6dKl6tq1q9PdVq6UkZGh8vJyJScn10r/QH1C2AFwxdavX6+HHnpIX3/9tQIDA9W7d28999xziouLc3dpV8XSpUs1YcIEderUSW+//fZFd7wBqBsIOwAAwNIYoAwAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACzt/wM5YFXqI8XqcgAAAABJRU5ErkJggg==\n" + }, + "metadata": {} + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "\n", + "\n", + "def plot_metrics(perf_metrics, current_optim_type):\n", + " df = pd.DataFrame.from_dict(perf_metrics, orient=\"index\")\n", + "\n", + " for idx in df.index:\n", + " df_opt = df.loc[idx]\n", + " # Add a dashed circle around the current optimization type\n", + " if idx == current_optim_type:\n", + " plt.scatter(\n", + " df_opt[\"time_avg_ms\"],\n", + " df_opt[\"accuracy\"] * 100,\n", + " alpha=0.5,\n", + " s=df_opt[\"size_mb\"],\n", + " label=idx,\n", + " marker=\"$\\u25CC$\",\n", + " )\n", + " else:\n", + " plt.scatter(\n", + " df_opt[\"time_avg_ms\"],\n", + " df_opt[\"accuracy\"] * 100,\n", + " s=df_opt[\"size_mb\"],\n", + " label=idx,\n", + " alpha=0.5,\n", + " )\n", + "\n", + " legend = plt.legend(bbox_to_anchor=(1, 1))\n", + " for handle in legend.legendHandles:\n", + " handle.set_sizes([20])\n", + "\n", + " plt.ylim(63, 95)\n", + " # Use the slowest model to define the x-axis range\n", + " xlim = int(perf_metrics[\"bge-small PyTorch\"][\"time_avg_ms\"] + 12)\n", + " plt.xlim(1, xlim)\n", + " plt.ylabel(\"Accuracy (%)\")\n", + " plt.xlabel(\"Average latency (ms)\")\n", + " plt.show()\n", + "\n", + "\n", + "plot_metrics(perf_metrics, \"bge-small\")" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## 4. Eval DistillBERT as refrence" + ], + "metadata": { + "id": "b-yd2LtbSbXj" + }, + "id": "b-yd2LtbSbXj" + }, + { + "cell_type": "code", + "source": [ + "from transformers import pipeline\n", + "\n", + "#hide_output\n", + "finetuned_model = \"distilbert-base-uncased-finetuned-sst-2-english\"\n", + "pipe = pipeline(\"text-classification\", model=finetuned_model)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 145, + "referenced_widgets": [ + "eb6e6d91566044f18e953d8df93ae3a8", + "29dbf37d879a4f05904b076378d190bf", + "1cfb1e7981264c328e02e49c4080c504", + "e1157a679f354eaba73d4fea3cbda6fe", + "a9eafe51348847b0be895b8502a21e56", + "f338c914be9641a4aee76971c0e9cf72", + "746f408b617a4974ab48665f5b0c6c01", + "561acc35f4e245e8adf81b1efa510fd2", + "85377644ee164abebcfb24c7fdeaab8a", + "b800f7b6b2fe448486f63a24066694aa", + "0661f3e76c9a490db89c29c35d8c0d1b", + "7612716749c14069bc5989f3ee09c00a", + "0e9bfe8c83a44537bb59c444744466ef", + "5e8641e8da904126a3680828c83774e8", + "2f99603c364b4af48ea25b8d43cd1941", + "26ae8872c49642c588330b34ff470609", + "a8ee2fe1c3d04082acdafbf39ba0076b", + "5e9bad63ccc14fe2b358b7c7f0bbae68", + "50597a7caa0242df82fafa89bf831827", + "48ba967e29ad45a1ac8f7ba659fb5a5f", + "788dc688a9e9449e9aeb431b06384d9f", + "c82bc76766fd420a9557c4dcd78cff0c", + "f52944fb4da049afae19d9d1f4a6b37a", + "9cee60b1935441e4a81d26cc5caa319e", + "4621e316677d44caab491fd8a7b1433f", + "39bea84d46004ecd9dae38e81c4bb2d8", + "5859b86234164b55aa97852b4368bd66", + "9fd724ea0d6947be92361604c9f318a7", + "996b172b2f434b5a95de4a839c2ae4ae", + "ff8e804c7b5e472187685d1fdcc5d0ff", + "764f957750064a7eb61d5e6b28e0e49d", + "d46affe5b0ff4914bc2504c2737c6d40", + "350d6653f3d947b8a125b051b36e9d4a", + "e9e9cb6c63c54831acf6922003883421", + "1d9e7ff2e4d74891a3f70798bb5266eb", + "55135c120cc94702be910b033f6df07d", + "a01975d1f8154082bd903bb55b3d49d6", + "26db4649e9924203912f92e0e1c994bb", + "2261900d8f594e7b965fe8d4d6264e35", + "14f43841b87646dd9c8eca2966488900", + "c7a0c077c06e4344bdffe4848de56e15", + "581e9a90552543188ec3c95b4cdab508", + "15b87e982dcf45139029b93eb6a7f857", + "dc42e38e51464a2095b179359da4c816" + ] + }, + "id": "3o6VdvFwvRUM", + "outputId": "7a853fe0-5d8f-4361-f99b-620f984d23dc" + }, + "id": "3o6VdvFwvRUM", + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Downloading (…)lve/main/config.json: 0%| | 0.00/629 [00:00:30: MatplotlibDeprecationWarning: The legendHandles attribute was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use legend_handles instead.\n", + " for handle in legend.legendHandles:\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAukAAAG2CAYAAADC9Bi0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABOKUlEQVR4nO3daXgUVf728buz0tkhIRsmISQQdlllG5GRKCCDbCIg/lmCgBBFVGAGFQU3BsYF0RkQZcJmEDdQHAUBZVNEZFcwQAQCkoCDJCGQPfW8yEOPLVsCSbqgv5/r6kuq6tSpX6fT9t2VU6cshmEYAgAAAGAaLo4uAAAAAIA9QjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMg4N6WfOnNG4ceMUFRUlq9Wq9u3ba+vWrbbtQ4cOlcVisXt07drVgRUDAAAAlc/NkQd/4IEH9MMPP2jRokUKDw/X4sWLFR8fr71796pWrVqSpK5duyopKcm2j6enp6PKBQAAAKqExTAMwxEHzs3Nla+vrz7++GN1797dtr5ly5bq1q2bnn/+eQ0dOlSZmZlavny5I0oEAAAAHMJhZ9KLiopUXFysatWq2a23Wq3atGmTbXndunUKDg5W9erVdfvtt+v5559XYGDgJfvNz89Xfn6+bbmkpES//fabAgMDZbFYKv6JAACACmcYhs6cOaPw8HC5uHAJHZyPw86kS1L79u3l4eGh5ORkhYSEaMmSJRoyZIhiY2OVkpKid999V15eXoqOjlZqaqqeeOIJ+fj4aPPmzXJ1db1on1OmTNHUqVOr+JkAAIDKcPToUd10002OLgOocg4N6ampqUpISNCGDRvk6uqqFi1aqF69etq2bZv27dt3Qfuff/5ZMTExWrNmjTp37nzRPv94Jj0rK0uRkZE6evSo/Pz8Ku25AACAipOdna2IiAhlZmbK39/f0eUAVc6hF47GxMRo/fr1Onv2rLKzsxUWFqb+/furTp06F21fp04dBQUF6eDBg5cM6Z6enhe9uNTPz4+QDgDAdYahqnBWphjk5e3trbCwMJ0+fVqrVq1Sz549L9ru2LFjOnXqlMLCwqq4QgAAAKDqOPRM+qpVq2QYhuLi4nTw4EFNmDBB9evX17Bhw5STk6OpU6eqb9++Cg0NVWpqqiZOnKjY2Fh16dLFkWUDAAAAlcqhZ9KzsrKUmJio+vXra/DgwfrTn/6kVatWyd3dXa6urtq9e7fuvvtu1atXT8OHD1fLli21ceNG5koHAADADc2hF45WhezsbPn7+ysrK4sx6QAAXCfK+vldXFyswsLCKqwMuHoeHh5lnlLUocNdAAAAroZhGMrIyFBmZqajSwHKzMXFRdHR0fLw8LhiW0I6AAC47pwP6MHBwfLy8mIWGJheSUmJjh8/rvT0dEVGRl7xd5aQDgAArivFxcW2gH65u5ADZlOzZk0dP35cRUVFcnd3v2xbU0zBCAAAUFbnx6B7eXk5uBKgfM4PcykuLr5iW0I6AAC4LjHEBdeb8vzOEtIBAAAAkyGkAwAAVIFOnTpp3Lhxji6jysyfP18BAQG25SlTpqhZs2YOq6cy1K5dWzNnzqyUvgnpAAAAcLjDhw/LYrHYHoGBgbrzzju1Y8eOMu1fu3Ztu/3/+Bg6dGjlPoEKxuwuAAAAMI01a9aoUaNGOnbsmMaOHatu3brpp59+sjsrfzFbt261XZD5zTffqG/fvkpJSbHdDMtqtZarjsLCwivOwFKZOJMOAACc1umzBTr037M6fbagSo5XVFSkhx56SP7+/goKCtLkyZP1+5u/p6enq3v37rJarYqOjlZycvIFQyoyMzP1wAMPqGbNmvLz89Ptt9+uXbt2Xfa469at0y233CJvb28FBASoQ4cOOnLkiKT/DUP597//rcjISPn4+GjMmDEqLi7WjBkzFBoaquDgYL3wwgt2fb7yyitq0qSJvL29FRERoTFjxignJ+eaf0aBgYEKDQ1Vq1at9NJLL+nEiRPasmWLnn32WTVu3PiC9s2aNdPkyZNVs2ZNhYaGKjQ0VDVq1JAkBQcH29YlJycrJiZGHh4eiouL06JFi+z6sVgsmj17tu6++255e3vbnu+KFSvUunVrVatWTUFBQerdu7fdfufOnVNCQoJ8fX0VGRmpuXPnXvPPQCKkAwAAJ5RXWKwPth3V9JU/6dXVKZq+8id9sO2o8gqvPDXetViwYIHc3Nz03Xff6bXXXtMrr7yit99+27Z98ODBOn78uNatW6cPP/xQc+fO1cmTJ+366Nevn06ePKnPP/9c27ZtU4sWLdS5c2f99ttvFz1mUVGRevXqpdtuu027d+/W5s2bNXLkSLuZRlJTU/X5559r5cqVWrJkiebNm6fu3bvr2LFjWr9+vaZPn66nnnpKW7Zsse3j4uKiWbNm6ccff9SCBQv05ZdfauLEiRX68zp/9rugoEAJCQnat2+ftm7datu+Y8cO7d69W8OGDbtsP8uWLdMjjzyixx9/XD/88INGjRqlYcOG6auvvrJrN2XKFPXu3Vt79uxRQkKC/vOf/6h379666667tGPHDq1du1a33HKL3T4vv/yyWrVqpR07dmjMmDEaPXq0UlJSrv3JGze4rKwsQ5KRlZXl6FIAAEAZXe7zOzc319i7d6+Rm5t71f2//32aMXLhVmPSh7uNaZ/tNSZ9uNsYuXCr8f73addS9mXddtttRoMGDYySkhLbur/+9a9GgwYNDMMwjH379hmSjK1bt9q2HzhwwJBkvPrqq4ZhGMbGjRsNPz8/Iy8vz67vmJgY480337zocU+dOmVIMtatW3fR7c8884zh5eVlZGdn29Z16dLFqF27tlFcXGxbFxcXZ0ybNu2Sz+/99983AgMDbctJSUmGv7+/3XFuvvnmS+5/6NAhQ5KxY8cOwzAM4/Tp00bv3r0NHx8fIyMjwzAMw+jWrZsxevRo2z4PP/yw0alTpwv6+uqrrwxJxunTpw3DMIz27dsbI0aMsGvTr18/46677rItSzLGjRtn16Zdu3bGoEGDLllzVFSUcf/999uWS0pKjODgYGP27NkXbV+e313OpAMAAKdy+myBvj98WoHenqrp6ylPN1fV9PVUoLenth0+XalDX9q2bWt3Brtdu3Y6cOCAiouLlZKSIjc3N7Vo0cK2PTY2VtWrV7ct79q1Szk5OQoMDJSPj4/tcejQIaWmpiotLc1u/YsvvqgaNWpo6NCh6tKli3r06KHXXntN6enpdnXVrl1bvr6+tuWQkBA1bNhQLi4udut+f1Z/zZo16ty5s2rVqiVfX1/93//9n06dOqVz585d08+offv28vHxUfXq1bVr1y4tXbpUISEhkqQRI0ZoyZIlysvLU0FBgZKTk5WQkHDFPvft26cOHTrYrevQoYP27dtnt65Vq1Z2yzt37lTnzp0v23fTpk1t/7ZYLAoNDb3grx9XgwtHAQCAU8nMLdS5giKFB9hfSOhnddPxzFxl5haqureHg6q7vJycHIWFhWndunUXbAsICFBAQIB27txpW3d+bHZSUpLGjh2rlStXaunSpXrqqae0evVqtW3bVpIuuEDSYrFcdF1JSYmk0plY/vKXv2j06NF64YUXVKNGDW3atEnDhw9XQUHBNd0NdunSpWrYsKECAwMvuFi0R48e8vT01LJly+Th4aHCwkLdc889V32sP/L29rZbLsvFppf7OV0LQjoAAHAqAVZ3eXm4KTu3SDV9XW3rs3OL5O3hpgBr5c3o8fsx3ZL07bffqm7dunJ1dVVcXJyKioq0Y8cOtWzZUpJ08OBBnT592ta+RYsWysjIkJubm2rXrn3RY8TGxl50ffPmzdW8eXNNmjRJ7dq1U3Jysi2kl9e2bdtUUlKil19+2Xa2/b333ruqvv4oIiJCMTExF93m5uamIUOGKCkpSR4eHhowYECZgnSDBg309ddfa8iQIbZ1X3/9tRo2bHjZ/Zo2baq1a9deccx7ZSCkAwAAp1Ld20OtalfX6r0nJJWeQc/OLdKps/m6o2FIpZ5FT0tL02OPPaZRo0Zp+/btev311/Xyyy9LkurXr6/4+HiNHDlSs2fPlru7ux5//HFZrVbbEJn4+Hi1a9dOvXr10owZM1SvXj0dP37cdoHjH4drSNKhQ4c0d+5c3X333QoPD1dKSooOHDigwYMHX/XziI2NVWFhoV5//XX16NFDX3/9tebMmXPV/ZXHAw88oAYNGkgqDdplMWHCBN17771q3ry54uPjtWLFCn300Udas2bNZfd75pln1LlzZ8XExGjAgAEqKirSZ599pr/+9a/X/DyuhDHpAADA6fylabjuaBgiwzB0PDNXhmHojoYh+kvT8Eo97uDBg5Wbm6tbbrlFiYmJeuSRRzRy5Ejb9oULFyokJEQdO3ZU7969NWLECPn6+qpatWqSSodSfPbZZ+rYsaOGDRumevXqacCAATpy5Iht3PYfeXl56aefflLfvn1Vr149jRw5UomJiRo1atRVP4+bb75Zr7zyiqZPn67GjRvrnXfe0bRp0666v/KoW7eu2rdvr/r166tNmzZl2qdXr1567bXX9NJLL6lRo0Z68803lZSUpE6dOl12v06dOun999/XJ598ombNmun222/Xd999VwHP4sos//9q1htWdna2/P39lZWVZZvMHgAAmNvlPr/z8vJ06NAhRUdH28Lr1Tp9tkCZuYUKsLqbchz6sWPHFBERYbtIE5JhGKpbt67GjBmjxx57zNHllEt5fncZ7gIAAJxWdW8PU4XzL7/8Ujk5OWrSpInS09M1ceJE1a5dWx07dnR0aabw66+/6t1331VGRoZDxolXJUI6AACASRQWFuqJJ57Qzz//LF9fX7Vv317vvPOOQ29PbybBwcEKCgrS3Llz7aamvBER0gEAAEyiS5cu6tKli6PLMK0bfJS2HS4cBQAAAEyGkA4AAACYDCEdAAAAMBlCOgAAAGAyhHQAAADAZAjpAAAAgMkQ0gEAABykU6dOGjdunG25du3amjlz5lX3N3/+fAUEBNiWp0yZombNmtmWhw4dql69el3y+LiyP/6MKwshHQAAwCS2bt2qkSNHlqntxQJ9//79tX///kqorPz++IXgcu0sFossFos8PDwUGxurZ599VkVFRVfcd/78+bZ9L/U4fPjwtT8ZB+BmRgAAOMLRrdJ/D0hnT0jFBZKnn+R/kxTZTvIOcnR1cJCaNWte0/5Wq1VWq7WCqrk6xcXFslgs5dqna9euSkpKUn5+vj777DMlJibK3d1dkyZNuux+/fv3V9euXW3Lffr0UePGjfXss8/a1pXnZ1pQUCAPD49y1V5ZOJMOAEBVOftfKS+r9N/7V0p73pMOf10a2A+ulXa/J51KlQxDOn1EKi50bL3OwDCk/Jwq+VmfPXtWgwcPlo+Pj8LCwvTyyy9f0Ob3Z8cNw9CUKVMUGRkpT09PhYeHa+zYsZJKh6kcOXJEjz76qO2MsXR1QzGKior00EMPyd/fX0FBQZo8ebLdnT3z8/M1fvx41apVS97e3mrTpo3WrVtn237+mJ988okaNmwoT09PJSQkaMGCBfr4449t9f1+nz/y9PRUaGiooqKiNHr0aMXHx+uTTz7R2bNn5efnpw8++MCu/fLly+Xt7a2ioiKFhobaHh4eHvLy8rItFxQUqE+fPvLx8ZGfn5/uvfdenThxwtbP+eFAb7/9tqKjo1WtWjVJUmZmpkaNGqWQkBBVq1ZNjRs31qeffmpXw6pVq9SgQQP5+Pioa9euSk9PL9fP/Uo4kw4AQGUrKZEOrZf2fSIF1ZPaPCjVaiFlpklu1SSLS+nZdA8fKaiu9NvP0uZ/Sv4R0s39Jb9wRz+DG9PJfdKPy6ScE5KnvxTZVoqNl9wq50zqhAkTtH79en388ccKDg7WE088oe3bt9uNGf+9Dz/8UK+++qreffddNWrUSBkZGdq1a5ck6aOPPtLNN9+skSNHasSIEddU14IFCzR8+HB99913+v777zVy5EhFRkba+n3ooYe0d+9evfvuuwoPD9eyZcvUtWtX7dmzR3Xr1pUknTt3TtOnT9fbb7+twMBAhYWFKTc3V9nZ2UpKSpIk1ahRo8w1Wa1WnTp1St7e3howYICSkpJ0zz332LafX/b19b1kHyUlJerZs6d8fHy0fv16FRUVKTExUf3797f7wnDw4EF9+OGH+uijj+Tq6qqSkhJ169ZNZ86c0eLFixUTE6O9e/fK1dXVts+5c+f00ksvadGiRXJxcdH999+v8ePH65133inzc7wSQjoAAJWpuFDa84F0YJV07jepqEDK/kWq82epZn3JK1BydZfyz0hF+ZJXjdKz6r+lSlnHpLMnpZbDpJr1HP1MbixZv0hb50lunpJ3cOm6nz6VivKkxn0q/HA5OTmaN2+eFi9erM6dO0sqDcc33XTTJfdJS0tTaGio4uPj5e7ursjISN1yyy2SSgOvq6urfH19FRoaek21RURE6NVXX5XFYlFcXJz27NmjV199VSNGjFBaWpqSkpKUlpam8PDSL4vjx4/XypUrlZSUpBdffFGSVFhYqH/961+6+eabbf1arVbl5+eXqz7DMLR27VqtWrVKDz/8sCTpgQceUPv27ZWenq6wsDCdPHlSn332mdasWXPZvtauXas9e/bo0KFDioiIkCQtXLhQjRo10tatW9W6dWtJpUNcFi5caBsW88UXX+i7777Tvn37VK9e6fuuTp06dn0XFhZqzpw5iomJkVT6Reb3Q2wqAsNdAACoLIYh7V0upXwmFeZKPiFS9G2Sd03Jw0sKjJGsAZKHt+QbKlWPKt0vvLl00y2Si2vp8Jetb5eedUfFOfKNVC1AumWkdPuTUscJUvVo6cSP/xuSVIFSU1NVUFCgNm3a2NbVqFFDcXFxl9ynX79+ys3NVZ06dTRixAgtW7asTBdTllfbtm3txpC3a9dOBw4cUHFxsfbs2aPi4mLVq1dPPj4+tsf69euVmppq28fDw0NNmza96ho+/fRT+fj4qFq1aurWrZv69++vKVOmSJJuueUWNWrUSAsWLJAkLV68WFFRUerYseNl+9y3b58iIiJsAV2SGjZsqICAAO3bt8+2Lioqym7c+s6dO3XTTTfZAvrFeHl52QK6JNuXh4pESAcAoLIUnpOO7yz9r1eg1Hq41LRf6dnbywmKlTqMlaI6lA6FyTkhZfxQJSU7jfRdUn72/74YuXmU/rUi52TptQMmEBERoZSUFP3rX/+S1WrVmDFj1LFjRxUWVt21Cjk5OXJ1ddW2bdu0c+dO22Pfvn167bXXbO2sVmu5Lxb9vT//+c/auXOnDhw4oNzcXC1YsEDe3t627Q888IDmz58vqXSoy7Bhw67peL/3++NIKtOFt+7u7nbLFovFbhx/RSCkA3BaJSUV+z9U4AIe3lK7h6T6f5Ga3Vc65rmswcLTV2o5RIrtXDrcpe4dlVurswm7uXRGndNHSpeLCqRf90s+wZUyu05MTIzc3d21ZcsW27rTp09fcbpEq9WqHj16aNasWVq3bp02b96sPXv2SCo9e11cXHzNtf2+Jkn69ttvVbduXbm6uqp58+YqLi7WyZMnFRsba/e40jCW8tTn7e2t2NhYRUZGys3twtHY999/v44cOaJZs2Zp7969GjJkyBX7bNCggY4ePaqjR4/a1u3du1eZmZlq2LDhJfdr2rSpjh075vCpLBmTDsBpFBaXKCXjjLYdOa20386qoKhEHm4uiqzhrZZR1RUX6it3V85doIL5hUmthl3dvp6+0i3XdlEgLiGqvXR0i/TdXMndq3Rd9i+lF45W86/ww/n4+Gj48OGaMGGCAgMDFRwcrCeffFIuLpf+f878+fNVXFysNm3ayMvLS4sXL5bValVUVOnZ/9q1a2vDhg0aMGCAPD09FRR0dV8u0tLS9Nhjj2nUqFHavn27Xn/9ddvMM/Xq1dOgQYM0ePBgvfzyy2revLl+/fVXrV27Vk2bNlX37t0v2W/t2rW1atUqpaSkKDAwUP7+/hecgS6r6tWrq0+fPpowYYLuvPPOy47lPy8+Pl5NmjTRoEGDNHPmTBUVFWnMmDG67bbb1KpVq0vud9ttt6ljx47q27evXnnlFcXGxuqnn36SxWKxm+6xsvFpBOCGZxiGdqSd1iurUzRnfaq2/HxKOXnFKi6RcvKKteXnU5qzPlWvrE7RjrTTji4XN4q0b6Ufl5eOKb9Whbml/e1MlnL5Ha0Q/rVKhx95eJdenFuUX/oXj/p/qbRD/uMf/9Ctt96qHj16KD4+Xn/605/UsmXLS7YPCAjQW2+9pQ4dOqhp06Zas2aNVqxYocDAQEnSs88+q8OHDysmJuaa5lcfPHiwcnNzdcsttygxMVGPPPKI3Q2VkpKSNHjwYD3++OOKi4tTr169tHXrVkVGRl623xEjRiguLk6tWrVSzZo19fXXX191jZI0fPhwFRQUKCEhoUztLRaLPv74Y1WvXl0dO3ZUfHy86tSpo6VLl15x3w8//FCtW7fWwIED1bBhQ02cOLFC/mpRHhajogfQmEx2drb8/f2VlZUlPz8/R5cDoIoZhqFNB/+rZdt/UWFJicL9rarm7npBu7zCYh3PypW7i4t6t6ilW+te2w1FAG18RTq8UQqsK3V+WvL0ufq+Ur+Uti+SSoqkWx8rvbD0Bne5z++8vDwdOnTIbl7rq2YYUsHZ0usEXK/uLC+qxqJFi/Too4/q+PHjprnhUHmV53eXM+kAbmg7j2Zq2fZf5OZqUZ0gn4sGdEmq5u6qOkE+cnO1aNmOX7TzaGbVFoobz9lfJcv//307P5ziann4lAZ0ScrLvra+YM9iKf0CRUA3rXPnzik1NVV///vfNWrUqOs2oJcXIR3ADauwuERf7M1QYUmJwvzLdpvsMH9r6X4/ZqiwuKSSK8QNyzD+dwdL92rSZcYdl4nr70JJCXchhXOZMWOG6tevr9DQUE2aNMnR5VQZQjqAG1ZKxhkd/S1X4WUM6OeF+1uV9ts5pWScqaTKcMOzWP53ZrYwr/SOo9eiuKA0+EuSC2d84VymTJmiwsJCrV27Vj4+1zBs7DpDSAdww9p25LRKSoxLDnG5lGruriopMbTtCBfo4Rp415SM/3+hWeG5a+urIKc09FssUjWurwKcAVMwArhhpf12Vr7Vru6so081N6X9do3BCs4tqr1Uo44U2uTaLhqVpMh2kru39NvPpXfFhCRV+M1jgMpWnt9ZQjqAG1JJiaGCohK5ulzdHelcXSwqKCpWSYkhl6vsA04usm3F9eVulSLblD5gm2v73LlzZbo7JGAWBQUFkiRX1yv/hZeQDuCG5OJikYebi3Lyrm5e2+ISQ14ebgR0XLvsdGn/SimorlT7T+XbNy9b2r20dBrH2h2YgeT/c3V1VUBAgE6ePClJ8vLyqrBbxAOVpaSkRL/++qu8vLwuelfVPyKkA7hhRdbw1pafTynUv/zzKOfkFalReMXfdRBOpuCstPkN6defJJ8QycVNimhTOrb8SvLPSNsXls61fnhj6bj2+ndVfs3XifO3pD8f1IHrgYuLiyIjI8v0pdKhIf3MmTOaPHmyli1bppMnT6p58+Z67bXX1Lp1a0ml43aeeeYZvfXWW8rMzFSHDh00e/Zs1a1b15FlA7hOtIyqrq2Hf1NeYXG5Lh7NKyyWi4tFLaOqV2J1cAruXlJ4MykzTTp3Sto6TzqdJjXqWXrznEv578HSM+gnfpCMEsk3TAprWmVlXw8sFovCwsIUHByswkKmpcT1wcPDQy5lnJLVoSH9gQce0A8//KBFixYpPDxcixcvVnx8vPbu3atatWppxowZmjVrlhYsWKDo6GhNnjxZXbp00d69e6/9DmMAbnhxob6KqGHVsdO5qhNU9gv3jmflKrKGl+JCfSuxOjgFi0Vq2EsqLpIOrJJyTkiH1ku120vWGlL2cck7sHQe9Lzs0qkWq0dJx3dIx76TXD2lwFipVYLkf5Ojn40pubq6lml8L3C9sRgOujQ6NzdXvr6++vjjj9W9e3fb+pYtW6pbt2567rnnFB4erscff1zjx4+XJGVlZSkkJETz58/XgAEDynScy91WGMCNb0faaS3afERurpYy3dAoPStXRSWGBrerrWYRAZVfIJxDSUlpON/3iRQUJ7UZJaV8Jv24XHKrJllcSgO6h4/050lS7mlp8z+lgAipaX/JL9zRz6DK8fkNZ+ewM+lFRUUqLi6+4Iy41WrVpk2bdOjQIWVkZCg+Pt62zd/fX23atNHmzZsvGdLz8/OVn59vW87O5vbJgDNrFhGgnPwiLdv+i37+b47C/a0XHfqSV1is41m5cnd1Ue/mtQjoqFguLlLMn0unYzw/3/kv26Wzv5aOUz+vIEf67wEp4hbpT+NKh7lwsSjglBwW0n19fdWuXTs999xzatCggUJCQrRkyRJt3rxZsbGxysjIkCSFhITY7RcSEmLbdjHTpk3T1KlTK7V2ANcPi8WiW+vWlG81d33xY4bSfjunkhJDPtXc5OpiUXGJoZy8Irm4WBRZw0t3NgoloKPyeAf9799x3aQaMdLZk1JxvuTpJ/lHSIExpSE+INJxdQJwOIeOSV+0aJESEhJUq1Ytubq6qkWLFho4cKC2bdt21X1OmjRJjz32mG05OztbERERFVEugOtYs4gANQr3U0rGGW07clppv51TQVGxvDzc1CjcXy2jqisu1FfurtyIGVXkplalDwC4CIeG9JiYGK1fv15nz55Vdna2wsLC1L9/f9WpU8c2tdKJEycUFhZm2+fEiRNq1qzZJfv09PSUp+dlrpgH4LTcXV3UuJa/GtcqnVqRGxUBAMzKFKeMvL29FRYWptOnT2vVqlXq2bOnoqOjFRoaqrVr19raZWdna8uWLWrXrp0DqwVwoyCgAwDMyqFn0letWiXDMBQXF6eDBw9qwoQJql+/voYNGyaLxaJx48bp+eefV926dW1TMIaHh6tXr16OLBsAAACoVA4N6VlZWZo0aZKOHTumGjVqqG/fvnrhhRfk7l56JfvEiRN19uxZjRw5UpmZmfrTn/6klStXMkc6AAAAbmgOmye9qjDPKgAA1x8+v+HsTDEmHQAAAMD/ENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZh4b04uJiTZ48WdHR0bJarYqJidFzzz0nwzBsbYYOHSqLxWL36Nq1qwOrBgAAACqXmyMPPn36dM2ePVsLFixQo0aN9P3332vYsGHy9/fX2LFjbe26du2qpKQk27Knp6cjygUAAACqhEND+jfffKOePXuqe/fukqTatWtryZIl+u677+zaeXp6KjQ01BElAgAAAFXOocNd2rdvr7Vr12r//v2SpF27dmnTpk3q1q2bXbt169YpODhYcXFxGj16tE6dOuWIcgEAAIAq4dAz6X/729+UnZ2t+vXry9XVVcXFxXrhhRc0aNAgW5uuXbuqT58+io6OVmpqqp544gl169ZNmzdvlqur6wV95ufnKz8/37acnZ1dJc8FAAAAqCgODenvvfee3nnnHSUnJ6tRo0bauXOnxo0bp/DwcA0ZMkSSNGDAAFv7Jk2aqGnTpoqJidG6devUuXPnC/qcNm2apk6dWmXPAQAAAKhoFuP3U6lUsYiICP3tb39TYmKibd3zzz+vxYsX66effrrkfjVr1tTzzz+vUaNGXbDtYmfSIyIilJWVJT8/v4p9AgAAoFJkZ2fL39+fz284LYeeST937pxcXOyHxbu6uqqkpOSS+xw7dkynTp1SWFjYRbd7enoy+wsAAACuaw4N6T169NALL7ygyMhINWrUSDt27NArr7yihIQESVJOTo6mTp2qvn37KjQ0VKmpqZo4caJiY2PVpUsXR5YOAAAAVBqHDnc5c+aMJk+erGXLlunkyZMKDw/XwIED9fTTT8vDw0O5ubnq1auXduzYoczMTIWHh+vOO+/Uc889p5CQkDIdgz+XAQBw/eHzG87OoSG9KvAmBwDg+sPnN5ydQ+dJBwAAAHAhQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGTcytO4pKRE69ev18aNG3XkyBGdO3dONWvWVPPmzRUfH6+IiIjKqhMAAABwGmU6k56bm6vnn39eERERuuuuu/T5558rMzNTrq6uOnjwoJ555hlFR0frrrvu0rffflvZNQMAAAA3tDKdSa9Xr57atWunt956S3fccYfc3d0vaHPkyBElJydrwIABevLJJzVixIgKLxYAAABwBhbDMIwrNdq3b58aNGhQpg4LCwuVlpammJiYay6uImRnZ8vf319ZWVny8/NzdDkAAKAM+PyGsyvTcJeyBnRJcnd3N01ABwAAAK5H5bpw9PeKior05ptvat26dSouLlaHDh2UmJioatWqVWR9AAAAgNO56pA+duxY7d+/X3369FFhYaEWLlyo77//XkuWLKnI+gAAAACnU+aQvmzZMvXu3du2/MUXXyglJUWurq6SpC5duqht27YVXyEAAADgZMp8M6N///vf6tWrl44fPy5JatGihR588EGtXLlSK1as0MSJE9W6detKKxQAAABwFmUO6StWrNDAgQPVqVMnvf7665o7d678/Pz05JNPavLkyYqIiFBycnJl1goAAAA4hTJNwfh7mZmZmjhxonbt2qU5c+aoefPmlVVbhWAKJwAArj98fsPZlflM+nkBAQGaO3eu/vGPf2jw4MGaMGGC8vLyKqM2AAAAwCmVOaSnpaXp3nvvVZMmTTRo0CDVrVtX27Ztk5eXl26++WZ9/vnnlVknAAAA4DTKPNylU6dOCg0N1dChQ7Vq1Sqlpqbqk08+kVR6R9JRo0YpNDRU7733XqUWXF78uQwAgOsPn99wdmWegvH777/Xrl27FBMToy5duig6Otq2rUGDBtqwYYPmzp1bKUUCAAAAzqTMIb1ly5Z6+umnNWTIEK1Zs0ZNmjS5oM3IkSMrtDgAAADAGZV5TPrChQuVn5+vRx99VL/88ovefPPNyqwLAAAAcFplPpMeFRWlDz74oDJrAQAAAKAynkk/e/ZsuTotb3sAAAAA/1OmkB4bG6u///3vSk9Pv2QbwzC0evVqdevWTbNmzaqwAgEAAABnU6bhLuvWrdMTTzyhKVOm6Oabb1arVq0UHh6uatWq6fTp09q7d682b94sNzc3TZo0SaNGjarsugEAAIAbVpnnSZdKb2j0/vvva+PGjTpy5Ihyc3MVFBSk5s2bq0uXLurWrZtcXV0rs95yY55VAACuP3x+w9mVK6Rfj3iTAwBw/eHzG86uzFMwAgAAAKgahHQAAADAZAjpAAAAgMkQ0gEAAACTIaQDAAAAJlPukF67dm09++yzSktLu+aDFxcXa/LkyYqOjpbValVMTIyee+45/X7CGcMw9PTTTyssLExWq1Xx8fE6cODANR8bAAAAMKtyh/Rx48bpo48+Up06dXTHHXfo3XffVX5+/lUdfPr06Zo9e7beeOMN7du3T9OnT9eMGTP0+uuv29rMmDFDs2bN0pw5c7RlyxZ5e3urS5cuysvLu6pjAgAAAGZ31fOkb9++XfPnz9eSJUtUXFys++67TwkJCWrRokWZ+/jLX/6ikJAQzZs3z7aub9++slqtWrx4sQzDUHh4uB5//HGNHz9ekpSVlaWQkBDNnz9fAwYMuOIxmGcVAIDrD5/fcHZXPSa9RYsWmjVrlo4fP65nnnlGb7/9tlq3bq1mzZrp3//+t8qS/du3b6+1a9dq//79kqRdu3Zp06ZN6tatmyTp0KFDysjIUHx8vG0ff39/tWnTRps3b75on/n5+crOzrZ7AAAAANcTt6vdsbCwUMuWLVNSUpJWr16ttm3bavjw4Tp27JieeOIJrVmzRsnJyZft429/+5uys7NVv359ubq6qri4WC+88IIGDRokScrIyJAkhYSE2O0XEhJi2/ZH06ZN09SpU6/2aQEAAAAOV+6Qvn37diUlJWnJkiVycXHR4MGD9eqrr6p+/fq2Nr1791br1q2v2Nd7772nd955R8nJyWrUqJF27typcePGKTw8XEOGDClvaZKkSZMm6bHHHrMtZ2dnKyIi4qr6AgAAAByh3CG9devWuuOOOzR79mz16tVL7u7uF7SJjo4u03jxCRMm6G9/+5utbZMmTXTkyBFNmzZNQ4YMUWhoqCTpxIkTCgsLs+134sQJNWvW7KJ9enp6ytPTs7xPCwAAADCNcof0n3/+WVFRUZdt4+3traSkpCv2de7cObm42A+Ld3V1VUlJiaTSsB8aGqq1a9faQnl2dra2bNmi0aNHl7d0AAAA4LpQ7pB+8uRJZWRkqE2bNnbrt2zZIldXV7Vq1arMffXo0UMvvPCCIiMj1ahRI+3YsUOvvPKKEhISJEkWi0Xjxo3T888/r7p16yo6OlqTJ09WeHi4evXqVd7SAQAAgOtCuWd3SUxM1NGjRy9Y/8svvygxMbFcfb3++uu65557NGbMGDVo0EDjx4/XqFGj9Nxzz9naTJw4UQ8//LBGjhyp1q1bKycnRytXrlS1atXKWzoAAABwXSj3POk+Pj7avXu36tSpY7f+0KFDatq0qc6cOVOhBV4r5lkFAOD6w+c3nF25z6R7enrqxIkTF6xPT0+Xm9tVz+gIAAAA4P8rd0i/8847NWnSJGVlZdnWZWZm6oknntAdd9xRocUBAAAAzqjcp75feukldezYUVFRUWrevLkkaefOnQoJCdGiRYsqvEAAAADA2ZQ7pNeqVUu7d+/WO++8o127dslqtWrYsGEaOHDgRedMBwAAAFA+VzWI3NvbWyNHjqzoWgAAAADoKkO6JO3du1dpaWkqKCiwW3/33Xdfc1EAAACAM7uqO4727t1be/bskcVi0fkZHC0WiySpuLi4YisEAAAAnEy5Z3d55JFHFB0drZMnT8rLy0s//vijNmzYoFatWmndunWVUCIAAADgXMp9Jn3z5s368ssvFRQUJBcXF7m4uOhPf/qTpk2bprFjx2rHjh2VUScAAADgNMp9Jr24uFi+vr6SpKCgIB0/flySFBUVpZSUlIqtDgAAAHBC5T6T3rhxY+3atUvR0dFq06aNZsyYIQ8PD82dO1d16tSpjBoBAAAAp1LukP7UU0/p7NmzkqRnn31Wf/nLX3TrrbcqMDBQS5curfACAQAAAGdjMc5Pz3INfvvtN1WvXt02w4uZZGdny9/fX1lZWfLz83N0OQAAoAz4/IazK9eY9MLCQrm5uemHH36wW1+jRg1TBnQAAADgelSukO7u7q7IyEjmQgcAAAAqUblnd3nyySf1xBNP6LfffquMegAAAACnV+4LR9944w0dPHhQ4eHhioqKkre3t9327du3V1hxAAAAgDMqd0jv1atXJZQBAAAA4LwKmd3FzLg6HACA6w+f33B25R6TDgAAAKBylXu4i4uLy2WnW2TmFwAAAODalDukL1u2zG65sLBQO3bs0IIFCzR16tQKKwwAAABwVhU2Jj05OVlLly7Vxx9/XBHdVRjGtAEAcP3h8xvOrsLGpLdt21Zr166tqO4AAAAAp1UhIT03N1ezZs1SrVq1KqI7AAAAwKmVe0x69erV7S4cNQxDZ86ckZeXlxYvXlyhxQEAAADOqNwh/dVXX7UL6S4uLqpZs6batGmj6tWrV2hxAAAAgDMqd0gfOnRoJZQBAAAA4Lxyj0lPSkrS+++/f8H6999/XwsWLKiQogAAAABnVu6QPm3aNAUFBV2wPjg4WC+++GKFFAUAAAA4s3KH9LS0NEVHR1+wPioqSmlpaRVSFAAAAODMyh3Sg4ODtXv37gvW79q1S4GBgRVSFAAAAODMyh3SBw4cqLFjx+qrr75ScXGxiouL9eWXX+qRRx7RgAEDKqNGAAAAwKmUe3aX5557TocPH1bnzp3l5la6e0lJiQYPHsyYdAAAAKACWAzDMK5mxwMHDmjnzp2yWq1q0qSJoqKiKrq2CpGdnS1/f39lZWXJz8/P0eUAAIAy4PMbzq7cZ9LPq1u3rurWrVuRtQAAAADQVYxJ79u3r6ZPn37B+hkzZqhfv34VUhQAAADgzMod0jds2KC77rrrgvXdunXThg0bKqQoAAAAwJmVO6Tn5OTIw8PjgvXu7u7Kzs6ukKIAAAAAZ1bukN6kSRMtXbr0gvXvvvuuGjZsWCFFAQAAAM6s3BeOTp48WX369FFqaqpuv/12SdLatWu1ZMkSvf/++xVeIAAAAOBsyh3Se/TooeXLl+vFF1/UBx98IKvVqqZNm2rNmjW67bbbKqNGAAAAwKlc9TzpF/PDDz+ocePGFdVdhWCeVQAArj98fsPZlXtM+h+dOXNGc+fO1S233KKbb765ImoCAAAAnNpVh/QNGzZo8ODBCgsL00svvaTbb79d3377bUXWBgAAADilco1Jz8jI0Pz58zVv3jxlZ2fr3nvvVX5+vpYvX87MLgAAAEAFKfOZ9B49eiguLk67d+/WzJkzdfz4cb3++uuVWRsAAADglMoc0j///HMNHz5cU6dOVffu3eXq6nrNB69du7YsFssFj8TERElSp06dLtj24IMPXvNxAQAAADMrc0jftGmTzpw5o5YtW6pNmzZ644039N///veaDr5161alp6fbHqtXr5Yk9evXz9ZmxIgRdm1mzJhxTccEAAAAzK7MIb1t27Z66623lJ6erlGjRundd99VeHi4SkpKtHr1ap05c6bcB69Zs6ZCQ0Ntj08//VQxMTF28617eXnZtWEaJgAAANzoyj27i7e3txISErRp0ybt2bNHjz/+uP7+978rODhYd99991UXUlBQoMWLFyshIUEWi8W2/p133lFQUJAaN26sSZMm6dy5c5ftJz8/X9nZ2XYPAAAA4HpyTfOkx8XFacaMGTp27JiWLFlyTYUsX75cmZmZGjp0qG3dfffdp8WLF+urr77SpEmTtGjRIt1///2X7WfatGny9/e3PSIiIq6pLgAAAKCqVegdR69Fly5d5OHhoRUrVlyyzZdffqnOnTvr4MGDiomJuWib/Px85efn25azs7MVERHBHcsAALiOcMdROLtyzZNeWY4cOaI1a9boo48+umy7Nm3aSNJlQ7qnp6c8PT0rvEYAAACgqlzTcJeKkpSUpODgYHXv3v2y7Xbu3ClJCgsLq4KqAAAAAMdw+Jn0kpISJSUlaciQIXJz+185qampSk5O1l133aXAwEDt3r1bjz76qDp27KimTZs6sGIAAACgcjk8pK9Zs0ZpaWlKSEiwW+/h4aE1a9Zo5syZOnv2rCIiItS3b1899dRTDqoUAAAAqBqmuXC0snDhCQAA1x8+v+HsTDEmHQAAAMD/ENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMg4N6bVr15bFYrngkZiYKEnKy8tTYmKiAgMD5ePjo759++rEiROOLBkAAACodA4N6Vu3blV6errtsXr1aklSv379JEmPPvqoVqxYoffff1/r16/X8ePH1adPH0eWDAAAAFQ6i2EYhqOLOG/cuHH69NNPdeDAAWVnZ6tmzZpKTk7WPffcI0n66aef1KBBA23evFlt27YtU5/Z2dny9/dXVlaW/Pz8KrN8AABQQfj8hrMzzZj0goICLV68WAkJCbJYLNq2bZsKCwsVHx9va1O/fn1FRkZq8+bNl+wnPz9f2dnZdg8AAADgemKakL58+XJlZmZq6NChkqSMjAx5eHgoICDArl1ISIgyMjIu2c+0adPk7+9ve0RERFRi1QAAAEDFM01Inzdvnrp166bw8PBr6mfSpEnKysqyPY4ePVpBFQIAAABVw83RBUjSkSNHtGbNGn300Ue2daGhoSooKFBmZqbd2fQTJ04oNDT0kn15enrK09OzMssFAAAAKpUpzqQnJSUpODhY3bt3t61r2bKl3N3dtXbtWtu6lJQUpaWlqV27do4oEwAAAKgSDj+TXlJSoqSkJA0ZMkRubv8rx9/fX8OHD9djjz2mGjVqyM/PTw8//LDatWtX5pldAAAAgOuRw0P6mjVrlJaWpoSEhAu2vfrqq3JxcVHfvn2Vn5+vLl266F//+pcDqgQAAACqjqnmSa8MzLMKAMD1h89vODtTjEkHAAAA8D+EdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJiMw0P6L7/8ovvvv1+BgYGyWq1q0qSJvv/+e9v2oUOHymKx2D26du3qwIoBAACAyuXmyIOfPn1aHTp00J///Gd9/vnnqlmzpg4cOKDq1avbtevatauSkpJsy56enlVdKgAAAFBlHBrSp0+froiICLsAHh0dfUE7T09PhYaGVmVpAAAAgMM4dLjLJ598olatWqlfv34KDg5W8+bN9dZbb13Qbt26dQoODlZcXJxGjx6tU6dOXbLP/Px8ZWdn2z0AAACA64lDQ/rPP/+s2bNnq27dulq1apVGjx6tsWPHasGCBbY2Xbt21cKFC7V27VpNnz5d69evV7du3VRcXHzRPqdNmyZ/f3/bIyIioqqeDgAAAFAhLIZhGI46uIeHh1q1aqVvvvnGtm7s2LHaunWrNm/efNF9fv75Z8XExGjNmjXq3LnzBdvz8/OVn59vW87OzlZERISysrLk5+dX8U8CAABUuOzsbPn7+/P5Dafl0DPpYWFhatiwod26Bg0aKC0t7ZL71KlTR0FBQTp48OBFt3t6esrPz8/uAQAAAFxPHBrSO3TooJSUFLt1+/fvV1RU1CX3OXbsmE6dOqWwsLDKLg8AAABwCIeG9EcffVTffvutXnzxRR08eFDJycmaO3euEhMTJUk5OTmaMGGCvv32Wx0+fFhr165Vz549FRsbqy5dujiydAAAAKDSODSkt27dWsuWLdOSJUvUuHFjPffcc5o5c6YGDRokSXJ1ddXu3bt19913q169eho+fLhatmypjRs3Mlc6AAAAblgOvXC0KnDhCQAA1x8+v+HsHHomHQAAAMCFCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGTdHF1DZDMOQJGVnZzu4EgAAUFbnP7fPf44DzuaGD+lnzpyRJEVERDi4EgAAUF6nTp2Sv7+/o8sAqpzFuMG/opaUlOj48ePy9fWVxWJxdDnXtezsbEVEROjo0aPy8/NzdDn4HV4b8+K1MTdeH/PKyspSZGSkTp8+rYCAAEeXA1S5G/5MuouLi2666SZHl3FD8fPz48PMpHhtzIvXxtx4fczLxYXL5+Cc+M0HAAAATIaQDgAAAJgMIR1l5unpqWeeeUaenp6OLgV/wGtjXrw25sbrY168NnB2N/yFowAAAMD1hjPpAAAAgMkQ0gEAAACTIaQDAAAAJkNIBwAAAEyGkI7LmjJliiwWi92jfv36ji7LaW3YsEE9evRQeHi4LBaLli9fbrfdMAw9/fTTCgsLk9VqVXx8vA4cOOCYYp3MlV6boUOHXvBe6tq1q2OKdTLTpk1T69at5evrq+DgYPXq1UspKSl2bfLy8pSYmKjAwED5+Piob9++OnHihIMqdh5leW06dep0wXvnwQcfdFDFQNUhpOOKGjVqpPT0dNtj06ZNji7JaZ09e1Y333yz/vnPf150+4wZMzRr1izNmTNHW7Zskbe3t7p06aK8vLwqrtT5XOm1kaSuXbvavZeWLFlShRU6r/Xr1ysxMVHffvutVq9ercLCQt155506e/asrc2jjz6qFStW6P3339f69et1/Phx9enTx4FVO4eyvDaSNGLECLv3zowZMxxUMVB13BxdAMzPzc1NoaGhji4Dkrp166Zu3bpddJthGJo5c6aeeuop9ezZU5K0cOFChYSEaPny5RowYEBVlup0LvfanOfp6cl7yQFWrlxptzx//nwFBwdr27Zt6tixo7KysjRv3jwlJyfr9ttvlyQlJSWpQYMG+vbbb9W2bVtHlO0UrvTanOfl5cV7B06HM+m4ogMHDig8PFx16tTRoEGDlJaW5uiScBGHDh1SRkaG4uPjbev8/f3Vpk0bbd682YGV4bx169YpODhYcXFxGj16tE6dOuXokpxSVlaWJKlGjRqSpG3btqmwsNDuvVO/fn1FRkby3qlif3xtznvnnXcUFBSkxo0ba9KkSTp37pwjygOqFGfScVlt2rTR/PnzFRcXp/T0dE2dOlW33nqrfvjhB/n6+jq6PPxORkaGJCkkJMRufUhIiG0bHKdr167q06ePoqOjlZqaqieeeELdunXT5s2b5erq6ujynEZJSYnGjRunDh06qHHjxpJK3zseHh4KCAiwa8t7p2pd7LWRpPvuu09RUVEKDw/X7t279de//lUpKSn66KOPHFgtUPkI6bis3//5vmnTpmrTpo2ioqL03nvvafjw4Q6sDLi+/H64UZMmTdS0aVPFxMRo3bp16ty5swMrcy6JiYn64YcfuLbGhC712owcOdL27yZNmigsLEydO3dWamqqYmJiqrpMoMow3AXlEhAQoHr16ungwYOOLgV/cH685h9npDhx4gRjOU2oTp06CgoK4r1UhR566CF9+umn+uqrr3TTTTfZ1oeGhqqgoECZmZl27XnvVJ1LvTYX06ZNG0nivYMbHiEd5ZKTk6PU1FSFhYU5uhT8QXR0tEJDQ7V27VrbuuzsbG3ZskXt2rVzYGW4mGPHjunUqVO8l6qAYRh66KGHtGzZMn355ZeKjo62296yZUu5u7vbvXdSUlKUlpbGe6eSXem1uZidO3dKEu8d3PAY7oLLGj9+vHr06KGoqCgdP35czzzzjFxdXTVw4EBHl+aUcnJy7M4eHTp0SDt37lSNGjUUGRmpcePG6fnnn1fdunUVHR2tyZMnKzw8XL169XJc0U7icq9NjRo1NHXqVPXt21ehoaFKTU3VxIkTFRsbqy5dujiwaueQmJio5ORkffzxx/L19bWNM/f395fVapW/v7+GDx+uxx57TDVq1JCfn58efvhhtWvXjpldKtmVXpvU1FQlJyfrrrvuUmBgoHbv3q1HH31UHTt2VNOmTR1cPVDJDOAy+vfvb4SFhRkeHh5GrVq1jP79+xsHDx50dFlO66uvvjIkXfAYMmSIYRiGUVJSYkyePNkICQkxPD09jc6dOxspKSmOLdpJXO61OXfunHHnnXcaNWvWNNzd3Y2oqChjxIgRRkZGhqPLdgoXe10kGUlJSbY2ubm5xpgxY4zq1asbXl5eRu/evY309HTHFe0krvTapKWlGR07djRq1KhheHp6GrGxscaECROMrKwsxxYOVAGLYRhGVX4pAAAAAHB5jEkHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMwlcOHD8tisdhu/X0jKigoUGxsrL755ptKO8acOXPUo0ePSusfAFC5COnAdWDz5s1ydXVV9+7dHV2KKQ0dOlS9evVydBllNmfOHEVHR6t9+/aVdoyEhARt375dGzdurLRjAAAqDyEduA7MmzdPDz/8sDZs2KDjx49X6rEMw1BRUVGlHsOZGYahN954Q8OHD6/U43h4eOi+++7TrFmzKvU4AIDKQUgHTC4nJ0dLly7V6NGj1b17d82fP9+27b777lP//v3t2hcWFiooKEgLFy6UJJWUlGjatGmKjo6W1WrVzTffrA8++MDWft26dbJYLPr888/VsmVLeXp6atOmTUpNTVXPnj0VEhIiHx8ftW7dWmvWrLE7Vnp6urp37y6r1aro6GglJyerdu3amjlzpq1NZmamHnjgAdWsWVN+fn66/fbbtWvXrjI//+LiYg0fPtxWf1xcnF577TXb9ilTpmjBggX6+OOPZbFYZLFYtG7dOknS0aNHde+99yogIEA1atRQz549dfjwYdu+58/Av/TSSwoLC1NgYKASExNVWFhoa5Ofn6+//vWvioiIkKenp2JjYzVv3jwZhqHY2Fi99NJLdvXu3LlTFotFBw8evOjz2bZtm1JTU+3+KnJ+iM97772nW2+9VVarVa1bt9b+/fu1detWtWrVSj4+PurWrZt+/fVXu9fulltukbe3twICAtShQwcdOXLEtr1Hjx765JNPlJubW+afNwDAJAwApjZv3jyjVatWhmEYxooVK4yYmBijpKTEMAzD+PTTTw2r1WqcOXPG1n7FihWG1Wo1srOzDcMwjOeff96oX7++sXLlSiM1NdVISkoyPD09jXXr1hmGYRhfffWVIclo2rSp8cUXXxgHDx40Tp06ZezcudOYM2eOsWfPHmP//v3GU089ZVSrVs04cuSI7Vjx8fFGs2bNjG+//dbYtm2bcdtttxlWq9V49dVX7dr06NHD2Lp1q7F//37j8ccfNwIDA41Tp05d9PkeOnTIkGTs2LHDMAzDKCgoMJ5++mlj69atxs8//2wsXrzY8PLyMpYuXWoYhmGcOXPGuPfee42uXbsa6enpRnp6upGfn28UFBQYDRo0MBISEozdu3cbe/fuNe677z4jLi7OyM/PNwzDMIYMGWL4+fkZDz74oLFv3z5jxYoVhpeXlzF37lxbPffee68RERFhfPTRR0ZqaqqxZs0a49133zUMwzBeeOEFo2HDhnb1jx071ujYseMlX89XXnnFqF+//kWf8/nXae/evUbbtm2Nli1bGp06dTI2bdpkbN++3YiNjTUefPBBwzAMo7Cw0PD39zfGjx9vHDx40Ni7d68xf/58u9fn7NmzhouLi/HVV19dsh4AgDkR0gGTa9++vTFz5kzDMEqDWVBQkC10nV9euHChrf3AgQON/v37G4ZhGHl5eYaXl5fxzTff2PU5fPhwY+DAgYZh/C+kL1++/Iq1NGrUyHj99dcNwzCMffv2GZKMrVu32rYfOHDAkGQL6Rs3bjT8/PyMvLw8u35iYmKMN99886LH+GNIv5jExESjb9++tuUhQ4YYPXv2tGuzaNEiIy4uzvaFxjAMIz8/37BarcaqVats+0VFRRlFRUW2Nv369bP9/FJSUgxJxurVqy9axy+//GK4uroaW7ZsMQyj9AtFUFCQMX/+/EvW/sgjjxi33377RZ/z22+/bVu3ZMkSQ5Kxdu1a27pp06YZcXFxhmEYxqlTpwxJti9bl1K9evXL1gMAMCeGuwAmlpKSou+++04DBw6UJLm5ual///6aN2+ebfnee+/VO++8I0k6e/asPv74Yw0aNEiSdPDgQZ07d0533HGHfHx8bI+FCxcqNTXV7litWrWyW87JydH48ePVoEEDBQQEyMfHR/v27VNaWpqtNjc3N7Vo0cK2T2xsrKpXr25b3rVrl3JychQYGGh3/EOHDl1w/Mv55z//qZYtW6pmzZry8fHR3LlzbXVcyq5du3Tw4EH5+vrajlujRg3l5eXZHbtRo0ZydXW1LYeFhenkyZOSSoeuuLq66rbbbrvoMcLDw9W9e3f9+9//liStWLFC+fn56tev3yXrys3NVbVq1S66rWnTprZ/h4SESJKaNGlit+58bTVq1NDQoUPVpUsX9ejRQ6+99prS09Mv6NNqtercuXOXrAcAYE5uji4AwKXNmzdPRUVFCg8Pt60zDEOenp5644035O/vr0GDBum2227TyZMntXr1almtVnXt2lVSadCWpP/85z+qVauWXd+enp52y97e3nbL48eP1+rVq/XSSy8pNjZWVqtV99xzjwoKCspcf05OjsLCwmxjxH8vICCgTH28++67Gj9+vF5++WW1a9dOvr6++sc//qEtW7Zc8dgtW7a0fYH5vZo1a9r+7e7ubrfNYrGopKREUmnAvZIHHnhA//d//6dXX31VSUlJ6t+/v7y8vC7ZPigoSHv27Lnott/XYrFYLrrufG2SlJSUpLFjx2rlypVaunSpnnrqKa1evVpt27a1tfntt9/sni8A4PpASAdMqqioSAsXLtTLL7+sO++8025br169tGTJEj344INq3769IiIitHTpUn3++efq16+fLdg1bNhQnp6eSktLu+TZ4Ev5+uuvNXToUPXu3VtSaej9/UWXcXFxKioq0o4dO9SyZUtJpWfuT58+bWvTokULZWRkyM3NTbVr176Kn0JpHe3bt9eYMWNs6/54Ft7Dw0PFxcV261q0aKGlS5cqODhYfn5+V3XsJk2aqKSkROvXr1d8fPxF29x1113y9vbW7NmztXLlSm3YsOGyfTZv3lyzZ8+WYRi2IH4tmjdvrubNm2vSpElq166dkpOTbSE9NTVVeXl5at68+TUfBwBQtRjuApjUp59+qtOnT2v48OFq3Lix3aNv3762IS9S6Swvc+bM0erVq21DXSTJ19dX48eP16OPPqoFCxYoNTVV27dv1+uvv64FCxZc9vh169bVRx99pJ07d2rXrl2677777M7i1q9fX/Hx8Ro5cqS+++477dixQyNHjpTVarWFz/j4eLVr1069evXSF198ocOHD+ubb77Rk08+qe+//75MP4e6devq+++/16pVq7R//35NnjxZW7dutWtTu3Zt7d69WykpKfrvf/+rwsJCDRo0SEFBQerZs6c2btyoQ4cOad26dRo7dqyOHTtWpmPXrl1bQ4YMUUJCgpYvX27r47333rO1cXV11dChQzVp0iTVrVtX7dq1u2yff/7zn5WTk6Mff/yxTDVcyqFDhzRp0iRt3rxZR44c0RdffKEDBw6oQYMGtjYbN25UnTp1FBMTc03HAgBUPUI6YFLz5s1TfHy8/P39L9jWt29fff/999q9e7ckadCgQdq7d69q1aqlDh062LV97rnnNHnyZE2bNk0NGjRQ165d9Z///EfR0dGXPf4rr7yi6tWrq3379urRo4e6dOliN/5ckhYuXKiQkBB17NhRvXv31ogRI+Tr62sbc22xWPTZZ5+pY8eOGjZsmOrVq6cBAwboyJEjtjHXVzJq1Cj16dNH/fv3V5s2bXTq1Cm7s+qSNGLECMXFxalVq1aqWbOmvv76a3l5eWnDhg2KjIxUnz591KBBAw0fPlx5eXnlOrM+e/Zs3XPPPRozZozq16+vESNG6OzZs3Zthg8froKCAg0bNuyK/QUGBqp3794XHYZTHl5eXvrpp5/Ut29f1atXTyNHjlRiYqJGjRpla7NkyRKNGDHimo4DAHAMi2EYhqOLAHBjOHbsmCIiIrRmzRp17tzZ0eVUmY0bN6pz5846evRomb587N69W3fccYdSU1Pl4+NTKTX9+OOPuv3227V///6LftEDAJgbIR3AVfvyyy+Vk5OjJk2aKD09XRMnTtQvv/yi/fv3X3BB5o0oPz9fv/76q4YMGaLQ0NBynR2fP3++WrZsaTd7S0Vas2aNiouL1aVLl0rpHwBQuQjpAK7aqlWr9Pjjj+vnn3+Wr6+v2rdvr5kzZyoqKsrRpVWJ+fPna/jw4WrWrJk++eSTC2bQAQDgahHSAQAAAJPhwlEAAADAZAjpAAAAgMkQ0gEAAACTIaQDAAAAJkNIBwAAAEyGkA4AAACYDCEdAAAAMBlCOgAAAGAyhHQAAADAZP4fAbK/kONKb2kAAAAASUVORK5CYII=\n" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## 5. Compressing further with Optimum ONNX and TensorrtExecutionProvider" + ], + "metadata": { + "id": "AiPUhOCNWRny" + }, + "id": "AiPUhOCNWRny" + }, + { + "cell_type": "code", + "source": [ + "#@title We'll be using Optimum's ONNX Runtime support with `CUDAExecutionProvider` [because it's fast while also supporting dynamic shapes](https://github.com/huggingface/optimum-benchmark/tree/main/examples/fast-mteb#notes)\n", + "\n", + "!pip install optimum[onnxruntime-gpu]" + ], + "metadata": { + "id": "NEnwnsEQWRn8", + "cellView": "form" + }, + "execution_count": null, + "outputs": [], + "id": "NEnwnsEQWRn8" + }, + { + "cell_type": "code", + "source": [ + "#@title [`optimum-cli`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/optimization#optimizing-a-model-during-the-onnx-export) makes it extremely easy to export a model to ONNX and apply SOTA graph optimizations / kernel fusions\n", + "\n", + "!optimum-cli export onnx \\\n", + " --model moshew/bge-small-en-v1.5_setfit-sst2-english \\\n", + " --task feature-extraction \\\n", + " --optimize O4 \\\n", + " --device cuda \\\n", + " bge_auto_opt_O4 # output folder" + ], + "metadata": { + "id": "hPqEcDi8WRn8" + }, + "execution_count": null, + "outputs": [], + "id": "hPqEcDi8WRn8" + }, + { + "cell_type": "code", + "source": [ + "onnx_path = Path(\"onnx\")" + ], + "metadata": { + "id": "XVt5Gm-nWRn8" + }, + "execution_count": null, + "outputs": [], + "id": "XVt5Gm-nWRn8" + }, + { + "cell_type": "code", + "source": [ + "class OnnxPerformanceBenchmark(PerformanceBenchmark):\n", + " def __init__(self, *args, model_path, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " self.model_path = model_path\n", + "\n", + " def compute_size(self):\n", + " size_mb = Path(self.model_path).stat().st_size / (1024 * 1024)\n", + " print(f\"Model size (MB) - {size_mb:.2f}\")\n", + " return {\"size_mb\": size_mb}\n", + "\n", + " def compute_accuracy(self):\n", + " preds = []\n", + " chunk_size = 100\n", + " for i in tqdm(range(0, len(self.dataset[\"text\"]), chunk_size)):\n", + " preds.extend(self.model.predict(self.dataset[\"text\"][i : i + chunk_size]))\n", + " labels = self.dataset[\"label\"]\n", + " accuracy = metric.compute(predictions=preds, references=labels)\n", + " print(f\"Accuracy on test set - {accuracy['accuracy']:.3f}\")\n", + " return accuracy" + ], + "metadata": { + "id": "8hvfl3xvlnEs" + }, + "id": "8hvfl3xvlnEs", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "#@title Based on the example given in [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5#using-huggingface-transformers)\n", + "import time\n", + "import torch\n", + "from transformers import AutoTokenizer\n", + "from optimum.onnxruntime import ORTModelForFeatureExtraction\n", + "\n", + "# Load model from HuggingFace Hub\n", + "tokenizer = AutoTokenizer.from_pretrained('/content/bge_auto_opt_O4')\n", + "#ort_model = ORTModelForFeatureExtraction.from_pretrained('/content/bge_auto_opt_O4', provider=\"CUDAExecutionProvider\")\n", + "ort_model = ORTModelForFeatureExtraction.from_pretrained('/content/bge_auto_opt_O4', provider=\"TensorrtExecutionProvider\")\n", + "\n", + "ort_model.save_pretrained(onnx_path)\n", + "tokenizer.save_pretrained(onnx_path)\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "IpoDwkPiWRn8", + "outputId": "b3d90e98-0d6c-4e55-cf6c-9dbf1b873d61" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "EP Error /onnxruntime_src/onnxruntime/core/session/provider_bridge_ort.cc:1193 onnxruntime::Provider& onnxruntime::ProviderLibrary::Get() [ONNXRuntimeError] : 1 : FAIL : Failed to load library libonnxruntime_providers_tensorrt.so with error: libnvinfer.so.8: cannot open shared object file: No such file or directory\n", + " when using ['TensorrtExecutionProvider', 'CUDAExecutionProvider']\n", + "Falling back to ['CUDAExecutionProvider', 'CPUExecutionProvider'] and retrying.\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "('onnx/tokenizer_config.json',\n", + " 'onnx/special_tokens_map.json',\n", + " 'onnx/vocab.txt',\n", + " 'onnx/added_tokens.json',\n", + " 'onnx/tokenizer.json')" + ] + }, + "metadata": {}, + "execution_count": 19 + } + ], + "id": "IpoDwkPiWRn8" + }, + { + "cell_type": "code", + "source": [ + "from setfit.exporters.utils import mean_pooling\n", + "from sklearn.linear_model import LogisticRegression\n", + "\n", + "\n", + "class OnnxSetFitModel_1:\n", + " def __init__(self, ort_model, tokenizer, model_head):\n", + " self.ort_model = ort_model\n", + " self.tokenizer = tokenizer\n", + " self.model_head = model_head\n", + "\n", + " def predict(self, inputs):\n", + " encoded_inputs = self.tokenizer(\n", + " inputs, padding=True, truncation=True, return_tensors=\"pt\"\n", + " ).to(\"cuda\")\n", + "\n", + " outputs = self.ort_model(**encoded_inputs)\n", + " embeddings = mean_pooling(\n", + " outputs[\"last_hidden_state\"], encoded_inputs[\"attention_mask\"]\n", + " )\n", + " return self.model_head.predict(torch.Tensor.cpu(embeddings))\n", + "\n", + " def __call__(self, inputs):\n", + " return self.predict(inputs)" + ], + "metadata": { + "id": "enaQpBF9WRn9" + }, + "execution_count": null, + "outputs": [], + "id": "enaQpBF9WRn9" + }, + { + "cell_type": "code", + "source": [ + "model = SetFitModel.from_pretrained(\"moshew/bge-small-en-v1.5_setfit-sst2-english\")\n", + "onnx_setfit_model = OnnxSetFitModel_1(ort_model, tokenizer, model.model_head)\n", + "onnx_setfit_model(test_dataset[\"text\"][:2])" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "19a9c233-d94a-4134-eb38-51a0c1e3b749", + "id": "qRviEk2WWRn9" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "array([0, 0])" + ] + }, + "metadata": {}, + "execution_count": 21 + } + ], + "id": "qRviEk2WWRn9" + }, + { + "cell_type": "code", + "source": [ + "pb = OnnxPerformanceBenchmark(\n", + " onnx_setfit_model,\n", + " test_dataset,\n", + " \"bge-small_TensorRT (optimum onnx)\",\n", + " model_path=\"onnx/model.onnx\",\n", + ")" + ], + "metadata": { + "id": "O8jpZ3gdWRn9" + }, + "execution_count": null, + "outputs": [], + "id": "O8jpZ3gdWRn9" + }, + { + "cell_type": "code", + "source": [ + "perf_metrics.update(pb.run_benchmark())\n", + "plot_metrics(perf_metrics, \"bge-small (optimum onnx)\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 574, + "referenced_widgets": [ + "12a4e79070444acbbb228aba4e34a25e", + "f8bfdfce5edc4c6397e22b90eea50817", + "edd2f3f9a691462dbfb703ca072fd644", + "70fd4d6c76d348fa82ac29e416c94f2c", + "8cefab222736437289c985ffbd6a1ed5", + "b3eaa19e1a97474995de388a44bc18f1", + "71a105f4761549fc898c0a4b03fbb043", + "d3c2759bf4324ad78eda1aa61d9c3fb0", + "37d8b6b1ea044b7680b715ec31b48d8e", + "dfbde307f9f84be1b413b47248410438", + "bc3d582d8e394397897f44efe51dd333" + ] + }, + "id": "tpjtxQQlZQPa", + "outputId": "515c2532-1f34-4abc-a016-dc20c10cf1db" + }, + "id": "tpjtxQQlZQPa", + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Model size (MB) - 65.93\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " 0%| | 0/9 [00:00:30: MatplotlibDeprecationWarning: The legendHandles attribute was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use legend_handles instead.\n", + " for handle in legend.legendHandles:\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2kAAAG2CAYAAADlf851AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABhBklEQVR4nO3deVyU5f7/8fewOuwCshkCguK+757UoxQux+OWaVm4JR6jzMrsWFpamSdPpWnfY3ky3M06pa1mamFqpuZaaiRkoglZKiCyyHL//uDn1OQGCswor+fjMY+a677u6/7MjDC+ve77uk2GYRgCAAAAANgFB1sXAAAAAAD4HSENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsiE1D2tmzZzVhwgSFhYXJbDarU6dO2rlzp2X7iBEjZDKZrB49e/a0YcUAAAAAULmcbHnw++67T999952WLl2qkJAQLVu2TDExMTp48KBq164tSerZs6cSExMt+7i6utqqXAAAAACodCbDMAxbHDgvL0+enp56//331adPH0t769at1atXLz333HMaMWKEMjMztWbNGluUCAAAAABVzmYzaUVFRSouLlaNGjWs2s1ms7Zs2WJ5npSUpICAANWsWVPdu3fXc889Jz8/v8uOW1BQoIKCAsvzkpISnT59Wn5+fjKZTBX/QgAAQIUzDENnz55VSEiIHBy4hB5A9WKzmTRJ6tSpk1xcXLRixQoFBgZq5cqVGj58uKKiopScnKy33npLbm5uioiIUGpqqp544gl5eHho27ZtcnR0vOSY06ZN0/Tp06v4lQAAgMpw7Ngx3XLLLbYuAwCqlE1DWmpqqkaNGqUvv/xSjo6OatWqlerXr69du3bp0KFDF/X/8ccfFRkZqQ0bNqhHjx6XHPPPM2lZWVmqU6eOjh07Ji8vr0p7LQAAoOJkZ2crNDRUmZmZ8vb2tnU5AFClbLpwSGRkpDZt2qRz584pOztbwcHBGjJkiOrWrXvJ/nXr1pW/v79SUlIuG9JcXV0vubiIl5cXIQ0AgBsMlyoAqI7s4iRvd3d3BQcH68yZM1q3bp369et3yX7Hjx/XqVOnFBwcXMUVAgAAAEDVsOlM2rp162QYhqKjo5WSkqLHHntMDRo00MiRI5WTk6Pp06dr0KBBCgoKUmpqqiZNmqSoqCjFxsbasmwAAAAAqDQ2nUnLyspSQkKCGjRooLi4OP3lL3/RunXr5OzsLEdHR+3fv19///vfVb9+fY0ePVqtW7fW5s2buVcaAAAAgJuWTRcOqQrZ2dny9vZWVlYW16QBAHCDKOv3d3FxsQoLC6uwMgAovwuTUGVl09MdAQAAroVhGMrIyFBmZqatSwGAMvHx8VFQUFCZFkQipAEAgBvOhYAWEBAgNzc3VoEEYLcMw1Bubq5OnjwpSWVaBJGQBgAAbijFxcWWgObn52frcgDgqsxmsyTp5MmTCggIuOqpj3axBD8AAEBZXbgGzc3NzcaVAEDZXfidVZbraAlpAADghsQpjgBuJOX5nUVIAwAAAAA7QkgDAACoAt26ddOECRNsXUaVWbRokXx8fCzPp02bphYtWtisnsoQHh6uOXPm2LoM3IQIaQAAALC5n376SSaTyfLw8/PT7bffrj179pRp//DwcKv9//wYMWJE5b4AoAKxuiMAAADsxoYNG9S4cWMdP35c48ePV69evfT9999bzcpdys6dO1VcXCxJ+uqrrzRo0CAlJydbboZ+YXW9siosLJSzs/M1vQbgejGTBgAAqq0z587ryG/ndObc+So5XlFRkR544AF5e3vL399fU6dOlWEYlu3p6enq06ePzGazIiIitGLFiotOqcvMzNR9992nWrVqycvLS927d9e+ffuueNykpCS1a9dO7u7u8vHxUefOnXX06FFJv5+G+Oabb6pOnTry8PDQ/fffr+LiYs2aNUtBQUEKCAjQjBkzrMZ8+eWX1bRpU7m7uys0NFT333+/cnJyrvs98vPzU1BQkNq0aaMXX3xRv/zyi7Zv365nnnlGTZo0uah/ixYtNHXqVNWqVUtBQUEKCgqSr6+vJCkgIMDStmLFCkVGRsrFxUXR0dFaunSp1Tgmk0nz58/X3//+d7m7u1te74cffqi2bduqRo0a8vf314ABA6z2y83N1ahRo+Tp6ak6depowYIF1/0eAIQ0AABQ7eQXFut/u47phU+/1+z1yXrh0+/1v13HlF9YXKnHXbx4sZycnLRjxw698sorevnll/XGG29YtsfFxenEiRNKSkrSu+++qwULFlhugHvB4MGDdfLkSa1du1a7du1Sq1at1KNHD50+ffqSxywqKlL//v3VtWtX7d+/X9u2bVN8fLzVSnOpqalau3atPv30U61cuVILFy5Unz59dPz4cW3atEkvvPCCpkyZou3bt1v2cXBw0Ny5c3XgwAEtXrxYn3/+uSZNmlSh79eF2a/z589r1KhROnTokHbu3GnZvmfPHu3fv18jR4684jirV6/WQw89pEcffVTfffedxo4dq5EjR+qLL76w6jdt2jQNGDBA3377rUaNGqWPP/5YAwYMUO/evbVnzx5t3LhR7dq1s9rnpZdeUps2bbRnzx7df//9GjdunJKTkyvoHUC1ZdzksrKyDElGVlaWrUsBAABldKXv77y8POPgwYNGXl7eNY//zjdpRvySncbkd/cbMz85aEx+d78Rv2Sn8c43addT9hV17drVaNiwoVFSUmJpe/zxx42GDRsahmEYhw4dMiQZO3futGw/fPiwIcmYPXu2YRiGsXnzZsPLy8vIz8+3GjsyMtJ4/fXXL3ncU6dOGZKMpKSkS25/+umnDTc3NyM7O9vSFhsba4SHhxvFxcWWtujoaGPmzJmXfX3vvPOO4efnZ3memJhoeHt7Wx2nefPml93/yJEjhiRjz549hmEYxpkzZ4wBAwYYHh4eRkZGhmEYhtGrVy9j3Lhxln0efPBBo1u3bheN9cUXXxiSjDNnzhiGYRidOnUyxowZY9Vn8ODBRu/evS3PJRkTJkyw6tOxY0dj2LBhl605LCzMuOeeeyzPS0pKjICAAGP+/PmX3QfVV3l+dzGTBgAAqpUz587rm5/OyM/dVbU8XeXq5Khanq7yc3fVrp/OVOqpjx06dLCawerYsaMOHz6s4uJiJScny8nJSa1atbJsj4qKUs2aNS3P9+3bp5ycHPn5+cnDw8PyOHLkiFJTU5WWlmbV/vzzz8vX11cjRoxQbGys+vbtq1deeUXp6elWdYWHh8vT09PyPDAwUI0aNZKDg4NV2x9n9TZs2KAePXqodu3a8vT01L333qtTp04pNzf3ut6jTp06ycPDQzVr1tS+ffu0atUqBQYGSpLGjBmjlStXKj8/X+fPn9eKFSs0atSoq4556NAhde7c2aqtc+fOOnTokFVbmzZtrJ7v3btXPXr0uOLYzZo1s/y/yWRSUFDQRbOfQHmxcAgAAKhWMvMKlXu+SCE+1gtJeJmddCIzT5l5harp7mKj6q4sJydHwcHBSkpKumibj4+PfHx8tHfvXkvbhWuzEhMTNX78eH366adatWqVpkyZovXr16tDhw6SdNECGSaT6ZJtJSUlkkpXYvzb3/6mcePGacaMGfL19dWWLVs0evRonT9/Xm5ubtf8GletWqVGjRrJz8/vosVC+vbtK1dXV61evVouLi4qLCzUHXfccc3H+jN3d3er52VZbORK7xNwrQhpAACgWvExO8vNxUnZeUWq5eloac/OK5K7i5N8zJW3ot8fr+mSpK+//lr16tWTo6OjoqOjVVRUpD179qh169aSpJSUFJ05c8bSv1WrVsrIyJCTk5PCw8MveYyoqKhLtrds2VItW7bU5MmT1bFjR61YscIS0spr165dKikp0UsvvWSZbXv77bevaaw/Cw0NVWRk5CW3OTk5afjw4UpMTJSLi4uGDh1apiDVsGFDbd26VcOHD7e0bd26VY0aNbrifs2aNdPGjRuves0bUNEIaQAAoFqp6e6iNuE1tf7gL5JKZ9Cy84p06lyBbmsUWKmzaGlpaXrkkUc0duxY7d69W/PmzdNLL70kSWrQoIFiYmIUHx+v+fPny9nZWY8++qjMZrPlFMmYmBh17NhR/fv316xZs1S/fn2dOHHCssDFn0/Xk6QjR45owYIF+vvf/66QkBAlJyfr8OHDiouLu+bXERUVpcLCQs2bN099+/bV1q1b9dprr13zeOVx3333qWHDhpJKg1ZZPPbYY7rzzjvVsmVLxcTE6MMPP9R7772nDRs2XHG/p59+Wj169FBkZKSGDh2qoqIiffLJJ3r88cev+3UAV8I1aQAAoNr5W7MQ3dYoUIZh6ERmngzD0G2NAvW3ZiGVety4uDjl5eWpXbt2SkhI0EMPPaT4+HjL9iVLligwMFBdunTRgAEDNGbMGHl6eqpGjRqSSk+l++STT9SlSxeNHDlS9evX19ChQ3X06FHLdVt/5ubmpu+//16DBg1S/fr1FR8fr4SEBI0dO/aaX0fz5s318ssv64UXXlCTJk20fPlyzZw585rHK4969eqpU6dOatCggdq3b1+mffr3769XXnlFL774oho3bqzXX39diYmJ6tat2xX369atm9555x198MEHatGihbp3764dO3ZUwKsArsxkGH+4OcdNKDs7W97e3srKyrLczBAAANi3K31/5+fn68iRI4qIiLCEl2t15tx5ZeYVysfsbJfXoR0/flyhoaGWRTogGYahevXq6f7779cjjzxi63KAMivP7y5OdwQAANVWTXcXuwpnn3/+uXJyctS0aVOlp6dr0qRJCg8PV5cuXWxdml349ddf9dZbbykjI4PrxHBTI6QBAADYicLCQj3xxBP68ccf5enpqU6dOmn58uUXrSBYXQUEBMjf318LFiywujUBcLMhpAEAANiJ2NhYxcbG2roMu3WTX6UDWLBwCAAAAADYEUIaAAAAANgRQhoAAAAA2BFCGgAAAADYEUIaAAAAANgRQhoAAAAA2BFCGgAAgI1069ZNEyZMsDwPDw/XnDlzrnm8RYsWycfHx/J82rRpatGiheX5iBEj1L9//8seH1f35/cYqAyENAAAqpJhSDm/ShnfSj8mSSkbSv+b8W1pO/eBqtZ27typ+Pj4MvW9VKAbMmSIfvjhh0qorPz+HAiv1M9kMslkMsnFxUVRUVF65plnVFRUdNV9Fy1aZNn3co+ffvrp+l8MUMW4mTUAAFWhMF9K3ycd/Uo6lSKdz5GMEkkmSYZkcpBcPCS/KCmskxTcXHKuYeuqUcVq1ap1XfubzWaZzeYKqubaFBcXy2QylWufnj17KjExUQUFBfrkk0+UkJAgZ2dnTZ48+Yr7DRkyRD179rQ8HzhwoJo0aaJnnnnG0lae9/T8+fNycXEpV+1AZWAmDQCAyvZbirT1Fenr/0gZ+yUX99IwFtBICmhY+l+/qNL2jP2l/ba+Ip1KtXXlN7/c06Xvc+7pSj/UuXPnFBcXJw8PDwUHB+ull166qM8fZ8cMw9C0adNUp04dubq6KiQkROPHj5dUepri0aNH9fDDD1tmjKRrOxWvqKhIDzzwgLy9veXv76+pU6fK+MOMbkFBgSZOnKjatWvL3d1d7du3V1JSkmX7hWN+8MEHatSokVxdXTVq1CgtXrxY77//vqW+P+7zZ66urgoKClJYWJjGjRunmJgYffDBBzp37py8vLz0v//9z6r/mjVr5O7urqKiIgUFBVkeLi4ucnNzszw/f/68Bg4cKA8PD3l5eenOO+/UL7/8Yhnnwumgb7zxhiIiIlSjRuk/jGRmZmrs2LEKDAxUjRo11KRJE3300UdWNaxbt04NGzaUh4eHevbsqfT09HK978CVMJMGAEBlMYzSUxm/e1fKz5Z8IySny8yOOThJ5pqlj6J86ZcDUtYxqckgqW43qZwzE7iKwjzpwGop7Wvp/LnSgFyng9R4gORcOTNRjz32mDZt2qT3339fAQEBeuKJJ7R7926ra8b+6N1339Xs2bP11ltvqXHjxsrIyNC+ffskSe+9956aN2+u+Ph4jRkz5rrqWrx4sUaPHq0dO3bom2++UXx8vOrUqWMZ94EHHtDBgwf11ltvKSQkRKtXr1bPnj317bffql69epKk3NxcvfDCC3rjjTfk5+en4OBg5eXlKTs7W4mJiZIkX1/fMtdkNpt16tQpubu7a+jQoUpMTNQdd9xh2X7huaen52XHKCkpUb9+/eTh4aFNmzapqKhICQkJGjJkiFVgTElJ0bvvvqv33ntPjo6OKikpUa9evXT27FktW7ZMkZGROnjwoBwdHS375Obm6sUXX9TSpUvl4OCge+65RxMnTtTy5cvL/BqBKyGkAQBQWX5MkvaukBxdpFoNyh60nGqU9s/+uXR/SYr8a6WVWS0dWC19/7HkHiB531Iaor//uHRbi7sr/HA5OTlauHChli1bph49ekgqDUe33HLLZfdJS0tTUFCQYmJi5OzsrDp16qhdu3aSSgOPo6OjPD09FRQUdF21hYaGavbs2TKZTIqOjta3336r2bNna8yYMUpLS1NiYqLS0tIUEhIiSZo4caI+/fRTJSYm6vnnn5ckFRYW6j//+Y+aN29uGddsNqugoKBc9RmGoY0bN2rdunV68MEHJUn33XefOnXqpPT0dAUHB+vkyZP65JNPtGHDhiuOtXHjRn377bc6cuSIQkNDJUlLlixR48aNtXPnTrVt21ZS6SmOS5YssZwW+dlnn2nHjh06dOiQ6tevL0mqW7eu1diFhYV67bXXFBkZKak0yP7xFEvgenG6IwAAleG3lNIZNEeX0hBQ3pkwk6l0P0eX0nF+S6mcOquj3NOlM2juAZJHQGko9ggofZ72daWc+piamqrz58+rffv2ljZfX19FR0dfdp/BgwcrLy9PdevW1ZgxY7R69eoyLaZRXh06dLC6hqxjx446fPiwiouL9e2336q4uFj169eXh4eH5bFp0yalpv5+Oq6Li4uaNWt2zTV89NFH8vDwUI0aNdSrVy8NGTJE06ZNkyS1a9dOjRs31uLFiyVJy5YtU1hYmLp06XLFMQ8dOqTQ0FBLQJOkRo0aycfHR4cOHbK0hYWFWV23tnfvXt1yyy2WgHYpbm5uloAmyRIegYpCSAMAoKIV5v9+iqNX7esby6t26TjfvVs6Lq5f3pnSUxxreFm31/Aqbc87Y5u6/iQ0NFTJycn6z3/+I7PZrPvvv19dunRRYWFhldWQk5MjR0dH7dq1S3v37rU8Dh06pFdeecXSz2w2l3uxkD/661//qr179+rw4cPKy8vT4sWL5e7ubtl+3333adGiRZJKT3UcOXLkdR3vj/54HEllWnjF2dnZ6rnJZLK6jg+4XoQ0ANVWSQlfqKgk6fukkwdLr0G73r9Imkyl45w8WLqoCK6fuWbpNWj52dbt+dml7eaaFX7IyMhIOTs7a/v27Za2M2fOXHW5fLPZrL59+2ru3LlKSkrStm3b9O2330oqnb0qLi6+7tr+WJMkff3116pXr54cHR3VsmVLFRcX6+TJk4qKirJ6XO00xvLU5+7urqioKNWpU0dOThdfjXPPPffo6NGjmjt3rg4ePKjhw4dfdcyGDRvq2LFjOnbsmKXt4MGDyszMVKNGjS67X7NmzXT8+HG7uZUBqieuSQNQbRQWlyg546x2HT2jtNPndL6oRC5ODqrj667WYTUVHeQpZ0f+7QrXyTBKl9k3OVx+kZDycqpRGtZ+2ird0pZFRK6Xm2/pIiEXrkGr4VUa0M6dlBr0Kd1ewTw8PDR69Gg99thj8vPzU0BAgJ588kk5OFz+d86iRYtUXFys9u3by83NTcuWLZPZbFZYWJik0pUgv/zySw0dOlSurq7y9/e/ptrS0tL0yCOPaOzYsdq9e7fmzZtnWXmyfv36GjZsmOLi4vTSSy+pZcuW+vXXX7Vx40Y1a9ZMffr0uey44eHhWrdunZKTk+Xn5ydvb++LZqDKqmbNmho4cKAee+wx3X777Ve8lu+CmJgYNW3aVMOGDdOcOXNUVFSk+++/X127dlWbNm0uu1/Xrl3VpUsXDRo0SC+//LKioqL0/fffy2QyWS33D1Qm/jYC4KZnGIb2pJ3Ry+uT9dqmVG3/8ZRy8otVXCLl5Bdr+4+n9NqmVL28Pll70uzjNCfcwM79VnofNPfru9/VRdwDSsfNPVWx41ZXjQeUBjKjWMo6XvrfBn1K2yvJv//9b916663q27evYmJi9Je//EWtW7e+bH8fHx/997//VefOndWsWTNt2LBBH374ofz8/CRJzzzzjH766SdFRkZe1/3V4uLilJeXp3bt2ikhIUEPPfSQ1Q21ExMTFRcXp0cffVTR0dHq37+/du7cqTp16lxx3DFjxig6Olpt2rRRrVq1tHXr1muuUZJGjx6t8+fPa9SoUWXqbzKZ9P7776tmzZrq0qWLYmJiVLduXa1ateqq+7777rtq27at7rrrLjVq1EiTJk2qkFlLoKxMxk1+Am12dra8vb2VlZUlLy+vq+9QQfKK8pR8OlmHzxxWblGu3JzcVK9mPUX7RsvsZNubTALViWEY2pLym1bv/lmFJSUK8TarhrPjRf3yC4t1IitPzg4OGtCqtm6tV8F/wUb1kfGt9OWLpfc9c6jAE1ZKikpDWpeJUlDTihvXTl3p+zs/P19Hjhyxuq/VNcs9XXoNmrlmpcygoeIsXbpUDz/8sE6cOMENp3FDKs/vLk53rAQHfjugT458ovRz6ZIhOTs6q7C4UNvStynYPVi9I3qrsX9jW5cJVAt7j2Vq9e6f5eRoUqivx2X71XB2VF1/D6Vn5Wn1np/lWcNZLUJ9qq5Q3DxyT0lGScUGNKl0PKOEmbSK5uZLOLNzubm5Sk9P17/+9S+NHTuWgIZqgdMdK9iB3w5oVfIq/Zr3q8I8wxRVM0phXv//v55h+i3vN61KXqUDvx2wdanATa+wuESfHcxQYUmJgr3LNoMd7G0u3e9AhgqLSyq5QtyUSookVdY1Y6b/Pz5QfcyaNUsNGjRQUFCQJk+ebOtygCpBSKtAeUV5+uTIJ8ovzleYZ5icHa0vjnV2dFYdzzoqKC7QJ0c+UV5Rno0qBaqH5IyzOnY6TyFlDGgXhHiblXY6V8kZZyupMtzUHJwkVdaVBEbFz9ABdm7atGkqLCzUxo0b5eFx+TMigJsJIa0CJZ9OVvq5dNV2r33Ze3eYTCaFuIco/Vy6kk8nV3GFQPWy6+gZlZQYl7wG7UpqODuqpMTQrqMsIoJr4OZXurJjRc94lRSVjuvmV7HjAgDsDiGtAh0+c9hyDdqVODs6S4aUkplSRZUB1VPa6XPyrHFtyz171HBS2uncCq4I1YJHkOTiIRVU8ExswdnScT2DK3ZcAIDdIaRVoNyi3KsGtAucHZ11rvBcJVcEVF8lJYbOF5XI0eHarg1ydDDpfFExN7xG+bn7l67seO7Xih333EnJP4qZNACoBghpFcjNyU2FxYVl6ltYXCh3Z/dKrgiovhwcTHJxclDxNYas4hJDLk6OcrjGkIdqzGSSwjqVrsRYlF8xYxbll94kO6wzN7IGgGqAkFaB6tWsJ5l01aBWWFwomaQon6gqqgyonur4uutsftn+4eTPcvKLVMfXrYIrQrUR3FwKaCSdPlIarq6HYZSOE9BICmpWMfUBAOyaTUPa2bNnNWHCBIWFhclsNqtTp07auXOnZbthGHrqqacUHBwss9msmJgYHT582IYVX1m0b7SC3YN14twJXe4e4YZh6MS5Ewp2D1a0b3QVVwhUL63DasrBwaT8wuJy7ZdfWCwHB5Nah9WspMpw03OuITUZJNXwkrJ/vr6xsn8uHafpHaXjAgBuejYNaffdd5/Wr1+vpUuX6ttvv9Xtt9+umJgY/fxz6RfarFmzNHfuXL322mvavn273N3dFRsbq/z8Cjp9pIKZnczqHdFbro6uSjubdtGMWmFxodLOpsnV0VW9I3rL7FS+ZcEBlE90kKdCfc06kVW+212cyMpTHV83RQd5VlJlqBb8o0qDWvF5Ket4+WfUDKN0v+LzpeP4RVZOnagy3bp104QJE2xdRpVZtGiRfHx8LM+nTZumFi1a2KyeG9m9996r559/vkqOZTKZtGbNmio51s1k6NCheumllypsPJuFtLy8PL377ruaNWuWunTpoqioKE2bNk1RUVGaP3++DMPQnDlzNGXKFPXr10/NmjXTkiVLdOLECbv+g9PYv7GGRA+Rv9lfR88eVcqZFB3NPqqUzBQdPXtU/mZ/DYkeosb+jW1dKnDTc3Z00O2NguTs4KD0Mga19Ky80v0aB8nZkTPCcZ3qdpNa3F16Hdmv35f9GrWi/NL+JlPp/nW7VWaVgF0JDw+XyWS67GPEiBG2LvGSunXrZqmxRo0aql+/vmbOnCnDMDRt2rQrvqbL3bpJkvbt26dPPvlE48ePr9B6Lxea09PT1atXrwo9VnUwZcoUzZgxQ1lZWRUyns3uiFlUVKTi4mLVqGF96obZbNaWLVt05MgRZWRkKCYmxrLN29tb7du317Zt2zR06NBLjltQUKCCggLL8+zs7Mp5AVfQ2L+x6vrUVfLpZKVkpuhc4Tm5O7sryidKDXwbqIYTp6sAVaVFqI9yCoq0evfP+vG3HIV4my9537T8wmKd+P8BbUDL2moR6lP1xeLmYzJJkX+VvEOl796VTh4sbXMPkFw9rW9MXVJUusz+uZOls2iBjUtPcWQGDdXMzp07VVxcepr6V199pUGDBik5OVleXl6SSv+uaE/Onz8vFxcXSdKYMWP0zDPPqKCgQJ9//rni4+Pl4+OjiRMn6h//+Idln7Zt2yo+Pl5jxoy56vjz5s3T4MGDq+xG3kFBQVVynJtNkyZNFBkZqWXLlikhIeG6x7PZPxN7enqqY8eOevbZZ3XixAkVFxdr2bJl2rZtm9LT05WRkSFJCgwMtNovMDDQsu1SZs6cKW9vb8sjNDS0Ul/H5ZidzGoR0EJ31L9DwxsP1x3171CLgBYENKCKmUwm3VqvluI6hSu0ppt+zszT4V/OKj0rTyfP5is9q/T5z5l5Cq3ppriO4bq1Xi1bl42bjX+U1PkhqWOCFNRcOn9OOpVSGtpOHir976mU0vag5qX9Oj9EQKsCmfmZOpp9VJn5mVVyvKKiIj3wwAPy9vaWv7+/pk6danUde3p6uvr06SOz2ayIiAitWLFC4eHhmjNnzu81Z2bqvvvuU61ateTl5aXu3btr3759VzxuUlKS2rVrJ3d3d/n4+Khz5846evSopN9nVN58803VqVNHHh4euv/++1VcXKxZs2YpKChIAQEBmjFjhtWYL7/8spo2bSp3d3eFhobq/vvvV05OznW/R7Vq1VJQUJCCgoLk6+srSQoICLC0JSUlqVWrVqpRo4bq1q2r6dOnq6jo95vHm0wmvfHGGxowYIDc3NxUr149ffDBB5btZ86c0bBhw1SrVi2ZzWbVq1dPiYmJlu3ffvutunfvLrPZLD8/P8XHx1u9rhEjRqh///6aMWOGQkJCFB39+xoDbm5uCgoKUlhYmEaOHKlmzZpp/fr18vDwsNQfFBQkR0dHeXp6WrVdSnFxsf73v/+pb9++Vu1nzpxRXFycatasKTc3N/Xq1ctq3YYLp5quWbNG9erVU40aNRQbG6tjx45Ztk+fPl379u2zzOQtWrTI8v5dOGvtp59+kslk0ttvv61bb71VZrNZbdu21Q8//KCdO3eqTZs28vDwUK9evfTrr7/fduRSp/b279/fahY0PDxczz33nOLi4uTh4aGwsDB98MEH+vXXX9WvXz95eHioWbNm+uabby753lyQlpZm6e/l5aU777xTv/zyi2X7hT/fS5cuVXh4uLy9vTV06FCdPfv7vSy7deum8ePHa9KkSfL19VVQUJCmTZtm2Z6UlCQXFxdt3rzZ0jZr1iwFBARYHatv37566623rlhvWdn0XJ6lS5fKMAzVrl1brq6umjt3ru666y45OFx7WZMnT1ZWVpblceEPI4DqrUWojx6+rb7+0TVS7ev6ybOGs5wcTPKs4az2df30j66Revi2+sygofI415BC20l/mSDdNl3qMlFqO1pqHVf63y4TS9v/MqG0H4uEVKr8ony9n/K+5uyeo//b83+as3uO3k95X/kVdduEy1i8eLGcnJy0Y8cOvfLKK3r55Zf1xhtvWLbHxcXpxIkTSkpK0rvvvqsFCxbo5MmTVmMMHjxYJ0+e1Nq1a7Vr1y61atVKPXr00OnTpy95zKKiIvXv319du3bV/v37tW3bNsXHx1udYpeamqq1a9fq008/1cqVK7Vw4UL16dNHx48f16ZNm/TCCy9oypQp2r59u2UfBwcHzZ07VwcOHNDixYv1+eefa9KkSRX8jlnbvHmz4uLi9NBDD+ngwYN6/fXXtWjRoosC5PTp03XnnXdq//796t27t4YNG2Z5f6ZOnaqDBw9q7dq1OnTokObPny9/f39J0rlz5xQbG6uaNWtq586deuedd7RhwwY98MADVuNv3LhRycnJWr9+vT766KOL6jQMQ5s3b9b3339vmWW7Fvv371dWVpbatGlj1T5ixAh98803+uCDD7Rt2zYZhqHevXursPD39RByc3M1Y8YMLVmyRFu3blVmZqblTLQhQ4bo0UcfVePGjZWenq709HQNGTLksnU8/fTTmjJlinbv3i0nJyfdfffdmjRpkl555RVt3rxZKSkpeuqpp8r9+mbPnq3OnTtrz5496tOnj+69917FxcXpnnvu0e7duxUZGam4uLjLLshXUlKifv366fTp09q0aZPWr1+vH3/88aLXkpqaqjVr1uijjz7SRx99pE2bNulf//qXVZ/FixfL3d1d27dv16xZs/TMM89o/fr1kn4Pnffee6+ysrK0Z88eTZ06VW+88YbVhFK7du20Y8cOq7P6rplhB3JycowTJ04YhmEYd955p9G7d28jNTXVkGTs2bPHqm+XLl2M8ePHl3nsrKwsQ5KRlZVVkSUDuAkUF5fYugQAl3Gl7++8vDzj4MGDRl5e3jWPv+bwGmP8xvHGM189Y7z8zcvGM189Y4zfON5Yc3jN9ZR9RV27djUaNmxolJT8/rvn8ccfNxo2bGgYhmEcOnTIkGTs3LnTsv3w4cOGJGP27NmGYRjG5s2bDS8vLyM/P99q7MjISOP111+/5HFPnTplSDKSkpIuuf3pp5823NzcjOzsbEtbbGysER4ebhQXF1vaoqOjjZkzZ1729b3zzjuGn5+f5XliYqLh7e1tdZzmzZtfdv9L+eKLLwxJxpkzZwzDMIwePXoYzz//vFWfpUuXGsHBwZbnkowpU6ZYnufk5BiSjLVr1xqGYRh9+/Y1Ro4cecnjLViwwKhZs6aRk5Njafv4448NBwcHIyMjwzAMwxg+fLgRGBhoFBQUWO3btWtXw9nZ2XB3dzecnZ0NSUaNGjWMrVu3XnScsLAwy2d6JatXrzYcHR2t/sz88MMPhiSrcX/77TfDbDYbb7/9tmEYpe+9JOPrr7+29Lnw52v79u2GYVz+85BkrF692jAMwzhy5IghyXjjjTcs21euXGlIMjZu3GhpmzlzphEdHW31Xjz00ENW4/br188YPny41Xtwzz33WJ6np6cbkoypU6da2rZt22ZIMtLT0y/5/nz22WeGo6OjkZaWZmk7cOCAIcnYsWOH5XX++c/3Y489ZrRv396q3r/85S9WY7dt29Z4/PHHLc8LCgqMFi1aGHfeeafRqFEjY8yYMRfVs2/fPkOS8dNPP12y3vL87rKLq+Ld3d0VHBysM2fOaN26derXr58iIiIUFBSkjRs3WvplZ2dr+/bt6tixow2rBXCz4EbVQPWUmZ+pPSf3yN/sLz+zn1wdXeVn9pO/2V97Tu6p1FMfO3ToYDWD1bFjRx0+fFjFxcVKTk6Wk5OTWrVqZdkeFRWlmjV/vx3Ivn37lJOTIz8/P3l4eFgeR44cUWpqqtLS0qzan3/+efn6+mrEiBGKjY1V37599corryg9Pd2qrvDwcHl6/r6ibWBgoBo1amR1dlNgYKDVrN6GDRvUo0cP1a5dW56enrr33nt16tQp5ebmVuh79kf79u3TM888Y/Uax4wZo/T0dKvjNmv2+z0F3d3d5eXlZal93Lhxeuutt9SiRQtNmjRJX331laXvoUOH1Lx5c7m7u1vaOnfurJKSEiUnJ1vamjZteskZsmHDhmnv3r3aunWrevXqpSeffFKdOnW65tebl5cnV1dXqz8zhw4dkpOTk9q3b29p8/PzU3R0tA4dOmRpc3JyUtu2bS3PGzRoIB8fH6s+ZfXH9/PCzFHTpk2t2v4841tR40q67NiHDh1SaGio1eVNjRo1uuh1/vnPd3Bw8EVj/rGWS/VxcXHR8uXL9e677yo/P1+zZ8++qJ4L10tWxM+AzRYOkaR169bJMAxFR0crJSVFjz32mBo0aKCRI0fKZDJpwoQJeu6551SvXj1FRERo6tSpCgkJUf/+/W1ZNgAAuIFlnc9SbmGugj2Crdo9XDyUcS5DWeez5FPDxzbFXUVOTo6Cg4OVlJR00TYfHx/5+Pho7969lrYL13QlJiZq/Pjx+vTTT7Vq1SpNmTJF69evV4cOHSRJzs7OVmOZTKZLtpWUlEgqvVbpb3/7m8aNG6cZM2bI19dXW7Zs0ejRo3X+/Hm5ublV4Kv+XU5OjqZPn66BAwdetO2Pi9FdqfZevXrp6NGj+uSTT7R+/Xr16NFDCQkJevHFF8tcxx9D3B95e3srKipKkvT2228rKipKHTp0sFoIrzz8/f2Vm5trtTiJLfzx/bwQGP/cduH9lUpPhTX+dIriH0/FLM+4kqzGvhZX+vNQnj4XAv3p06d1+vTpi/4cXDiltlat67+23aYzaVlZWUpISFCDBg0UFxenv/zlL1q3bp3lTZo0aZIefPBBxcfHq23btsrJydGnn3560YqQAAAAZeXt4i03ZzflnLde5CLnfI7MTmZ5u3hX2rH/eE2XJH399deqV6+eHB0dFR0draKiIu3Zs8eyPSUlRWfOnLE8b9WqlTIyMuTk5KSoqCirh7+//0XtF0KaJLVs2VKTJ0/WV199pSZNmmjFihXX/Dp27dqlkpISvfTSS+rQoYPq16+vEydOXPN4ZdWqVSslJydf9NqjoqLKtaZBrVq1NHz4cC1btkxz5szRggULJEkNGzbUvn37dO7cOUvfrVu3ysHBwWqBkLLw8PDQQw89pIkTJ172mqqrubBE/sGDBy1tDRs2VFFRkdWfpVOnTik5OVmNGjWytBUVFVktupGcnKzMzEw1bNhQUunM0IVVNCtarVq1rGZri4uL9d1331X4cRo2bKhjx45ZrUFx8OBBZWZmWr0XFSE1NVUPP/yw/vvf/6p9+/YaPnz4RSHuu+++0y233GK5xvF62DSk3XnnnUpNTVVBQYHS09P16quvytv791+MJpNJzzzzjDIyMpSfn68NGzaofv36NqwYAADc6Hxq+KhlQEv9lvebTuWdUkFxgU7lndJveb+pZUDLSp1FS0tL0yOPPKLk5GStXLlS8+bN00MPPSSp9HS0mJgYxcfHa8eOHdqzZ4/i4+NlNpstMwoxMTHq2LGj+vfvr88++0w//fSTvvrqKz355JOXXQXvyJEjmjx5srZt26ajR4/qs88+0+HDhy1/Wb8WUVFRKiws1Lx58/Tjjz9q6dKleu211655vLJ66qmntGTJEk2fPl0HDhzQoUOH9NZbb2nKlCnlGuP9999XSkqKDhw4oI8++sjyXgwbNkw1atTQ8OHD9d133+mLL77Qgw8+qHvvvfeiFcfLYuzYsfrhhx/07rvvlntfqTTstGrVSlu2bLG01atXT/369dOYMWO0ZcsW7du3T/fcc49q166tfv36Wfo5OzvrwQcf1Pbt27Vr1y6NGDFCHTp0ULt27SSVngJ45MgR7d27V7/99lvFLHbx/3Xv3l0ff/yxPv74Y33//fcaN26cMjMzK2z8C2JiYtS0aVMNGzZMu3fv1o4dOxQXF6euXbtetNjK9SguLtY999yj2NhYjRw5UomJidq/f/9FN6/evHmzbr/99go5pl1ckwYAAFCVYsNj1b1OdxUbxco4l6Fio1jd63RXbHhspR43Li5OeXl5ateunRISEvTQQw8pPj7esn3JkiUKDAxUly5dNGDAAI0ZM0aenp6Ws4hMJpM++eQTdenSRSNHjlT9+vU1dOhQHT169LIhws3NTd9//70GDRqk+vXrKz4+XgkJCRo7duw1v47mzZvr5Zdf1gsvvKAmTZpo+fLlmjlz5jWPV1axsbH66KOP9Nlnn6lt27bq0KGDZs+erbCwsDKP4eLiosmTJ6tZs2bq0qWLHB0dLcumu7m5ad26dTp9+rTatm2rO+64Qz169NCrr756TfX6+voqLi5O06ZNu+ZT9u677z4tX77cqi0xMVGtW7fW3/72N3Xs2FGGYeiTTz6xOmXPzc1Njz/+uO6++2517txZHh4eWrVqlWX7oEGD1LNnT/31r39VrVq1tHLlymuq71JGjRql4cOHWwJT3bp19de//rXCxr/AZDLp/fffV82aNdWlSxfFxMSobt26Vq+zIsyYMUNHjx7V66+/Lqn0erUFCxZoypQplttf5Ofna82aNWW6911ZmIxrnX+9QWRnZ8vb21tZWVmWmyACAAD7dqXv7/z8fB05ckQRERHXfQlEZn6mss5nydvF2y6vQzt+/LhCQ0Mti3Sg+snLy1N0dLRWrVpV5sXzFi1apAkTJlTK7BUubf78+Vq9erU+++yzy/Ypz+8umy4cAgAAYEs+NXzsKpx9/vnnysnJUdOmTZWenq5JkyYpPDxcXbp0sXVpsBGz2awlS5bot99+s3UpuAJnZ2fNmzevwsYjpAEAANiJwsJCPfHEE/rxxx/l6empTp06afny5RetPHej8/DwuOy2tWvX6tZbb63Cauxft27dbF0CruK+++6r0PE43REAANidqjrdEbaRkpJy2W21a9e23G8KuJlwuiMAAADs1oV7iQG4NFZ3BAAAN6Sb/GQgADeZ8vzOIqQBAIAbyoXrs3Jzc21cCQCU3YXfWWW5xpTTHQEAwA3F0dFRPj4+OnnypKTS+0FduNkzANgbwzCUm5urkydPysfHR46Ojlfdh5AGAABuOEFBQZJkCWoAYO98fHwsv7uuhpAGAABuOCaTScHBwQoICFBhYaGtywGAK3J2di7TDNoFhDQAAHDDcnR0LNdffADgRsDCIQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgR2wa0oqLizV16lRFRETIbDYrMjJSzz77rAzDsPQZMWKETCaT1aNnz542rBoAAAAAKo+TLQ/+wgsvaP78+Vq8eLEaN26sb775RiNHjpS3t7fGjx9v6dezZ08lJiZanru6utqiXAAAAACodDYNaV999ZX69eunPn36SJLCw8O1cuVK7dixw6qfq6urgoKCbFEiAAAAAFQpm57u2KlTJ23cuFE//PCDJGnfvn3asmWLevXqZdUvKSlJAQEBio6O1rhx43Tq1ClblAsAAAAAlc6mM2n//Oc/lZ2drQYNGsjR0VHFxcWaMWOGhg0bZunTs2dPDRw4UBEREUpNTdUTTzyhXr16adu2bXJ0dLxozIKCAhUUFFieZ2dnV8lrAQAAAICKYNOQ9vbbb2v58uVasWKFGjdurL1792rChAkKCQnR8OHDJUlDhw619G/atKmaNWumyMhIJSUlqUePHheNOXPmTE2fPr3KXgMAAAAAVCST8celFKtYaGio/vnPfyohIcHS9txzz2nZsmX6/vvvL7tfrVq19Nxzz2ns2LEXbbvUTFpoaKiysrLk5eVVsS8AAABUiuzsbHl7e/P9DaBasulMWm5urhwcrC+Lc3R0VElJyWX3OX78uE6dOqXg4OBLbnd1dWX1RwAAAAA3LJuGtL59+2rGjBmqU6eOGjdurD179ujll1/WqFGjJEk5OTmaPn26Bg0apKCgIKWmpmrSpEmKiopSbGysLUsHAAAAgEph09Mdz549q6lTp2r16tU6efKkQkJCdNddd+mpp56Si4uL8vLy1L9/f+3Zs0eZmZkKCQnR7bffrmeffVaBgYFlOganSwAAcOPh+xtAdWbTkFYV+CUPAMCNh+9vANWZTe+TBgAAAACwRkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA74lSeziUlJdq0aZM2b96so0ePKjc3V7Vq1VLLli0VExOj0NDQyqoTAAAAAKqFMs2k5eXl6bnnnlNoaKh69+6ttWvXKjMzU46OjkpJSdHTTz+tiIgI9e7dW19//XVl1wwAAAAAN60yzaTVr19fHTt21H//+1/ddtttcnZ2vqjP0aNHtWLFCg0dOlRPPvmkxowZU+HFAgAAAMDNzmQYhnG1TocOHVLDhg3LNGBhYaHS0tIUGRl53cVVhOzsbHl7eysrK0teXl62LgcAAJQB398AqrMyne5Y1oAmSc7OznYT0AAAAADgRlOuhUP+qKioSK+//rqSkpJUXFyszp07KyEhQTVq1KjI+gAAAACgWrnmkDZ+/Hj98MMPGjhwoAoLC7VkyRJ98803WrlyZUXWBwAAAADVSplD2urVqzVgwADL888++0zJyclydHSUJMXGxqpDhw4VXyEAAAAAVCNlvpn1m2++qf79++vEiROSpFatWukf//iHPv30U3344YeaNGmS2rZtW2mFAgAAAEB1UOaQ9uGHH+quu+5St27dNG/ePC1YsEBeXl568sknNXXqVIWGhmrFihWVWSsAAAAA3PTKtAT/H2VmZmrSpEnat2+fXnvtNbVs2bKyaqsQLOELAMCNh+9vANVZmWfSLvDx8dGCBQv073//W3FxcXrssceUn59fGbUBAAAAQLVT5pCWlpamO++8U02bNtWwYcNUr1497dq1S25ubmrevLnWrl1bmXUCAAAAQLVQ5tMdu3XrpqCgII0YMULr1q1TamqqPvjgA0nSoUOHNHbsWAUFBentt9+u1ILLi9MlAAC48fD9DaA6K/MS/N9884327dunyMhIxcbGKiIiwrKtYcOG+vLLL7VgwYJKKRIAAAAAqosyh7TWrVvrqaee0vDhw7VhwwY1bdr0oj7x8fEVWhwAAAAAVDdlviZtyZIlKigo0MMPP6yff/5Zr7/+emXWBQAAAADVUpln0sLCwvS///2vMmsBAAAAgGqvTDNp586dK9eg5e0PAAAAAChVppAWFRWlf/3rX0pPT79sH8MwtH79evXq1Utz586tsAIBAAAAoDop0+mOSUlJeuKJJzRt2jQ1b95cbdq0UUhIiGrUqKEzZ87o4MGD2rZtm5ycnDR58mSNHTu2susGAAAAgJtSme+TJpXe0Pqdd97R5s2bdfToUeXl5cnf318tW7ZUbGysevXqJUdHx8qst9y4zwoAADcevr8BVGflCmk3In7JAwBw4+H7G0B1VuYl+AEAAAAAlY+QBgAAAAB2hJAGAAAAAHaEkAYAAAAAdoSQBgAAAAB2pNwhLTw8XM8884zS0tKu++DFxcWaOnWqIiIiZDabFRkZqWeffVZ/XHDSMAw99dRTCg4OltlsVkxMjA4fPnzdxwYAAAAAe1TukDZhwgS99957qlu3rm677Ta99dZbKigouKaDv/DCC5o/f75effVVHTp0SC+88IJmzZqlefPmWfrMmjVLc+fO1Wuvvabt27fL3d1dsbGxys/Pv6ZjAgAAAIA9u+b7pO3evVuLFi3SypUrVVxcrLvvvlujRo1Sq1atyjzG3/72NwUGBmrhwoWWtkGDBslsNmvZsmUyDEMhISF69NFHNXHiRElSVlaWAgMDtWjRIg0dOvSqx+A+KwAA3Hj4/gZQnV3zNWmtWrXS3LlzdeLECT399NN644031LZtW7Vo0UJvvvmmypL9OnXqpI0bN+qHH36QJO3bt09btmxRr169JElHjhxRRkaGYmJiLPt4e3urffv22rZt2yXHLCgoUHZ2ttUDAAAAAG4UTte6Y2FhoVavXq3ExEStX79eHTp00OjRo3X8+HE98cQT2rBhg1asWHHFMf75z38qOztbDRo0kKOjo4qLizVjxgwNGzZMkpSRkSFJCgwMtNovMDDQsu3PZs6cqenTp1/rywIAAAAAmyp3SNu9e7cSExO1cuVKOTg4KC4uTrNnz1aDBg0sfQYMGKC2bdteday3335by5cv14oVK9S4cWPt3btXEyZMUEhIiIYPH17e0iRJkydP1iOPPGJ5np2drdDQ0GsaCwAAAACqWrlDWtu2bXXbbbdp/vz56t+/v5ydnS/qExERUabrxR577DH985//tPRt2rSpjh49qpkzZ2r48OEKCgqSJP3yyy8KDg627PfLL7+oRYsWlxzT1dVVrq6u5X1ZAAAAAGAXyh3SfvzxR4WFhV2xj7u7uxITE686Vm5urhwcrC+Lc3R0VElJiaTSsBcUFKSNGzdaQll2dra2b9+ucePGlbd0AAAAALB75Q5pJ0+eVEZGhtq3b2/Vvn37djk6OqpNmzZlHqtv376aMWOG6tSpo8aNG2vPnj16+eWXNWrUKEmSyWTShAkT9Nxzz6levXqKiIjQ1KlTFRISov79+5e3dAAAAACwe+Ve3TEhIUHHjh27qP3nn39WQkJCucaaN2+e7rjjDt1///1q2LChJk6cqLFjx+rZZ5+19Jk0aZIefPBBxcfHq23btsrJydGnn36qGjVqlLd0AAAAALB75b5PmoeHh/bv36+6detatR85ckTNmjXT2bNnK7TA68V9VgAAuPHw/Q2gOiv3TJqrq6t++eWXi9rT09Pl5HTNK/oDAAAAAHQNIe3222/X5MmTlZWVZWnLzMzUE088odtuu61CiwMAAACA6qbcU18vvviiunTporCwMLVs2VKStHfvXgUGBmrp0qUVXiAAAAAAVCflDmm1a9fW/v37tXz5cu3bt09ms1kjR47UXXfddcl7pgEAAAAAyu6aLiJzd3dXfHx8RdcCAAAAANXeNa/0cfDgQaWlpen8+fNW7X//+9+vuygAAAAAqK7KHdJ+/PFHDRgwQN9++61MJpMurOBvMpkkScXFxRVbIQAAAABUI+Ve3fGhhx5SRESETp48KTc3Nx04cEBffvml2rRpo6SkpEooEQAAAACqj3LPpG3btk2ff/65/P395eDgIAcHB/3lL3/RzJkzNX78eO3Zs6cy6gQAAACAaqHcM2nFxcXy9PSUJPn7++vEiROSpLCwMCUnJ1dsdQAAAABQzZR7Jq1Jkybat2+fIiIi1L59e82aNUsuLi5asGCB6tatWxk1AgAAAEC1Ue6QNmXKFJ07d06S9Mwzz+hvf/ubbr31Vvn5+WnVqlUVXiAAAAAAVCcm48LyjNfh9OnTqlmzpmWFR3uSnZ0tb29vZWVlycvLy9blAACAMuD7G0B1Vq5r0goLC+Xk5KTvvvvOqt3X19cuAxoAAAAA3GjKFdKcnZ1Vp04d7oUGAAAAAJWk3Ks7Pvnkk3riiSd0+vTpyqgHAAAAAKq1ci8c8uqrryolJUUhISEKCwuTu7u71fbdu3dXWHEAAAAAUN2UO6T179+/EsoAAAAAAEgVtLqjPWN1KAAAbjx8fwOozsp9TRoAAAAAoPKU+3RHBweHKy63z8qPAAAAAHDtyh3SVq9ebfW8sLBQe/bs0eLFizV9+vQKKwwAAAAAqqMKuyZtxYoVWrVqld5///2KGK7CcE47AAA3Hr6/AVRnFXZNWocOHbRx48aKGg4AAAAAqqUKCWl5eXmaO3euateuXRHDAQAAAEC1Ve5r0mrWrGm1cIhhGDp79qzc3Ny0bNmyCi0OAAAAAKqbcoe02bNnW4U0BwcH1apVS+3bt1fNmjUrtDgAAAAAqG7KHdJGjBhRCWUAAAAAAKRruCYtMTFR77zzzkXt77zzjhYvXlwhRQEAAABAdVXukDZz5kz5+/tf1B4QEKDnn3++QooCAAAAgOqq3CEtLS1NERERF7WHhYUpLS2tQooCAAAAgOqq3CEtICBA+/fvv6h937598vPzq5CiAAAAAKC6KndIu+uuuzR+/Hh98cUXKi4uVnFxsT7//HM99NBDGjp0aGXUCAAAAADVRrlXd3z22Wf1008/qUePHnJyKt29pKREcXFxXJMGAAAAANfJZBiGcS07Hj58WHv37pXZbFbTpk0VFhZW0bVViOzsbHl7eysrK0teXl62LgcAAJQB398AqrNyz6RdUK9ePdWrV68iawEAAACAaq/c16QNGjRIL7zwwkXts2bN0uDBgyukKAAAAACorsod0r788kv17t37ovZevXrpyy+/rJCiAAAAAKC6KndIy8nJkYuLy0Xtzs7Oys7OrpCiAAAAAKC6KndIa9q0qVatWnVR+1tvvaVGjRpVSFEAAAAAUF2Ve+GQqVOnauDAgUpNTVX37t0lSRs3btTKlSv1zjvvVHiBAAAAAFCdlDuk9e3bV2vWrNHzzz+v//3vfzKbzWrWrJk2bNigrl27VkaNAAAAAFBtXPN90i7lu+++U5MmTSpquArBfVYAALjx8P0NoDor9zVpf3b27FktWLBA7dq1U/PmzSuiJgAAAACotq45pH355ZeKi4tTcHCwXnzxRXXv3l1ff/11RdYGAAAAANVOua5Jy8jI0KJFi7Rw4UJlZ2frzjvvVEFBgdasWcPKjgAAAABQAco8k9a3b19FR0dr//79mjNnjk6cOKF58+ZVZm0AAAAAUO2UOaStXbtWo0eP1vTp09WnTx85Ojpe98HDw8NlMpkueiQkJEiSunXrdtG2f/zjH9d9XAAAAACwV2UOaVu2bNHZs2fVunVrtW/fXq+++qp+++236zr4zp07lZ6ebnmsX79ekjR48GBLnzFjxlj1mTVr1nUdEwAAAADsWZlDWocOHfTf//5X6enpGjt2rN566y2FhISopKRE69ev19mzZ8t98Fq1aikoKMjy+OijjxQZGWl1vzU3NzerPizDCwAAAOBmVu7VHd3d3TVq1Cht2bJF3377rR599FH961//UkBAgP7+979fcyHnz5/XsmXLNGrUKJlMJkv78uXL5e/vryZNmmjy5MnKzc294jgFBQXKzs62egAAAADAjeK67pMWHR2tWbNm6fjx41q5cuV1FbJmzRplZmZqxIgRlra7775by5Yt0xdffKHJkydr6dKluueee644zsyZM+Xt7W15hIaGXlddAAAAAFCVTIZhGLYuQpJiY2Pl4uKiDz/88LJ9Pv/8c/Xo0UMpKSmKjIy8ZJ+CggIVFBRYnmdnZys0NFRZWVmcKgkAwA0iOztb3t7efH8DqJbKdZ+0ynL06FFt2LBB77333hX7tW/fXpKuGNJcXV3l6upa4TUCAAAAQFW4rtMdK0piYqICAgLUp0+fK/bbu3evJCk4OLgKqgIAAACAqmfzmbSSkhIlJiZq+PDhcnL6vZzU1FStWLFCvXv3lp+fn/bv36+HH35YXbp0UbNmzWxYMQAAAABUHpuHtA0bNigtLU2jRo2yandxcdGGDRs0Z84cnTt3TqGhoRo0aJCmTJlio0oBAAAAoPLZzcIhlYULjwEAuPHw/Q2gOrOLa9IAAAAAAKUIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgR2wa0sLDw2UymS56JCQkSJLy8/OVkJAgPz8/eXh4aNCgQfrll19sWTIAAAAAVCqbhrSdO3cqPT3d8li/fr0kafDgwZKkhx9+WB9++KHeeecdbdq0SSdOnNDAgQNtWTIAAAAAVCqTYRiGrYu4YMKECfroo490+PBhZWdnq1atWlqxYoXuuOMOSdL333+vhg0batu2berQoUOZxszOzpa3t7eysrLk5eVVmeUDAIAKwvc3gOrMbq5JO3/+vJYtW6ZRo0bJZDJp165dKiwsVExMjKVPgwYNVKdOHW3btu2y4xQUFCg7O9vqAQAAAAA3CrsJaWvWrFFmZqZGjBghScrIyJCLi4t8fHys+gUGBiojI+Oy48ycOVPe3t6WR2hoaCVWDQAAAAAVy25C2sKFC9WrVy+FhIRc1ziTJ09WVlaW5XHs2LEKqhAAAAAAKp+TrQuQpKNHj2rDhg167733LG1BQUE6f/68MjMzrWbTfvnlFwUFBV12LFdXV7m6ulZmuQAAAABQaexiJi0xMVEBAQHq06ePpa1169ZydnbWxo0bLW3JyclKS0tTx44dbVEmAAAAAFQ6m8+klZSUKDExUcOHD5eT0+/leHt7a/To0XrkkUfk6+srLy8vPfjgg+rYsWOZV3YEAAAAgBuNzUPahg0blJaWplGjRl20bfbs2XJwcNCgQYNUUFCg2NhY/ec//7FBlQAAAABQNezqPmmVgfusAABw4+H7G0B1ZhfXpAEAAAAAShHSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCO2Dyk/fzzz7rnnnvk5+cns9mspk2b6ptvvrFsHzFihEwmk9WjZ8+eNqwYAAAAACqPky0PfubMGXXu3Fl//etftXbtWtWqVUuHDx9WzZo1rfr17NlTiYmJlueurq5VXSoAAAAAVAmbhrQXXnhBoaGhVgEsIiLion6urq4KCgqqytIAAAAAwCZserrjBx98oDZt2mjw4MEKCAhQy5Yt9d///veifklJSQoICFB0dLTGjRunU6dOXXbMgoICZWdnWz0AAAAA4EZh05D2448/av78+apXr57WrVuncePGafz48Vq8eLGlT8+ePbVkyRJt3LhRL7zwgjZt2qRevXqpuLj4kmPOnDlT3t7elkdoaGhVvRwAAAAAuG4mwzAMWx3cxcVFbdq00VdffWVpGz9+vHbu3Klt27Zdcp8ff/xRkZGR2rBhg3r06HHR9oKCAhUUFFieZ2dnKzQ0VFlZWfLy8qr4FwEAACpcdna2vL29+f4GUC3ZdCYtODhYjRo1smpr2LCh0tLSLrtP3bp15e/vr5SUlEtud3V1lZeXl9UDAAAAAG4UNg1pnTt3VnJyslXbDz/8oLCwsMvuc/z4cZ06dUrBwcGVXR4AAAAAVDmbhrSHH35YX3/9tZ5//nmlpKRoxYoVWrBggRISEiRJOTk5euyxx/T111/rp59+0saNG9WvXz9FRUUpNjbWlqUDAAAAQKWwaUhr27atVq9erZUrV6pJkyZ69tlnNWfOHA0bNkyS5OjoqP379+vvf/+76tevr9GjR6t169bavHkz90oDAAAAcFOy6cIhVYELjwEAuPHw/Q2gOrPpTBoAAAAAwBohDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOyIk60LqGyGYUiSsrOzbVwJAAAoqwvf2xe+xwGgOrnpQ9rZs2clSaGhoTauBAAAlNepU6fk7e1t6zIAoEqZjJv8n6hKSkp04sQJeXp6ymQy2bqcG1p2drZCQ0N17NgxeXl52boc/AGfjf3is7FvfD72KysrS3Xq1NGZM2fk4+Nj63IAoErd9DNpDg4OuuWWW2xdxk3Fy8uLv8zYKT4b+8VnY9/4fOyXgwOXzwOofvjNBwAAAAB2hJAGAAAAAHaEkIYyc3V11dNPPy1XV1dbl4I/4bOxX3w29o3Px37x2QCozm76hUMAAAAA4EbCTBoAAAAA2BFCGgAAAADYEUIaAAAAANgRQhoAAAAA2BFCGq5o2rRpMplMVo8GDRrYuqxq68svv1Tfvn0VEhIik8mkNWvWWG03DENPPfWUgoODZTabFRMTo8OHD9um2Grmap/NiBEjLvpZ6tmzp22KrWZmzpyptm3bytPTUwEBAerfv7+Sk5Ot+uTn5yshIUF+fn7y8PDQoEGD9Msvv9io4uqjLJ9Nt27dLvrZ+cc//mGjigGgahDScFWNGzdWenq65bFlyxZbl1RtnTt3Ts2bN9f//d//XXL7rFmzNHfuXL322mvavn273N3dFRsbq/z8/CqutPq52mcjST179rT6WVq5cmUVVlh9bdq0SQkJCfr666+1fv16FRYW6vbbb9e5c+csfR5++GF9+OGHeuedd7Rp0yadOHFCAwcOtGHV1UNZPhtJGjNmjNXPzqxZs2xUMQBUDSdbFwD75+TkpKCgIFuXAUm9evVSr169LrnNMAzNmTNHU6ZMUb9+/SRJS5YsUWBgoNasWaOhQ4dWZanVzpU+mwtcXV35WbKBTz/91Or5okWLFBAQoF27dqlLly7KysrSwoULtWLFCnXv3l2SlJiYqIYNG+rrr79Whw4dbFF2tXC1z+YCNzc3fnYAVCvMpOGqDh8+rJCQENWtW1fDhg1TWlqarUvCJRw5ckQZGRmKiYmxtHl7e6t9+/batm2bDSvDBUlJSQoICFB0dLTGjRunU6dO2bqkaikrK0uS5OvrK0natWuXCgsLrX52GjRooDp16vCzU8X+/NlcsHz5cvn7+6tJkyaaPHmycnNzbVEeAFQZZtJwRe3bt9eiRYsUHR2t9PR0TZ8+Xbfeequ+++47eXp62ro8/EFGRoYkKTAw0Ko9MDDQsg2207NnTw0cOFARERFKTU3VE088oV69emnbtm1ydHS0dXnVRklJiSZMmKDOnTurSZMmkkp/dlxcXOTj42PVl5+dqnWpz0aS7r77boWFhSkkJET79+/X448/ruTkZL333ns2rBYAKhchDVf0x9O3mjVrpvbt2yssLExvv/22Ro8ebcPKgBvLH083bdq0qZo1a6bIyEglJSWpR48eNqyseklISNB3333HtbV26HKfTXx8vOX/mzZtquDgYPXo0UOpqamKjIys6jIBoEpwuiPKxcfHR/Xr11dKSoqtS8GfXLhe488r0v3yyy9cy2GH6tatK39/f36WqtADDzygjz76SF988YVuueUWS3tQUJDOnz+vzMxMq/787FSdy302l9K+fXtJ4mcHwE2NkIZyycnJUWpqqoKDg21dCv4kIiJCQUFB2rhxo6UtOztb27dvV8eOHW1YGS7l+PHjOnXqFD9LVcAwDD3wwANavXq1Pv/8c0VERFhtb926tZydna1+dpKTk5WWlsbPTiW72mdzKXv37pUkfnYA3NQ43RFXNHHiRPXt21dhYWE6ceKEnn76aTk6Ouquu+6ydWnVUk5OjtW/Hh85ckR79+6Vr6+v6tSpowkTJui5555TvXr1FBERoalTpyokJET9+/e3XdHVxJU+G19fX02fPl2DBg1SUFCQUlNTNWnSJEVFRSk2NtaGVVcPCQkJWrFihd5//315enparjPz9vaW2WyWt7e3Ro8erUceeUS+vr7y8vLSgw8+qI4dO7KyYyW72meTmpqqFStWqHfv3vLz89P+/fv18MMPq0uXLmrWrJmNqweASmQAVzBkyBAjODjYcHFxMWrXrm0MGTLESElJsXVZ1dYXX3xhSLroMXz4cMMwDKOkpMSYOnWqERgYaLi6uho9evQwkpOTbVt0NXGlzyY3N9e4/fbbjVq1ahnOzs5GWFiYMWbMGCMjI8PWZVcLl/pcJBmJiYmWPnl5ecb9999v1KxZ03BzczMGDBhgpKen267oauJqn01aWprRpUsXw9fX13B1dTWioqKMxx57zMjKyrJt4QBQyUyGYRhVGQoBAAAAAJfHNWkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQDsyk8//SSTyaS9e/faupRKc/78eUVFRemrr76qtGO89tpr6tu3b6WNDwAAKg8hDbgBbNu2TY6OjurTp4+tS7FLI0aMUP/+/W1dRpm99tprioiIUKdOnSrtGKNGjdLu3bu1efPmSjsGAACoHIQ04AawcOFCPfjgg/ryyy914sSJSj2WYRgqKiqq1GNUZ4Zh6NVXX9Xo0aMr9TguLi66++67NXfu3Eo9DgAAqHiENMDO5eTkaNWqVRo3bpz69OmjRYsWWbbdfffdGjJkiFX/wsJC+fv7a8mSJZKkkpISzZw5UxERETKbzWrevLn+97//WfonJSXJZDJp7dq1at26tVxdXbVlyxalpqaqX79+CgwMlIeHh9q2basNGzZYHSs9PV19+vSR2WxWRESEVqxYofDwcM2ZM8fSJzMzU/fdd59q1aolLy8vde/eXfv27Svz6y8uLtbo0aMt9UdHR+uVV16xbJ82bZoWL16s999/XyaTSSaTSUlJSZKkY8eO6c4775SPj498fX3Vr18//fTTT5Z9L8zAvfjiiwoODpafn58SEhJUWFho6VNQUKDHH39coaGhcnV1VVRUlBYuXCjDMBQVFaUXX3zRqt69e/fKZDIpJSXlkq9n165dSk1NtZoVvXCK59tvv61bb71VZrNZbdu21Q8//KCdO3eqTZs28vDwUK9evfTrr79afXbt2rWTu7u7fHx81LlzZx09etSyvW/fvvrggw+Ul5dX5vcbAADYHiENsHNvv/22GjRooOjoaN1zzz168803ZRiGJGnYsGH68MMPlZOTY+m/bt065ebmasCAAZKkmTNnasmSJXrttdd04MABPfzww7rnnnu0adMmq+P885//1L/+9S8dOnRIzZo1U05Ojnr37q2NGzdqz5496tmzp/r27au0tDTLPnFxcTpx4oSSkpL07rvvasGCBTp58qTVuIMHD9bJkye1du1a7dq1S61atVKPHj10+vTpMr3+kpIS3XLLLXrnnXd08OBBPfXUU3riiSf09ttvS5ImTpyoO++8Uz179lR6errS09PVqVMnFRYWKjY2Vp6entq8ebO2bt0qDw8P9ezZU+fPn7eM/8UXXyg1NVVffPGFFi9erEWLFlkF4bi4OK1cuVJz587VoUOH9Prrr8vDw0Mmk0mjRo1SYmKiVb2JiYnq0qWLoqKiLvl6Nm/erPr168vT0/OibU8//bSmTJmi3bt3y8nJSXfffbcmTZqkV155RZs3b1ZKSoqeeuopSVJRUZH69++vrl27av/+/dq2bZvi4+NlMpks47Vp00ZFRUXavn17md5rAABgJwwAdq1Tp07GnDlzDMMwjMLCQsPf39/44osvrJ4vWbLE0v+uu+4yhgwZYhiGYeTn5xtubm7GV199ZTXm6NGjjbvuusswDMP44osvDEnGmjVrrlpL48aNjXnz5hmGYRiHDh0yJBk7d+60bD98+LAhyZg9e7ZhGIaxefNmw8vLy8jPz7caJzIy0nj99dcveYwjR44Ykow9e/Zcto6EhARj0KBBlufDhw83+vXrZ9Vn6dKlRnR0tFFSUmJpKygoMMxms7Fu3TrLfmFhYUZRUZGlz+DBgy3vX3JysiHJWL9+/SXr+Pnnnw1HR0dj+/bthmEYxvnz5w1/f39j0aJFl639oYceMrp3737J1/zGG29Y2lauXGlIMjZu3GhpmzlzphEdHW0YhmGcOnXKkGQkJSVd9liGYRg1a9a8Yj0AAMD+MJMG2LHk5GTt2LFDd911lyTJyclJQ4YM0cKFCy3P77zzTi1fvlySdO7cOb3//vsaNmyYJCklJUW5ubm67bbb5OHhYXksWbJEqampVsdq06aN1fOcnBxNnDhRDRs2lI+Pjzw8PHTo0CHLTFpycrKcnJzUqlUryz5RUVGqWbOm5fm+ffuUk5MjPz8/q+MfOXLkouNfyf/93/+pdevWqlWrljw8PLRgwQKrGb1L2bdvn1JSUuTp6Wk5rq+vr/Lz862O3bhxYzk6OlqeBwcHW2YD9+7dK0dHR3Xt2vWSxwgJCVGfPn305ptvSpI+/PBDFRQUaPDgwZetKy8vTzVq1LjktmbNmln+PzAwUJLUtGlTq7YLtfn6+mrEiBGKjY1V37599corryg9Pf2iMc1ms3Jzcy9bDwAAsD9Oti4AwOUtXLhQRUVFCgkJsbQZhiFXV1e9+uqr8vb21rBhw9S1a1edPHlS69evl9lsVs+ePSXJchrkxx9/rNq1a1uN7erqavXc3d3d6vnEiRO1fv16vfjii4qKipLZbNYdd9xhdarg1eTk5Cg4ONhyjdgf+fj4lGmMt956SxMnTtRLL72kjh07ytPTU//+97+vegpfTk6OWrdubQmwf1SrVi3L/zs7O1ttM5lMKikpkVQacK7mvvvu07333qvZs2crMTFRQ4YMkZub22X7+/v769tvv73ktj/WcuG0xT+3XahNKj21cvz48fr000+1atUqTZkyRevXr1eHDh0sfU6fPm31egEAgP0jpAF2qqioSEuWLNFLL72k22+/3Wpb//79tXLlSv3jH/9Qp06dFBoaqlWrVmnt2rUaPHiw5S/2jRo1kqurq9LS0i47G3Q5W7du1YgRIyzXtuXk5FgtuhEdHa2ioiLt2bNHrVu3llQ6c3fmzBlLn1atWikjI0NOTk4KDw+/hnehtI5OnTrp/vvvt7T9eRbOxcVFxcXFVm2tWrXSqlWrFBAQIC8vr2s6dtOmTVVSUqJNmzYpJibmkn169+4td3d3zZ8/X59++qm+/PLLK47ZsmVLzZ8/X4ZhWF0/dq1atmypli1bavLkyerYsaNWrFhhCWmpqanKz89Xy5Ytr/s4AACg6nC6I2CnPvroI505c0ajR49WkyZNrB6DBg2ynPIola7y+Nprr2n9+vWWUx0lydPTUxMnTtTDDz+sxYsXKzU1Vbt379a8efO0ePHiKx6/Xr16eu+997R3717t27dPd999t9UsToMGDRQTE6P4+Hjt2LFDe/bsUXx8vMxmsyV8xMTEqGPHjurfv78+++wz/fTTT/rqq6/05JNP6ptvvinT+1CvXj198803WrdunX744QdNnTpVO3futOoTHh6u/fv3Kzk5Wb/99psKCws1bNgw+fv7q1+/ftq8ebOOHDmipKQkjR8/XsePHy/TscPDwzV8+HCNGjVKa9assYxxYdESSXJ0dNSIESM0efJk1atXTx07drzimH/961+Vk5OjAwcOlKmGyzly5IgmT56sbdu26ejRo/rss890+PBhNWzY0NJn8+bNqlu3riIjI6/rWAAAoGoR0gA7tXDhQsXExMjb2/uibYMGDdI333yj/fv3Sypd5fHgwYOqXbu2OnfubNX32Wef1dSpUzVz5kw1bNhQPXv21Mcff6yIiIgrHv/ll19WzZo11alTJ/Xt21exsbFW159J0pIlSxQYGKguXbpowIABGjNmjDw9PS3XXJlMJn3yySfq0qWLRo4cqfr162vo0KE6evSo5Zqrqxk7dqwGDhyoIUOGqH379jp16pTVrJokjRkzRtHR0WrTpo1q1aqlrVu3ys3NTV9++aXq1KmjgQMHqmHDhho9erTy8/PLNbM2f/583XHHHbr//vvVoEEDjRkzRufOnbPqM3r0aJ0/f14jR4686nh+fn4aMGDAJU/DLA83Nzd9//33GjRokOrXr6/4+HglJCRo7Nixlj4rV67UmDFjrus4AACg6pkM4/+v5Q0A1+n48eMKDQ3Vhg0b1KNHD1uXU2U2b96sHj166NixY2UKn/v379dtt92m1NRUeXh4VEpNBw4cUPfu3fXDDz9cMugDAAD7RUgDcM0+//xz5eTkqGnTpkpPT9ekSZP0888/64cffrhoQY6bUUFBgX799VcNHz5cQUFB5ZodW7RokVq3bm21emNF2rBhg4qLixUbG1sp4wMAgMpDSANwzdatW6dHH31UP/74ozw9PdWpUyfNmTNHYWFhti6tSixatEijR49WixYt9MEHH1y0giYAAMC1IKQBAAAAgB1h4RAAAAAAsCOENAAAAACwI4Q0AAAAALAjhDQAAAAAsCOENAAAAACwI4Q0AAAAALAjhDQAAAAAsCOENAAAAACwI4Q0AAAAALAj/w9Ovd8N32EWsgAAAABJRU5ErkJggg==\n" + }, + "metadata": {} + } + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + }, + "colab": { + "provenance": [], + "collapsed_sections": [ + "ffbea843-2e86-4f14-961c-b8895f9de77d", + "1402c1ba-aa7f-4b0b-9db5-1e6f0d301e70", + "b-yd2LtbSbXj", + "AiPUhOCNWRny" + ], + "machine_shape": "hm", + "gpuType": "T4" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "c22db575e86b45229ab97eb5f5193fe1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "VBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "VBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "VBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1da311f97a57441988a622d5b019fa5b", + "IPY_MODEL_458a3ee5ff8d44fca9ca6f4888d79174", + "IPY_MODEL_09d3ebe126e2484d8cc50653f7d8f8fb", + "IPY_MODEL_f867a0c844b547e8bd1d294245c4fa13", + "IPY_MODEL_1993399b93224ec6865eaa01b6a73d8b" + ], + "layout": "IPY_MODEL_b35a241e895c41ca8be0f334615df0a3" + } + }, + "1da311f97a57441988a622d5b019fa5b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_99ef6333ea314434a1dd3ef05fa0e42b", + "placeholder": "​", + "style": "IPY_MODEL_ac1617fabea24f8a912e7cb6bb9abca4", + "value": "

Copy a token from your Hugging Face\ntokens page and paste it below.
Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file.
" + } + }, + "458a3ee5ff8d44fca9ca6f4888d79174": { + "model_module": "@jupyter-widgets/controls", + "model_name": "PasswordModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "PasswordModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "PasswordView", + "continuous_update": true, + "description": "Token:", + "description_tooltip": null, + "disabled": false, + "layout": "IPY_MODEL_108c04b373e243218924ab3b101e8908", + "placeholder": "​", + "style": "IPY_MODEL_16019f898e814155ae1c27ab2475565a", + "value": "" + } + }, + "09d3ebe126e2484d8cc50653f7d8f8fb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "CheckboxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "CheckboxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "CheckboxView", + "description": "Add token as git credential?", + "description_tooltip": null, + "disabled": false, + "indent": true, + "layout": "IPY_MODEL_d1df847b4fbe4ab992c64d5d2262cbcf", + "style": "IPY_MODEL_eeb36e1cf46b46328725481ce4b9a23b", + "value": true + } + }, + "f867a0c844b547e8bd1d294245c4fa13": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ButtonModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ButtonModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ButtonView", + "button_style": "", + "description": "Login", + "disabled": false, + "icon": "", + "layout": "IPY_MODEL_58c9a6969edf4b5ab636c0fdcb9b5b5b", + "style": "IPY_MODEL_abe2bd79c6574e5aafa865bd18f14221", + "tooltip": "" + } + }, + "1993399b93224ec6865eaa01b6a73d8b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_317f736683394e5cb974117fd8f5e7fe", + "placeholder": "​", + "style": "IPY_MODEL_a34b80886d56489cb82c36ba1415841c", + "value": "\nPro Tip: If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks.
" + } + }, + "b35a241e895c41ca8be0f334615df0a3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": "center", + "align_self": null, + "border": null, + "bottom": null, + "display": "flex", + "flex": null, + "flex_flow": "column", + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "50%" + } + }, + "99ef6333ea314434a1dd3ef05fa0e42b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ac1617fabea24f8a912e7cb6bb9abca4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "108c04b373e243218924ab3b101e8908": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "16019f898e814155ae1c27ab2475565a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d1df847b4fbe4ab992c64d5d2262cbcf": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "eeb36e1cf46b46328725481ce4b9a23b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "58c9a6969edf4b5ab636c0fdcb9b5b5b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "abe2bd79c6574e5aafa865bd18f14221": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ButtonStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ButtonStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "button_color": null, + "font_weight": "" + } + }, + "317f736683394e5cb974117fd8f5e7fe": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a34b80886d56489cb82c36ba1415841c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "25405341ff8949ad8da33761ed6f66fc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_494db21467a242abae3367fe33ae22e2", + "IPY_MODEL_5d7cbb32534a4237bc92aac8ea7b5f6a", + "IPY_MODEL_d9e0294296c744d19d89a99199de3498" + ], + "layout": "IPY_MODEL_c79628e8178041688dd3bf655160e36c" + } + }, + "494db21467a242abae3367fe33ae22e2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4254f2566bb84309bfc1f44485e07fd2", + "placeholder": "​", + "style": "IPY_MODEL_d1b2a7d252ae4a8992dc66578aad690b", + "value": "Downloading builder script: 100%" + } + }, + "5d7cbb32534a4237bc92aac8ea7b5f6a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_147f2cbac9414145a9d25c4c7279098c", + "max": 4203, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_f9a6c4b4ff744f7f9f88e9d9db1476ce", + "value": 4203 + } + }, + "d9e0294296c744d19d89a99199de3498": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9378346306f241de9a7820cb4d3083a6", + "placeholder": "​", + "style": "IPY_MODEL_f8f367707cfb4c51a06a96dfc73a5f8c", + "value": " 4.20k/4.20k [00:00<00:00, 367kB/s]" + } + }, + "c79628e8178041688dd3bf655160e36c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4254f2566bb84309bfc1f44485e07fd2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d1b2a7d252ae4a8992dc66578aad690b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "147f2cbac9414145a9d25c4c7279098c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f9a6c4b4ff744f7f9f88e9d9db1476ce": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "9378346306f241de9a7820cb4d3083a6": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f8f367707cfb4c51a06a96dfc73a5f8c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "af6c3c3e2bc240cab72aaa1e3f924925": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5e23e345354845b6aa8bb09240ed6121", + "IPY_MODEL_447b76e1644548e18c5a2c7252acd6fa", + "IPY_MODEL_e84886bc35784d45b07f12278d85a051" + ], + "layout": "IPY_MODEL_b1a98e3cdb1c4065824791888dae60dc" + } + }, + "5e23e345354845b6aa8bb09240ed6121": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8aac54d700ab440a95c1f1027316c7ab", + "placeholder": "​", + "style": "IPY_MODEL_aa7b1c90b7504bad8dcb81ad14aeabba", + "value": "Downloading readme: 100%" + } + }, + "447b76e1644548e18c5a2c7252acd6fa": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4903060e8ef24a039941fed7bfd249c1", + "max": 378, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d9c1fcbb414345988cc5dbb18bf0090f", + "value": 378 + } + }, + "e84886bc35784d45b07f12278d85a051": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_69dace5d98fb4916a3d8afbe1b95c291", + "placeholder": "​", + "style": "IPY_MODEL_4c37b8182e1d42c784564c8f592434a7", + "value": " 378/378 [00:00<00:00, 34.2kB/s]" + } + }, + "b1a98e3cdb1c4065824791888dae60dc": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8aac54d700ab440a95c1f1027316c7ab": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aa7b1c90b7504bad8dcb81ad14aeabba": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4903060e8ef24a039941fed7bfd249c1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d9c1fcbb414345988cc5dbb18bf0090f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "69dace5d98fb4916a3d8afbe1b95c291": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c37b8182e1d42c784564c8f592434a7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "82b80be2ae3444318c0b371696a0e7e4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c89f41dfdc11499790eb7b2898421788", + "IPY_MODEL_4e6a444b8787437bad081a0c601db38f", + "IPY_MODEL_15896bcd72ed4f2b83164986e4cf6a89" + ], + "layout": "IPY_MODEL_5a0fb282d639488ca6542405582552a4" + } + }, + "c89f41dfdc11499790eb7b2898421788": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_883a493d3294437ca0d5f575dd567b6a", + "placeholder": "​", + "style": "IPY_MODEL_59b070f51c7247699cb1d08ff826983e", + "value": "Downloading data files: 100%" + } + }, + "4e6a444b8787437bad081a0c601db38f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6140854720f242169351615b62d69ea2", + "max": 3, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_8763052dfc0f49a8a43b01e483c72efa", + "value": 3 + } + }, + "15896bcd72ed4f2b83164986e4cf6a89": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4a7727656e3c4a6c99620c462af236f2", + "placeholder": "​", + "style": "IPY_MODEL_7c99a33d66444ad6801e1ef45282ca77", + "value": " 3/3 [00:01<00:00, 1.98it/s]" + } + }, + "5a0fb282d639488ca6542405582552a4": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "883a493d3294437ca0d5f575dd567b6a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "59b070f51c7247699cb1d08ff826983e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6140854720f242169351615b62d69ea2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8763052dfc0f49a8a43b01e483c72efa": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "4a7727656e3c4a6c99620c462af236f2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7c99a33d66444ad6801e1ef45282ca77": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8ca983f1de9a42f6ac889638260ee546": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c61f32ac7513430fb4baf02ea477e0ca", + "IPY_MODEL_a1538109caf4497d82bc7a5161dbc132", + "IPY_MODEL_bf62c8aab16d4bcb956ef7aff31a712a" + ], + "layout": "IPY_MODEL_5dfa9e86e0e84c939bfee8a3a81e9f03" + } + }, + "c61f32ac7513430fb4baf02ea477e0ca": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9ab181750904882abebcd2ea17ade22", + "placeholder": "​", + "style": "IPY_MODEL_b1617dc6515a437b81e21ddcc8e212cf", + "value": "Downloading data: 100%" + } + }, + "a1538109caf4497d82bc7a5161dbc132": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2b2b57b77c514370919c0ed60101b491", + "max": 1071431, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_db866bc03c2440c2acc06b90295d8dc4", + "value": 1071431 + } + }, + "bf62c8aab16d4bcb956ef7aff31a712a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fd08faeb036f49179c9c409575b6829e", + "placeholder": "​", + "style": "IPY_MODEL_b8d05609e0c2462c9c29909b72e4ce4d", + "value": " 1.07M/1.07M [00:00<00:00, 3.39MB/s]" + } + }, + "5dfa9e86e0e84c939bfee8a3a81e9f03": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b9ab181750904882abebcd2ea17ade22": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b1617dc6515a437b81e21ddcc8e212cf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2b2b57b77c514370919c0ed60101b491": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "db866bc03c2440c2acc06b90295d8dc4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "fd08faeb036f49179c9c409575b6829e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b8d05609e0c2462c9c29909b72e4ce4d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a168593bb357422e9e5707a5180e26f5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a2414abd5de84802b6c30255d3aca777", + "IPY_MODEL_212c74bfd17b4b41b8c03b42d33672d5", + "IPY_MODEL_a62f55c8225641e69d21e5bdb072cb2f" + ], + "layout": "IPY_MODEL_869e78300e044d889fb3eb70ec6c3498" + } + }, + "a2414abd5de84802b6c30255d3aca777": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dbaba5bf8be84be2b41a3b66461716c9", + "placeholder": "​", + "style": "IPY_MODEL_728fef012809452fa4edde3ecd114fd2", + "value": "Downloading data: 100%" + } + }, + "212c74bfd17b4b41b8c03b42d33672d5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_32344c7f62a246a08b804b3b2697b858", + "max": 136292, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e9c9fbb0955548a1a95a06e1a50a4bba", + "value": 136292 + } + }, + "a62f55c8225641e69d21e5bdb072cb2f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0e00b9fb00b645ebbbd4444edfacaad7", + "placeholder": "​", + "style": "IPY_MODEL_fe79098c2b044816b8bc12cf5181d0b6", + "value": " 136k/136k [00:00<00:00, 255kB/s]" + } + }, + "869e78300e044d889fb3eb70ec6c3498": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "dbaba5bf8be84be2b41a3b66461716c9": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "728fef012809452fa4edde3ecd114fd2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "32344c7f62a246a08b804b3b2697b858": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9c9fbb0955548a1a95a06e1a50a4bba": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "0e00b9fb00b645ebbbd4444edfacaad7": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fe79098c2b044816b8bc12cf5181d0b6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fcb883d5a42f44d28f5c9024b2c7cb3c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_88a2de0b046443858bcf64207436ef61", + "IPY_MODEL_d5ce1c09377d4d65abe05a43b7248ef2", + "IPY_MODEL_4c1558b7aa3a4df0888bbbf53f7ace2e" + ], + "layout": "IPY_MODEL_4498e6513e374d1db7d4fdf3e19db8b2" + } + }, + "88a2de0b046443858bcf64207436ef61": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e90fd1f7a97f471d8d0e1002ea662fb5", + "placeholder": "​", + "style": "IPY_MODEL_a9e4fa39283f46ed97f005c170860e1e", + "value": "Downloading data: 100%" + } + }, + "d5ce1c09377d4d65abe05a43b7248ef2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ccaf3c7cbc404bcdb2df8766cafc6ca3", + "max": 281037, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_5ba057f74d6b4803aa10a274214516e1", + "value": 281037 + } + }, + "4c1558b7aa3a4df0888bbbf53f7ace2e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_03ee849ef4ce489cb38d7d1ef197c7f8", + "placeholder": "​", + "style": "IPY_MODEL_83a4acbc55fd4d829118a8623e886c34", + "value": " 281k/281k [00:00<00:00, 506kB/s]" + } + }, + "4498e6513e374d1db7d4fdf3e19db8b2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e90fd1f7a97f471d8d0e1002ea662fb5": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a9e4fa39283f46ed97f005c170860e1e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ccaf3c7cbc404bcdb2df8766cafc6ca3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5ba057f74d6b4803aa10a274214516e1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "03ee849ef4ce489cb38d7d1ef197c7f8": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "83a4acbc55fd4d829118a8623e886c34": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "be18b797b8f94720b8bbbef4ed6f8294": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_c5525330a50a492fa3ab4f5953add4a5", + "IPY_MODEL_b16f4c82e60644dcb3377816d368e350", + "IPY_MODEL_2e98dbfb059f45a29253745b04ce7571" + ], + "layout": "IPY_MODEL_cad57d29986d4326ab150f5b4d2c5410" + } + }, + "c5525330a50a492fa3ab4f5953add4a5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_92fd04a760f4480ebdc1716f26af1fea", + "placeholder": "​", + "style": "IPY_MODEL_198efd15631d4c7d8ef37368e261ccc6", + "value": "Extracting data files: 100%" + } + }, + "b16f4c82e60644dcb3377816d368e350": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_04a5bc70b79c487a9f766d6b799710f0", + "max": 3, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3ee22e5d88404568a798c33422eac2a1", + "value": 3 + } + }, + "2e98dbfb059f45a29253745b04ce7571": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_98e214e341144e518975429816963527", + "placeholder": "​", + "style": "IPY_MODEL_7e6cb607d3a94c72b57b985c19167a73", + "value": " 3/3 [00:00<00:00, 142.19it/s]" + } + }, + "cad57d29986d4326ab150f5b4d2c5410": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "92fd04a760f4480ebdc1716f26af1fea": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "198efd15631d4c7d8ef37368e261ccc6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "04a5bc70b79c487a9f766d6b799710f0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3ee22e5d88404568a798c33422eac2a1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "98e214e341144e518975429816963527": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7e6cb607d3a94c72b57b985c19167a73": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7b3fa3ef1631414caa526a500f10548b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_412d2001df7b478ca4ab15da3e1e5387", + "IPY_MODEL_a8ce332fc16140c88cf982276f3f8f2b", + "IPY_MODEL_f0d2bda0937f4e4bac51f7bf98de5188" + ], + "layout": "IPY_MODEL_673b1fcb790044c99d8b965b20ba5530" + } + }, + "412d2001df7b478ca4ab15da3e1e5387": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e24a51f9c8af474c91dbc82675757eee", + "placeholder": "​", + "style": "IPY_MODEL_8d03a24919f4489a8f026a95a0d98a29", + "value": "Generating train split: " + } + }, + "a8ce332fc16140c88cf982276f3f8f2b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_93ffb9e29cd54ce1b397a5eed8b1ceef", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_04228d1cb9094b6ba5c90d072ded66e1", + "value": 1 + } + }, + "f0d2bda0937f4e4bac51f7bf98de5188": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_09ffa024197c4654a080ea720281c685", + "placeholder": "​", + "style": "IPY_MODEL_c141d1452e08451baa5ab8a69f38bc80", + "value": " 6920/0 [00:00<00:00, 163091.53 examples/s]" + } + }, + "673b1fcb790044c99d8b965b20ba5530": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e24a51f9c8af474c91dbc82675757eee": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8d03a24919f4489a8f026a95a0d98a29": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "93ffb9e29cd54ce1b397a5eed8b1ceef": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "20px" + } + }, + "04228d1cb9094b6ba5c90d072ded66e1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "09ffa024197c4654a080ea720281c685": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c141d1452e08451baa5ab8a69f38bc80": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5f9b9c8ed9764beba4ca6f99326deacd": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b228adde847e4601b78b16554cb1060b", + "IPY_MODEL_f07333bb1174419b95d09a306e0969c4", + "IPY_MODEL_ae97fac586b94d3fbb448c37cd350538" + ], + "layout": "IPY_MODEL_fbd20e710796474fa92fd91c4f80da3e" + } + }, + "b228adde847e4601b78b16554cb1060b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3e6f0aea7fd044fab818aa0fb7517e1f", + "placeholder": "​", + "style": "IPY_MODEL_d5a4f8aecc5d43c29697b0a9052e339a", + "value": "Generating validation split: " + } + }, + "f07333bb1174419b95d09a306e0969c4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_eeaf377dcfea4954940b0da7f87cf008", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_31fef98c822544e3923aa51018d5d3cc", + "value": 1 + } + }, + "ae97fac586b94d3fbb448c37cd350538": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_45e6bfbf3c67469a974d4bd4d9b91595", + "placeholder": "​", + "style": "IPY_MODEL_ff710d6f22d049d6b20d96a3d4ae082b", + "value": " 872/0 [00:00<00:00, 42874.28 examples/s]" + } + }, + "fbd20e710796474fa92fd91c4f80da3e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3e6f0aea7fd044fab818aa0fb7517e1f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d5a4f8aecc5d43c29697b0a9052e339a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "eeaf377dcfea4954940b0da7f87cf008": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "20px" + } + }, + "31fef98c822544e3923aa51018d5d3cc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "45e6bfbf3c67469a974d4bd4d9b91595": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ff710d6f22d049d6b20d96a3d4ae082b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c16bb3d240f64a6dbfbdbd91b7fa27ca": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_50806132e381403d95214aa1ea8dcdda", + "IPY_MODEL_3c7feaf56edc404782bf6e7c1e02a31d", + "IPY_MODEL_de812255fa0044e7bb1f6d30f264203e" + ], + "layout": "IPY_MODEL_8e7325aae85441bfb5613a4a3bc9bd13" + } + }, + "50806132e381403d95214aa1ea8dcdda": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9930de3e02d946689e717f6821bf5bd0", + "placeholder": "​", + "style": "IPY_MODEL_3f102e34df7b4be2a85a6db22bbc2d88", + "value": "Generating test split: " + } + }, + "3c7feaf56edc404782bf6e7c1e02a31d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c9fe4932ebb6450195d0c5a936cd34b8", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2b3d9ab563104e8db5259079d4610760", + "value": 1 + } + }, + "de812255fa0044e7bb1f6d30f264203e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b1c9975483d04fd49a32a941846b3c19", + "placeholder": "​", + "style": "IPY_MODEL_88455de695bb4339abcafc29adb1e427", + "value": " 1821/0 [00:00<00:00, 82528.28 examples/s]" + } + }, + "8e7325aae85441bfb5613a4a3bc9bd13": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9930de3e02d946689e717f6821bf5bd0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3f102e34df7b4be2a85a6db22bbc2d88": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c9fe4932ebb6450195d0c5a936cd34b8": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": "20px" + } + }, + "2b3d9ab563104e8db5259079d4610760": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b1c9975483d04fd49a32a941846b3c19": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "88455de695bb4339abcafc29adb1e427": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "dc146e9b07ba4c7b90fe5118cb5aa0a9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_99a79a8f99724020b4d66e895cc9a772", + "IPY_MODEL_9f9d6787f0ea4acc8b9428513c92bcc5", + "IPY_MODEL_4500ae3ea21b4499b62262ef01cb0a24" + ], + "layout": "IPY_MODEL_a0b1fb701a5243c4bfaa02b19c7f347e" + } + }, + "99a79a8f99724020b4d66e895cc9a772": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d6d0461b752e4d95a7ee368f783c7908", + "placeholder": "​", + "style": "IPY_MODEL_6d06a18ec5e54eafb47df9ddd595a6c1", + "value": "Downloading (…)lve/main/config.json: 100%" + } + }, + "9f9d6787f0ea4acc8b9428513c92bcc5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bcac97d19e044d25acab914b12ada9f7", + "max": 748, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a0e50ee4759b421598d3e78d523642fa", + "value": 748 + } + }, + "4500ae3ea21b4499b62262ef01cb0a24": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_eae6863c13ed4179a19862dd9f8dd78e", + "placeholder": "​", + "style": "IPY_MODEL_8040a35914dc47519ea680c14a621903", + "value": " 748/748 [00:00<00:00, 66.5kB/s]" + } + }, + "a0b1fb701a5243c4bfaa02b19c7f347e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d6d0461b752e4d95a7ee368f783c7908": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6d06a18ec5e54eafb47df9ddd595a6c1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bcac97d19e044d25acab914b12ada9f7": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a0e50ee4759b421598d3e78d523642fa": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "eae6863c13ed4179a19862dd9f8dd78e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8040a35914dc47519ea680c14a621903": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "399418e7502445b98d7192e24ee94351": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_3a13cdf78533435b963cb43ad9fa5965", + "IPY_MODEL_380eca1553d144eb849dff5857015b02", + "IPY_MODEL_77da5e6e677a4d5d864184907cf3ec38" + ], + "layout": "IPY_MODEL_00ae52637a424295ac0d3330fb6d1744" + } + }, + "3a13cdf78533435b963cb43ad9fa5965": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d4d33ec061bc4901aaa088f2b9be67b3", + "placeholder": "​", + "style": "IPY_MODEL_d460b19e61ee446388a6e2abca5da9d0", + "value": "Downloading (…)831ea/.gitattributes: 100%" + } + }, + "380eca1553d144eb849dff5857015b02": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_adf02a94243f44ffa376f72c76bcc6ee", + "max": 1519, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_2e80bdbdbb6f4895ab7823dd9d0c7264", + "value": 1519 + } + }, + "77da5e6e677a4d5d864184907cf3ec38": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d6a0b910a6064ec296805253ff7d4353", + "placeholder": "​", + "style": "IPY_MODEL_32bd5b2693a14cbf8b2934d0809d75a7", + "value": " 1.52k/1.52k [00:00<00:00, 141kB/s]" + } + }, + "00ae52637a424295ac0d3330fb6d1744": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d4d33ec061bc4901aaa088f2b9be67b3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d460b19e61ee446388a6e2abca5da9d0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "adf02a94243f44ffa376f72c76bcc6ee": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2e80bdbdbb6f4895ab7823dd9d0c7264": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d6a0b910a6064ec296805253ff7d4353": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "32bd5b2693a14cbf8b2934d0809d75a7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4995e67d7019480f9b638c86e1890434": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f8b3819f461548138f5f944a62f3975a", + "IPY_MODEL_98ab5169667a4655b5690d3b7ef20163", + "IPY_MODEL_4f35804a8a634ff58049168aecccb5c5" + ], + "layout": "IPY_MODEL_d9cf86bf5cb141f9b9b9788630aeb628" + } + }, + "f8b3819f461548138f5f944a62f3975a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4c32f663032c40c3995507e02bbe2a8b", + "placeholder": "​", + "style": "IPY_MODEL_3def1ccbd0f2496190c05cbf1f6a6d56", + "value": "Downloading (…)_Pooling/config.json: 100%" + } + }, + "98ab5169667a4655b5690d3b7ef20163": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8abb3e4520474d5f84eb2985d4945935", + "max": 190, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_97c6990e29774cbca80699adde648e47", + "value": 190 + } + }, + "4f35804a8a634ff58049168aecccb5c5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_64001a83afb24177b6c3648a453bb06a", + "placeholder": "​", + "style": "IPY_MODEL_712b0a9c7db04af18680fe57b12a6399", + "value": " 190/190 [00:00<00:00, 18.2kB/s]" + } + }, + "d9cf86bf5cb141f9b9b9788630aeb628": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c32f663032c40c3995507e02bbe2a8b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3def1ccbd0f2496190c05cbf1f6a6d56": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8abb3e4520474d5f84eb2985d4945935": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "97c6990e29774cbca80699adde648e47": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "64001a83afb24177b6c3648a453bb06a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "712b0a9c7db04af18680fe57b12a6399": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c6c88bd7a68044cc8b6c79666bd3f604": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_3719de4ae5ac4ed58a359c713c6351bf", + "IPY_MODEL_1ce663c3d8ac43f38e9a1cdb660fc328", + "IPY_MODEL_0f752900bf484d8bb36f0c3f6a81c65e" + ], + "layout": "IPY_MODEL_f8a0d2ea13294358988c6ef6849e8745" + } + }, + "3719de4ae5ac4ed58a359c713c6351bf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c620f73097504444aba0d692f0b1698f", + "placeholder": "​", + "style": "IPY_MODEL_9ad9981dd31b41fc9792a95dc3ca096f", + "value": "Downloading (…)1c222831ea/README.md: 100%" + } + }, + "1ce663c3d8ac43f38e9a1cdb660fc328": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a7aa45bdc2a24b18adbacf20921eb344", + "max": 2222, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_abab565070934031b0112ef27ac01d7b", + "value": 2222 + } + }, + "0f752900bf484d8bb36f0c3f6a81c65e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ce2b600a91da4b4e829f4f4e96d863cd", + "placeholder": "​", + "style": "IPY_MODEL_745525d6b9c24149bfc54e7f10dd3540", + "value": " 2.22k/2.22k [00:00<00:00, 212kB/s]" + } + }, + "f8a0d2ea13294358988c6ef6849e8745": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c620f73097504444aba0d692f0b1698f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9ad9981dd31b41fc9792a95dc3ca096f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a7aa45bdc2a24b18adbacf20921eb344": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "abab565070934031b0112ef27ac01d7b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "ce2b600a91da4b4e829f4f4e96d863cd": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "745525d6b9c24149bfc54e7f10dd3540": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9c7ef01f9d6e4561a7822a063edfc26a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_3d37acbb3b68444dbd68849e46d55a61", + "IPY_MODEL_e25e63d49be94535a41a6425796e8093", + "IPY_MODEL_bb1219f567c24b0e84231b30b0d4670e" + ], + "layout": "IPY_MODEL_81907ab8a98b4763b26d23caf1f7252d" + } + }, + "3d37acbb3b68444dbd68849e46d55a61": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1a4a4cae5ef948ca950d3ac9b4ff9947", + "placeholder": "​", + "style": "IPY_MODEL_30a0606700fc424b8bd30a194002ee03", + "value": "Downloading (…)222831ea/config.json: 100%" + } + }, + "e25e63d49be94535a41a6425796e8093": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ca995a79154848c6b0d64214832cf29c", + "max": 748, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_327f0119d2934623bb31eec9afb92717", + "value": 748 + } + }, + "bb1219f567c24b0e84231b30b0d4670e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a246c6dc9d6c44579bcabed1fd63e052", + "placeholder": "​", + "style": "IPY_MODEL_da4e4803b1ee43899aa7c6cf5bce3c0e", + "value": " 748/748 [00:00<00:00, 67.7kB/s]" + } + }, + "81907ab8a98b4763b26d23caf1f7252d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1a4a4cae5ef948ca950d3ac9b4ff9947": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30a0606700fc424b8bd30a194002ee03": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ca995a79154848c6b0d64214832cf29c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "327f0119d2934623bb31eec9afb92717": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a246c6dc9d6c44579bcabed1fd63e052": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "da4e4803b1ee43899aa7c6cf5bce3c0e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0f2fc48655bf4039ae0f82c1aed4ecd6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_62250f331dc541ebbb347c9bf5cb89e7", + "IPY_MODEL_40c6fb85a6454b7080ba69307c4c05d9", + "IPY_MODEL_4d48891cfe0749dba0e2b4ed3aec7b81" + ], + "layout": "IPY_MODEL_15a8d50ddfa74c87aa64cd59d6708275" + } + }, + "62250f331dc541ebbb347c9bf5cb89e7": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5cc27f55bf5544cd9441e2f222183a9b", + "placeholder": "​", + "style": "IPY_MODEL_9eae3d0f8d6545bdaafca20189423455", + "value": "Downloading (…)ce_transformers.json: 100%" + } + }, + "40c6fb85a6454b7080ba69307c4c05d9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5eddb4176e6f4f9ba8a210af56faaeaa", + "max": 124, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d244786bc24147bf85682feaa17aab16", + "value": 124 + } + }, + "4d48891cfe0749dba0e2b4ed3aec7b81": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2c0644b1f1714c2bbdcf143684ec8f9b", + "placeholder": "​", + "style": "IPY_MODEL_5a01c037c4074fbbb86597a3c77e9ba8", + "value": " 124/124 [00:00<00:00, 9.36kB/s]" + } + }, + "15a8d50ddfa74c87aa64cd59d6708275": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5cc27f55bf5544cd9441e2f222183a9b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9eae3d0f8d6545bdaafca20189423455": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5eddb4176e6f4f9ba8a210af56faaeaa": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d244786bc24147bf85682feaa17aab16": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2c0644b1f1714c2bbdcf143684ec8f9b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5a01c037c4074fbbb86597a3c77e9ba8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cd1768dee5c248948510056b2509901c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_77693468af0e4e049a034586c91fbd8e", + "IPY_MODEL_76750d33664b40069f5f326c4e2ea1bb", + "IPY_MODEL_dcc1295b86c44aa3aef9ca95bf1de1f0" + ], + "layout": "IPY_MODEL_05c33c32b81d4b4999b9528ee26b61f5" + } + }, + "77693468af0e4e049a034586c91fbd8e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ee07c09131ca4d918262a8356f48be1d", + "placeholder": "​", + "style": "IPY_MODEL_9d1bdad0bca1488ea52b1b25023b1264", + "value": "Downloading model_head.pkl: 100%" + } + }, + "76750d33664b40069f5f326c4e2ea1bb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8b8c2dbc3bf5407db755cdf32a238833", + "max": 3919, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_9271a7f8736f4c6d932197eff416edeb", + "value": 3919 + } + }, + "dcc1295b86c44aa3aef9ca95bf1de1f0": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_43614ce55026485cb6cccb9164f2f9f3", + "placeholder": "​", + "style": "IPY_MODEL_668f3676156d4da4ab628634dd6788a1", + "value": " 3.92k/3.92k [00:00<00:00, 340kB/s]" + } + }, + "05c33c32b81d4b4999b9528ee26b61f5": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ee07c09131ca4d918262a8356f48be1d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9d1bdad0bca1488ea52b1b25023b1264": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8b8c2dbc3bf5407db755cdf32a238833": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9271a7f8736f4c6d932197eff416edeb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "43614ce55026485cb6cccb9164f2f9f3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "668f3676156d4da4ab628634dd6788a1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "aa8820f46e7d46138e5d772dc52b9944": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5a2e9682ea2c42dd980d465f91236b82", + "IPY_MODEL_110d200e8f454daf98575056aa9e27f1", + "IPY_MODEL_edfec7c81c6e4ed19da71dde6f0689a6" + ], + "layout": "IPY_MODEL_0d86d77967fb4c009ad6988aefed07d4" + } + }, + "5a2e9682ea2c42dd980d465f91236b82": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_54720c63b00d4515a49a00d469e0e81b", + "placeholder": "​", + "style": "IPY_MODEL_f505a56350ea42618c3ea7a60dc017ec", + "value": "Downloading pytorch_model.bin: 100%" + } + }, + "110d200e8f454daf98575056aa9e27f1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1a77f99e83d047f482eb3506aab7c61c", + "max": 133507174, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4f0cff377f414596b7d6d61256896be1", + "value": 133507174 + } + }, + "edfec7c81c6e4ed19da71dde6f0689a6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d95b5633c5e74bdab7791f427537ac12", + "placeholder": "​", + "style": "IPY_MODEL_a1d201343b324c4386c044aa32d0d076", + "value": " 134M/134M [00:03<00:00, 53.3MB/s]" + } + }, + "0d86d77967fb4c009ad6988aefed07d4": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "54720c63b00d4515a49a00d469e0e81b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f505a56350ea42618c3ea7a60dc017ec": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1a77f99e83d047f482eb3506aab7c61c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4f0cff377f414596b7d6d61256896be1": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d95b5633c5e74bdab7791f427537ac12": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a1d201343b324c4386c044aa32d0d076": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4ce6dfd9090b433982962b6c23332bc9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d5b25ac6c2b540d29583dbf3cf118ee5", + "IPY_MODEL_9cdfee8f567a4c459cd229e9641b6d37", + "IPY_MODEL_39cf3db0efda4676bf5ec7aad7ca37be" + ], + "layout": "IPY_MODEL_40390da1ecb24416b031a72af0ff82ab" + } + }, + "d5b25ac6c2b540d29583dbf3cf118ee5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1e06fe633c2c4ed79e219c3541e6740f", + "placeholder": "​", + "style": "IPY_MODEL_f468411955b648fb8b472f4aafa26765", + "value": "Downloading (…)nce_bert_config.json: 100%" + } + }, + "9cdfee8f567a4c459cd229e9641b6d37": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a64b96e1935e46339b02ef664f693c6d", + "max": 52, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_55c23dad05c14e0fa5255ea3cc3d774f", + "value": 52 + } + }, + "39cf3db0efda4676bf5ec7aad7ca37be": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_401fb1beebf042c18160a7b5eac59841", + "placeholder": "​", + "style": "IPY_MODEL_b48520d0aa7d40339cf7f52105d8c396", + "value": " 52.0/52.0 [00:00<00:00, 4.85kB/s]" + } + }, + "40390da1ecb24416b031a72af0ff82ab": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1e06fe633c2c4ed79e219c3541e6740f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f468411955b648fb8b472f4aafa26765": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a64b96e1935e46339b02ef664f693c6d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "55c23dad05c14e0fa5255ea3cc3d774f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "401fb1beebf042c18160a7b5eac59841": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b48520d0aa7d40339cf7f52105d8c396": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "33c005a5f12448ceb283f358d2cf4adc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6770b60a09804294a8b4f966888db00f", + "IPY_MODEL_e54de119f490455e8c4fc164975b4e3e", + "IPY_MODEL_f92f2aa2c16d49a993ccfc2bc2a6c405" + ], + "layout": "IPY_MODEL_3b4dc1919db346b8bcd384cc1261438d" + } + }, + "6770b60a09804294a8b4f966888db00f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_55039a4737164130ab1ee6cb29ffa786", + "placeholder": "​", + "style": "IPY_MODEL_df30813516c644dab488bd09a4aa1a86", + "value": "Downloading (…)cial_tokens_map.json: 100%" + } + }, + "e54de119f490455e8c4fc164975b4e3e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_88d26f1a452c4733bc1427e56e62f8dc", + "max": 125, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_865fe252fd594d7d927365e14573cd1e", + "value": 125 + } + }, + "f92f2aa2c16d49a993ccfc2bc2a6c405": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c6d7fbc357e2468f94811a2a0249cb01", + "placeholder": "​", + "style": "IPY_MODEL_2bdd98ffe4e942f08f6d81fd61dad7ee", + "value": " 125/125 [00:00<00:00, 10.5kB/s]" + } + }, + "3b4dc1919db346b8bcd384cc1261438d": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "55039a4737164130ab1ee6cb29ffa786": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "df30813516c644dab488bd09a4aa1a86": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "88d26f1a452c4733bc1427e56e62f8dc": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "865fe252fd594d7d927365e14573cd1e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c6d7fbc357e2468f94811a2a0249cb01": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2bdd98ffe4e942f08f6d81fd61dad7ee": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2fe48b1d327544b78aee28e1f871d7fc": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_44880c7b99764d14add9f31903bea9ce", + "IPY_MODEL_e16088132c1c4ba7886449e35f338372", + "IPY_MODEL_52f05d88a6254d398eca1196f5e031b2" + ], + "layout": "IPY_MODEL_9a66a3c87df14440b547ac9772899b38" + } + }, + "44880c7b99764d14add9f31903bea9ce": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3514155994664dca8a9e1cfd136d1f32", + "placeholder": "​", + "style": "IPY_MODEL_1bed6c4ad69a48b19180c1e6e7af6123", + "value": "Downloading (…)831ea/tokenizer.json: 100%" + } + }, + "e16088132c1c4ba7886449e35f338372": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e079d8a609e745a4a701e6b998119610", + "max": 711649, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6b81b872347b484d8624422f729c357f", + "value": 711649 + } + }, + "52f05d88a6254d398eca1196f5e031b2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7316bc0130fa40359ad26efba5b22b3e", + "placeholder": "​", + "style": "IPY_MODEL_391b543200884ccd85dbc4ea89b8917d", + "value": " 712k/712k [00:00<00:00, 4.35MB/s]" + } + }, + "9a66a3c87df14440b547ac9772899b38": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3514155994664dca8a9e1cfd136d1f32": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1bed6c4ad69a48b19180c1e6e7af6123": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e079d8a609e745a4a701e6b998119610": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6b81b872347b484d8624422f729c357f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7316bc0130fa40359ad26efba5b22b3e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "391b543200884ccd85dbc4ea89b8917d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "047f10baf53c4023af28934a847021e2": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1fae7f1a5d9c492296f505877445ef0b", + "IPY_MODEL_65d3ca6bc3e44628b4ad7527d4c02294", + "IPY_MODEL_67782dd666ee4eaab8960e46826677fa" + ], + "layout": "IPY_MODEL_32c40d3b63c74f0da5d64bed6d216f2c" + } + }, + "1fae7f1a5d9c492296f505877445ef0b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0520ac5668a445c585d9476f83ec7088", + "placeholder": "​", + "style": "IPY_MODEL_4a84b8b89dc443999b362a06c762569b", + "value": "Downloading (…)okenizer_config.json: 100%" + } + }, + "65d3ca6bc3e44628b4ad7527d4c02294": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e7f9835056ce44cf974a987ff571aaf5", + "max": 1270, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1a2f951678b54010a96e393738c4a2f5", + "value": 1270 + } + }, + "67782dd666ee4eaab8960e46826677fa": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_54eac4acad6a4f44b9839442866b1641", + "placeholder": "​", + "style": "IPY_MODEL_7fc2664adffe40e295fb14bffc3c53e3", + "value": " 1.27k/1.27k [00:00<00:00, 120kB/s]" + } + }, + "32c40d3b63c74f0da5d64bed6d216f2c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0520ac5668a445c585d9476f83ec7088": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4a84b8b89dc443999b362a06c762569b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e7f9835056ce44cf974a987ff571aaf5": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1a2f951678b54010a96e393738c4a2f5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "54eac4acad6a4f44b9839442866b1641": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7fc2664adffe40e295fb14bffc3c53e3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e2f4617e834644d0b3211e9a1d964bbe": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_009b5b89fa4947319825034d458786ff", + "IPY_MODEL_7062248300d049fcb6aab73529535864", + "IPY_MODEL_7cecaeee3d2a44ed844bce978daef136" + ], + "layout": "IPY_MODEL_45efd6ebc60645dd83797c225184a92c" + } + }, + "009b5b89fa4947319825034d458786ff": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_819fd904bf4d44a09eebdc05bbf8d39b", + "placeholder": "​", + "style": "IPY_MODEL_e00957ae947f4ca298aa9563ceb50446", + "value": "Downloading (…)1c222831ea/vocab.txt: 100%" + } + }, + "7062248300d049fcb6aab73529535864": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b69d0bc437b449ce803e56d97a2d57f0", + "max": 231508, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_b86c025377c64e218717075bcdbcac33", + "value": 231508 + } + }, + "7cecaeee3d2a44ed844bce978daef136": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9c0e0201bca498eb65eddae56412b1b", + "placeholder": "​", + "style": "IPY_MODEL_e5e361b8bdf7435cb92149993df8ddf9", + "value": " 232k/232k [00:00<00:00, 2.83MB/s]" + } + }, + "45efd6ebc60645dd83797c225184a92c": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "819fd904bf4d44a09eebdc05bbf8d39b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e00957ae947f4ca298aa9563ceb50446": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b69d0bc437b449ce803e56d97a2d57f0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b86c025377c64e218717075bcdbcac33": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b9c0e0201bca498eb65eddae56412b1b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e5e361b8bdf7435cb92149993df8ddf9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5572121bcbd94107929ed5dff76a6cbd": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5fbb7145a5f74e88bfeadc30f6ea96ef", + "IPY_MODEL_5a10ccd98b724b7a862b343e1e0010c9", + "IPY_MODEL_a548ed5b6a3e42fbb4d9e288a97e9b27" + ], + "layout": "IPY_MODEL_11bef2742d6a4f53a37640ab7ca84c49" + } + }, + "5fbb7145a5f74e88bfeadc30f6ea96ef": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_097d394b44cb4e318a11f955d0c63ef2", + "placeholder": "​", + "style": "IPY_MODEL_9de476dd14c94060b50ed96c70120747", + "value": "Downloading (…)22831ea/modules.json: 100%" + } + }, + "5a10ccd98b724b7a862b343e1e0010c9": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fdfdce50a81f451aa2dbeb02fe226a5f", + "max": 349, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_f2ca6bddd40a468396c6b0fa24d92785", + "value": 349 + } + }, + "a548ed5b6a3e42fbb4d9e288a97e9b27": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7602b79489c6406eb087f045ecef9e31", + "placeholder": "​", + "style": "IPY_MODEL_55ef22f22dbc44d59ce1abd5bed5d174", + "value": " 349/349 [00:00<00:00, 29.3kB/s]" + } + }, + "11bef2742d6a4f53a37640ab7ca84c49": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "097d394b44cb4e318a11f955d0c63ef2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9de476dd14c94060b50ed96c70120747": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fdfdce50a81f451aa2dbeb02fe226a5f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f2ca6bddd40a468396c6b0fa24d92785": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7602b79489c6406eb087f045ecef9e31": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "55ef22f22dbc44d59ce1abd5bed5d174": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c24ded7ea4674b9ab404a541f82f5e23": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_987d0f782cc84d24a723f0640c2c6b7b", + "IPY_MODEL_5bbe003313d541b196b62765ffc8cdeb", + "IPY_MODEL_c69a869a500b4a31bd82166a5f5320c6" + ], + "layout": "IPY_MODEL_88bf04ac3b9448c9b4bd1f1f381fa4f3" + } + }, + "987d0f782cc84d24a723f0640c2c6b7b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5f0725d5ecd2452ab2dabbb9a6809329", + "placeholder": "​", + "style": "IPY_MODEL_2322b2da29634875b67851042da03a9a", + "value": "Downloading model_head.pkl: 100%" + } + }, + "5bbe003313d541b196b62765ffc8cdeb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cfa79548ffee42949bf10d718321adfd", + "max": 3919, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4a776c2df1d249f8a61f9663092de047", + "value": 3919 + } + }, + "c69a869a500b4a31bd82166a5f5320c6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1223136c2471432e84b1d700d9dcee72", + "placeholder": "​", + "style": "IPY_MODEL_9e7eaa3b76a54091b189c3465248460b", + "value": " 3.92k/3.92k [00:00<00:00, 333kB/s]" + } + }, + "88bf04ac3b9448c9b4bd1f1f381fa4f3": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5f0725d5ecd2452ab2dabbb9a6809329": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2322b2da29634875b67851042da03a9a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "cfa79548ffee42949bf10d718321adfd": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4a776c2df1d249f8a61f9663092de047": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1223136c2471432e84b1d700d9dcee72": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9e7eaa3b76a54091b189c3465248460b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "eb6e6d91566044f18e953d8df93ae3a8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_29dbf37d879a4f05904b076378d190bf", + "IPY_MODEL_1cfb1e7981264c328e02e49c4080c504", + "IPY_MODEL_e1157a679f354eaba73d4fea3cbda6fe" + ], + "layout": "IPY_MODEL_a9eafe51348847b0be895b8502a21e56" + } + }, + "29dbf37d879a4f05904b076378d190bf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f338c914be9641a4aee76971c0e9cf72", + "placeholder": "​", + "style": "IPY_MODEL_746f408b617a4974ab48665f5b0c6c01", + "value": "Downloading (…)lve/main/config.json: 100%" + } + }, + "1cfb1e7981264c328e02e49c4080c504": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_561acc35f4e245e8adf81b1efa510fd2", + "max": 629, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_85377644ee164abebcfb24c7fdeaab8a", + "value": 629 + } + }, + "e1157a679f354eaba73d4fea3cbda6fe": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b800f7b6b2fe448486f63a24066694aa", + "placeholder": "​", + "style": "IPY_MODEL_0661f3e76c9a490db89c29c35d8c0d1b", + "value": " 629/629 [00:00<00:00, 56.8kB/s]" + } + }, + "a9eafe51348847b0be895b8502a21e56": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f338c914be9641a4aee76971c0e9cf72": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "746f408b617a4974ab48665f5b0c6c01": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "561acc35f4e245e8adf81b1efa510fd2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "85377644ee164abebcfb24c7fdeaab8a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b800f7b6b2fe448486f63a24066694aa": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0661f3e76c9a490db89c29c35d8c0d1b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7612716749c14069bc5989f3ee09c00a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_0e9bfe8c83a44537bb59c444744466ef", + "IPY_MODEL_5e8641e8da904126a3680828c83774e8", + "IPY_MODEL_2f99603c364b4af48ea25b8d43cd1941" + ], + "layout": "IPY_MODEL_26ae8872c49642c588330b34ff470609" + } + }, + "0e9bfe8c83a44537bb59c444744466ef": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a8ee2fe1c3d04082acdafbf39ba0076b", + "placeholder": "​", + "style": "IPY_MODEL_5e9bad63ccc14fe2b358b7c7f0bbae68", + "value": "Downloading model.safetensors: 100%" + } + }, + "5e8641e8da904126a3680828c83774e8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_50597a7caa0242df82fafa89bf831827", + "max": 267832558, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_48ba967e29ad45a1ac8f7ba659fb5a5f", + "value": 267832558 + } + }, + "2f99603c364b4af48ea25b8d43cd1941": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_788dc688a9e9449e9aeb431b06384d9f", + "placeholder": "​", + "style": "IPY_MODEL_c82bc76766fd420a9557c4dcd78cff0c", + "value": " 268M/268M [00:00<00:00, 476MB/s]" + } + }, + "26ae8872c49642c588330b34ff470609": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a8ee2fe1c3d04082acdafbf39ba0076b": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5e9bad63ccc14fe2b358b7c7f0bbae68": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "50597a7caa0242df82fafa89bf831827": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "48ba967e29ad45a1ac8f7ba659fb5a5f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "788dc688a9e9449e9aeb431b06384d9f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c82bc76766fd420a9557c4dcd78cff0c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f52944fb4da049afae19d9d1f4a6b37a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9cee60b1935441e4a81d26cc5caa319e", + "IPY_MODEL_4621e316677d44caab491fd8a7b1433f", + "IPY_MODEL_39bea84d46004ecd9dae38e81c4bb2d8" + ], + "layout": "IPY_MODEL_5859b86234164b55aa97852b4368bd66" + } + }, + "9cee60b1935441e4a81d26cc5caa319e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9fd724ea0d6947be92361604c9f318a7", + "placeholder": "​", + "style": "IPY_MODEL_996b172b2f434b5a95de4a839c2ae4ae", + "value": "Downloading (…)okenizer_config.json: 100%" + } + }, + "4621e316677d44caab491fd8a7b1433f": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ff8e804c7b5e472187685d1fdcc5d0ff", + "max": 48, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_764f957750064a7eb61d5e6b28e0e49d", + "value": 48 + } + }, + "39bea84d46004ecd9dae38e81c4bb2d8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d46affe5b0ff4914bc2504c2737c6d40", + "placeholder": "​", + "style": "IPY_MODEL_350d6653f3d947b8a125b051b36e9d4a", + "value": " 48.0/48.0 [00:00<00:00, 4.45kB/s]" + } + }, + "5859b86234164b55aa97852b4368bd66": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9fd724ea0d6947be92361604c9f318a7": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "996b172b2f434b5a95de4a839c2ae4ae": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ff8e804c7b5e472187685d1fdcc5d0ff": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "764f957750064a7eb61d5e6b28e0e49d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d46affe5b0ff4914bc2504c2737c6d40": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "350d6653f3d947b8a125b051b36e9d4a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e9e9cb6c63c54831acf6922003883421": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1d9e7ff2e4d74891a3f70798bb5266eb", + "IPY_MODEL_55135c120cc94702be910b033f6df07d", + "IPY_MODEL_a01975d1f8154082bd903bb55b3d49d6" + ], + "layout": "IPY_MODEL_26db4649e9924203912f92e0e1c994bb" + } + }, + "1d9e7ff2e4d74891a3f70798bb5266eb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2261900d8f594e7b965fe8d4d6264e35", + "placeholder": "​", + "style": "IPY_MODEL_14f43841b87646dd9c8eca2966488900", + "value": "Downloading (…)solve/main/vocab.txt: 100%" + } + }, + "55135c120cc94702be910b033f6df07d": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c7a0c077c06e4344bdffe4848de56e15", + "max": 231508, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_581e9a90552543188ec3c95b4cdab508", + "value": 231508 + } + }, + "a01975d1f8154082bd903bb55b3d49d6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_15b87e982dcf45139029b93eb6a7f857", + "placeholder": "​", + "style": "IPY_MODEL_dc42e38e51464a2095b179359da4c816", + "value": " 232k/232k [00:00<00:00, 2.79MB/s]" + } + }, + "26db4649e9924203912f92e0e1c994bb": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2261900d8f594e7b965fe8d4d6264e35": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "14f43841b87646dd9c8eca2966488900": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c7a0c077c06e4344bdffe4848de56e15": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "581e9a90552543188ec3c95b4cdab508": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "15b87e982dcf45139029b93eb6a7f857": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "dc42e38e51464a2095b179359da4c816": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "12a4e79070444acbbb228aba4e34a25e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f8bfdfce5edc4c6397e22b90eea50817", + "IPY_MODEL_edd2f3f9a691462dbfb703ca072fd644", + "IPY_MODEL_70fd4d6c76d348fa82ac29e416c94f2c" + ], + "layout": "IPY_MODEL_8cefab222736437289c985ffbd6a1ed5" + } + }, + "f8bfdfce5edc4c6397e22b90eea50817": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b3eaa19e1a97474995de388a44bc18f1", + "placeholder": "​", + "style": "IPY_MODEL_71a105f4761549fc898c0a4b03fbb043", + "value": "100%" + } + }, + "edd2f3f9a691462dbfb703ca072fd644": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d3c2759bf4324ad78eda1aa61d9c3fb0", + "max": 9, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_37d8b6b1ea044b7680b715ec31b48d8e", + "value": 9 + } + }, + "70fd4d6c76d348fa82ac29e416c94f2c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dfbde307f9f84be1b413b47248410438", + "placeholder": "​", + "style": "IPY_MODEL_bc3d582d8e394397897f44efe51dd333", + "value": " 9/9 [00:00<00:00, 19.41it/s]" + } + }, + "8cefab222736437289c985ffbd6a1ed5": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b3eaa19e1a97474995de388a44bc18f1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "71a105f4761549fc898c0a4b03fbb043": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d3c2759bf4324ad78eda1aa61d9c3fb0": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "37d8b6b1ea044b7680b715ec31b48d8e": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "dfbde307f9f84be1b413b47248410438": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bc3d582d8e394397897f44efe51dd333": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + } + } + }, + "accelerator": "GPU" + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From 7c3feeda54b5be9ee0a3c1c1e7f8dcd2faf8691c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 13:34:42 +0100 Subject: [PATCH 062/183] Don't scale gradients during evaluation --- src/setfit/trainer.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 9b1b2333..d8828f46 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -601,7 +601,7 @@ def _train_sentence_transformer( self.control = self.log(args, metrics) eval_loss = None - if self.control.should_evaluate and eval_dataloader: + if self.control.should_evaluate and eval_dataloader is not None: eval_loss = self._evaluate_with_loss(model_body, eval_dataloader, args, loss_func) learning_rate = scheduler_obj.get_last_lr()[0] metrics = {"eval_embedding_loss": round(eval_loss, 4), "learning_rate": learning_rate} @@ -654,12 +654,8 @@ def _evaluate_with_loss( loss_func: nn.Module, ) -> float: model_body.eval() - - if args.use_amp: - scaler = torch.cuda.amp.GradScaler() - losses = [] - for data in tqdm(iter(eval_dataloader), leave=False, disable=not args.show_progress_bar): + for data in tqdm(iter(eval_dataloader), total=len(eval_dataloader), leave=False, disable=not args.show_progress_bar): features, labels = data labels = labels.to(model_body._target_device) features = list(map(lambda batch: batch_to_device(batch, model_body._target_device), features)) @@ -668,7 +664,7 @@ def _evaluate_with_loss( with autocast(): loss_value = loss_func(features, labels) - losses.append(scaler.scale(loss_value).item()) + losses.append(loss_value.item()) else: losses.append(loss_func(features, labels).item()) From cdc8979e0c641bd2324662a081462f5085900738 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 13:34:59 +0100 Subject: [PATCH 063/183] Use evaluation_strategy="steps" if eval_steps is set --- src/setfit/training_args.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 73fd95f8..5336bf4c 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -13,6 +13,9 @@ from transformers.training_args import default_logdir from transformers.utils import is_torch_available +from . import logging + +logger = logging.get_logger(__name__) @dataclass class TrainingArguments: @@ -257,6 +260,10 @@ def __post_init__(self) -> None: self.logging_strategy = IntervalStrategy(self.logging_strategy) self.evaluation_strategy = IntervalStrategy(self.evaluation_strategy) + if self.eval_steps is not None and self.evaluation_strategy == IntervalStrategy.NO: + logger.info("Using `evaluation_strategy=\"steps\"` as `eval_steps` is defined.") + self.evaluation_strategy = IntervalStrategy.STEPS + # eval_steps has to be defined and non-zero, fallbacks to logging_steps if the latter is non-zero if self.evaluation_strategy == IntervalStrategy.STEPS and (self.eval_steps is None or self.eval_steps == 0): if self.logging_steps > 0: From e0401674fde5532f2d0076774da6663527e5620f Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 13:35:42 +0100 Subject: [PATCH 064/183] Run formatting --- src/setfit/sampler.py | 2 +- src/setfit/trainer.py | 6 ++++-- src/setfit/trainer_distillation.py | 4 ++-- src/setfit/training_args.py | 4 +++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/setfit/sampler.py b/src/setfit/sampler.py index 1bea2e78..0eceba1e 100644 --- a/src/setfit/sampler.py +++ b/src/setfit/sampler.py @@ -2,8 +2,8 @@ from typing import Generator, Iterable, List, Optional import numpy as np -from sentence_transformers import InputExample import torch +from sentence_transformers import InputExample from torch.utils.data import IterableDataset from . import logging diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index d8828f46..848796b6 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -374,7 +374,7 @@ def train( parameters.extend(self.dataset_to_parameters(dataset)) self.train_embeddings(*parameters, args=args) - training_parameters = parameters[:len(parameters) // 2] if self.eval_dataset else parameters + training_parameters = parameters[: len(parameters) // 2] if self.eval_dataset else parameters self.train_classifier(*training_parameters, args=args) def dataset_to_parameters(self, dataset: Dataset) -> List[Iterable]: @@ -655,7 +655,9 @@ def _evaluate_with_loss( ) -> float: model_body.eval() losses = [] - for data in tqdm(iter(eval_dataloader), total=len(eval_dataloader), leave=False, disable=not args.show_progress_bar): + for data in tqdm( + iter(eval_dataloader), total=len(eval_dataloader), leave=False, disable=not args.show_progress_bar + ): features, labels = data labels = labels.to(model_body._target_device) features = list(map(lambda batch: batch_to_device(batch, model_body._target_device), features)) diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index 2bc2d629..da5ec4b8 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -1,9 +1,9 @@ import warnings from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Union -from datasets import Dataset import torch -from sentence_transformers import losses, util, InputExample +from datasets import Dataset +from sentence_transformers import InputExample, losses, util from torch import nn from torch.utils.data import DataLoader diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 5336bf4c..9ed24fb7 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -15,8 +15,10 @@ from . import logging + logger = logging.get_logger(__name__) + @dataclass class TrainingArguments: """ @@ -261,7 +263,7 @@ def __post_init__(self) -> None: self.evaluation_strategy = IntervalStrategy(self.evaluation_strategy) if self.eval_steps is not None and self.evaluation_strategy == IntervalStrategy.NO: - logger.info("Using `evaluation_strategy=\"steps\"` as `eval_steps` is defined.") + logger.info('Using `evaluation_strategy="steps"` as `eval_steps` is defined.') self.evaluation_strategy = IntervalStrategy.STEPS # eval_steps has to be defined and non-zero, fallbacks to logging_steps if the latter is non-zero From d2f24896dc8b95ce21686518047a99ff6bf5bdcd Mon Sep 17 00:00:00 2001 From: Tom Aarsen <37621491+tomaarsen@users.noreply.github.com> Date: Thu, 9 Nov 2023 21:22:03 +0100 Subject: [PATCH 065/183] Implement SetFit for ABSA from Intel Labs (#6) * Initial version for SetFit ABSA * Create complete test suite for ABSA (100%,90%,96%) Only push_to_hub is not under test * Run formatting * Allow initializing models with different span_context * Remove resolved TODO * Raise error if args is the wrong type * Update column mapping, allow partial maps * Remove model_init from ABSA Trainer, not used * Split train into train_aspect and train_polarity And reformat * Prefix logs with aspect/polarity when training * Add ABSA-specific model cards * If spaCy doesn't agree with the start/end, just ignore those cases * If there are no aspects, just return * Elaborate on the required columns in the datasets * Add Absa to the WIP docs --- .github/workflows/tests.yml | 2 + docs/source/en/api/main.mdx | 4 + docs/source/en/api/trainer.mdx | 6 +- setup.py | 12 +- src/setfit/__init__.py | 1 + src/setfit/modeling.py | 41 ++-- src/setfit/span/__init__.py | 3 + src/setfit/span/aspect_extractor.py | 34 +++ src/setfit/span/model_card_template.md | 64 +++++ src/setfit/span/modeling.py | 292 +++++++++++++++++++++++ src/setfit/span/trainer.py | 316 +++++++++++++++++++++++++ src/setfit/trainer.py | 154 +++++++----- tests/conftest.py | 23 +- tests/span/test_modeling.py | 78 ++++++ tests/span/test_trainer.py | 75 ++++++ tests/test_trainer.py | 32 ++- 16 files changed, 1052 insertions(+), 85 deletions(-) create mode 100644 src/setfit/span/__init__.py create mode 100644 src/setfit/span/aspect_extractor.py create mode 100644 src/setfit/span/model_card_template.md create mode 100644 src/setfit/span/modeling.py create mode 100644 src/setfit/span/trainer.py create mode 100644 tests/span/test_modeling.py create mode 100644 tests/span/test_trainer.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index afdcf1ec..243c1306 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,6 +40,8 @@ jobs: run: | python -m pip install --no-cache-dir --upgrade pip python -m pip install --no-cache-dir ${{ matrix.requirements }} + python -m spacy download en_core_web_lg + python -m spacy download en_core_web_sm if: steps.restore-cache.outputs.cache-hit != 'true' - name: Install the checked-out setfit diff --git a/docs/source/en/api/main.mdx b/docs/source/en/api/main.mdx index ac2b77e4..a65b3db4 100644 --- a/docs/source/en/api/main.mdx +++ b/docs/source/en/api/main.mdx @@ -6,3 +6,7 @@ # SetFitHead [[autodoc]] SetFitHead + +# AbsaModel + +[[autodoc]] AbsaModel \ No newline at end of file diff --git a/docs/source/en/api/trainer.mdx b/docs/source/en/api/trainer.mdx index 4b605dc8..3e3d39d1 100644 --- a/docs/source/en/api/trainer.mdx +++ b/docs/source/en/api/trainer.mdx @@ -5,4 +5,8 @@ # DistillationTrainer -[[autodoc]] DistillationTrainer \ No newline at end of file +[[autodoc]] DistillationTrainer + +# AbsaTrainer + +[[autodoc]] AbsaTrainer \ No newline at end of file diff --git a/setup.py b/setup.py index dcd5a8ea..7079d145 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,18 @@ MAINTAINER_EMAIL = "lewis@huggingface.co" INTEGRATIONS_REQUIRE = ["optuna"] -REQUIRED_PKGS = ["datasets>=2.3.0", "sentence-transformers>=2.2.1", "evaluate>=0.3.0"] +REQUIRED_PKGS = [ + "datasets>=2.3.0", + "sentence-transformers>=2.2.1", + "evaluate>=0.3.0", + "huggingface_hub>=0.11.0", + "scikit-learn", +] +ABSA_REQUIRE = ["spacy"] QUALITY_REQUIRE = ["black", "flake8", "isort", "tabulate"] ONNX_REQUIRE = ["onnxruntime", "onnx", "skl2onnx"] OPENVINO_REQUIRE = ["hummingbird-ml<0.4.9", "openvino==2022.3.0"] -TESTS_REQUIRE = ["pytest", "pytest-cov"] + ONNX_REQUIRE + OPENVINO_REQUIRE +TESTS_REQUIRE = ["pytest", "pytest-cov"] + ONNX_REQUIRE + OPENVINO_REQUIRE + ABSA_REQUIRE DOCS_REQUIRE = ["hf-doc-builder>=0.3.0"] EXTRAS_REQUIRE = { "optuna": INTEGRATIONS_REQUIRE, @@ -23,6 +30,7 @@ "onnx": ONNX_REQUIRE, "openvino": ONNX_REQUIRE + OPENVINO_REQUIRE, "docs": DOCS_REQUIRE, + "absa": ABSA_REQUIRE, } diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index c36d630d..f131eee0 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -4,6 +4,7 @@ from .data import get_templated_dataset, sample_dataset from .modeling import SetFitHead, SetFitModel +from .span import AbsaModel, AbsaTrainer, AspectExtractor, AspectModel, PolarityModel from .trainer import SetFitTrainer, Trainer from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer from .training_args import TrainingArguments diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 0662d2d3..793b2c72 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -17,6 +17,7 @@ import requests import torch from huggingface_hub import PyTorchModelHubMixin, hf_hub_download +from huggingface_hub.utils import validate_hf_hub_args from sentence_transformers import SentenceTransformer, models from sklearn.linear_model import LogisticRegression from sklearn.multiclass import OneVsRestClassifier @@ -74,14 +75,14 @@ ```bibtex @article{{https://doi.org/10.48550/arxiv.2209.11055, -doi = {{10.48550/ARXIV.2209.11055}}, -url = {{https://arxiv.org/abs/2209.11055}}, -author = {{Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}}, -keywords = {{Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences}}, -title = {{Efficient Few-Shot Learning Without Prompts}}, -publisher = {{arXiv}}, -year = {{2022}}, -copyright = {{Creative Commons Attribution 4.0 International}} + doi = {{10.48550/ARXIV.2209.11055}}, + url = {{https://arxiv.org/abs/2209.11055}}, + author = {{Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}}, + keywords = {{Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences}}, + title = {{Efficient Few-Shot Learning Without Prompts}}, + publisher = {{arXiv}}, + year = {{2022}}, + copyright = {{Creative Commons Attribution 4.0 International}} }} ``` """ @@ -246,7 +247,6 @@ class SetFitModel(PyTorchModelHubMixin): model_body: Optional[SentenceTransformer] = (None,) model_head: Optional[Union[SetFitHead, LogisticRegression]] = None multi_target_strategy: Optional[str] = None - l2_weight: float = 1e-2 normalize_embeddings: bool = False @property @@ -372,7 +372,7 @@ def _prepare_optimizer( l2_weight: float, ) -> torch.optim.Optimizer: body_learning_rate = body_learning_rate or head_learning_rate - l2_weight = l2_weight or self.l2_weight + l2_weight = l2_weight or 1e-2 optimizer = torch.optim.AdamW( [ { @@ -519,6 +519,15 @@ def predict_proba( outputs = self.model_head.predict_proba(embeddings) return self._output_type_conversion(outputs, as_numpy=as_numpy) + @property + def device(self) -> torch.device: + """Get the Torch device that this model is on. + + Returns: + torch.device: The device that the model is on. + """ + return self.model_body.device + def to(self, device: Union[str, torch.device]) -> "SetFitModel": """Move this SetFitModel to `device`, and then return `self`. This method does not copy. @@ -589,6 +598,7 @@ def _save_pretrained(self, save_directory: Union[Path, str]) -> None: joblib.dump(self.model_head, str(Path(save_directory) / MODEL_HEAD_NAME)) @classmethod + @validate_hf_hub_args def _from_pretrained( cls, model_id: str, @@ -598,13 +608,13 @@ def _from_pretrained( proxies: Optional[Dict] = None, resume_download: Optional[bool] = None, local_files_only: Optional[bool] = None, - use_auth_token: Optional[Union[bool, str]] = None, + token: Optional[Union[bool, str]] = None, multi_target_strategy: Optional[str] = None, use_differentiable_head: bool = False, normalize_embeddings: bool = False, **model_kwargs, ) -> "SetFitModel": - model_body = SentenceTransformer(model_id, cache_folder=cache_dir, use_auth_token=use_auth_token) + model_body = SentenceTransformer(model_id, cache_folder=cache_dir, use_auth_token=token) target_device = model_body._target_device model_body.to(target_device) # put `model_body` on the target device @@ -628,7 +638,7 @@ def _from_pretrained( force_download=force_download, proxies=proxies, resume_download=resume_download, - use_auth_token=use_auth_token, + token=token, local_files_only=local_files_only, ) except requests.exceptions.RequestException: @@ -641,7 +651,7 @@ def _from_pretrained( if model_head_file is not None: model_head = joblib.load(model_head_file) else: - head_params = model_kwargs.get("head_params", {}) + head_params = model_kwargs.pop("head_params", {}) if use_differentiable_head: if multi_target_strategy is None: use_multitarget = False @@ -677,9 +687,12 @@ def _from_pretrained( else: model_head = clf + # Remove the `transformers` config + model_kwargs.pop("config", None) return cls( model_body=model_body, model_head=model_head, multi_target_strategy=multi_target_strategy, normalize_embeddings=normalize_embeddings, + **model_kwargs, ) diff --git a/src/setfit/span/__init__.py b/src/setfit/span/__init__.py new file mode 100644 index 00000000..7fc6f9db --- /dev/null +++ b/src/setfit/span/__init__.py @@ -0,0 +1,3 @@ +from .aspect_extractor import AspectExtractor +from .modeling import AbsaModel, AspectModel, PolarityModel +from .trainer import AbsaTrainer diff --git a/src/setfit/span/aspect_extractor.py b/src/setfit/span/aspect_extractor.py new file mode 100644 index 00000000..096b9bb6 --- /dev/null +++ b/src/setfit/span/aspect_extractor.py @@ -0,0 +1,34 @@ +from typing import TYPE_CHECKING, List, Tuple + + +if TYPE_CHECKING: + from spacy.tokens import Doc + + +class AspectExtractor: + def __init__(self, spacy_model: str) -> None: + super().__init__() + import spacy + + self.nlp = spacy.load(spacy_model) + + def find_groups(self, aspect_mask: List[bool]): + start = None + for idx, flag in enumerate(aspect_mask): + if flag: + if start is None: + start = idx + else: + if start is not None: + yield slice(start, idx) + start = None + if start is not None: + yield slice(start, idx) + + def __call__(self, texts: List[str]) -> Tuple[List["Doc"], List[slice]]: + aspects_list = [] + docs = list(self.nlp.pipe(texts)) + for doc in docs: + aspect_mask = [token.pos_ in ("NOUN", "PROPN") for token in doc] + aspects_list.append(list(self.find_groups(aspect_mask))) + return docs, aspects_list diff --git a/src/setfit/span/model_card_template.md b/src/setfit/span/model_card_template.md new file mode 100644 index 00000000..31ec618f --- /dev/null +++ b/src/setfit/span/model_card_template.md @@ -0,0 +1,64 @@ +--- +license: apache-2.0 +tags: +- setfit +- sentence-transformers +- absa +- token-classification +pipeline_tag: token-classification +--- + +# {{ model_name | default("SetFit ABSA Model", true) }} + +This is a [SetFit ABSA model](https://github.com/huggingface/setfit) that can be used for Aspect Based Sentiment Analysis (ABSA). \ +In particular, this model is in charge of {{ "filtering aspect span candidates" if is_aspect else "classifying aspect polarities"}}. +It has been trained using SetFit, an efficient few-shot learning technique that involves: + +1. Fine-tuning a [Sentence Transformer](https://www.sbert.net) with contrastive learning. +2. Training a classification head with features from the fine-tuned Sentence Transformer. + +This model was trained within the context of a larger system for ABSA, which looks like so: + +1. Use a spaCy model to select possible aspect span candidates. +2. {{ "**" if is_aspect else "" }}Use {{ "this" if is_aspect else "a" }} SetFit model to filter these possible aspect span candidates.{{ "**" if is_aspect else "" }} +3. {{ "**" if not is_aspect else "" }}Use {{ "this" if not is_aspect else "a" }} SetFit model to classify the filtered aspect span candidates.{{ "**" if not is_aspect else "" }} + +## Usage + +To use this model for inference, first install the SetFit library: + +```bash +pip install setfit +``` + +You can then run inference as follows: + +```python +from setfit import AbsaModel + +# Download from Hub and run inference +model = AbsaModel.from_pretrained( + "{{ aspect_model }}", + "{{ polarity_model }}", +) +# Run inference +preds = model([ + "The best pizza outside of Italy and really tasty.", + "The food here is great but the service is terrible", +]) +``` + +## BibTeX entry and citation info + +```bibtex +@article{https://doi.org/10.48550/arxiv.2209.11055, + doi = {10.48550/ARXIV.2209.11055}, + url = {https://arxiv.org/abs/2209.11055}, + author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}, + keywords = {Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {Efficient Few-Shot Learning Without Prompts}, + publisher = {arXiv}, + year = {2022}, + copyright = {Creative Commons Attribution 4.0 International} +} +``` \ No newline at end of file diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py new file mode 100644 index 00000000..02b0b1dd --- /dev/null +++ b/src/setfit/span/modeling.py @@ -0,0 +1,292 @@ +import json +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import requests +import torch +from huggingface_hub import hf_hub_download +from huggingface_hub.utils import SoftTemporaryDirectory, validate_hf_hub_args +from jinja2 import Environment, FileSystemLoader + +from .. import logging +from ..modeling import SetFitModel +from .aspect_extractor import AspectExtractor + + +if TYPE_CHECKING: + from spacy.tokens import Doc + +logger = logging.get_logger(__name__) + +CONFIG_NAME = "config_span_setfit.json" + + +@dataclass +class SpanSetFitModel(SetFitModel): + span_context: int = 0 + + def prepend_aspects(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[str]: + for doc, aspects in zip(docs, aspects_list): + for aspect_slice in aspects: + aspect = doc[max(aspect_slice.start - self.span_context, 0) : aspect_slice.stop + self.span_context] + # TODO: Investigate performance difference of different formats + yield aspect.text + ":" + doc.text + + def __call__(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[bool]: + inputs_list = list(self.prepend_aspects(docs, aspects_list)) + preds = self.predict(inputs_list, as_numpy=True) + iter_preds = iter(preds) + return [[next(iter_preds) for _ in aspects] for aspects in aspects_list] + + @classmethod + @validate_hf_hub_args + def _from_pretrained( + cls, + model_id: str, + span_context: Optional[int] = None, + revision: Optional[str] = None, + cache_dir: Optional[str] = None, + force_download: Optional[bool] = None, + proxies: Optional[Dict] = None, + resume_download: Optional[bool] = None, + local_files_only: Optional[bool] = None, + token: Optional[Union[bool, str]] = None, + **model_kwargs, + ) -> "SpanSetFitModel": + config_file: Optional[str] = None + if os.path.isdir(model_id): + if CONFIG_NAME in os.listdir(model_id): + config_file = os.path.join(model_id, CONFIG_NAME) + else: + try: + config_file = hf_hub_download( + repo_id=model_id, + filename=CONFIG_NAME, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + token=token, + local_files_only=local_files_only, + ) + except requests.exceptions.RequestException: + pass + + if config_file is not None: + with open(config_file, "r", encoding="utf-8") as f: + config = json.load(f) + model_kwargs.update(config) + + if span_context is not None: + model_kwargs["span_context"] = span_context + + return super(SpanSetFitModel, cls)._from_pretrained( + model_id, + revision, + cache_dir, + force_download, + proxies, + resume_download, + local_files_only, + token, + **model_kwargs, + ) + + def _save_pretrained(self, save_directory: Union[Path, str]) -> None: + path = os.path.join(save_directory, CONFIG_NAME) + with open(path, "w") as f: + json.dump({"span_context": self.span_context}, f, indent=2) + + super()._save_pretrained(save_directory) + + def create_model_card(self, path: str, model_name: Optional[str] = None) -> None: + """Creates and saves a model card for a SetFit model. + + Args: + path (str): The path to save the model card to. + model_name (str, *optional*): The name of the model. Defaults to `SetFit Model`. + """ + if not os.path.exists(path): + os.makedirs(path) + + # If the model_path is a folder that exists locally, i.e. when create_model_card is called + # via push_to_hub, and the path is in a temporary folder, then we only take the last two + # directories + model_path = Path(model_name) + if model_path.exists() and Path(tempfile.gettempdir()) in model_path.resolve().parents: + model_name = "/".join(model_path.parts[-2:]) + + environment = Environment(loader=FileSystemLoader(Path(__file__).parent)) + template = environment.get_template("model_card_template.md") + is_aspect = isinstance(self, AspectModel) + aspect_model = "setfit-absa-aspect" + polarity_model = "setfit-absa-polarity" + if model_name is not None: + if is_aspect: + aspect_model = model_name + if model_name.endswith("-aspect"): + polarity_model = model_name[: -len("-aspect")] + "-polarity" + else: + polarity_model = model_name + if model_name.endswith("-polarity"): + aspect_model = model_name[: -len("-polarity")] + "-aspect" + + model_card_content = template.render( + model_name=model_name, is_aspect=is_aspect, aspect_model=aspect_model, polarity_model=polarity_model + ) + with open(os.path.join(path, "README.md"), "w", encoding="utf-8") as f: + f.write(model_card_content) + + +class AspectModel(SpanSetFitModel): + # TODO: Assumes binary SetFitModel with 0 == no aspect, 1 == aspect + def __call__(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[bool]: + sentence_preds = super().__call__(docs, aspects_list) + return [ + [aspect for aspect, pred in zip(aspects, preds) if pred == 1] + for aspects, preds in zip(aspects_list, sentence_preds) + ] + + +@dataclass +class PolarityModel(SpanSetFitModel): + span_context: int = 3 + + +@dataclass +class AbsaModel: + aspect_extractor: AspectExtractor + aspect_model: AspectModel + polarity_model: PolarityModel + + def predict(self, inputs: Union[str, List[str]]) -> List[Dict[str, Any]]: + is_str = isinstance(inputs, str) + inputs_list = [inputs] if is_str else inputs + docs, aspects_list = self.aspect_extractor(inputs_list) + if sum(aspects_list, []) == []: + return aspects_list + + aspects_list = self.aspect_model(docs, aspects_list) + if sum(aspects_list, []) == []: + return aspects_list + + polarity_list = self.polarity_model(docs, aspects_list) + outputs = [] + for docs, aspects, polarities in zip(docs, aspects_list, polarity_list): + outputs.append( + [ + {"span": docs[aspect_slice].text, "polarity": polarity} + for aspect_slice, polarity in zip(aspects, polarities) + ] + ) + return outputs if not is_str else outputs[0] + + @property + def device(self) -> torch.device: + return self.aspect_model.device + + def to(self, device: Union[str, torch.device]) -> "AbsaModel": + self.aspect_model.to(device) + self.polarity_model.to(device) + + def __call__(self, inputs: Union[str, List[str]]) -> List[Dict[str, Any]]: + return self.predict(inputs) + + def save_pretrained( + self, + save_directory: Union[str, Path], + polarity_save_directory: Optional[Union[str, Path]] = None, + push_to_hub: bool = False, + **kwargs, + ) -> None: + if polarity_save_directory is None: + base_save_directory = Path(save_directory) + save_directory = base_save_directory.parent / (base_save_directory.name + "-aspect") + polarity_save_directory = base_save_directory.parent / (base_save_directory.name + "-polarity") + self.aspect_model.save_pretrained(save_directory=save_directory, push_to_hub=push_to_hub, **kwargs) + self.polarity_model.save_pretrained(save_directory=polarity_save_directory, push_to_hub=push_to_hub, **kwargs) + + @classmethod + def from_pretrained( + cls, + model_id: str, + polarity_model_id: Optional[str] = None, + spacy_model: Optional[str] = "en_core_web_lg", + span_contexts: Tuple[Optional[int], Optional[int]] = (None, None), + force_download: bool = False, + resume_download: bool = False, + proxies: Optional[Dict] = None, + token: Optional[Union[str, bool]] = None, + cache_dir: Optional[str] = None, + local_files_only: bool = False, + use_differentiable_head: bool = False, + normalize_embeddings: bool = False, + **model_kwargs, + ) -> "AbsaModel": + revision = None + if len(model_id.split("@")) == 2: + model_id, revision = model_id.split("@") + aspect_model = AspectModel.from_pretrained( + model_id, + span_context=span_contexts[0], + revision=revision, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + token=token, + cache_dir=cache_dir, + local_files_only=local_files_only, + use_differentiable_head=use_differentiable_head, + normalize_embeddings=normalize_embeddings, + **model_kwargs, + ) + if polarity_model_id: + model_id = polarity_model_id + revision = None + if len(model_id.split("@")) == 2: + model_id, revision = model_id.split("@") + polarity_model = PolarityModel.from_pretrained( + model_id, + span_context=span_contexts[1], + revision=revision, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + token=token, + cache_dir=cache_dir, + local_files_only=local_files_only, + use_differentiable_head=use_differentiable_head, + normalize_embeddings=normalize_embeddings, + **model_kwargs, + ) + + aspect_extractor = AspectExtractor(spacy_model=spacy_model) + + return cls(aspect_extractor, aspect_model, polarity_model) + + def push_to_hub(self, repo_id: str, polarity_repo_id: Optional[str] = None, **kwargs) -> None: + if "/" not in repo_id: + raise ValueError( + '`repo_id` must be a full repository ID, including organisation, e.g. "tomaarsen/setfit-absa-restaurant".' + ) + if polarity_repo_id is not None and "/" not in polarity_repo_id: + raise ValueError( + '`polarity_repo_id` must be a full repository ID, including organisation, e.g. "tomaarsen/setfit-absa-restaurant".' + ) + commit_message = kwargs.pop("commit_message", "Add SetFit ABSA model") + + # Push the files to the repo in a single commit + with SoftTemporaryDirectory() as tmp_dir: + save_directory = Path(tmp_dir) / repo_id + polarity_save_directory = None if polarity_repo_id is None else Path(tmp_dir) / polarity_repo_id + self.save_pretrained( + save_directory=save_directory, + polarity_save_directory=polarity_save_directory, + push_to_hub=True, + commit_message=commit_message, + **kwargs, + ) diff --git a/src/setfit/span/trainer.py b/src/setfit/span/trainer.py new file mode 100644 index 00000000..477cddf8 --- /dev/null +++ b/src/setfit/span/trainer.py @@ -0,0 +1,316 @@ +from collections import defaultdict +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import optuna +from datasets import Dataset +from transformers.trainer_callback import TrainerCallback + +from setfit.span.modeling import AbsaModel, AspectModel, PolarityModel, SpanSetFitModel +from setfit.training_args import TrainingArguments + +from .. import logging +from ..trainer import ColumnMappingMixin, Trainer + + +logger = logging.get_logger(__name__) + + +class AbsaTrainer(ColumnMappingMixin): + """Trainer to train a SetFit ABSA model. + + Args: + model (`AbsaModel`): + The AbsaModel model to train. + args (`TrainingArguments`, *optional*): + The training arguments to use. If `polarity_args` is not defined, then `args` is used for both + the aspect and the polarity model. + polarity_args (`TrainingArguments`, *optional*): + The training arguments to use for the polarity model. If not defined, `args` is used for both + the aspect and the polarity model. + train_dataset (`Dataset`): + The training dataset. The dataset must have "text", "span", "label" and "ordinal" columns. + eval_dataset (`Dataset`, *optional*): + The evaluation dataset. The dataset must have "text", "span", "label" and "ordinal" columns. + metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): + The metric to use for evaluation. If a string is provided, we treat it as the metric + name and load it with default settings. + If a callable is provided, it must take two arguments (`y_pred`, `y_test`). + metric_kwargs (`Dict[str, Any]`, *optional*): + Keyword arguments passed to the evaluation function if `metric` is an evaluation string like "f1". + For example useful for providing an averaging strategy for computing f1 in a multi-label setting. + callbacks: (`List[~transformers.TrainerCallback]`, *optional*): + A list of callbacks to customize the training loop. Will add those to the list of default callbacks + detailed in [here](https://huggingface.co/docs/transformers/main/en/main_classes/callback). + If you want to remove one of the default callbacks used, use the `Trainer.remove_callback()` method. + column_mapping (`Dict[str, str]`, *optional*): + A mapping from the column names in the dataset to the column names expected by the model. + The expected format is a dictionary with the following format: + `{"text_column_name": "text", "span_column_name": "span", "label_column_name: "label", "ordinal_column_name": "ordinal"}`. + """ + + _REQUIRED_COLUMNS = {"text", "span", "label", "ordinal"} + + def __init__( + self, + model: AbsaModel, + args: Optional[TrainingArguments] = None, + polarity_args: Optional[TrainingArguments] = None, + train_dataset: Optional["Dataset"] = None, + eval_dataset: Optional["Dataset"] = None, + metric: Union[str, Callable[["Dataset", "Dataset"], Dict[str, float]]] = "accuracy", + metric_kwargs: Optional[Dict[str, Any]] = None, + callbacks: Optional[List[TrainerCallback]] = None, + column_mapping: Optional[Dict[str, str]] = None, + ) -> None: + self.model = model + self.aspect_extractor = model.aspect_extractor + + if train_dataset is not None and column_mapping: + train_dataset = self._apply_column_mapping(train_dataset, column_mapping) + aspect_train_dataset, polarity_train_dataset = self.preprocess_dataset( + model.aspect_model, model.polarity_model, train_dataset + ) + if eval_dataset is not None and column_mapping: + eval_dataset = self._apply_column_mapping(eval_dataset, column_mapping) + aspect_eval_dataset, polarity_eval_dataset = self.preprocess_dataset( + model.aspect_model, model.polarity_model, eval_dataset + ) + + self.aspect_trainer = Trainer( + model.aspect_model, + args=args, + train_dataset=aspect_train_dataset, + eval_dataset=aspect_eval_dataset, + metric=metric, + metric_kwargs=metric_kwargs, + callbacks=callbacks, + ) + self.aspect_trainer._set_logs_mapper( + {"eval_embedding_loss": "eval_aspect_embedding_loss", "embedding_loss": "aspect_embedding_loss"} + ) + self.polarity_trainer = Trainer( + model.polarity_model, + args=polarity_args or args, + train_dataset=polarity_train_dataset, + eval_dataset=polarity_eval_dataset, + metric=metric, + metric_kwargs=metric_kwargs, + callbacks=callbacks, + ) + self.polarity_trainer._set_logs_mapper( + {"eval_embedding_loss": "eval_polarity_embedding_loss", "embedding_loss": "polarity_embedding_loss"} + ) + + def preprocess_dataset( + self, aspect_model: AspectModel, polarity_model: PolarityModel, dataset: Dataset + ) -> Dataset: + if dataset is None: + return dataset, dataset + + # Group by "text" + grouped_data = defaultdict(list) + for sample in dataset: + text = sample.pop("text") + grouped_data[text].append(sample) + + def index_ordinal(text: str, target: str, ordinal: int) -> Tuple[int, int]: + find_from = 0 + for _ in range(ordinal + 1): + start_idx = text.index(target, find_from) + find_from = start_idx + 1 + return start_idx, start_idx + len(target) + + docs, aspects_list = self.aspect_extractor(grouped_data.keys()) + intersected_aspect_list = [] + polarity_labels = [] + aspect_labels = [] + for doc, aspects, text in zip(docs, aspects_list, grouped_data): + gold_aspects = [] + gold_polarity_labels = [] + for annotation in grouped_data[text]: + try: + start, end = index_ordinal(text, annotation["span"], annotation["ordinal"]) + except ValueError: + logger.info( + f"The ordinal of {annotation['ordinal']} for span {annotation['span']!r} in {text!r} is too high. " + "Skipping this sample." + ) + continue + + gold_aspect_span = doc.char_span(start, end) + if gold_aspect_span is None: + continue + gold_aspects.append(slice(gold_aspect_span.start, gold_aspect_span.end)) + gold_polarity_labels.append(annotation["label"]) + + # The Aspect model uses all predicted aspects, with labels depending on whether + # the predicted aspects are indeed true/gold aspects. + aspect_labels.extend([aspect in gold_aspects for aspect in aspects]) + + # The Polarity model uses the intersection of pred and gold aspects, with labels for the gold label. + intersected_aspects = [] + for gold_aspect, gold_label in zip(gold_aspects, gold_polarity_labels): + if gold_aspect in aspects: + intersected_aspects.append(gold_aspect) + polarity_labels.append(gold_label) + intersected_aspect_list.append(intersected_aspects) + + aspect_texts = list(aspect_model.prepend_aspects(docs, aspects_list)) + polarity_texts = list(polarity_model.prepend_aspects(docs, intersected_aspect_list)) + return Dataset.from_dict({"text": aspect_texts, "label": aspect_labels}), Dataset.from_dict( + {"text": polarity_texts, "label": polarity_labels} + ) + + def train( + self, + args: Optional[TrainingArguments] = None, + polarity_args: Optional[TrainingArguments] = None, + trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, + **kwargs, + ) -> None: + """ + Main training entry point. + + Args: + args (`TrainingArguments`, *optional*): + Temporarily change the aspect training arguments for this training call. + polarity_args (`TrainingArguments`, *optional*): + Temporarily change the polarity training arguments for this training call. + trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): + The trial run or the hyperparameter dictionary for hyperparameter search. + """ + self.train_aspect(args=args, trial=trial, **kwargs) + self.train_polarity(args=polarity_args, trial=trial, **kwargs) + + def train_aspect( + self, + args: Optional[TrainingArguments] = None, + trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, + **kwargs, + ) -> None: + """ + Train the aspect model only. + + Args: + args (`TrainingArguments`, *optional*): + Temporarily change the aspect training arguments for this training call. + trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): + The trial run or the hyperparameter dictionary for hyperparameter search. + """ + self.aspect_trainer.train(args=args, trial=trial, **kwargs) + + def train_polarity( + self, + args: Optional[TrainingArguments] = None, + trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, + **kwargs, + ) -> None: + """ + Train the polarity model only. + + Args: + args (`TrainingArguments`, *optional*): + Temporarily change the aspect training arguments for this training call. + trial (`optuna.Trial` or `Dict[str, Any]`, *optional*): + The trial run or the hyperparameter dictionary for hyperparameter search. + """ + self.polarity_trainer.train(args=args, trial=trial, **kwargs) + + def add_callback(self, callback): + """ + Add a callback to the current list of [`~transformer.TrainerCallback`]. + + Args: + callback (`type` or [`~transformer.TrainerCallback`]): + A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + first case, will instantiate a member of that class. + """ + self.aspect_trainer.add_callback(callback) + self.polarity_trainer.add_callback(callback) + + def pop_callback(self, callback): + """ + Remove a callback from the current list of [`~transformer.TrainerCallback`] and returns it. + + If the callback is not found, returns `None` (and no error is raised). + + Args: + callback (`type` or [`~transformer.TrainerCallback`]): + A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + first case, will pop the first member of that class found in the list of callbacks. + + Returns: + [`Tuple[~transformer.TrainerCallback]`]: The callbacks removed from the aspect and polarity trainers, if found. + """ + return self.aspect_trainer.pop_callback(callback), self.polarity_trainer.pop_callback(callback) + + def remove_callback(self, callback): + """ + Remove a callback from the current list of [`~transformer.TrainerCallback`]. + + Args: + callback (`type` or [`~transformer.TrainerCallback`]): + A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + first case, will remove the first member of that class found in the list of callbacks. + """ + self.aspect_trainer.remove_callback(callback) + self.polarity_trainer.remove_callback(callback) + + def push_to_hub(self, repo_id: str, polarity_repo_id: Optional[str] = None, **kwargs) -> None: + """Upload model checkpoint to the Hub using `huggingface_hub`. + + See the full list of parameters for your `huggingface_hub` version in the\ + [huggingface_hub documentation](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.ModelHubMixin.push_to_hub). + + Args: + repo_id (`str`): + The full repository ID to push to, e.g. `"tomaarsen/setfit-aspect"`. + repo_id (`str`): + The full repository ID to push to, e.g. `"tomaarsen/setfit-sst2"`. + config (`dict`, *optional*): + Configuration object to be saved alongside the model weights. + commit_message (`str`, *optional*): + Message to commit while pushing. + private (`bool`, *optional*, defaults to `False`): + Whether the repository created should be private. + api_endpoint (`str`, *optional*): + The API endpoint to use when pushing the model to the hub. + token (`str`, *optional*): + The token to use as HTTP bearer authorization for remote files. + If not set, will use the token set when logging in with + `transformers-cli login` (stored in `~/.huggingface`). + branch (`str`, *optional*): + The git branch on which to push the model. This defaults to + the default branch as specified in your repository, which + defaults to `"main"`. + create_pr (`boolean`, *optional*): + Whether or not to create a Pull Request from `branch` with that commit. + Defaults to `False`. + allow_patterns (`List[str]` or `str`, *optional*): + If provided, only files matching at least one pattern are pushed. + ignore_patterns (`List[str]` or `str`, *optional*): + If provided, files matching any of the patterns are not pushed. + """ + return self.model.push_to_hub(repo_id=repo_id, polarity_repo_id=polarity_repo_id, **kwargs) + + def evaluate(self, dataset: Optional[Dataset] = None) -> Dict[str, Dict[str, float]]: + """ + Computes the metrics for a given classifier. + + Args: + dataset (`Dataset`, *optional*): + The dataset to compute the metrics on. If not provided, will use the evaluation dataset passed via + the `eval_dataset` argument at `Trainer` initialization. + + Returns: + `Dict[str, Dict[str, float]]`: The evaluation metrics. + """ + aspect_eval_dataset = polarity_eval_dataset = None + if dataset: + aspect_eval_dataset, polarity_eval_dataset = self.preprocess_dataset( + self.model.aspect_model, self.model.polarity_model, dataset + ) + return { + "aspect": self.aspect_trainer.evaluate(aspect_eval_dataset), + "polarity": self.polarity_trainer.evaluate(polarity_eval_dataset), + } diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 848796b6..baecd053 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -12,6 +12,7 @@ from sentence_transformers.datasets import SentenceLabelDataset from sentence_transformers.losses.BatchHardTripletLoss import BatchHardTripletLossDistanceFunction from sentence_transformers.util import batch_to_device +from sklearn.preprocessing import LabelEncoder from torch import nn from torch.cuda.amp import autocast from torch.utils.data import DataLoader @@ -68,7 +69,70 @@ DEFAULT_PROGRESS_CALLBACK = NotebookProgressCallback -class Trainer: +class ColumnMappingMixin: + _REQUIRED_COLUMNS = {"text", "label"} + + def _validate_column_mapping(self, dataset: "Dataset") -> None: + """ + Validates the provided column mapping against the dataset. + """ + column_names = set(dataset.column_names) + if self.column_mapping is None and not self._REQUIRED_COLUMNS.issubset(column_names): + # Issue #226: load_dataset will automatically assign points to "train" if no split is specified + if column_names == {"train"} and isinstance(dataset, DatasetDict): + raise ValueError( + "SetFit expected a Dataset, but it got a DatasetDict with the split ['train']. " + "Did you mean to select the training split with dataset['train']?" + ) + elif isinstance(dataset, DatasetDict): + raise ValueError( + f"SetFit expected a Dataset, but it got a DatasetDict with the splits {sorted(column_names)}. " + "Did you mean to select one of these splits from the dataset?" + ) + else: + raise ValueError( + f"SetFit expected the dataset to have the columns {sorted(self._REQUIRED_COLUMNS)}, " + f"but only the columns {sorted(column_names)} were found. " + "Either make sure these columns are present, or specify which columns to use with column_mapping in Trainer." + ) + if self.column_mapping is not None: + missing_columns = self._REQUIRED_COLUMNS.difference(self.column_mapping.values()) + if missing_columns: + raise ValueError( + f"The following columns are missing from the column mapping: {missing_columns}. Please provide a mapping for all required columns." + ) + if not set(self.column_mapping.keys()).issubset(column_names): + raise ValueError( + f"The column mapping expected the columns {sorted(self.column_mapping.keys())} in the dataset, " + f"but the dataset had the columns {sorted(column_names)}." + ) + + def _apply_column_mapping(self, dataset: "Dataset", column_mapping: Dict[str, str]) -> "Dataset": + """ + Applies the provided column mapping to the dataset, renaming columns accordingly. + Extra features not in the column mapping are prefixed with `"feat_"`. + """ + dataset = dataset.rename_columns( + { + **column_mapping, + **{ + col: f"feat_{col}" + for col in dataset.column_names + if col not in column_mapping and col not in self._REQUIRED_COLUMNS + }, + } + ) + dset_format = dataset.format + dataset = dataset.with_format( + type=dset_format["type"], + columns=dataset.column_names, + output_all_columns=dset_format["output_all_columns"], + **dset_format["format_kwargs"], + ) + return dataset + + +class Trainer(ColumnMappingMixin): """Trainer to train a SetFit model. Args: @@ -91,14 +155,16 @@ class Trainer: metric_kwargs (`Dict[str, Any]`, *optional*): Keyword arguments passed to the evaluation function if `metric` is an evaluation string like "f1". For example useful for providing an averaging strategy for computing f1 in a multi-label setting. + callbacks: (`List[~transformers.TrainerCallback]`, *optional*): + A list of callbacks to customize the training loop. Will add those to the list of default callbacks + detailed in [here](https://huggingface.co/docs/transformers/main/en/main_classes/callback). + If you want to remove one of the default callbacks used, use the `Trainer.remove_callback()` method. column_mapping (`Dict[str, str]`, *optional*): A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: `{"text_column_name": "text", "label_column_name: "label"}`. """ - _REQUIRED_COLUMNS = {"text", "label"} - def __init__( self, model: Optional["SetFitModel"] = None, @@ -111,6 +177,8 @@ def __init__( callbacks: Optional[List[TrainerCallback]] = None, column_mapping: Optional[Dict[str, str]] = None, ) -> None: + if args is not None and not isinstance(args, TrainingArguments): + raise ValueError("`args` must be a `TrainingArguments` instance imported from `setfit`.") self.args = args or TrainingArguments() self.train_dataset = train_dataset self.eval_dataset = eval_dataset @@ -118,6 +186,7 @@ def __init__( self.metric = metric self.metric_kwargs = metric_kwargs self.column_mapping = column_mapping + self.logs_mapper = {} # Seed must be set before instantiating the model when using model_init. set_seed(12) @@ -184,61 +253,6 @@ def remove_callback(self, callback): """ self.callback_handler.remove_callback(callback) - def _validate_column_mapping(self, dataset: "Dataset") -> None: - """ - Validates the provided column mapping against the dataset. - """ - column_names = set(dataset.column_names) - if self.column_mapping is None and not self._REQUIRED_COLUMNS.issubset(column_names): - # Issue #226: load_dataset will automatically assign points to "train" if no split is specified - if column_names == {"train"} and isinstance(dataset, DatasetDict): - raise ValueError( - "SetFit expected a Dataset, but it got a DatasetDict with the split ['train']. " - "Did you mean to select the training split with dataset['train']?" - ) - elif isinstance(dataset, DatasetDict): - raise ValueError( - f"SetFit expected a Dataset, but it got a DatasetDict with the splits {sorted(column_names)}. " - "Did you mean to select one of these splits from the dataset?" - ) - else: - raise ValueError( - f"SetFit expected the dataset to have the columns {sorted(self._REQUIRED_COLUMNS)}, " - f"but only the columns {sorted(column_names)} were found. " - "Either make sure these columns are present, or specify which columns to use with column_mapping in Trainer." - ) - if self.column_mapping is not None: - missing_columns = self._REQUIRED_COLUMNS.difference(self.column_mapping.values()) - if missing_columns: - raise ValueError( - f"The following columns are missing from the column mapping: {missing_columns}. Please provide a mapping for all required columns." - ) - if not set(self.column_mapping.keys()).issubset(column_names): - raise ValueError( - f"The column mapping expected the columns {sorted(self.column_mapping.keys())} in the dataset, " - f"but the dataset had the columns {sorted(column_names)}." - ) - - def _apply_column_mapping(self, dataset: "Dataset", column_mapping: Dict[str, str]) -> "Dataset": - """ - Applies the provided column mapping to the dataset, renaming columns accordingly. - Extra features not in the column mapping are prefixed with `"feat_"`. - """ - dataset = dataset.rename_columns( - { - **column_mapping, - **{col: f"feat_{col}" for col in dataset.column_names if col not in column_mapping}, - } - ) - dset_format = dataset.format - dataset = dataset.with_format( - type=dset_format["type"], - columns=dataset.column_names, - output_all_columns=dset_format["output_all_columns"], - **dset_format["format_kwargs"], - ) - return dataset - def apply_hyperparameters(self, params: Dict[str, Any], final_model: bool = False) -> None: """Applies a dictionary of hyperparameters to both the trainer and the model @@ -329,7 +343,7 @@ def train( args: Optional[TrainingArguments] = None, trial: Optional[Union["optuna.Trial", Dict[str, Any]]] = None, **kwargs, - ): + ) -> None: """ Main training entry point. @@ -478,6 +492,7 @@ def log(self, args: TrainingArguments, logs: Dict[str, float]) -> None: logs (`Dict[str, float]`): The values to log. """ + logs = {self.logs_mapper.get(key, key): value for key, value in logs.items()} if self.state.epoch is not None: logs["epoch"] = round(self.state.epoch, 2) @@ -485,6 +500,14 @@ def log(self, args: TrainingArguments, logs: Dict[str, float]) -> None: self.state.log_history.append(output) return self.callback_handler.on_log(args, self.state, self.control, logs) + def _set_logs_mapper(self, logs_mapper: Dict[str, str]) -> None: + """Set the logging mapper. + + Args: + logs_mapper (str): The logging mapper, e.g. {"eval_embedding_loss": "eval_aspect_embedding_loss"}. + """ + self.logs_mapper = logs_mapper + def _train_sentence_transformer( self, model_body: SentenceTransformer, @@ -732,6 +755,8 @@ def evaluate(self, dataset: Optional[Dataset] = None) -> Dict[str, float]: """ eval_dataset = dataset or self.eval_dataset + if eval_dataset is None: + raise ValueError("No evaluation dataset provided to `Trainer.evaluate` nor the `Trainer` initialzation.") self._validate_column_mapping(eval_dataset) if self.column_mapping is not None: @@ -746,6 +771,13 @@ def evaluate(self, dataset: Optional[Dataset] = None) -> Dict[str, float]: if isinstance(y_pred, torch.Tensor): y_pred = y_pred.cpu() + # Normalize string outputs + if y_test and isinstance(y_test[0], str): + encoder = LabelEncoder() + encoder.fit(list(y_test) + list(y_pred)) + y_test = encoder.transform(y_test) + y_pred = encoder.transform(y_pred) + if isinstance(self.metric, str): metric_config = "multilabel" if self.model.multi_target_strategy is not None else None metric_fn = evaluate.load(self.metric, config_name=metric_config) @@ -843,7 +875,7 @@ def push_to_hub(self, repo_id: str, **kwargs) -> str: Args: repo_id (`str`): - The full repository ID to push to, e.g. `"tomaarsen/setfit_sst2"`. + The full repository ID to push to, e.g. `"tomaarsen/setfit-sst2"`. config (`dict`, *optional*): Configuration object to be saved alongside the model weights. commit_message (`str`, *optional*): @@ -873,7 +905,7 @@ def push_to_hub(self, repo_id: str, **kwargs) -> str: """ if "/" not in repo_id: raise ValueError( - '`repo_id` must be a full repository ID, including organisation, e.g. "tomaarsen/setfit_sst2".' + '`repo_id` must be a full repository ID, including organisation, e.g. "tomaarsen/setfit-sst2".' ) commit_message = kwargs.pop("commit_message", "Add SetFit model") return self.model.push_to_hub(repo_id, commit_message=commit_message, **kwargs) diff --git a/tests/conftest.py b/tests/conftest.py index acf5b825..11051223 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,29 @@ import pytest +from datasets import Dataset -from setfit import SetFitModel +from setfit import AbsaModel, SetFitModel @pytest.fixture() def model() -> SetFitModel: return SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + + +@pytest.fixture() +def absa_model() -> AbsaModel: + return AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + + +@pytest.fixture() +def absa_dataset() -> Dataset: + texts = [ + "It is about food and ambiance, and imagine how dreadful it will be it we only had to listen to an idle engine.", + "It is about food and ambiance, and imagine how dreadful it will be it we only had to listen to an idle engine.", + "Food is great and inexpensive.", + "Good bagels and good cream cheese.", + "Good bagels and good cream cheese.", + ] + spans = ["food", "ambiance", "Food", "bagels", "cream cheese"] + labels = ["negative", "negative", "positive", "positive", "positive"] + ordinals = [0, 0, 0, 0, 0] + return Dataset.from_dict({"text": texts, "span": spans, "label": labels, "ordinal": ordinals}) diff --git a/tests/span/test_modeling.py b/tests/span/test_modeling.py new file mode 100644 index 00000000..02fd7c3e --- /dev/null +++ b/tests/span/test_modeling.py @@ -0,0 +1,78 @@ +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest +import torch + +from setfit import AbsaModel +from setfit.span.aspect_extractor import AspectExtractor +from setfit.span.modeling import AspectModel, PolarityModel + + +def test_loading(): + model = AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + assert isinstance(model, AbsaModel) + assert isinstance(model.aspect_extractor, AspectExtractor) + assert isinstance(model.aspect_model, AspectModel) + assert isinstance(model.polarity_model, PolarityModel) + + model = AbsaModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2@6c91e73a51599e35bd1145dfdcd3289215225009", + "sentence-transformers/paraphrase-albert-small-v2", + ) + assert isinstance(model, AbsaModel) + + model = AbsaModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", + "sentence-transformers/paraphrase-albert-small-v2@6c91e73a51599e35bd1145dfdcd3289215225009", + ) + assert isinstance(model, AbsaModel) + + with pytest.raises(OSError): + model = AbsaModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", spacy_model="not_a_spacy_model" + ) + + model = AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", normalize_embeddings=True) + assert model.aspect_model.normalize_embeddings + assert model.polarity_model.normalize_embeddings + + aspect_model = AspectModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", span_context=12) + assert aspect_model.span_context == 12 + polarity_model = PolarityModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", span_context=12) + assert polarity_model.span_context == 12 + + model = AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", span_contexts=(12, None)) + assert model.aspect_model.span_context == 12 + assert model.polarity_model.span_context == 3 # <- default + + +def test_save_load(absa_model: AbsaModel) -> None: + absa_model.polarity_model.span_context = 5 + + with TemporaryDirectory() as tmp_dir: + tmp_dir = str(Path(tmp_dir) / "model") + absa_model.save_pretrained(tmp_dir) + assert (Path(tmp_dir + "-aspect") / "config_span_setfit.json").exists() + assert (Path(tmp_dir + "-polarity") / "config_span_setfit.json").exists() + + fresh_model = AbsaModel.from_pretrained(tmp_dir + "-aspect", tmp_dir + "-polarity") + assert fresh_model.polarity_model.span_context == 5 + + with TemporaryDirectory() as aspect_tmp_dir: + with TemporaryDirectory() as polarity_tmp_dir: + absa_model.save_pretrained(aspect_tmp_dir, polarity_tmp_dir) + assert (Path(aspect_tmp_dir) / "config_span_setfit.json").exists() + assert (Path(polarity_tmp_dir) / "config_span_setfit.json").exists() + + fresh_model = AbsaModel.from_pretrained(aspect_tmp_dir, polarity_tmp_dir) + assert fresh_model.polarity_model.span_context == 5 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA must be available to move a model between devices") +def test_to(absa_model: AbsaModel) -> None: + assert absa_model.device.type == "cuda" + absa_model.to("cpu") + assert absa_model.device.type == "cpu" + assert absa_model.aspect_model.device.type == "cpu" + assert absa_model.polarity_model.device.type == "cpu" diff --git a/tests/span/test_trainer.py b/tests/span/test_trainer.py new file mode 100644 index 00000000..f89044dc --- /dev/null +++ b/tests/span/test_trainer.py @@ -0,0 +1,75 @@ +from datasets import Dataset +from transformers import TrainerCallback + +from setfit import AbsaTrainer +from setfit.span.modeling import AbsaModel + + +def test_trainer(absa_model: AbsaModel, absa_dataset: Dataset) -> None: + trainer = AbsaTrainer(absa_model, train_dataset=absa_dataset, eval_dataset=absa_dataset) + trainer.train() + + metrics = trainer.evaluate() + assert "aspect" in metrics + assert "polarity" in metrics + assert "accuracy" in metrics["aspect"] + assert "accuracy" in metrics["polarity"] + assert metrics["aspect"]["accuracy"] > 0.0 + assert metrics["polarity"]["accuracy"] > 0.0 + new_metrics = trainer.evaluate(absa_dataset) + assert metrics == new_metrics + + predict = absa_model.predict("Best pizza outside of Italy and really tasty.") + assert {"span": "pizza", "polarity": "positive"} in predict + predict = absa_model.predict(["Best pizza outside of Italy and really tasty.", "This is another sentence"]) + assert isinstance(predict, list) and len(predict) == 2 and isinstance(predict[0], list) + predict = absa_model(["Best pizza outside of Italy and really tasty.", "This is another sentence"]) + assert isinstance(predict, list) and len(predict) == 2 and isinstance(predict[0], list) + + +def test_trainer_callbacks(absa_model: AbsaModel) -> None: + trainer = AbsaTrainer(absa_model) + assert len(trainer.aspect_trainer.callback_handler.callbacks) >= 2 + callback_names = {callback.__class__.__name__ for callback in trainer.aspect_trainer.callback_handler.callbacks} + assert {"DefaultFlowCallback", "ProgressCallback"} <= callback_names + + class TestCallback(TrainerCallback): + pass + + callback = TestCallback() + trainer.add_callback(callback) + assert len(trainer.aspect_trainer.callback_handler.callbacks) == len(callback_names) + 1 + assert len(trainer.polarity_trainer.callback_handler.callbacks) == len(callback_names) + 1 + assert trainer.aspect_trainer.callback_handler.callbacks[-1] == callback + assert trainer.polarity_trainer.callback_handler.callbacks[-1] == callback + + assert trainer.pop_callback(callback) == (callback, callback) + trainer.add_callback(callback) + assert trainer.aspect_trainer.callback_handler.callbacks[-1] == callback + assert trainer.polarity_trainer.callback_handler.callbacks[-1] == callback + trainer.remove_callback(callback) + assert callback not in trainer.aspect_trainer.callback_handler.callbacks + assert callback not in trainer.polarity_trainer.callback_handler.callbacks + + +def test_train_ordinal_too_high(absa_model: AbsaModel) -> None: + absa_dataset = Dataset.from_dict( + { + "text": [ + "It is about food and ambiance, and imagine how dreadful it will be it we only had to listen to an idle engine." + ], + "span": ["food"], + "label": ["negative"], + "ordinal": [1], + } + ) + AbsaTrainer(absa_model, train_dataset=absa_dataset) + # TODO: Capture warning and test against it. + + +def test_train_column_mapping(absa_model: AbsaModel, absa_dataset: Dataset) -> None: + absa_dataset = absa_dataset.rename_columns({"text": "sentence", "span": "aspect"}) + trainer = AbsaTrainer( + absa_model, train_dataset=absa_dataset, column_mapping={"sentence": "text", "aspect": "span"} + ) + trainer.train() diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 2c699ea2..8eee4d57 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -1,15 +1,17 @@ import os -import pathlib import re import tempfile +from pathlib import Path from unittest import TestCase import evaluate import pytest import torch from datasets import Dataset, load_dataset +from pytest import LogCaptureFixture from sentence_transformers import losses from transformers import TrainerCallback +from transformers import TrainingArguments as TransformersTrainingArguments from transformers.testing_utils import require_optuna from transformers.utils.hp_naming import TrialShortNamer @@ -132,7 +134,7 @@ def test_trainer_raises_error_when_dataset_not_split(self): def test_trainer_raises_error_when_dataset_is_dataset_dict_with_train(self): """Verify that a useful error is raised if we pass an unsplit dataset with only a `train` split to the trainer.""" with tempfile.TemporaryDirectory() as tmpdirname: - path = pathlib.Path(tmpdirname) / "test_dataset_dict_with_train.csv" + path = Path(tmpdirname) / "test_dataset_dict_with_train.csv" path.write_text("label,text\n1,good\n0,terrible\n") dataset = load_dataset("csv", data_files=str(path)) trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) @@ -534,20 +536,20 @@ def test_trainer_warn_freeze(model: SetFitModel): trainer.freeze() -def test_train_with_kwargs(model: SetFitModel): +def test_train_with_kwargs(model: SetFitModel) -> None: train_dataset = Dataset.from_dict({"text": ["positive sentence", "negative sentence"], "label": [1, 0]}) trainer = Trainer(model, train_dataset=train_dataset) with pytest.warns(DeprecationWarning, match="`Trainer.train` does not accept keyword arguments anymore."): trainer.train(num_epochs=5) -def test_train_no_dataset(model: SetFitModel): +def test_train_no_dataset(model: SetFitModel) -> None: trainer = Trainer(model) with pytest.raises(ValueError, match="Training requires a `train_dataset` given to the `Trainer` initialization."): trainer.train() -def test_train_amp_save(model: SetFitModel, tmp_path): +def test_train_amp_save(model: SetFitModel, tmp_path: Path) -> None: args = TrainingArguments(output_dir=tmp_path, use_amp=True, save_steps=5, num_epochs=5) dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2]}) trainer = Trainer(model, args=args, train_dataset=dataset, eval_dataset=dataset) @@ -556,7 +558,7 @@ def test_train_amp_save(model: SetFitModel, tmp_path): assert os.listdir(tmp_path) == ["step_5"] -def test_train_load_best(model: SetFitModel, tmp_path, caplog): +def test_train_load_best(model: SetFitModel, tmp_path: Path, caplog: LogCaptureFixture) -> None: args = TrainingArguments( output_dir=tmp_path, save_steps=5, @@ -571,3 +573,21 @@ def test_train_load_best(model: SetFitModel, tmp_path, caplog): trainer.train() assert any("Load pretrained SentenceTransformer" in text for _, _, text in caplog.record_tuples) + + +def test_evaluate_with_strings(model: SetFitModel) -> None: + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": ["positive", "positive", "negative"]}) + trainer = Trainer(model, train_dataset=dataset, eval_dataset=dataset) + trainer.train() + metrics = trainer.evaluate() + assert "accuracy" in metrics + + +def test_trainer_wrong_args(model: SetFitModel, tmp_path: Path) -> None: + args = TransformersTrainingArguments(output_dir=tmp_path) + dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2]}) + expected = "`args` must be a `TrainingArguments` instance imported from `setfit`." + with pytest.raises(ValueError, match=expected): + Trainer(model, args=args) + with pytest.raises(ValueError, match=expected): + Trainer(model, dataset) From 5c4569db8b6f1954781080cbe4785ece1ebb424d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 21:32:32 +0100 Subject: [PATCH 066/183] Import optuna under TYPE_CHECKING --- src/setfit/span/trainer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/setfit/span/trainer.py b/src/setfit/span/trainer.py index 477cddf8..f5647322 100644 --- a/src/setfit/span/trainer.py +++ b/src/setfit/span/trainer.py @@ -1,7 +1,6 @@ from collections import defaultdict -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union -import optuna from datasets import Dataset from transformers.trainer_callback import TrainerCallback @@ -12,6 +11,9 @@ from ..trainer import ColumnMappingMixin, Trainer +if TYPE_CHECKING: + import optuna + logger = logging.get_logger(__name__) From ceeb7256ae44be2b2a8ca8fa4c0c3c58c251a0d5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 21:39:48 +0100 Subject: [PATCH 067/183] Remove unused import, reformat --- src/setfit/span/trainer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/setfit/span/trainer.py b/src/setfit/span/trainer.py index f5647322..1d362616 100644 --- a/src/setfit/span/trainer.py +++ b/src/setfit/span/trainer.py @@ -4,7 +4,7 @@ from datasets import Dataset from transformers.trainer_callback import TrainerCallback -from setfit.span.modeling import AbsaModel, AspectModel, PolarityModel, SpanSetFitModel +from setfit.span.modeling import AbsaModel, AspectModel, PolarityModel from setfit.training_args import TrainingArguments from .. import logging @@ -88,7 +88,10 @@ def __init__( callbacks=callbacks, ) self.aspect_trainer._set_logs_mapper( - {"eval_embedding_loss": "eval_aspect_embedding_loss", "embedding_loss": "aspect_embedding_loss"} + { + "eval_embedding_loss": "eval_aspect_embedding_loss", + "embedding_loss": "aspect_embedding_loss", + } ) self.polarity_trainer = Trainer( model.polarity_model, @@ -100,7 +103,10 @@ def __init__( callbacks=callbacks, ) self.polarity_trainer._set_logs_mapper( - {"eval_embedding_loss": "eval_polarity_embedding_loss", "embedding_loss": "polarity_embedding_loss"} + { + "eval_embedding_loss": "eval_polarity_embedding_loss", + "embedding_loss": "polarity_embedding_loss", + } ) def preprocess_dataset( From 5c669b5add5370cd1273389539b012c1a5e8a58f Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 21:42:17 +0100 Subject: [PATCH 068/183] Add MANIFEST.in with model_card_template --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..69617566 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include src/setfit/span/model_card_template.md \ No newline at end of file From 8e201e5d4ef50e0980f284fa90238bcb62b80257 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 21:43:21 +0100 Subject: [PATCH 069/183] Don't require transformers TrainingArgs in tests As it requires accelerate to be updated --- tests/test_trainer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 8eee4d57..c5654524 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -11,7 +11,6 @@ from pytest import LogCaptureFixture from sentence_transformers import losses from transformers import TrainerCallback -from transformers import TrainingArguments as TransformersTrainingArguments from transformers.testing_utils import require_optuna from transformers.utils.hp_naming import TrialShortNamer @@ -584,10 +583,7 @@ def test_evaluate_with_strings(model: SetFitModel) -> None: def test_trainer_wrong_args(model: SetFitModel, tmp_path: Path) -> None: - args = TransformersTrainingArguments(output_dir=tmp_path) dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2]}) expected = "`args` must be a `TrainingArguments` instance imported from `setfit`." - with pytest.raises(ValueError, match=expected): - Trainer(model, args=args) with pytest.raises(ValueError, match=expected): Trainer(model, dataset) From 6ae5045a186465348a58994b249a57d5cf5d5441 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 21:47:20 +0100 Subject: [PATCH 070/183] Update URLs in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7079d145..c3ea792c 100644 --- a/setup.py +++ b/setup.py @@ -54,8 +54,8 @@ def combine_requirements(base_keys): long_description_content_type="text/markdown", maintainer=MAINTAINER, maintainer_email=MAINTAINER_EMAIL, - url="https://github.com/SetFit/setfit", - download_url="https://github.com/SetFit/setfit/tags", + url="https://github.com/huggingface/setfit", + download_url="https://github.com/huggingface/setfit/tags", license="Apache 2.0", package_dir={"": "src"}, packages=find_packages("src"), From ecaabb47d923630e1db4deb06ab6e75ba885bce2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 21:57:16 +0100 Subject: [PATCH 071/183] Increase min hf_hub version to 0.12.0 for SoftTemporaryDirectory --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c3ea792c..d7918a99 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ "datasets>=2.3.0", "sentence-transformers>=2.2.1", "evaluate>=0.3.0", - "huggingface_hub>=0.11.0", + "huggingface_hub>=0.12.0", "scikit-learn", ] ABSA_REQUIRE = ["spacy"] From 4e79397679b14bdc2742614f03e33416682cf301 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 22:04:02 +0100 Subject: [PATCH 072/183] Include MANIFEST.in data via `include_package_data=True` --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d7918a99..3d1f8433 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ def combine_requirements(base_keys): license="Apache 2.0", package_dir={"": "src"}, packages=find_packages("src"), + include_package_data=True, install_requires=REQUIRED_PKGS, extras_require=EXTRAS_REQUIRE, classifiers=[ From 65aff3215095b9f0576ca2ff0627b488e964cd52 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 22:29:19 +0100 Subject: [PATCH 073/183] Use kwargs instead of args in super call --- src/setfit/span/modeling.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index 02b0b1dd..f25a72c1 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -85,14 +85,14 @@ def _from_pretrained( model_kwargs["span_context"] = span_context return super(SpanSetFitModel, cls)._from_pretrained( - model_id, - revision, - cache_dir, - force_download, - proxies, - resume_download, - local_files_only, - token, + model_id=model_id, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, **model_kwargs, ) From eeeac55c07126a39099a70e51d4fdb202cc4ed29 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 9 Nov 2023 22:39:00 +0100 Subject: [PATCH 074/183] Use v0.13.0 as min. version as huggingface/huggingface_hub#1315 solved an issue that was causing failures --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3d1f8433..bdc32252 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ "datasets>=2.3.0", "sentence-transformers>=2.2.1", "evaluate>=0.3.0", - "huggingface_hub>=0.12.0", + "huggingface_hub>=0.13.0", "scikit-learn", ] ABSA_REQUIRE = ["spacy"] From 3214f1bbad2692249748393f177f8e15e86ed5d6 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 10 Nov 2023 11:30:08 +0100 Subject: [PATCH 075/183] Use en_core_web_sm for tests --- tests/conftest.py | 2 +- tests/span/test_modeling.py | 26 +++++++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 11051223..f92a81d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ def model() -> SetFitModel: @pytest.fixture() def absa_model() -> AbsaModel: - return AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + return AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm") @pytest.fixture() diff --git a/tests/span/test_modeling.py b/tests/span/test_modeling.py index 02fd7c3e..4a2caec0 100644 --- a/tests/span/test_modeling.py +++ b/tests/span/test_modeling.py @@ -10,7 +10,7 @@ def test_loading(): - model = AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + model = AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm") assert isinstance(model, AbsaModel) assert isinstance(model.aspect_extractor, AspectExtractor) assert isinstance(model.aspect_model, AspectModel) @@ -19,12 +19,14 @@ def test_loading(): model = AbsaModel.from_pretrained( "sentence-transformers/paraphrase-albert-small-v2@6c91e73a51599e35bd1145dfdcd3289215225009", "sentence-transformers/paraphrase-albert-small-v2", + spacy_model="en_core_web_sm", ) assert isinstance(model, AbsaModel) model = AbsaModel.from_pretrained( "sentence-transformers/paraphrase-albert-small-v2", "sentence-transformers/paraphrase-albert-small-v2@6c91e73a51599e35bd1145dfdcd3289215225009", + spacy_model="en_core_web_sm", ) assert isinstance(model, AbsaModel) @@ -33,16 +35,24 @@ def test_loading(): "sentence-transformers/paraphrase-albert-small-v2", spacy_model="not_a_spacy_model" ) - model = AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", normalize_embeddings=True) + model = AbsaModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm", normalize_embeddings=True + ) assert model.aspect_model.normalize_embeddings assert model.polarity_model.normalize_embeddings - aspect_model = AspectModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", span_context=12) + aspect_model = AspectModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm", span_context=12 + ) assert aspect_model.span_context == 12 - polarity_model = PolarityModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", span_context=12) + polarity_model = PolarityModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm", span_context=12 + ) assert polarity_model.span_context == 12 - model = AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", span_contexts=(12, None)) + model = AbsaModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm", span_contexts=(12, None) + ) assert model.aspect_model.span_context == 12 assert model.polarity_model.span_context == 3 # <- default @@ -56,7 +66,9 @@ def test_save_load(absa_model: AbsaModel) -> None: assert (Path(tmp_dir + "-aspect") / "config_span_setfit.json").exists() assert (Path(tmp_dir + "-polarity") / "config_span_setfit.json").exists() - fresh_model = AbsaModel.from_pretrained(tmp_dir + "-aspect", tmp_dir + "-polarity") + fresh_model = AbsaModel.from_pretrained( + tmp_dir + "-aspect", tmp_dir + "-polarity", spacy_model="en_core_web_sm" + ) assert fresh_model.polarity_model.span_context == 5 with TemporaryDirectory() as aspect_tmp_dir: @@ -65,7 +77,7 @@ def test_save_load(absa_model: AbsaModel) -> None: assert (Path(aspect_tmp_dir) / "config_span_setfit.json").exists() assert (Path(polarity_tmp_dir) / "config_span_setfit.json").exists() - fresh_model = AbsaModel.from_pretrained(aspect_tmp_dir, polarity_tmp_dir) + fresh_model = AbsaModel.from_pretrained(aspect_tmp_dir, polarity_tmp_dir, spacy_model="en_core_web_sm") assert fresh_model.polarity_model.span_context == 5 From 2b78bb05dea12ba4f827c9b082393c7b3e0b061d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 10 Nov 2023 11:39:01 +0100 Subject: [PATCH 076/183] Remove incorrect spacy_model from AspectModel/PolarityModel --- tests/span/test_modeling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/span/test_modeling.py b/tests/span/test_modeling.py index 4a2caec0..a2998e97 100644 --- a/tests/span/test_modeling.py +++ b/tests/span/test_modeling.py @@ -42,11 +42,11 @@ def test_loading(): assert model.polarity_model.normalize_embeddings aspect_model = AspectModel.from_pretrained( - "sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm", span_context=12 + "sentence-transformers/paraphrase-albert-small-v2", span_context=12 ) assert aspect_model.span_context == 12 polarity_model = PolarityModel.from_pretrained( - "sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm", span_context=12 + "sentence-transformers/paraphrase-albert-small-v2", span_context=12 ) assert polarity_model.span_context == 12 From b68f655dae56bc9ddf86426eb1a9966dd6ea2339 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 10 Nov 2023 11:55:57 +0100 Subject: [PATCH 077/183] Rerun formatting --- tests/span/test_modeling.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/span/test_modeling.py b/tests/span/test_modeling.py index a2998e97..0bc3ccb8 100644 --- a/tests/span/test_modeling.py +++ b/tests/span/test_modeling.py @@ -41,13 +41,9 @@ def test_loading(): assert model.aspect_model.normalize_embeddings assert model.polarity_model.normalize_embeddings - aspect_model = AspectModel.from_pretrained( - "sentence-transformers/paraphrase-albert-small-v2", span_context=12 - ) + aspect_model = AspectModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", span_context=12) assert aspect_model.span_context == 12 - polarity_model = PolarityModel.from_pretrained( - "sentence-transformers/paraphrase-albert-small-v2", span_context=12 - ) + polarity_model = PolarityModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", span_context=12) assert polarity_model.span_context == 12 model = AbsaModel.from_pretrained( From d85f0d97cf75a3c5d84219787e39e87fd87fd99a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 10 Nov 2023 12:14:23 +0100 Subject: [PATCH 078/183] Run CI on pre branch & workflow dispatch --- .github/workflows/quality.yml | 3 +++ .github/workflows/tests.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 9ced4d45..b3cdcd6b 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -5,9 +5,12 @@ on: branches: - main - v*-release + - v*-pre pull_request: branches: - main + - v*-pre + workflow_dispatch: jobs: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 243c1306..45dccb7f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,9 +5,12 @@ on: branches: - main - v*-release + - v*-pre pull_request: branches: - main + - v*-pre + workflow_dispatch: jobs: From 81952bfcb395acde74a1125c67d040a69664167e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 10 Nov 2023 13:36:12 +0100 Subject: [PATCH 079/183] Set development version to 1.0.0.dev0 --- setup.py | 2 +- src/setfit/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bdc32252..16980b84 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def combine_requirements(base_keys): setup( name="setfit", - version="0.8.0.dev0", + version="1.0.0.dev0", description="Efficient few-shot learning with Sentence Transformers", long_description=README_TEXT, long_description_content_type="text/markdown", diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index f131eee0..9540a68a 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.8.0.dev0" +__version__ = "1.0.0.dev0" import warnings From 5b763615ba71b75bc5216a5e7a9d7f33031cef11 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 10 Nov 2023 14:34:25 +0100 Subject: [PATCH 080/183] Extend training argument tests --- src/setfit/training_args.py | 18 ++---------------- tests/test_training_args.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 9ed24fb7..bf79d2ce 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -117,7 +117,7 @@ class TrainingArguments: logging_first_step (`bool`, *optional*, defaults to `False`): Whether to log and evaluate the first `global_step` or not. - logging_steps (`int`, *optional*, defaults to 50): + logging_steps (`int`, defaults to 50): Number of update steps between two logs if `logging_strategy="steps"`. evaluation_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`): The evaluation strategy to adopt during training. Possible values are: @@ -284,20 +284,6 @@ def __post_init__(self) -> None: f"strategy: {self.evaluation_strategy}\n- Save strategy: {self.save_strategy}" ) if self.evaluation_strategy == IntervalStrategy.STEPS and self.save_steps % self.eval_steps != 0: - if self.eval_steps < 1 or self.save_steps < 1: - if not (self.eval_steps < 1 and self.save_steps < 1): - raise ValueError( - "`load_best_model_at_end` requires the saving steps to be a multiple of the evaluation " - "steps, which cannot get guaranteed when mixing ratio and absolute steps for save_steps" - f"{self.save_steps} and eval_steps {self.eval_steps}." - ) - # Work around floating point precision issues - LARGE_MULTIPLIER = 1_000_000 - if (self.save_steps * LARGE_MULTIPLIER) % (self.eval_steps * LARGE_MULTIPLIER) != 0: - raise ValueError( - "`load_best_model_at_end` requires the saving steps to be a multiple of the evaluation " - f"steps, but found {self.save_steps}, which is not a multiple of {self.eval_steps}." - ) raise ValueError( "`load_best_model_at_end` requires the saving steps to be a round multiple of the evaluation " f"steps, but found {self.save_steps}, which is not a round multiple of {self.eval_steps}." @@ -305,7 +291,7 @@ def __post_init__(self) -> None: # logging_steps must be non-zero for logging_strategy that is other than 'no' if self.logging_strategy == IntervalStrategy.STEPS and self.logging_steps == 0: - raise ValueError(f"logging strategy {self.logging_strategy} requires non-zero --logging_steps") + raise ValueError(f"Logging strategy {self.logging_strategy} requires non-zero `logging_steps`") def to_dict(self) -> Dict[str, Any]: # filter out fields that are defined as field(init=False) diff --git a/tests/test_training_args.py b/tests/test_training_args.py index a573e10e..8e10653d 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -3,6 +3,7 @@ import pytest from setfit.training_args import TrainingArguments +from transformers import IntervalStrategy class TestTrainingArguments(TestCase): @@ -111,3 +112,33 @@ def test_learning_rates(self): self.assertEqual(args.body_embedding_learning_rate, learning_rate_B) self.assertEqual(args.body_classifier_learning_rate, learning_rate_A) self.assertEqual(args.head_learning_rate, learning_rate_C) + + def test_report_to(self): + args = TrainingArguments(report_to="none") + self.assertEqual(args.report_to, []) + args = TrainingArguments(report_to=["none"]) + self.assertEqual(args.report_to, []) + args = TrainingArguments(report_to="hello") + self.assertEqual(args.report_to, ["hello"]) + + def test_eval_steps_without_eval_strat(self): + args = TrainingArguments(eval_steps=5) + self.assertEqual(args.evaluation_strategy, IntervalStrategy.STEPS) + + def test_eval_strat_steps_without_eval_steps(self): + args = TrainingArguments(evaluation_strategy="steps") + self.assertEqual(args.eval_steps, args.logging_steps) + with self.assertRaises(ValueError): + TrainingArguments(evaluation_strategy="steps", logging_steps=0, logging_strategy="no") + + def test_load_best_model(self): + with self.assertRaises(ValueError): + TrainingArguments(load_best_model_at_end=True, evaluation_strategy="steps", save_strategy="epoch") + with self.assertRaises(ValueError): + TrainingArguments(load_best_model_at_end=True, evaluation_strategy="steps", save_strategy="steps", eval_steps=100, save_steps=50) + # No error: save_steps is a round multiple of eval_steps + TrainingArguments(load_best_model_at_end=True, evaluation_strategy="steps", save_strategy="steps", eval_steps=50, save_steps=100) + + def test_logging_steps_zero(self): + with self.assertRaises(ValueError): + TrainingArguments(logging_strategy="steps", logging_steps=0) \ No newline at end of file From 54b5d55799314cad646a25fd1e6d333e8daa7638 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 14 Nov 2023 10:26:03 +0100 Subject: [PATCH 081/183] Only create evaluation dataloader if eval_strat is set --- src/setfit/trainer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index baecd053..45def287 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -26,6 +26,7 @@ TrainerCallback, TrainerControl, TrainerState, + IntervalStrategy, ) from transformers.trainer_utils import ( HPSearchBackend, @@ -418,7 +419,7 @@ def train_embeddings( self.state.save_steps = args.save_steps train_dataloader, loss_func, batch_size = self.get_dataloader(x_train, y_train, args=args) - if x_eval is not None: + if x_eval is not None and args.evaluation_strategy != IntervalStrategy.NO: eval_dataloader, _, _ = self.get_dataloader(x_eval, y_eval, args=args) else: eval_dataloader = None From 47887132bc46beb7373b4fe272c4f5a59ddad12e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 14 Nov 2023 10:26:10 +0100 Subject: [PATCH 082/183] Run formatting --- src/setfit/trainer.py | 2 +- tests/test_training_args.py | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 45def287..5fdd784d 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -21,12 +21,12 @@ from transformers.trainer_callback import ( CallbackHandler, DefaultFlowCallback, + IntervalStrategy, PrinterCallback, ProgressCallback, TrainerCallback, TrainerControl, TrainerState, - IntervalStrategy, ) from transformers.trainer_utils import ( HPSearchBackend, diff --git a/tests/test_training_args.py b/tests/test_training_args.py index 8e10653d..993db7df 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -1,9 +1,9 @@ from unittest import TestCase import pytest +from transformers import IntervalStrategy from setfit.training_args import TrainingArguments -from transformers import IntervalStrategy class TestTrainingArguments(TestCase): @@ -124,7 +124,7 @@ def test_report_to(self): def test_eval_steps_without_eval_strat(self): args = TrainingArguments(eval_steps=5) self.assertEqual(args.evaluation_strategy, IntervalStrategy.STEPS) - + def test_eval_strat_steps_without_eval_steps(self): args = TrainingArguments(evaluation_strategy="steps") self.assertEqual(args.eval_steps, args.logging_steps) @@ -135,10 +135,22 @@ def test_load_best_model(self): with self.assertRaises(ValueError): TrainingArguments(load_best_model_at_end=True, evaluation_strategy="steps", save_strategy="epoch") with self.assertRaises(ValueError): - TrainingArguments(load_best_model_at_end=True, evaluation_strategy="steps", save_strategy="steps", eval_steps=100, save_steps=50) + TrainingArguments( + load_best_model_at_end=True, + evaluation_strategy="steps", + save_strategy="steps", + eval_steps=100, + save_steps=50, + ) # No error: save_steps is a round multiple of eval_steps - TrainingArguments(load_best_model_at_end=True, evaluation_strategy="steps", save_strategy="steps", eval_steps=50, save_steps=100) - + TrainingArguments( + load_best_model_at_end=True, + evaluation_strategy="steps", + save_strategy="steps", + eval_steps=50, + save_steps=100, + ) + def test_logging_steps_zero(self): with self.assertRaises(ValueError): - TrainingArguments(logging_strategy="steps", logging_steps=0) \ No newline at end of file + TrainingArguments(logging_strategy="steps", logging_steps=0) From 74a5b7cf36803c1ecb68ded867d1dff145280b7a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 14 Nov 2023 11:08:55 +0100 Subject: [PATCH 083/183] max_steps isn't optional --- src/setfit/training_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index bf79d2ce..716fb5e9 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -35,7 +35,7 @@ class TrainingArguments: Set the number of epochs the embedding and classifier training phases respectively, or set both if an integer is provided. Note that the number of epochs for the classifier is only used with a differentiable PyTorch head. - max_steps (`int`, *optional*, defaults to `-1`): + max_steps (`int`, defaults to `-1`): If set to a positive number, the total number of training steps to perform. Overrides `num_epochs`. The training may stop before reaching the set number of steps when all data is exhausted. sampling_strategy (`str`, defaults to `"oversampling"`): From 7ef5bbc5819c30f9f5194bed5530e252b1aa5c88 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 15 Nov 2023 11:59:38 +0100 Subject: [PATCH 084/183] Fix indentation of docstring --- src/setfit/data.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/setfit/data.py b/src/setfit/data.py index 2d9cd5f8..1441d2fa 100644 --- a/src/setfit/data.py +++ b/src/setfit/data.py @@ -44,9 +44,9 @@ def get_templated_dataset( Args: dataset (`Dataset`, *optional*): A Dataset to add templated examples to. candidate_labels (`List[str]`, *optional*): The list of candidate - labels to be fed into the template to construct examples. + labels to be fed into the template to construct examples. reference_dataset (`str`, *optional*): A dataset to take labels - from, if `candidate_labels` is not supplied. + from, if `candidate_labels` is not supplied. template (`str`, *optional*, defaults to `"This sentence is {}"`): The template used to turn each label into a synthetic training example. This template must include a {} for the candidate label to be inserted into the template. @@ -54,16 +54,16 @@ def get_templated_dataset( candidate label "sports", this would produce an example "This sentence is sports". sample_size (`int`, *optional*, defaults to 2): The number of examples to make for - each candidate label. + each candidate label. text_column (`str`, *optional*, defaults to `"text"`): The name of the column - containing the text of the examples. + containing the text of the examples. label_column (`str`, *optional*, defaults to `"label"`): The name of the column - in `dataset` containing the labels of the examples. + in `dataset` containing the labels of the examples. multi_label (`bool`, *optional*, defaults to `False`): Whether or not multiple - candidate labels can be true. + candidate labels can be true. label_names_column (`str`, *optional*, defaults to "label_text"): The name of the - label column in the `reference_dataset`, to be used in case there is no ClassLabel - feature for the label column. + label column in the `reference_dataset`, to be used in case there is no ClassLabel + feature for the label column. Returns: `Dataset`: A copy of the input Dataset with templated examples added. From ca3030fa828677b3663299a86b51ae365df93a00 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 15 Nov 2023 12:01:03 +0100 Subject: [PATCH 085/183] Apply fixes for HPO --- src/setfit/trainer.py | 12 +++++---- src/setfit/training_args.py | 49 +++++++++++++++++++++++-------------- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 5fdd784d..b8fd5a06 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -267,6 +267,8 @@ def apply_hyperparameters(self, params: Dict[str, Any], final_model: bool = Fals else: self.args = TrainingArguments.from_dict(params, ignore_extra=True) + # Seed must be set before instantiating the model when using model_init. + set_seed(self.args.seed) self.model = self.model_init(params) if final_model: self.model_init = None @@ -363,14 +365,11 @@ def train( stacklevel=2, ) - args = args or self.args or TrainingArguments() - - # Seed must be set before instantiating the model when using model_init. - set_seed(args.seed) - if trial: # Trial and model initialization self._hp_search_setup(trial) # sets trainer parameters and initializes model + args = args or self.args or TrainingArguments() + if self.train_dataset is None: raise ValueError( f"Training requires a `train_dataset` given to the `{self.__class__.__name__}` initialization." @@ -417,6 +416,9 @@ def train_embeddings( self.state.logging_steps = args.logging_steps self.state.eval_steps = args.eval_steps self.state.save_steps = args.save_steps + # Reset the state + self.state.global_step = 0 + self.state.total_flos = 0 train_dataloader, loss_func, batch_size = self.get_dataloader(x_train, y_train, args=args) if x_eval is not None and args.evaluation_strategy != IntervalStrategy.NO: diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 716fb5e9..b74442b1 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -23,6 +23,10 @@ class TrainingArguments: """ TrainingArguments is the subset of the arguments which relate to the training loop itself. + Note that training with SetFit consists of two phases behind the scenes: **finetuning embeddings** and + **training a classification head**. As a result, some of the training arguments can be tuples, + where the two values are used for each of the two phases, respectively. The second value is often only + used when training the model was loaded using `use_differentiable_head=True`. Parameters: output_dir (`str`, defaults to `"checkpoints"`): @@ -63,7 +67,8 @@ class TrainingArguments: Note that the body learning rate for the classifier is only used with a differentiable PyTorch head *and* if `end_to_end=True`. head_learning_rate (`float`, defaults to `1e-2`): - Set the learning rate for the head for the classifier training phase. + Set the learning rate for the head for the classifier training phase. Only used with a + differentiable PyTorch head. loss (`nn.Module`, defaults to `CosineSimilarityLoss`): The loss function to use for contrastive training of the embedding training phase. distance_metric (`Callable`, defaults to `BatchHardTripletLossDistanceFunction.cosine_distance`): @@ -160,14 +165,10 @@ class TrainingArguments: # batch_size is only used to conveniently set `embedding_batch_size` and `classifier_batch_size` # which are used in practice batch_size: Union[int, Tuple[int, int]] = field(default=(16, 2), repr=False) - embedding_batch_size: int = None - classifier_batch_size: int = None # num_epochs is only used to conveniently set `embedding_num_epochs` and `classifier_num_epochs` # which are used in practice num_epochs: Union[int, Tuple[int, int]] = field(default=(1, 16), repr=False) - embedding_num_epochs: int = None - classifier_num_epochs: int = None max_steps: int = -1 @@ -177,8 +178,6 @@ class TrainingArguments: # As with batch_size and num_epochs, the first value in the tuple is the learning rate # for the embeddings step, while the second value is the learning rate for the classifier step. body_learning_rate: Union[float, Tuple[float, float]] = field(default=(2e-5, 1e-5), repr=False) - body_embedding_learning_rate: float = None - body_classifier_learning_rate: float = None head_learning_rate: float = 1e-2 # Loss-related arguments @@ -222,27 +221,15 @@ def __post_init__(self) -> None: # Set `self.embedding_batch_size` and `self.classifier_batch_size` using values from `self.batch_size` if isinstance(self.batch_size, int): self.batch_size = (self.batch_size, self.batch_size) - if self.embedding_batch_size is None: - self.embedding_batch_size = self.batch_size[0] - if self.classifier_batch_size is None: - self.classifier_batch_size = self.batch_size[1] # Set `self.embedding_num_epochs` and `self.classifier_num_epochs` using values from `self.num_epochs` if isinstance(self.num_epochs, int): self.num_epochs = (self.num_epochs, self.num_epochs) - if self.embedding_num_epochs is None: - self.embedding_num_epochs = self.num_epochs[0] - if self.classifier_num_epochs is None: - self.classifier_num_epochs = self.num_epochs[1] # Set `self.body_embedding_learning_rate` and `self.body_classifier_learning_rate` using # values from `self.body_learning_rate` if isinstance(self.body_learning_rate, float): self.body_learning_rate = (self.body_learning_rate, self.body_learning_rate) - if self.body_embedding_learning_rate is None: - self.body_embedding_learning_rate = self.body_learning_rate[0] - if self.body_classifier_learning_rate is None: - self.body_classifier_learning_rate = self.body_learning_rate[1] if self.warmup_proportion < 0.0 or self.warmup_proportion > 1.0: raise ValueError( @@ -293,6 +280,30 @@ def __post_init__(self) -> None: if self.logging_strategy == IntervalStrategy.STEPS and self.logging_steps == 0: raise ValueError(f"Logging strategy {self.logging_strategy} requires non-zero `logging_steps`") + @property + def embedding_batch_size(self) -> int: + return self.batch_size[0] + + @property + def classifier_batch_size(self) -> int: + return self.batch_size[1] + + @property + def embedding_num_epochs(self) -> int: + return self.num_epochs[0] + + @property + def classifier_num_epochs(self) -> int: + return self.num_epochs[1] + + @property + def body_embedding_learning_rate(self) -> float: + return self.body_learning_rate[0] + + @property + def body_classifier_learning_rate(self) -> float: + return self.body_learning_rate[1] + def to_dict(self) -> Dict[str, Any]: # filter out fields that are defined as field(init=False) return {field.name: getattr(self, field.name) for field in fields(self) if field.init} From f114572c8bce53bd5b0acab72fd46dc360994e00 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 15 Nov 2023 16:35:06 +0100 Subject: [PATCH 086/183] Remove outdated tests --- tests/test_training_args.py | 48 ------------------------------------- 1 file changed, 48 deletions(-) diff --git a/tests/test_training_args.py b/tests/test_training_args.py index 993db7df..391e7c16 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -31,16 +31,6 @@ def test_batch_sizes(self): self.assertEqual(args.embedding_batch_size, batch_size_A) self.assertEqual(args.classifier_batch_size, batch_size_B) - args = TrainingArguments(batch_size=(batch_size_A, batch_size_B), embedding_batch_size=batch_size_C) - self.assertEqual(args.batch_size, (batch_size_A, batch_size_B)) - self.assertEqual(args.embedding_batch_size, batch_size_C) - self.assertEqual(args.classifier_batch_size, batch_size_B) - - args = TrainingArguments(batch_size=batch_size_A, embedding_batch_size=batch_size_C) - self.assertEqual(args.batch_size, (batch_size_A, batch_size_A)) - self.assertEqual(args.embedding_batch_size, batch_size_C) - self.assertEqual(args.classifier_batch_size, batch_size_A) - def test_num_epochs(self): num_epochs_A = 12 num_epochs_B = 4 @@ -56,16 +46,6 @@ def test_num_epochs(self): self.assertEqual(args.embedding_num_epochs, num_epochs_A) self.assertEqual(args.classifier_num_epochs, num_epochs_B) - args = TrainingArguments(num_epochs=(num_epochs_A, num_epochs_B), embedding_num_epochs=num_epochs_C) - self.assertEqual(args.num_epochs, (num_epochs_A, num_epochs_B)) - self.assertEqual(args.embedding_num_epochs, num_epochs_C) - self.assertEqual(args.classifier_num_epochs, num_epochs_B) - - args = TrainingArguments(num_epochs=num_epochs_A, embedding_num_epochs=num_epochs_C) - self.assertEqual(args.num_epochs, (num_epochs_A, num_epochs_A)) - self.assertEqual(args.embedding_num_epochs, num_epochs_C) - self.assertEqual(args.classifier_num_epochs, num_epochs_A) - def test_learning_rates(self): learning_rate_A = 1e-2 learning_rate_B = 1e-3 @@ -85,34 +65,6 @@ def test_learning_rates(self): self.assertEqual(args.body_classifier_learning_rate, learning_rate_B) self.assertEqual(args.head_learning_rate, base.head_learning_rate) - args = TrainingArguments( - body_learning_rate=(learning_rate_A, learning_rate_B), head_learning_rate=learning_rate_C - ) - self.assertEqual(args.body_learning_rate, (learning_rate_A, learning_rate_B)) - self.assertEqual(args.body_embedding_learning_rate, learning_rate_A) - self.assertEqual(args.body_classifier_learning_rate, learning_rate_B) - self.assertEqual(args.head_learning_rate, learning_rate_C) - - args = TrainingArguments( - body_learning_rate=learning_rate_A, - body_embedding_learning_rate=learning_rate_B, - head_learning_rate=learning_rate_C, - ) - # Perhaps not ideal, but body_learning_rate is never used directly: - self.assertEqual(args.body_learning_rate, (learning_rate_A, learning_rate_A)) - self.assertEqual(args.body_embedding_learning_rate, learning_rate_B) - self.assertEqual(args.body_classifier_learning_rate, learning_rate_A) - self.assertEqual(args.head_learning_rate, learning_rate_C) - - args = TrainingArguments( - body_classifier_learning_rate=learning_rate_A, - body_embedding_learning_rate=learning_rate_B, - head_learning_rate=learning_rate_C, - ) - self.assertEqual(args.body_embedding_learning_rate, learning_rate_B) - self.assertEqual(args.body_classifier_learning_rate, learning_rate_A) - self.assertEqual(args.head_learning_rate, learning_rate_C) - def test_report_to(self): args = TrainingArguments(report_to="none") self.assertEqual(args.report_to, []) From 8d118d55074e33c601eec35227a609e20fd909f7 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 11:45:12 +0100 Subject: [PATCH 087/183] Use SetFitModel as the model in CallbackHandler Instead of SentenceTransformer --- src/setfit/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index b8fd5a06..98d8ffdf 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -209,7 +209,7 @@ def __init__( callbacks = default_callbacks if callbacks is None else default_callbacks + callbacks # TODO: Observe optimizer and scheduler by wrapping SentenceTransformer._get_scheduler self.callback_handler = CallbackHandler( - callbacks, self.model.model_body, self.model.model_body.tokenizer, None, None + callbacks, self.model, self.model.model_body.tokenizer, None, None ) self.state = TrainerState() self.control = TrainerControl() From b964238c6aeef0c6e15a0ac8b6711644d8a27e7a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 11:46:09 +0100 Subject: [PATCH 088/183] Correctly set the total training steps based on args.max_steps Also pass optimizer/scheduler/train/eval dataloader to the CallbackHandler during training --- src/setfit/trainer.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 98d8ffdf..269b7401 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -426,7 +426,10 @@ def train_embeddings( else: eval_dataloader = None - total_train_steps = len(train_dataloader) * args.embedding_num_epochs + if args.max_steps > 0: + total_train_steps = args.max_steps + else: + total_train_steps = len(train_dataloader) * args.embedding_num_epochs logger.info("***** Running training *****") logger.info(f" Num examples = {len(train_dataloader)}") logger.info(f" Num epochs = {args.embedding_num_epochs}") @@ -541,6 +544,7 @@ def _train_sentence_transformer( else: self.state.max_steps = len(train_dataloader) * args.embedding_num_epochs self.control = self.callback_handler.on_train_begin(args, self.state, self.control) + steps_per_epoch = len(train_dataloader) if args.use_amp: scaler = torch.cuda.amp.GradScaler() @@ -553,9 +557,6 @@ def _train_sentence_transformer( if eval_dataloader: eval_dataloader.collate_fn = model_body.smart_batching_collate - steps_per_epoch = len(train_dataloader) - num_train_steps = int(steps_per_epoch * args.embedding_num_epochs) - # Prepare optimizers param_optimizer = list(loss_func.named_parameters()) @@ -570,8 +571,12 @@ def _train_sentence_transformer( optimizer = torch.optim.AdamW(optimizer_grouped_parameters, **{"lr": args.body_embedding_learning_rate}) scheduler_obj = model_body._get_scheduler( - optimizer, scheduler="WarmupLinear", warmup_steps=warmup_steps, t_total=num_train_steps + optimizer, scheduler="WarmupLinear", warmup_steps=warmup_steps, t_total=self.state.max_steps ) + self.callback_handler.optimizer = optimizer + self.callback_handler.lr_scheduler = scheduler_obj + self.callback_handler.train_dataloader = train_dataloader + self.callback_handler.eval_dataloader = eval_dataloader data_iterator = iter(train_dataloader) skip_scheduler = False From 2f06847a12075a04b68029b7360512ff8069f6bd Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 11:46:18 +0100 Subject: [PATCH 089/183] Add missing comma --- src/setfit/training_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index b74442b1..ce667b42 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -53,7 +53,7 @@ class TrainingArguments: number of positive/ negative sentence pairs). The default is set to `"oversampling"`, ensuring all sentence pairs are drawn at least once. - Alternatively setting `num_iterations` will override this argument and determine the number + Alternatively, setting `num_iterations` will override this argument and determine the number of generated sentence pairs. num_iterations (`int`, *optional*): If not set the `sampling_strategy` will determine the number of sentence pairs to generate. From fcb38fcc4f9ca320dcc441e812f9362413f775f0 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 11:46:29 +0100 Subject: [PATCH 090/183] Capitalize first letter of sentence --- notebooks/zero-shot-classification.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/notebooks/zero-shot-classification.ipynb b/notebooks/zero-shot-classification.ipynb index 8a0f0737..509eeb03 100644 --- a/notebooks/zero-shot-classification.ipynb +++ b/notebooks/zero-shot-classification.ipynb @@ -297,7 +297,7 @@ "id": "85a0d010-389a-4bc1-aaef-4814c0c96e45", "metadata": {}, "source": [ - "we can see that each input takes the form of the template and has a corresponding label associated with it. \n", + "We can see that each input takes the form of the template and has a corresponding label associated with it. \n", "\n", "Let's not train a SetFit model on these examples!" ] @@ -562,7 +562,7 @@ "id": "01c7c784-3896-483f-bc60-43373a38b4ed", "metadata": {}, "source": [ - "we can use the `str2int()` function from the `label` column to convert them to integers. " + "We can use the `str2int()` function from the `label` column to convert them to integers. " ] }, { @@ -988,7 +988,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAswAAAGdCAYAAAAG6yXVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8mklEQVR4nO3deVxV1f7/8fcBZFAGJxQoBIdwNkHTtJwxp0zNcriVoWldrWzStK+SUzmlpmWDM3brSpZm3lJTuel1SlPBVMiBK2KFQ06Imijs3x9e988juOUgehBfz8djPx6cfdZe+7MXlG8Wa+9jMwzDEAAAAIBcuTi7AAAAAKAwIzADAAAAFgjMAAAAgAUCMwAAAGCBwAwAAABYIDADAAAAFgjMAAAAgAUCMwAAAGDBzdkFAHe67Oxs/fHHH/Lx8ZHNZnN2OQAAIA8Mw9CZM2cUFBQkFxfrOWQCM3CT/vjjDwUHBzu7DAAAkA+HDh3Svffea9mGwAzcJB8fH0mX/4Pz9fV1cjUAACAv0tPTFRwcbP47boXADNykK8swfH19CcwAANxh8rKckpv+AAAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACw4ObsAoCiotaIH+TiUdzZZQAAUGSkjO/g7BIkMcMMAAAAWCIwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwFwI2m01LlizJ17HR0dF6/vnnC7agu0jz5s316quvmq8ffPBBLVq0yHkFAQCAQofAfJVjx46pf//+qlChgjw8PBQQEKA2bdpow4YNee5j5MiRqlu3bo79oaGhstlsdtu9994rSUpLS1O7du0kSSkpKbLZbEpISLjhuQ4fPqxp06Zp2LBhea6vKLg25Bak4cOHa+jQocrOzr4l/QMAgDsPgfkqXbt2VXx8vObPn6+9e/dq6dKlat68uY4fP14g/Y8ePVppaWnmFh8fL0kKCAiQh4eHw/3Nnj1bjRs3VkhISIHUB6ldu3Y6c+aMli9f7uxSAABAIUFg/p9Tp05p3bp1mjBhglq0aKGQkBA1aNBAb731lh577DG7dn379pW/v798fX3VsmVL7dixQ5IUExOjUaNGaceOHeYsckxMjHmsj4+PAgICzM3f31+S/ZKMihUrSpLCw8Nls9nUvHnz69YcGxurjh072u1bsWKFHn74YZUsWVJlypTRo48+quTkZPP9NWvWyGaz6dSpU+a+hIQE2Ww2paSkmPtmzZql4OBgFS9eXF26dNGUKVNUsmRJ8/0rM+lz585VhQoV5O3trQEDBigrK0sTJ05UQECAypUrp3fffTfHOF9v/K7u9x//+IdCQ0Pl5+enHj166MyZM5KkqKgorV27VtOmTTPH+Erdu3btUrt27eTt7a3y5cvrmWee0Z9//mn2ffbsWfXq1Uve3t4KDAzU5MmTc4ypq6ur2rdvr9jY2OuOOwAAuLsQmP/H29tb3t7eWrJkiS5cuHDddk8++aSOHj2q5cuXa9u2bYqIiFCrVq104sQJde/eXW+88YZq1qxpziJ3797doTq2bNkiSVq9erXS0tK0ePHiXNudOHFCiYmJql+/vt3+s2fP6vXXX9fWrVsVFxcnFxcXdenSxaElBhs2bNDf//53vfLKK0pISFDr1q1zBF9JSk5O1vLly7VixQotWLBAc+bMUYcOHfTbb79p7dq1mjBhgoYPH67Nmzebx1iN39X9LlmyRN99952+++47rV27VuPHj5ckTZs2TY0aNVK/fv3MMQ4ODtapU6fUsmVLhYeHa+vWrVqxYoWOHDmibt26mf0OHjxYa9eu1bfffquVK1dqzZo12r59e47ratCggdatW3fd8blw4YLS09PtNgAAUHS5ObuAwsLNzU0xMTHq16+fPv30U0VERKhZs2bq0aOH6tSpI0lav369tmzZoqNHj5pLKCZNmqQlS5bo66+/1vPPPy9vb2+5ubkpICAgxzmGDBmi4cOHm6/Hjh2rgQMH2rW5MutcpkyZXPu4IjU1VYZhKCgoyG5/165d7V7PnTtX/v7+SkxMVK1atfI0Fh9++KHatWunQYMGSZLCwsK0ceNGfffdd3btsrOzNXfuXPn4+KhGjRpq0aKF9uzZo2XLlsnFxUVVq1bVhAkT9OOPP6phw4Z5Gr8r/cbExMjHx0eS9MwzzyguLk7vvvuu/Pz85O7uruLFi9uNz/Tp0xUeHq6xY8faXXtwcLD27t2roKAgzZkzR59//rlatWolSZo/f765jvxqQUFBOnTokLKzs+XikvN3ynHjxmnUqFF5GksAAHDnY4b5Kl27dtUff/yhpUuXqm3btlqzZo0iIiLMZRU7duxQRkaGypQpY85Ie3t768CBA3bLHq5n8ODBSkhIMLdevXrlu9bz589Lkjw9Pe3279u3Tz179lSlSpXk6+ur0NBQSZcDdl7t2bNHDRo0sNt37Wvp8o2MV0KtJJUvX141atSwC5nly5fX0aNHJeV9/K7tNzAw0Ozjenbs2KEff/zRrt9q1apJujxjnZycrMzMTDVs2NA8pnTp0qpatWqOvry8vJSdnX3dvzS89dZbOn36tLkdOnTIsjYAAHBnY4b5Gp6enmrdurVat26t6Oho9e3bVyNGjFBUVJQyMjIUGBioNWvW5Dju6vW911O2bFlVqVKlQOosW7asJOnkyZPmrLQkdezYUSEhIZo1a5aCgoKUnZ2tWrVqKTMzU5LMMGsYhnnMxYsX81VDsWLF7F7bbLZc911ZDpLX8bPq43oyMjLUsWNHTZgwIcd7gYGB2r9/v+XxVztx4oRKlCghLy+vXN/38PDI102aAADgzkRgvoEaNWqYN+RFRETo8OHDcnNzM2dur+Xu7q6srKx8n8/d3V2SbthH5cqV5evrq8TERIWFhUmSjh8/rj179mjWrFlq0qSJpMvLSK52JVynpaWpVKlSkpTjEXZVq1bVzz//bLfv2tf5kZfxy4vcxjgiIkKLFi1SaGio3Nxy/lhXrlxZxYoV0+bNm1WhQgVJl3/Z2Lt3r5o1a2bXdteuXQoPD893fQAAoGhhScb/HD9+XC1bttTnn3+uX375RQcOHNBXX32liRMnqlOnTpKkyMhINWrUSJ07d9bKlSuVkpKijRs3atiwYdq6dauky8sJDhw4oISEBP3555+WNxDmply5cvLy8jJvWjt9+nSu7VxcXBQZGWkXiEuVKqUyZcpo5syZ2r9/v/7973/r9ddftzuuSpUqCg4O1siRI7Vv3z59//33OZ4W8fLLL2vZsmWaMmWK9u3bpxkzZmj58uWy2WwOXcu18jJ+eREaGqrNmzcrJSVFf/75p7Kzs/Xiiy/qxIkT6tmzp37++WclJyfrhx9+UO/evZWVlSVvb28999xzGjx4sP79739r165dioqKynWN8rp16/TII4/c1LUCAICig8D8P97e3mrYsKHef/99NW3aVLVq1VJ0dLT69eun6dOnS7q8NGDZsmVq2rSpevfurbCwMPXo0UMHDx5U+fLlJV1eB922bVu1aNFC/v7+WrBggUN1uLm56YMPPtCMGTMUFBRkhvXc9O3bV7GxseZyBRcXF8XGxmrbtm2qVauWXnvtNb333nt2xxQrVkwLFizQr7/+qjp16mjChAl655137No89NBD+vTTTzVlyhTdf//9WrFihV577bUc66UdlZfxy4tBgwbJ1dVVNWrUkL+/v1JTUxUUFKQNGzYoKytLjzzyiGrXrq1XX31VJUuWNEPxe++9pyZNmqhjx46KjIzUww8/rHr16tn1/fvvv2vjxo3q3bv3TV0rAAAoOmzG1YtZcUcxDEMNGzbUa6+9pp49e97Sc/Xr10+//vqr5ePWioIhQ4bo5MmTmjlzZp6PSU9Pl5+fn4JfXSgXj+K3sDoAAO4uKeM73LK+r/z7ffr0afn6+lq2ZYb5Dmaz2TRz5kxdunSpwPueNGmSduzYof379+vDDz/U/Pnz9eyzzxb4eQqbcuXKacyYMc4uAwAAFCLc9HeHq1u3rurWrVvg/W7ZskUTJ07UmTNnVKlSJX3wwQfq27dvgZ+nsHnjjTecXQIAAChkCMzI1cKFC51dAgAAQKHAkgwAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAgpuzCwCKil2j2sjX19fZZQAAgALGDDMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABgwc3ZBQBFRa0RP8jFo7izywAAFGEp4zs4u4S7EjPMAAAAgAUCMwAAAGCBwAwAAABYIDADAAAAFgjMAAAAgAUCMwAAAGCBwAwAAABYIDADAAAAFvL8wSXh4eGy2Wx5art9+/Z8FwQAAAAUJnkOzJ07dza//uuvv/Txxx+rRo0aatSokSTpp59+0u7duzVgwIACLxIAAABwljwH5hEjRphf9+3bVwMHDtSYMWNytDl06FDBVQcAAAA4Wb7WMH/11Vfq1atXjv1PP/20Fi1adNNFAQAAAIVFvgKzl5eXNmzYkGP/hg0b5OnpedNFAQAAAIVFnpdkXO3VV19V//79tX37djVo0ECStHnzZs2dO1fR0dEFWiAAAADgTPkKzEOHDlWlSpU0bdo0ff7555Kk6tWra968eerWrVuBFggAAAA4U74CsyR169aNcAwAAIAiL9+BWZK2bdumpKQkSVLNmjUVHh5eIEUBAAAAhUW+AvPRo0fVo0cPrVmzRiVLlpQknTp1Si1atFBsbKz8/f0LskYAAADAafL1lIyXX35ZZ86c0e7du3XixAmdOHFCu3btUnp6ugYOHFjQNQIAAABOk68Z5hUrVmj16tWqXr26ua9GjRr66KOP9MgjjxRYcQAAAICz5WuGOTs7W8WKFcuxv1ixYsrOzr7pogAAAIDCIl+BuWXLlnrllVf0xx9/mPt+//13vfbaa2rVqlWBFQcAAAA4W74C8/Tp05Wenq7Q0FBVrlxZlStXVsWKFZWenq4PP/ywoGsEAAAAnCZfa5iDg4O1fft2rV69Wr/++qukyx9cEhkZWaDFAQAAAM6W7+cw22w2tW7dWq1bty7IegAAAIBCJd+BOS4uTnFxcTp69GiOG/3mzp1704Uhb2w2m7755ht17tzZ4WOjo6N15MgRzZw5s+ALu4WioqJ06tQpLVmypMD7fvDBBzV48GB17dq1wPsGAAB3pnytYR41apQeeeQRxcXF6c8//9TJkyfttrvRsWPH1L9/f1WoUEEeHh4KCAhQmzZttGHDhjz3MXLkSNWtWzfH/tDQUNlsNrvt3nvvlSSlpaWpXbt2kqSUlBTZbDYlJCTc8FyHDx/WtGnTNGzYsDzXV5TExMSYH7pzteHDh2vo0KE87QUAAJjyNcP86aefKiYmRs8880xB13PH6tq1qzIzMzV//nxVqlRJR44cUVxcnI4fP14g/Y8ePVr9+vUzX7u6ukqSAgIC8tXf7Nmz1bhxY4WEhBRIfbnJzMyUu7v7Lev/VmjXrp369u2r5cuXq0OHDs4uBwAAFAL5mmHOzMxU48aNC7qWO9apU6e0bt06TZgwQS1atFBISIgaNGigt956S4899phdu759+8rf31++vr5q2bKlduzYIenyjOeoUaO0Y8cOcxY5JibGPNbHx0cBAQHmduXjx202m7k0oWLFipKk8PBw2Ww2NW/e/Lo1x8bGqmPHjubrK7PT125X97F+/Xo1adJEXl5eCg4O1sCBA3X27Fnz/dDQUI0ZM0a9evWSr6+vnn/+eUnSokWLVLNmTXl4eCg0NFSTJ0++4Zh+/fXXql27try8vFSmTBlFRkbanUuSJk2apMDAQJUpU0YvvviiLl68aL538uRJ9erVS6VKlVLx4sXVrl077du3T5K0Zs0a9e7dW6dPnzavc+TIkZIu/yLSvn17xcbGXre2CxcuKD093W4DAABFV74Cc9++ffXPf/6zoGu5Y3l7e8vb21tLlizRhQsXrtvuySef1NGjR7V8+XJt27ZNERERatWqlU6cOKHu3bvrjTfeUM2aNZWWlqa0tDR1797doTq2bNkiSVq9erXS0tK0ePHiXNudOHFCiYmJql+/vrkvODjYPG9aWpri4+NVpkwZNW3aVJKUnJystm3bqmvXrvrll1/05Zdfav369XrppZfs+p40aZLuv/9+xcfHKzo6Wtu2bVO3bt3Uo0cP7dy5UyNHjlR0dLTdLwPXSktLU8+ePdWnTx8lJSVpzZo1evzxx2UYhtnmxx9/VHJysn788UfNnz9fMTExdn1GRUVp69atWrp0qTZt2iTDMNS+fXtdvHhRjRs31tSpU+Xr62te76BBg8xjGzRooHXr1l23vnHjxsnPz8/cgoODr9sWAADc+WzG1SnEwuuvv25+nZ2drfnz56tOnTqqU6dOjk/9mzJlSsFWeQdYtGiR+vXrp/PnzysiIkLNmjVTjx49VKdOHUmXZ2c7dOigo0ePysPDwzyuSpUqevPNN/X8889r5MiRWrJkSY41yKGhoUpLS7Mb57Fjx2rgwIF2N/2lpKSoYsWKio+Pz3Ut9BUJCQkKDw9XampqrmHvr7/+UvPmzeXv769vv/1WLi4u6tu3r1xdXTVjxgyz3fr169WsWTOdPXtWnp6eCg0NVXh4uL755huzzVNPPaVjx45p5cqV5r4333xT33//vXbv3p1rfdu3b1e9evWUkpKS65KRqKgorVmzRsnJyebSlG7dusnFxUWxsbHat2+fwsLCtGHDBvMvIcePH1dwcLDmz5+vJ598UjExMXr11Vd16tSpHP0vXbpUXbp00cWLF+XikvN3ygsXLtj9YpSenq7g4GAFv7pQLh7Fc70mAAAKQsp4lgsWlPT0dPn5+en06dPy9fW1bJvnNczx8fF2r68Esl27djleYRHUtWtXdejQQevWrdNPP/2k5cuXa+LEiZo9e7aioqK0Y8cOZWRkqEyZMnbHnT9/XsnJyTfsf/DgwYqKijJfly1bNt+1nj9/XpLk6emZ6/t9+vTRmTNntGrVKjMw7tixQ7/88ou++OILs51hGMrOztaBAwdUvXp1SbKbtZakpKQkderUyW7fQw89pKlTpyorK0sbN240b1qUpBkzZqhHjx5q1aqVateurTZt2uiRRx7RE088oVKlSpntatasaYZlSQoMDNTOnTvNc7q5ualhw4bm+2XKlFHVqlWVlJR0w/Hx8vJSdna2Lly4IC8vrxzve3h42P3SAwAAirY8B+Yff/zxVtZRJHh6eprPpo6Ojlbfvn01YsQIRUVFKSMjQ4GBgVqzZk2O43J7WsO1ypYtqypVqhRInVfC9smTJ8210Fe88847+uGHH7Rlyxb5+PiY+zMyMvTCCy9o4MCBOfqrUKGC+XWJEiUcqqV+/fp2M+rly5eXq6urVq1apY0bN2rlypX68MMPNWzYMG3evNlcp33tXzVsNluBPdnixIkTKlGiRK5hGQAA3H3ytYb5ygzktc6ePas+ffrcdFFFRY0aNcwb1SIiInT48GG5ubmpSpUqdtuVAOvu7q6srKx8n+/KEylu1EflypXl6+urxMREu/2LFi3S6NGjtXDhQlWuXNnuvYiICCUmJuaovUqVKpZPwqhevXqOR+tt2LBBYWFhcnV1lZeXl11fV0K6zWbTQw89pFGjRik+Pl7u7u52Sz2sVK9eXZcuXdLmzZvNfcePH9eePXtUo0YNSdZjvWvXLoWHh+fpXAAAoOjLV2CeP3+++Wf9q50/f16fffbZTRd1pzl+/Lhatmypzz//XL/88osOHDigr776ShMnTjSXI0RGRqpRo0bq3LmzVq5cqZSUFG3cuFHDhg3T1q1bJV1eq3zgwAElJCTozz//tLyBMDflypWTl5eXVqxYoSNHjuj06dO5tnNxcVFkZKTWr19v7tu1a5d69eqlIUOGqGbNmjp8+LAOHz6sEydOSJKGDBmijRs36qWXXlJCQoL27dunb7/9NsdNf9d64403FBcXpzFjxmjv3r2aP3++pk+fbneT3bU2b96ssWPHauvWrUpNTdXixYt17Ngxc9nHjdx3333q1KmT+vXrp/Xr12vHjh16+umndc8995jfj9DQUGVkZJjPEj937px5/Lp16/TII4/k6VwAAKDocygwp6en6/Tp0zIMQ2fOnLF7rNbJkye1bNkylStX7lbVWmh5e3urYcOGev/999W0aVPVqlVL0dHR6tevn6ZPny7p8ozpsmXL1LRpU/Xu3VthYWHq0aOHDh48qPLly0u6vA66bdu2atGihfz9/bVgwQKH6nBzc9MHH3ygGTNmKCgoKMfa4av17dtXsbGx5jKGrVu36ty5c3rnnXcUGBhobo8//rgkqU6dOlq7dq327t2rJk2aKDw8XG+//baCgoIsa4qIiNDChQsVGxurWrVq6e2339bo0aPt1mNfy9fXV//5z3/Uvn17hYWFafjw4Zo8ebLdWucbmTdvnurVq6dHH31UjRo1kmEYWrZsmbmUo3Hjxvr73/+u7t27y9/fXxMnTpQk/f7779q4caN69+6d53MBAICiLc9PyZAuz0zabLbrd2azadSoUXftp8fdSQzDUMOGDfXaa6+pZ8+ezi6n0BgyZIhOnjzp0MeFX7nLlqdkAABuNZ6SUXBuyVMypMs3/hmGoZYtW2rRokUqXbq0+Z67u7tCQkJuOOOIwsFms2nmzJnmkyVwWbly5eweoQgAAODQDPMVBw8eVIUKFSxnm4G7BTPMAIDbhRnmguPIDHO+bvoLCQnR+vXr9fTTT6tx48b6/fffJUn/+Mc/7G4kAwAAAO50+QrMixYtUps2beTl5aXt27ebT3M4ffq0xo4dW6AFAgAAAM6Ur8D8zjvv6NNPP9WsWbPsPkDioYce0vbt2wusOAAAAMDZ8hWY9+zZo6ZNm+bY7+fnp1OnTt1sTQAAAEChka/AHBAQoP379+fYv379elWqVOmmiwIAAAAKi3wF5n79+umVV17R5s2bZbPZ9Mcff+iLL77QoEGD1L9//4KuEQAAAHAah57DfMXQoUOVnZ2tVq1a6dy5c2ratKk8PDw0aNAgvfzyywVdIwAAAOA0+QrMNptNw4YN0+DBg7V//35lZGSoRo0a8vb2Luj6AAAAAKdyKDD36dMnT+3mzp2br2IAAACAwsahwBwTE6OQkBCFh4crHx8QCAAAANxxHArM/fv314IFC3TgwAH17t1bTz/9tEqXLn2ragMAAACczqGnZHz00UdKS0vTm2++qX/9618KDg5Wt27d9MMPPzDjDAAAgCLJ4cfKeXh4qGfPnlq1apUSExNVs2ZNDRgwQKGhocrIyLgVNQIAAABOk6/nMJsHu7jIZrPJMAxlZWUVVE0AAABAoeFwYL5w4YIWLFig1q1bKywsTDt37tT06dOVmprKY+UAAABQ5Dh009+AAQMUGxur4OBg9enTRwsWLFDZsmVvVW0AAACA0zkUmD/99FNVqFBBlSpV0tq1a7V27dpc2y1evLhAigMAAACczaHA3KtXL9lstltVCwAAAFDoOPzBJQAAAMDd5KaekgEAAAAUdQRmAAAAwIJDSzIAXN+uUW3k6+vr7DIAAEABY4YZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsODm7AKAoqLWiB/k4lHc2WUAuEuljO/g7BKAIosZZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgfkahw8fVuvWrVWiRAmVLFnS2eXcUsePH1e5cuWUkpLi7FIckpKSIpvNpoSEhALve8WKFapbt66ys7MLvG8AAHBnclpgttlsltvIkSOdUtf777+vtLQ0JSQkaO/evU6p4XZ599131alTJ4WGhjq7FKcIDQ3V1KlT7fa1bdtWxYoV0xdffOGcogAAQKHj5qwTp6WlmV9/+eWXevvtt7Vnzx5zn7e3t/m1YRjKysqSm9utLzc5OVn16tXTfffdl+8+MjMz5e7uXoBVWbt48aKKFSvm0DHnzp3TnDlz9MMPP9yiqqSsrCzZbDa5uNxZf8iIiorSBx98oGeeecbZpQAAgELAaUkmICDA3Pz8/GSz2czXv/76q3x8fLR8+XLVq1dPHh4eWr9+vZKTk9WpUyeVL19e3t7eeuCBB7R69Wq7fkNDQzV27Fj16dNHPj4+qlChgmbOnGm+n5mZqZdeekmBgYHy9PRUSEiIxo0bZx67aNEiffbZZ7LZbIqKipIkpaamqlOnTvL29pavr6+6deumI0eOmH2OHDlSdevW1ezZs1WxYkV5enpKujyLPmPGDD366KMqXry4qlevrk2bNmn//v1q3ry5SpQoocaNGys5OdnuGr799ltFRETI09NTlSpV0qhRo3Tp0iXzfZvNpk8++USPPfaYSpQooXfffVcnT57UU089JX9/f3l5eem+++7TvHnzrjv+y5Ytk4eHhx588EFzX1RUVK6z/WvWrJEkXbhwQYMGDdI999yjEiVKqGHDhuZ7khQTE6OSJUtq6dKlqlGjhjw8PJSamqqTJ0+qV69eKlWqlIoXL6527dpp3759lj8febme//73v2rRooWKFy+u+++/X5s2bbJ7f9GiRapZs6Y8PDwUGhqqyZMnm+81b95cBw8e1GuvvWZe5xUdO3bU1q1bc3xfAADA3alQT/0NHTpU48ePV1JSkurUqaOMjAy1b99ecXFxio+PV9u2bdWxY0elpqbaHTd58mTVr19f8fHxGjBggPr372/OXn/wwQdaunSpFi5cqD179uiLL74wlyT8/PPPatu2rbp166a0tDRNmzZN2dnZ6tSpk06cOKG1a9dq1apV+u9//6vu3bvbnXP//v1atGiRFi9ebLe2dsyYMerVq5cSEhJUrVo1/e1vf9MLL7ygt956S1u3bpVhGHrppZfM9uvWrVOvXr30yiuvKDExUTNmzFBMTIzeffddu/ONHDlSXbp00c6dO9WnTx9FR0crMTFRy5cvV1JSkj755BOVLVv2umO7bt061atXz27ftGnTlJaWZm6vvPKKypUrp2rVqkmSXnrpJW3atEmxsbH65Zdf9OSTT6pt27Z24ffcuXOaMGGCZs+erd27d6tcuXKKiorS1q1btXTpUm3atEmGYah9+/a6ePHidevLy/UMGzZMgwYNUkJCgsLCwtSzZ0/zF4tt27apW7du6tGjh3bu3KmRI0cqOjpaMTExkqTFixfr3nvv1ejRo83rvaJChQoqX7681q1bl2ttFy5cUHp6ut0GAACKLqctyciL0aNHq3Xr1ubr0qVL6/777zdfjxkzRt98842WLl1qFzrbt2+vAQMGSJKGDBmi999/Xz/++KOqVq2q1NRU3XfffXr44Ydls9kUEhJiHufv7y8PDw95eXkpICBAkrRq1Srt3LlTBw4cUHBwsCTps88+U82aNfXzzz/rgQcekHR55vqzzz6Tv7+/3TX07t1b3bp1M2tp1KiRoqOj1aZNG0nSK6+8ot69e5vtR40apaFDh+rZZ5+VJFWqVEljxozRm2++qREjRpjt/va3v9kdl5qaqvDwcNWvX1+Sbrgu+eDBgwoKCrLb5+fnJz8/P0mXA+WMGTO0evVqBQQEKDU1VfPmzVNqaqp53KBBg7RixQrNmzdPY8eOlXR5ecjHH39sfp/27dunpUuXasOGDWrcuLEk6YsvvlBwcLCWLFmiJ598Mtf68nI9gwYNUocOHcxxq1mzpvbv369q1appypQpatWqlaKjoyVJYWFhSkxM1HvvvaeoqCiVLl1arq6u8vHxMb/XVwsKCtLBgwdzrW3cuHEaNWrU9QcXAAAUKYV6hvlKWLoiIyNDgwYNUvXq1VWyZEl5e3srKSkpxwxznTp1zK+vLPU4evSopMvLDhISElS1alUNHDhQK1eutKwhKSlJwcHBZliWpBo1aqhkyZJKSkoy94WEhOQIy9fWUr58eUlS7dq17fb99ddf5izljh07NHr0aHl7e5tbv379lJaWpnPnzl13bPr376/Y2FjVrVtXb775pjZu3Gh5XefPnzeXjlwrPj5ezzzzjKZPn66HHnpIkrRz505lZWUpLCzMrra1a9faLV1wd3e3u+akpCS5ubmpYcOG5r4yZcqoatWq5vi1a9fO7K9mzZp5vp6rzxMYGChJ5vc5KSnJrP2Khx56SPv27VNWVpbl2EiSl5eX3Xhf7a233tLp06fN7dChQzfsDwAA3LkK9QxziRIl7F4PGjRIq1at0qRJk1SlShV5eXnpiSeeUGZmpl27a2+As9ls5mPCIiIidODAAS1fvlyrV69Wt27dFBkZqa+//rpAa82tlivrZHPbd6W+jIwMjRo1So8//niOvq4OuNeer127djp48KCWLVumVatWqVWrVnrxxRc1adKkXOsqW7asTp48mWP/4cOH9dhjj6lv37567rnnzP0ZGRlydXXVtm3b5OrqanfM1Tdoenl52a0HzovZs2fr/Pnzkv7/2OTleqzG8WadOHEi11+AJMnDw0MeHh4Fch4AAFD4FerAfK0NGzYoKipKXbp0kXQ5xOXnGcK+vr7q3r27unfvrieeeEJt27bViRMnVLp06Rxtq1evrkOHDunQoUPmLHNiYqJOnTqlGjVq3NT15CYiIkJ79uxRlSpVHD7W399fzz77rJ599lk1adJEgwcPvm5gDg8P1+eff26376+//lKnTp3MJQ3Xts/KytLRo0fVpEmTPNdUvXp1Xbp0SZs3bzaXZBw/flx79uwxx++ee+656evJ7bwbNmyw27dhwwaFhYWZgd/d3T3X2ea//vpLycnJCg8Pz/N1AgCAouuOCsz33XefFi9erI4dO8pmsyk6OtrhGcUpU6YoMDBQ4eHhcnFx0VdffaWAgIDrfkhJZGSkateuraeeekpTp07VpUuXNGDAADVr1izHsoiC8Pbbb+vRRx9VhQoV9MQTT8jFxUU7duzQrl279M4771geV69ePdWsWVMXLlzQd999p+rVq1+3fZs2bfTWW2/p5MmTKlWqlCTphRde0KFDhxQXF6djx46ZbUuXLq2wsDA99dRT6tWrlyZPnqzw8HAdO3ZMcXFxqlOnjrmW+Fr33XefOnXqpH79+mnGjBny8fHR0KFDdc8996hTp04Fdj3XeuONN/TAAw9ozJgx6t69uzZt2qTp06fr448/NtuEhobqP//5j3r06CEPDw/zpsKffvpJHh4eatSoUZ7PBwAAiq5CvYb5WlOmTFGpUqXUuHFjdezYUW3atFFERIRDffj4+GjixImqX7++HnjgAaWkpGjZsmXXfVawzWbTt99+q1KlSqlp06aKjIxUpUqV9OWXXxbEJeXQpk0bfffdd1q5cqUeeOABPfjgg3r//fftbk7Mjbu7u9566y3VqVNHTZs2laurq2JjY6/bvnbt2oqIiNDChQvNfWvXrlVaWppq1KihwMBAc7uyfnjevHnq1auX3njjDVWtWlWdO3fWzz//rAoVKljWNm/ePNWrV0+PPvqoGjVqJMMwtGzZMstnRzt6Pde6cm2xsbGqVauW3n77bY0ePdp8VKB0+abSlJQUVa5c2W75xYIFC/TUU0+pePHieT4fAAAoumyGYRjOLgLO8f3332vw4MHatWvXHffhIrfKn3/+qapVq2rr1q2qWLFino5JT0+Xn5+fgl9dKBcPQjYA50gZn/tf+gDk7sq/36dPn5avr69l2ztqSQYKVocOHbRv3z79/vvvdk8BuZulpKTo448/znNYBgAARR+B+S736quvOruEQqV+/fq3ZG06AAC4c/F3eAAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALLg5uwAAAIDCLDs7W5mZmc4uAw4qVqyYXF1dC6QvAjMAAMB1ZGZm6sCBA8rOznZ2KciHkiVLKiAgQDab7ab6ITADBWTXqDby9fV1dhkAgAJiGIbS0tLk6uqq4OBgubiwkvVOYRiGzp07p6NHj0qSAgMDb6o/AjMAAEAuLl26pHPnzikoKEjFixd3djlwkJeXlyTp6NGjKleu3E0tz+BXJQAAgFxkZWVJktzd3Z1cCfLryi86Fy9evKl+CMwAAAAWbnb9K5ynoL53BGYAAADAAoEZAAAAsMBNfwAAAA4IHfr9bT1fyvgO+Tpu06ZNevjhh9W2bVt9//3trbmoYYYZAACgCJozZ45efvll/ec//9Eff/zhtDqKwoe+EJgBAACKmIyMDH355Zfq37+/OnTooJiYGLv3//Wvf+mBBx6Qp6enypYtqy5dupjvXbhwQUOGDFFwcLA8PDxUpUoVzZkzR5IUExOjkiVL2vW1ZMkSu5vrRo4cqbp162r27NmqWLGiPD09JUkrVqzQww8/rJIlS6pMmTJ69NFHlZycbNfXb7/9pp49e6p06dIqUaKE6tevr82bNyslJUUuLi7aunWrXfupU6cqJCTkln+wDIEZAACgiFm4cKGqVaumqlWr6umnn9bcuXNlGIYk6fvvv1eXLl3Uvn17xcfHKy4uTg0aNDCP7dWrlxYsWKAPPvhASUlJmjFjhry9vR06//79+7Vo0SItXrxYCQkJkqSzZ8/q9ddf19atWxUXFycXFxd16dLFDLsZGRlq1qyZfv/9dy1dulQ7duzQm2++qezsbIWGhioyMlLz5s2zO8+8efMUFRV1yz9UhjXMAAAARcycOXP09NNPS5Latm2r06dPa+3atWrevLneffdd9ejRQ6NGjTLb33///ZKkvXv3auHChVq1apUiIyMlSZUqVXL4/JmZmfrss8/k7+9v7uvatatdm7lz58rf31+JiYmqVauW/vnPf+rYsWP6+eefVbp0aUlSlSpVzPZ9+/bV3//+d02ZMkUeHh7avn27du7cqW+//dbh+hzFDDMAAEARsmfPHm3ZskU9e/aUJLm5ual79+7msoqEhAS1atUq12MTEhLk6uqqZs2a3VQNISEhdmFZkvbt26eePXuqUqVK8vX1VWhoqCQpNTXVPHd4eLgZlq/VuXNnubq66ptvvpF0eXlIixYtzH5uJWaYAQAAipA5c+bo0qVLCgoKMvcZhiEPDw9Nnz7d/Mjo3Fi9J0kuLi7m0o4rcvsUvRIlSuTY17FjR4WEhGjWrFkKCgpSdna2atWqZd4UeKNzu7u7q1evXpo3b54ef/xx/fOf/9S0adMsjykozDADAAAUEZcuXdJnn32myZMnKyEhwdx27NihoKAgLViwQHXq1FFcXFyux9euXVvZ2dlau3Ztru/7+/vrzJkzOnv2rLnvyhplK8ePH9eePXs0fPhwtWrVStWrV9fJkyft2tSpU0cJCQk6ceLEdfvp27evVq9erY8//liXLl3S448/fsNzFwRmmAEAAIqI7777TidPntRzzz0nPz8/u/e6du2qOXPm6L333lOrVq1UuXJl9ejRQ5cuXdKyZcs0ZMgQhYaG6tlnn1WfPn30wQcf6P7779fBgwd19OhRdevWTQ0bNlTx4sX1f//3fxo4cKA2b96c4wkcuSlVqpTKlCmjmTNnKjAwUKmpqRo6dKhdm549e2rs2LHq3Lmzxo0bp8DAQMXHxysoKEiNGjWSJFWvXl0PPvighgwZoj59+txwVrqgMMMMAABQRMyZM0eRkZE5wrJ0OTBv3bpVpUuX1ldffaWlS5eqbt26atmypbZs2WK2++STT/TEE09owIABqlatmvr162fOKJcuXVqff/65li1bptq1a2vBggUaOXLkDetycXFRbGystm3bplq1aum1117Te++9Z9fG3d1dK1euVLly5dS+fXvVrl1b48ePl6urq1275557TpmZmerTp08+Rih/bMa1C1EAOCQ9PV1+fn46ffq0fH19nV0OAKCA/PXXXzpw4IDds4ThfGPGjNFXX32lX3755YZtrb6Hjvz7zQwzAAAACr2MjAzt2rVL06dP18svv3xbz01gBgAAQKH30ksvqV69emrevPltXY4hcdMfAAAA7gAxMTF5usHwVmCGGQAAALBAYAYAAAAsEJgBAAAs8ECxO1d2dnaB9MMaZgAAgFwUK1ZMNptNx44dk7+/v2w2m7NLQh4ZhqHMzEwdO3ZMLi4ucnd3v6n+CMwAAAC5cHV11b333qvffvtNKSkpzi4H+VC8eHFVqFBBLi43t6iCwAwAAHAd3t7euu+++3Tx4kVnlwIHubq6ys3NrUD+MkBgBgAAsODq6prj45lxd+GmPwAAAMACgRkAAACwQGAGAAAALLCGGbhJV57PmZ6e7uRKAABAXl35dzsvz9kmMAM36fjx45Kk4OBgJ1cCAAAcdebMGfn5+Vm2ITADN6l06dKSpNTU1Bv+B4fL0tPTFRwcrEOHDsnX19fZ5dwRGDPHMWaOY8wcx5g5rrCMmWEYOnPmjIKCgm7YlsAM3KQrD0P38/Pjf5YO8vX1ZcwcxJg5jjFzHGPmOMbMcYVhzPI60cVNfwAAAIAFAjMAAABggcAM3CQPDw+NGDFCHh4ezi7ljsGYOY4xcxxj5jjGzHGMmePuxDGzGXl5lgYAAABwl2KGGQAAALBAYAYAAAAsEJgBAAAACwRmAAAAwAKBGciDjz76SKGhofL09FTDhg21ZcsWy/ZfffWVqlWrJk9PT9WuXVvLli27TZUWHo6M2e7du9W1a1eFhobKZrNp6tSpt6/QQsSRMZs1a5aaNGmiUqVKqVSpUoqMjLzhz2VR5MiYLV68WPXr11fJkiVVokQJ1a1bV//4xz9uY7WFg6P/P7siNjZWNptNnTt3vrUFFkKOjFlMTIxsNpvd5unpeRurLRwc/Tk7deqUXnzxRQUGBsrDw0NhYWGF699OA4Cl2NhYw93d3Zg7d66xe/duo1+/fkbJkiWNI0eO5Np+w4YNhqurqzFx4kQjMTHRGD58uFGsWDFj586dt7ly53F0zLZs2WIMGjTIWLBggREQEGC8//77t7fgQsDRMfvb3/5mfPTRR0Z8fLyRlJRkREVFGX5+fsZvv/12myt3HkfH7McffzQWL15sJCYmGvv37zemTp1quLq6GitWrLjNlTuPo2N2xYEDB4x77rnHaNKkidGpU6fbU2wh4eiYzZs3z/D19TXS0tLM7fDhw7e5audydMwuXLhg1K9f32jfvr2xfv1648CBA8aaNWuMhISE21z59RGYgRto0KCB8eKLL5qvs7KyjKCgIGPcuHG5tu/WrZvRoUMHu30NGzY0XnjhhVtaZ2Hi6JhdLSQk5K4MzDczZoZhGJcuXTJ8fHyM+fPn36oSC52bHTPDMIzw8HBj+PDht6K8Qik/Y3bp0iWjcePGxuzZs41nn332rgvMjo7ZvHnzDD8/v9tUXeHk6Jh98sknRqVKlYzMzMzbVaLDWJIBWMjMzNS2bdsUGRlp7nNxcVFkZKQ2bdqU6zGbNm2yay9Jbdq0uW77oiY/Y3a3K4gxO3funC5evKjSpUvfqjILlZsdM8MwFBcXpz179qhp06a3stRCI79jNnr0aJUrV07PPffc7SizUMnvmGVkZCgkJETBwcHq1KmTdu/efTvKLRTyM2ZLly5Vo0aN9OKLL6p8+fKqVauWxo4dq6ysrNtV9g0RmAELf/75p7KyslS+fHm7/eXLl9fhw4dzPebw4cMOtS9q8jNmd7uCGLMhQ4YoKCgoxy9rRVV+x+z06dPy9vaWu7u7OnTooA8//FCtW7e+1eUWCvkZs/Xr12vOnDmaNWvW7Six0MnPmFWtWlVz587Vt99+q88//1zZ2dlq3Lixfvvtt9tRstPlZ8z++9//6uuvv1ZWVpaWLVum6OhoTZ48We+8887tKDlP3JxdAADg5owfP16xsbFas2bNXXlzkSN8fHyUkJCgjIwMxcXF6fXXX1elSpXUvHlzZ5dW6Jw5c0bPPPOMZs2apbJlyzq7nDtGo0aN1KhRI/N148aNVb16dc2YMUNjxoxxYmWFV3Z2tsqVK6eZM2fK1dVV9erV0++//6733ntPI0aMcHZ5kgjMgKWyZcvK1dVVR44csdt/5MgRBQQE5HpMQECAQ+2LmvyM2d3uZsZs0qRJGj9+vFavXq06dercyjILlfyOmYuLi6pUqSJJqlu3rpKSkjRu3Li7IjA7OmbJyclKSUlRx44dzX3Z2dmSJDc3N+3Zs0eVK1e+tUU7WUH8/6xYsWIKDw/X/v37b0WJhU5+xiwwMFDFihWTq6urua969eo6fPiwMjMz5e7ufktrzguWZAAW3N3dVa9ePcXFxZn7srOzFRcXZzeDcLVGjRrZtZekVatWXbd9UZOfMbvb5XfMJk6cqDFjxmjFihWqX7/+7Si10Cion7Ps7GxduHDhVpRY6Dg6ZtWqVdPOnTuVkJBgbo899phatGihhIQEBQcH387ynaIgfs6ysrK0c+dOBQYG3qoyC5X8jNlDDz2k/fv3m7+QSdLevXsVGBhYKMKyJB4rB9xIbGys4eHhYcTExBiJiYnG888/b5QsWdJ8TNAzzzxjDB061Gy/YcMGw83NzZg0aZKRlJRkjBgx4q58rJwjY3bhwgUjPj7eiI+PNwIDA41BgwYZ8fHxxr59+5x1Cbedo2M2fvx4w93d3fj666/tHl915swZZ13CbefomI0dO9ZYuXKlkZycbCQmJhqTJk0y3NzcjFmzZjnrEm47R8fsWnfjUzIcHbNRo0YZP/zwg5GcnGxs27bN6NGjh+Hp6Wns3r3bWZdw2zk6ZqmpqYaPj4/x0ksvGXv27DG+++47o1y5csY777zjrEvIgcAM5MGHH35oVKhQwXB3dzcaNGhg/PTTT+Z7zZo1M5599lm79gsXLjTCwsIMd3d3o2bNmsb3339/myt2PkfG7MCBA4akHFuzZs1uf+FO5MiYhYSE5DpmI0aMuP2FO5EjYzZs2DCjSpUqhqenp1GqVCmjUaNGRmxsrBOqdi5H/392tbsxMBuGY2P26quvmm3Lly9vtG/f3ti+fbsTqnYuR3/ONm7caDRs2NDw8PAwKlWqZLz77rvGpUuXbnPV12czDMNw1uw2AAAAUNixhhkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMDC/wP65VIMye3TlgAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAswAAAGdCAYAAAAG6yXVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8mklEQVR4nO3deVxV1f7/8fcBZFAGJxQoBIdwNkHTtJwxp0zNcriVoWldrWzStK+SUzmlpmWDM3brSpZm3lJTuel1SlPBVMiBK2KFQ06Imijs3x9e988juOUgehBfz8djPx6cfdZe+7MXlG8Wa+9jMwzDEAAAAIBcuTi7AAAAAKAwIzADAAAAFgjMAAAAgAUCMwAAAGCBwAwAAABYIDADAAAAFgjMAAAAgAUCMwAAAGDBzdkFAHe67Oxs/fHHH/Lx8ZHNZnN2OQAAIA8Mw9CZM2cUFBQkFxfrOWQCM3CT/vjjDwUHBzu7DAAAkA+HDh3Svffea9mGwAzcJB8fH0mX/4Pz9fV1cjUAACAv0tPTFRwcbP47boXADNykK8swfH19CcwAANxh8rKckpv+AAAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACw4ObsAoCiotaIH+TiUdzZZQAAUGSkjO/g7BIkMcMMAAAAWCIwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwFwI2m01LlizJ17HR0dF6/vnnC7agu0jz5s316quvmq8ffPBBLVq0yHkFAQCAQofAfJVjx46pf//+qlChgjw8PBQQEKA2bdpow4YNee5j5MiRqlu3bo79oaGhstlsdtu9994rSUpLS1O7du0kSSkpKbLZbEpISLjhuQ4fPqxp06Zp2LBhea6vKLg25Bak4cOHa+jQocrOzr4l/QMAgDsPgfkqXbt2VXx8vObPn6+9e/dq6dKlat68uY4fP14g/Y8ePVppaWnmFh8fL0kKCAiQh4eHw/3Nnj1bjRs3VkhISIHUB6ldu3Y6c+aMli9f7uxSAABAIUFg/p9Tp05p3bp1mjBhglq0aKGQkBA1aNBAb731lh577DG7dn379pW/v798fX3VsmVL7dixQ5IUExOjUaNGaceOHeYsckxMjHmsj4+PAgICzM3f31+S/ZKMihUrSpLCw8Nls9nUvHnz69YcGxurjh072u1bsWKFHn74YZUsWVJlypTRo48+quTkZPP9NWvWyGaz6dSpU+a+hIQE2Ww2paSkmPtmzZql4OBgFS9eXF26dNGUKVNUsmRJ8/0rM+lz585VhQoV5O3trQEDBigrK0sTJ05UQECAypUrp3fffTfHOF9v/K7u9x//+IdCQ0Pl5+enHj166MyZM5KkqKgorV27VtOmTTPH+Erdu3btUrt27eTt7a3y5cvrmWee0Z9//mn2ffbsWfXq1Uve3t4KDAzU5MmTc4ypq6ur2rdvr9jY2OuOOwAAuLsQmP/H29tb3t7eWrJkiS5cuHDddk8++aSOHj2q5cuXa9u2bYqIiFCrVq104sQJde/eXW+88YZq1qxpziJ3797doTq2bNkiSVq9erXS0tK0ePHiXNudOHFCiYmJql+/vt3+s2fP6vXXX9fWrVsVFxcnFxcXdenSxaElBhs2bNDf//53vfLKK0pISFDr1q1zBF9JSk5O1vLly7VixQotWLBAc+bMUYcOHfTbb79p7dq1mjBhgoYPH67Nmzebx1iN39X9LlmyRN99952+++47rV27VuPHj5ckTZs2TY0aNVK/fv3MMQ4ODtapU6fUsmVLhYeHa+vWrVqxYoWOHDmibt26mf0OHjxYa9eu1bfffquVK1dqzZo12r59e47ratCggdatW3fd8blw4YLS09PtNgAAUHS5ObuAwsLNzU0xMTHq16+fPv30U0VERKhZs2bq0aOH6tSpI0lav369tmzZoqNHj5pLKCZNmqQlS5bo66+/1vPPPy9vb2+5ubkpICAgxzmGDBmi4cOHm6/Hjh2rgQMH2rW5MutcpkyZXPu4IjU1VYZhKCgoyG5/165d7V7PnTtX/v7+SkxMVK1atfI0Fh9++KHatWunQYMGSZLCwsK0ceNGfffdd3btsrOzNXfuXPn4+KhGjRpq0aKF9uzZo2XLlsnFxUVVq1bVhAkT9OOPP6phw4Z5Gr8r/cbExMjHx0eS9MwzzyguLk7vvvuu/Pz85O7uruLFi9uNz/Tp0xUeHq6xY8faXXtwcLD27t2roKAgzZkzR59//rlatWolSZo/f765jvxqQUFBOnTokLKzs+XikvN3ynHjxmnUqFF5GksAAHDnY4b5Kl27dtUff/yhpUuXqm3btlqzZo0iIiLMZRU7duxQRkaGypQpY85Ie3t768CBA3bLHq5n8ODBSkhIMLdevXrlu9bz589Lkjw9Pe3279u3Tz179lSlSpXk6+ur0NBQSZcDdl7t2bNHDRo0sNt37Wvp8o2MV0KtJJUvX141atSwC5nly5fX0aNHJeV9/K7tNzAw0Ozjenbs2KEff/zRrt9q1apJujxjnZycrMzMTDVs2NA8pnTp0qpatWqOvry8vJSdnX3dvzS89dZbOn36tLkdOnTIsjYAAHBnY4b5Gp6enmrdurVat26t6Oho9e3bVyNGjFBUVJQyMjIUGBioNWvW5Dju6vW911O2bFlVqVKlQOosW7asJOnkyZPmrLQkdezYUSEhIZo1a5aCgoKUnZ2tWrVqKTMzU5LMMGsYhnnMxYsX81VDsWLF7F7bbLZc911ZDpLX8bPq43oyMjLUsWNHTZgwIcd7gYGB2r9/v+XxVztx4oRKlCghLy+vXN/38PDI102aAADgzkRgvoEaNWqYN+RFRETo8OHDcnNzM2dur+Xu7q6srKx8n8/d3V2SbthH5cqV5evrq8TERIWFhUmSjh8/rj179mjWrFlq0qSJpMvLSK52JVynpaWpVKlSkpTjEXZVq1bVzz//bLfv2tf5kZfxy4vcxjgiIkKLFi1SaGio3Nxy/lhXrlxZxYoV0+bNm1WhQgVJl3/Z2Lt3r5o1a2bXdteuXQoPD893fQAAoGhhScb/HD9+XC1bttTnn3+uX375RQcOHNBXX32liRMnqlOnTpKkyMhINWrUSJ07d9bKlSuVkpKijRs3atiwYdq6dauky8sJDhw4oISEBP3555+WNxDmply5cvLy8jJvWjt9+nSu7VxcXBQZGWkXiEuVKqUyZcpo5syZ2r9/v/7973/r9ddftzuuSpUqCg4O1siRI7Vv3z59//33OZ4W8fLLL2vZsmWaMmWK9u3bpxkzZmj58uWy2WwOXcu18jJ+eREaGqrNmzcrJSVFf/75p7Kzs/Xiiy/qxIkT6tmzp37++WclJyfrhx9+UO/evZWVlSVvb28999xzGjx4sP79739r165dioqKynWN8rp16/TII4/c1LUCAICig8D8P97e3mrYsKHef/99NW3aVLVq1VJ0dLT69eun6dOnS7q8NGDZsmVq2rSpevfurbCwMPXo0UMHDx5U+fLlJV1eB922bVu1aNFC/v7+WrBggUN1uLm56YMPPtCMGTMUFBRkhvXc9O3bV7GxseZyBRcXF8XGxmrbtm2qVauWXnvtNb333nt2xxQrVkwLFizQr7/+qjp16mjChAl655137No89NBD+vTTTzVlyhTdf//9WrFihV577bUc66UdlZfxy4tBgwbJ1dVVNWrUkL+/v1JTUxUUFKQNGzYoKytLjzzyiGrXrq1XX31VJUuWNEPxe++9pyZNmqhjx46KjIzUww8/rHr16tn1/fvvv2vjxo3q3bv3TV0rAAAoOmzG1YtZcUcxDEMNGzbUa6+9pp49e97Sc/Xr10+//vqr5ePWioIhQ4bo5MmTmjlzZp6PSU9Pl5+fn4JfXSgXj+K3sDoAAO4uKeM73LK+r/z7ffr0afn6+lq2ZYb5Dmaz2TRz5kxdunSpwPueNGmSduzYof379+vDDz/U/Pnz9eyzzxb4eQqbcuXKacyYMc4uAwAAFCLc9HeHq1u3rurWrVvg/W7ZskUTJ07UmTNnVKlSJX3wwQfq27dvgZ+nsHnjjTecXQIAAChkCMzI1cKFC51dAgAAQKHAkgwAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAgpuzCwCKil2j2sjX19fZZQAAgALGDDMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABggcAMAAAAWCAwAwAAABYIzAAAAIAFAjMAAABgwc3ZBQBFRa0RP8jFo7izywAAFGEp4zs4u4S7EjPMAAAAgAUCMwAAAGCBwAwAAABYIDADAAAAFgjMAAAAgAUCMwAAAGCBwAwAAABYIDADAAAAFvL8wSXh4eGy2Wx5art9+/Z8FwQAAAAUJnkOzJ07dza//uuvv/Txxx+rRo0aatSokSTpp59+0u7duzVgwIACLxIAAABwljwH5hEjRphf9+3bVwMHDtSYMWNytDl06FDBVQcAAAA4Wb7WMH/11Vfq1atXjv1PP/20Fi1adNNFAQAAAIVFvgKzl5eXNmzYkGP/hg0b5OnpedNFAQAAAIVFnpdkXO3VV19V//79tX37djVo0ECStHnzZs2dO1fR0dEFWiAAAADgTPkKzEOHDlWlSpU0bdo0ff7555Kk6tWra968eerWrVuBFggAAAA4U74CsyR169aNcAwAAIAiL9+BWZK2bdumpKQkSVLNmjUVHh5eIEUBAAAAhUW+AvPRo0fVo0cPrVmzRiVLlpQknTp1Si1atFBsbKz8/f0LskYAAADAafL1lIyXX35ZZ86c0e7du3XixAmdOHFCu3btUnp6ugYOHFjQNQIAAABOk68Z5hUrVmj16tWqXr26ua9GjRr66KOP9MgjjxRYcQAAAICz5WuGOTs7W8WKFcuxv1ixYsrOzr7pogAAAIDCIl+BuWXLlnrllVf0xx9/mPt+//13vfbaa2rVqlWBFQcAAAA4W74C8/Tp05Wenq7Q0FBVrlxZlStXVsWKFZWenq4PP/ywoGsEAAAAnCZfa5iDg4O1fft2rV69Wr/++qukyx9cEhkZWaDFAQAAAM6W7+cw22w2tW7dWq1bty7IegAAAIBCJd+BOS4uTnFxcTp69GiOG/3mzp1704Uhb2w2m7755ht17tzZ4WOjo6N15MgRzZw5s+ALu4WioqJ06tQpLVmypMD7fvDBBzV48GB17dq1wPsGAAB3pnytYR41apQeeeQRxcXF6c8//9TJkyfttrvRsWPH1L9/f1WoUEEeHh4KCAhQmzZttGHDhjz3MXLkSNWtWzfH/tDQUNlsNrvt3nvvlSSlpaWpXbt2kqSUlBTZbDYlJCTc8FyHDx/WtGnTNGzYsDzXV5TExMSYH7pzteHDh2vo0KE87QUAAJjyNcP86aefKiYmRs8880xB13PH6tq1qzIzMzV//nxVqlRJR44cUVxcnI4fP14g/Y8ePVr9+vUzX7u6ukqSAgIC8tXf7Nmz1bhxY4WEhBRIfbnJzMyUu7v7Lev/VmjXrp369u2r5cuXq0OHDs4uBwAAFAL5mmHOzMxU48aNC7qWO9apU6e0bt06TZgwQS1atFBISIgaNGigt956S4899phdu759+8rf31++vr5q2bKlduzYIenyjOeoUaO0Y8cOcxY5JibGPNbHx0cBAQHmduXjx202m7k0oWLFipKk8PBw2Ww2NW/e/Lo1x8bGqmPHjubrK7PT125X97F+/Xo1adJEXl5eCg4O1sCBA3X27Fnz/dDQUI0ZM0a9evWSr6+vnn/+eUnSokWLVLNmTXl4eCg0NFSTJ0++4Zh+/fXXql27try8vFSmTBlFRkbanUuSJk2apMDAQJUpU0YvvviiLl68aL538uRJ9erVS6VKlVLx4sXVrl077du3T5K0Zs0a9e7dW6dPnzavc+TIkZIu/yLSvn17xcbGXre2CxcuKD093W4DAABFV74Cc9++ffXPf/6zoGu5Y3l7e8vb21tLlizRhQsXrtvuySef1NGjR7V8+XJt27ZNERERatWqlU6cOKHu3bvrjTfeUM2aNZWWlqa0tDR1797doTq2bNkiSVq9erXS0tK0ePHiXNudOHFCiYmJql+/vrkvODjYPG9aWpri4+NVpkwZNW3aVJKUnJystm3bqmvXrvrll1/05Zdfav369XrppZfs+p40aZLuv/9+xcfHKzo6Wtu2bVO3bt3Uo0cP7dy5UyNHjlR0dLTdLwPXSktLU8+ePdWnTx8lJSVpzZo1evzxx2UYhtnmxx9/VHJysn788UfNnz9fMTExdn1GRUVp69atWrp0qTZt2iTDMNS+fXtdvHhRjRs31tSpU+Xr62te76BBg8xjGzRooHXr1l23vnHjxsnPz8/cgoODr9sWAADc+WzG1SnEwuuvv25+nZ2drfnz56tOnTqqU6dOjk/9mzJlSsFWeQdYtGiR+vXrp/PnzysiIkLNmjVTjx49VKdOHUmXZ2c7dOigo0ePysPDwzyuSpUqevPNN/X8889r5MiRWrJkSY41yKGhoUpLS7Mb57Fjx2rgwIF2N/2lpKSoYsWKio+Pz3Ut9BUJCQkKDw9XampqrmHvr7/+UvPmzeXv769vv/1WLi4u6tu3r1xdXTVjxgyz3fr169WsWTOdPXtWnp6eCg0NVXh4uL755huzzVNPPaVjx45p5cqV5r4333xT33//vXbv3p1rfdu3b1e9evWUkpKS65KRqKgorVmzRsnJyebSlG7dusnFxUWxsbHat2+fwsLCtGHDBvMvIcePH1dwcLDmz5+vJ598UjExMXr11Vd16tSpHP0vXbpUXbp00cWLF+XikvN3ygsXLtj9YpSenq7g4GAFv7pQLh7Fc70mAAAKQsp4lgsWlPT0dPn5+en06dPy9fW1bJvnNczx8fF2r68Esl27djleYRHUtWtXdejQQevWrdNPP/2k5cuXa+LEiZo9e7aioqK0Y8cOZWRkqEyZMnbHnT9/XsnJyTfsf/DgwYqKijJfly1bNt+1nj9/XpLk6emZ6/t9+vTRmTNntGrVKjMw7tixQ7/88ou++OILs51hGMrOztaBAwdUvXp1SbKbtZakpKQkderUyW7fQw89pKlTpyorK0sbN240b1qUpBkzZqhHjx5q1aqVateurTZt2uiRRx7RE088oVKlSpntatasaYZlSQoMDNTOnTvNc7q5ualhw4bm+2XKlFHVqlWVlJR0w/Hx8vJSdna2Lly4IC8vrxzve3h42P3SAwAAirY8B+Yff/zxVtZRJHh6eprPpo6Ojlbfvn01YsQIRUVFKSMjQ4GBgVqzZk2O43J7WsO1ypYtqypVqhRInVfC9smTJ8210Fe88847+uGHH7Rlyxb5+PiY+zMyMvTCCy9o4MCBOfqrUKGC+XWJEiUcqqV+/fp2M+rly5eXq6urVq1apY0bN2rlypX68MMPNWzYMG3evNlcp33tXzVsNluBPdnixIkTKlGiRK5hGQAA3H3ytYb5ygzktc6ePas+ffrcdFFFRY0aNcwb1SIiInT48GG5ubmpSpUqdtuVAOvu7q6srKx8n+/KEylu1EflypXl6+urxMREu/2LFi3S6NGjtXDhQlWuXNnuvYiICCUmJuaovUqVKpZPwqhevXqOR+tt2LBBYWFhcnV1lZeXl11fV0K6zWbTQw89pFGjRik+Pl7u7u52Sz2sVK9eXZcuXdLmzZvNfcePH9eePXtUo0YNSdZjvWvXLoWHh+fpXAAAoOjLV2CeP3+++Wf9q50/f16fffbZTRd1pzl+/Lhatmypzz//XL/88osOHDigr776ShMnTjSXI0RGRqpRo0bq3LmzVq5cqZSUFG3cuFHDhg3T1q1bJV1eq3zgwAElJCTozz//tLyBMDflypWTl5eXVqxYoSNHjuj06dO5tnNxcVFkZKTWr19v7tu1a5d69eqlIUOGqGbNmjp8+LAOHz6sEydOSJKGDBmijRs36qWXXlJCQoL27dunb7/9NsdNf9d64403FBcXpzFjxmjv3r2aP3++pk+fbneT3bU2b96ssWPHauvWrUpNTdXixYt17Ngxc9nHjdx3333q1KmT+vXrp/Xr12vHjh16+umndc8995jfj9DQUGVkZJjPEj937px5/Lp16/TII4/k6VwAAKDocygwp6en6/Tp0zIMQ2fOnLF7rNbJkye1bNkylStX7lbVWmh5e3urYcOGev/999W0aVPVqlVL0dHR6tevn6ZPny7p8ozpsmXL1LRpU/Xu3VthYWHq0aOHDh48qPLly0u6vA66bdu2atGihfz9/bVgwQKH6nBzc9MHH3ygGTNmKCgoKMfa4av17dtXsbGx5jKGrVu36ty5c3rnnXcUGBhobo8//rgkqU6dOlq7dq327t2rJk2aKDw8XG+//baCgoIsa4qIiNDChQsVGxurWrVq6e2339bo0aPt1mNfy9fXV//5z3/Uvn17hYWFafjw4Zo8ebLdWucbmTdvnurVq6dHH31UjRo1kmEYWrZsmbmUo3Hjxvr73/+u7t27y9/fXxMnTpQk/f7779q4caN69+6d53MBAICiLc9PyZAuz0zabLbrd2azadSoUXftp8fdSQzDUMOGDfXaa6+pZ8+ezi6n0BgyZIhOnjzp0MeFX7nLlqdkAABuNZ6SUXBuyVMypMs3/hmGoZYtW2rRokUqXbq0+Z67u7tCQkJuOOOIwsFms2nmzJnmkyVwWbly5eweoQgAAODQDPMVBw8eVIUKFSxnm4G7BTPMAIDbhRnmguPIDHO+bvoLCQnR+vXr9fTTT6tx48b6/fffJUn/+Mc/7G4kAwAAAO50+QrMixYtUps2beTl5aXt27ebT3M4ffq0xo4dW6AFAgAAAM6Ur8D8zjvv6NNPP9WsWbPsPkDioYce0vbt2wusOAAAAMDZ8hWY9+zZo6ZNm+bY7+fnp1OnTt1sTQAAAEChka/AHBAQoP379+fYv379elWqVOmmiwIAAAAKi3wF5n79+umVV17R5s2bZbPZ9Mcff+iLL77QoEGD1L9//4KuEQAAAHAah57DfMXQoUOVnZ2tVq1a6dy5c2ratKk8PDw0aNAgvfzyywVdIwAAAOA0+QrMNptNw4YN0+DBg7V//35lZGSoRo0a8vb2Luj6AAAAAKdyKDD36dMnT+3mzp2br2IAAACAwsahwBwTE6OQkBCFh4crHx8QCAAAANxxHArM/fv314IFC3TgwAH17t1bTz/9tEqXLn2ragMAAACczqGnZHz00UdKS0vTm2++qX/9618KDg5Wt27d9MMPPzDjDAAAgCLJ4cfKeXh4qGfPnlq1apUSExNVs2ZNDRgwQKGhocrIyLgVNQIAAABOk6/nMJsHu7jIZrPJMAxlZWUVVE0AAABAoeFwYL5w4YIWLFig1q1bKywsTDt37tT06dOVmprKY+UAAABQ5Dh009+AAQMUGxur4OBg9enTRwsWLFDZsmVvVW0AAACA0zkUmD/99FNVqFBBlSpV0tq1a7V27dpc2y1evLhAigMAAACczaHA3KtXL9lstltVCwAAAFDoOPzBJQAAAMDd5KaekgEAAAAUdQRmAAAAwIJDSzIAXN+uUW3k6+vr7DIAAEABY4YZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsEBgBgAAACwQmAEAAAALBGYAAADAAoEZAAAAsODm7AKAoqLWiB/k4lHc2WUAuEuljO/g7BKAIosZZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgfkahw8fVuvWrVWiRAmVLFnS2eXcUsePH1e5cuWUkpLi7FIckpKSIpvNpoSEhALve8WKFapbt66ys7MLvG8AAHBnclpgttlsltvIkSOdUtf777+vtLQ0JSQkaO/evU6p4XZ599131alTJ4WGhjq7FKcIDQ3V1KlT7fa1bdtWxYoV0xdffOGcogAAQKHj5qwTp6WlmV9/+eWXevvtt7Vnzx5zn7e3t/m1YRjKysqSm9utLzc5OVn16tXTfffdl+8+MjMz5e7uXoBVWbt48aKKFSvm0DHnzp3TnDlz9MMPP9yiqqSsrCzZbDa5uNxZf8iIiorSBx98oGeeecbZpQAAgELAaUkmICDA3Pz8/GSz2czXv/76q3x8fLR8+XLVq1dPHh4eWr9+vZKTk9WpUyeVL19e3t7eeuCBB7R69Wq7fkNDQzV27Fj16dNHPj4+qlChgmbOnGm+n5mZqZdeekmBgYHy9PRUSEiIxo0bZx67aNEiffbZZ7LZbIqKipIkpaamqlOnTvL29pavr6+6deumI0eOmH2OHDlSdevW1ezZs1WxYkV5enpKujyLPmPGDD366KMqXry4qlevrk2bNmn//v1q3ry5SpQoocaNGys5OdnuGr799ltFRETI09NTlSpV0qhRo3Tp0iXzfZvNpk8++USPPfaYSpQooXfffVcnT57UU089JX9/f3l5eem+++7TvHnzrjv+y5Ytk4eHhx588EFzX1RUVK6z/WvWrJEkXbhwQYMGDdI999yjEiVKqGHDhuZ7khQTE6OSJUtq6dKlqlGjhjw8PJSamqqTJ0+qV69eKlWqlIoXL6527dpp3759lj8febme//73v2rRooWKFy+u+++/X5s2bbJ7f9GiRapZs6Y8PDwUGhqqyZMnm+81b95cBw8e1GuvvWZe5xUdO3bU1q1bc3xfAADA3alQT/0NHTpU48ePV1JSkurUqaOMjAy1b99ecXFxio+PV9u2bdWxY0elpqbaHTd58mTVr19f8fHxGjBggPr372/OXn/wwQdaunSpFi5cqD179uiLL74wlyT8/PPPatu2rbp166a0tDRNmzZN2dnZ6tSpk06cOKG1a9dq1apV+u9//6vu3bvbnXP//v1atGiRFi9ebLe2dsyYMerVq5cSEhJUrVo1/e1vf9MLL7ygt956S1u3bpVhGHrppZfM9uvWrVOvXr30yiuvKDExUTNmzFBMTIzeffddu/ONHDlSXbp00c6dO9WnTx9FR0crMTFRy5cvV1JSkj755BOVLVv2umO7bt061atXz27ftGnTlJaWZm6vvPKKypUrp2rVqkmSXnrpJW3atEmxsbH65Zdf9OSTT6pt27Z24ffcuXOaMGGCZs+erd27d6tcuXKKiorS1q1btXTpUm3atEmGYah9+/a6ePHidevLy/UMGzZMgwYNUkJCgsLCwtSzZ0/zF4tt27apW7du6tGjh3bu3KmRI0cqOjpaMTExkqTFixfr3nvv1ejRo83rvaJChQoqX7681q1bl2ttFy5cUHp6ut0GAACKLqctyciL0aNHq3Xr1ubr0qVL6/777zdfjxkzRt98842WLl1qFzrbt2+vAQMGSJKGDBmi999/Xz/++KOqVq2q1NRU3XfffXr44Ydls9kUEhJiHufv7y8PDw95eXkpICBAkrRq1Srt3LlTBw4cUHBwsCTps88+U82aNfXzzz/rgQcekHR55vqzzz6Tv7+/3TX07t1b3bp1M2tp1KiRoqOj1aZNG0nSK6+8ot69e5vtR40apaFDh+rZZ5+VJFWqVEljxozRm2++qREjRpjt/va3v9kdl5qaqvDwcNWvX1+Sbrgu+eDBgwoKCrLb5+fnJz8/P0mXA+WMGTO0evVqBQQEKDU1VfPmzVNqaqp53KBBg7RixQrNmzdPY8eOlXR5ecjHH39sfp/27dunpUuXasOGDWrcuLEk6YsvvlBwcLCWLFmiJ598Mtf68nI9gwYNUocOHcxxq1mzpvbv369q1appypQpatWqlaKjoyVJYWFhSkxM1HvvvaeoqCiVLl1arq6u8vHxMb/XVwsKCtLBgwdzrW3cuHEaNWrU9QcXAAAUKYV6hvlKWLoiIyNDgwYNUvXq1VWyZEl5e3srKSkpxwxznTp1zK+vLPU4evSopMvLDhISElS1alUNHDhQK1eutKwhKSlJwcHBZliWpBo1aqhkyZJKSkoy94WEhOQIy9fWUr58eUlS7dq17fb99ddf5izljh07NHr0aHl7e5tbv379lJaWpnPnzl13bPr376/Y2FjVrVtXb775pjZu3Gh5XefPnzeXjlwrPj5ezzzzjKZPn66HHnpIkrRz505lZWUpLCzMrra1a9faLV1wd3e3u+akpCS5ubmpYcOG5r4yZcqoatWq5vi1a9fO7K9mzZp5vp6rzxMYGChJ5vc5KSnJrP2Khx56SPv27VNWVpbl2EiSl5eX3Xhf7a233tLp06fN7dChQzfsDwAA3LkK9QxziRIl7F4PGjRIq1at0qRJk1SlShV5eXnpiSeeUGZmpl27a2+As9ls5mPCIiIidODAAS1fvlyrV69Wt27dFBkZqa+//rpAa82tlivrZHPbd6W+jIwMjRo1So8//niOvq4OuNeer127djp48KCWLVumVatWqVWrVnrxxRc1adKkXOsqW7asTp48mWP/4cOH9dhjj6lv37567rnnzP0ZGRlydXXVtm3b5OrqanfM1Tdoenl52a0HzovZs2fr/Pnzkv7/2OTleqzG8WadOHEi11+AJMnDw0MeHh4Fch4AAFD4FerAfK0NGzYoKipKXbp0kXQ5xOXnGcK+vr7q3r27unfvrieeeEJt27bViRMnVLp06Rxtq1evrkOHDunQoUPmLHNiYqJOnTqlGjVq3NT15CYiIkJ79uxRlSpVHD7W399fzz77rJ599lk1adJEgwcPvm5gDg8P1+eff26376+//lKnTp3MJQ3Xts/KytLRo0fVpEmTPNdUvXp1Xbp0SZs3bzaXZBw/flx79uwxx++ee+656evJ7bwbNmyw27dhwwaFhYWZgd/d3T3X2ea//vpLycnJCg8Pz/N1AgCAouuOCsz33XefFi9erI4dO8pmsyk6OtrhGcUpU6YoMDBQ4eHhcnFx0VdffaWAgIDrfkhJZGSkateuraeeekpTp07VpUuXNGDAADVr1izHsoiC8Pbbb+vRRx9VhQoV9MQTT8jFxUU7duzQrl279M4771geV69ePdWsWVMXLlzQd999p+rVq1+3fZs2bfTWW2/p5MmTKlWqlCTphRde0KFDhxQXF6djx46ZbUuXLq2wsDA99dRT6tWrlyZPnqzw8HAdO3ZMcXFxqlOnjrmW+Fr33XefOnXqpH79+mnGjBny8fHR0KFDdc8996hTp04Fdj3XeuONN/TAAw9ozJgx6t69uzZt2qTp06fr448/NtuEhobqP//5j3r06CEPDw/zpsKffvpJHh4eatSoUZ7PBwAAiq5CvYb5WlOmTFGpUqXUuHFjdezYUW3atFFERIRDffj4+GjixImqX7++HnjgAaWkpGjZsmXXfVawzWbTt99+q1KlSqlp06aKjIxUpUqV9OWXXxbEJeXQpk0bfffdd1q5cqUeeOABPfjgg3r//fftbk7Mjbu7u9566y3VqVNHTZs2laurq2JjY6/bvnbt2oqIiNDChQvNfWvXrlVaWppq1KihwMBAc7uyfnjevHnq1auX3njjDVWtWlWdO3fWzz//rAoVKljWNm/ePNWrV0+PPvqoGjVqJMMwtGzZMstnRzt6Pde6cm2xsbGqVauW3n77bY0ePdp8VKB0+abSlJQUVa5c2W75xYIFC/TUU0+pePHieT4fAAAoumyGYRjOLgLO8f3332vw4MHatWvXHffhIrfKn3/+qapVq2rr1q2qWLFino5JT0+Xn5+fgl9dKBcPQjYA50gZn/tf+gDk7sq/36dPn5avr69l2ztqSQYKVocOHbRv3z79/vvvdk8BuZulpKTo448/znNYBgAARR+B+S736quvOruEQqV+/fq3ZG06AAC4c/F3eAAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALLg5uwAAAIDCLDs7W5mZmc4uAw4qVqyYXF1dC6QvAjMAAMB1ZGZm6sCBA8rOznZ2KciHkiVLKiAgQDab7ab6ITADBWTXqDby9fV1dhkAgAJiGIbS0tLk6uqq4OBgubiwkvVOYRiGzp07p6NHj0qSAgMDb6o/AjMAAEAuLl26pHPnzikoKEjFixd3djlwkJeXlyTp6NGjKleu3E0tz+BXJQAAgFxkZWVJktzd3Z1cCfLryi86Fy9evKl+CMwAAAAWbnb9K5ynoL53BGYAAADAAoEZAAAAsMBNfwAAAA4IHfr9bT1fyvgO+Tpu06ZNevjhh9W2bVt9//3trbmoYYYZAACgCJozZ45efvll/ec//9Eff/zhtDqKwoe+EJgBAACKmIyMDH355Zfq37+/OnTooJiYGLv3//Wvf+mBBx6Qp6enypYtqy5dupjvXbhwQUOGDFFwcLA8PDxUpUoVzZkzR5IUExOjkiVL2vW1ZMkSu5vrRo4cqbp162r27NmqWLGiPD09JUkrVqzQww8/rJIlS6pMmTJ69NFHlZycbNfXb7/9pp49e6p06dIqUaKE6tevr82bNyslJUUuLi7aunWrXfupU6cqJCTkln+wDIEZAACgiFm4cKGqVaumqlWr6umnn9bcuXNlGIYk6fvvv1eXLl3Uvn17xcfHKy4uTg0aNDCP7dWrlxYsWKAPPvhASUlJmjFjhry9vR06//79+7Vo0SItXrxYCQkJkqSzZ8/q9ddf19atWxUXFycXFxd16dLFDLsZGRlq1qyZfv/9dy1dulQ7duzQm2++qezsbIWGhioyMlLz5s2zO8+8efMUFRV1yz9UhjXMAAAARcycOXP09NNPS5Latm2r06dPa+3atWrevLneffdd9ejRQ6NGjTLb33///ZKkvXv3auHChVq1apUiIyMlSZUqVXL4/JmZmfrss8/k7+9v7uvatatdm7lz58rf31+JiYmqVauW/vnPf+rYsWP6+eefVbp0aUlSlSpVzPZ9+/bV3//+d02ZMkUeHh7avn27du7cqW+//dbh+hzFDDMAAEARsmfPHm3ZskU9e/aUJLm5ual79+7msoqEhAS1atUq12MTEhLk6uqqZs2a3VQNISEhdmFZkvbt26eePXuqUqVK8vX1VWhoqCQpNTXVPHd4eLgZlq/VuXNnubq66ptvvpF0eXlIixYtzH5uJWaYAQAAipA5c+bo0qVLCgoKMvcZhiEPDw9Nnz7d/Mjo3Fi9J0kuLi7m0o4rcvsUvRIlSuTY17FjR4WEhGjWrFkKCgpSdna2atWqZd4UeKNzu7u7q1evXpo3b54ef/xx/fOf/9S0adMsjykozDADAAAUEZcuXdJnn32myZMnKyEhwdx27NihoKAgLViwQHXq1FFcXFyux9euXVvZ2dlau3Ztru/7+/vrzJkzOnv2rLnvyhplK8ePH9eePXs0fPhwtWrVStWrV9fJkyft2tSpU0cJCQk6ceLEdfvp27evVq9erY8//liXLl3S448/fsNzFwRmmAEAAIqI7777TidPntRzzz0nPz8/u/e6du2qOXPm6L333lOrVq1UuXJl9ejRQ5cuXdKyZcs0ZMgQhYaG6tlnn1WfPn30wQcf6P7779fBgwd19OhRdevWTQ0bNlTx4sX1f//3fxo4cKA2b96c4wkcuSlVqpTKlCmjmTNnKjAwUKmpqRo6dKhdm549e2rs2LHq3Lmzxo0bp8DAQMXHxysoKEiNGjWSJFWvXl0PPvighgwZoj59+txwVrqgMMMMAABQRMyZM0eRkZE5wrJ0OTBv3bpVpUuX1ldffaWlS5eqbt26atmypbZs2WK2++STT/TEE09owIABqlatmvr162fOKJcuXVqff/65li1bptq1a2vBggUaOXLkDetycXFRbGystm3bplq1aum1117Te++9Z9fG3d1dK1euVLly5dS+fXvVrl1b48ePl6urq1275557TpmZmerTp08+Rih/bMa1C1EAOCQ9PV1+fn46ffq0fH19nV0OAKCA/PXXXzpw4IDds4ThfGPGjNFXX32lX3755YZtrb6Hjvz7zQwzAAAACr2MjAzt2rVL06dP18svv3xbz01gBgAAQKH30ksvqV69emrevPltXY4hcdMfAAAA7gAxMTF5usHwVmCGGQAAALBAYAYAAAAsEJgBAAAs8ECxO1d2dnaB9MMaZgAAgFwUK1ZMNptNx44dk7+/v2w2m7NLQh4ZhqHMzEwdO3ZMLi4ucnd3v6n+CMwAAAC5cHV11b333qvffvtNKSkpzi4H+VC8eHFVqFBBLi43t6iCwAwAAHAd3t7euu+++3Tx4kVnlwIHubq6ys3NrUD+MkBgBgAAsODq6prj45lxd+GmPwAAAMACgRkAAACwQGAGAAAALLCGGbhJV57PmZ6e7uRKAABAXl35dzsvz9kmMAM36fjx45Kk4OBgJ1cCAAAcdebMGfn5+Vm2ITADN6l06dKSpNTU1Bv+B4fL0tPTFRwcrEOHDsnX19fZ5dwRGDPHMWaOY8wcx5g5rrCMmWEYOnPmjIKCgm7YlsAM3KQrD0P38/Pjf5YO8vX1ZcwcxJg5jjFzHGPmOMbMcYVhzPI60cVNfwAAAIAFAjMAAABggcAM3CQPDw+NGDFCHh4ezi7ljsGYOY4xcxxj5jjGzHGMmePuxDGzGXl5lgYAAABwl2KGGQAAALBAYAYAAAAsEJgBAAAACwRmAAAAwAKBGciDjz76SKGhofL09FTDhg21ZcsWy/ZfffWVqlWrJk9PT9WuXVvLli27TZUWHo6M2e7du9W1a1eFhobKZrNp6tSpt6/QQsSRMZs1a5aaNGmiUqVKqVSpUoqMjLzhz2VR5MiYLV68WPXr11fJkiVVokQJ1a1bV//4xz9uY7WFg6P/P7siNjZWNptNnTt3vrUFFkKOjFlMTIxsNpvd5unpeRurLRwc/Tk7deqUXnzxRQUGBsrDw0NhYWGF699OA4Cl2NhYw93d3Zg7d66xe/duo1+/fkbJkiWNI0eO5Np+w4YNhqurqzFx4kQjMTHRGD58uFGsWDFj586dt7ly53F0zLZs2WIMGjTIWLBggREQEGC8//77t7fgQsDRMfvb3/5mfPTRR0Z8fLyRlJRkREVFGX5+fsZvv/12myt3HkfH7McffzQWL15sJCYmGvv37zemTp1quLq6GitWrLjNlTuPo2N2xYEDB4x77rnHaNKkidGpU6fbU2wh4eiYzZs3z/D19TXS0tLM7fDhw7e5audydMwuXLhg1K9f32jfvr2xfv1648CBA8aaNWuMhISE21z59RGYgRto0KCB8eKLL5qvs7KyjKCgIGPcuHG5tu/WrZvRoUMHu30NGzY0XnjhhVtaZ2Hi6JhdLSQk5K4MzDczZoZhGJcuXTJ8fHyM+fPn36oSC52bHTPDMIzw8HBj+PDht6K8Qik/Y3bp0iWjcePGxuzZs41nn332rgvMjo7ZvHnzDD8/v9tUXeHk6Jh98sknRqVKlYzMzMzbVaLDWJIBWMjMzNS2bdsUGRlp7nNxcVFkZKQ2bdqU6zGbNm2yay9Jbdq0uW77oiY/Y3a3K4gxO3funC5evKjSpUvfqjILlZsdM8MwFBcXpz179qhp06a3stRCI79jNnr0aJUrV07PPffc7SizUMnvmGVkZCgkJETBwcHq1KmTdu/efTvKLRTyM2ZLly5Vo0aN9OKLL6p8+fKqVauWxo4dq6ysrNtV9g0RmAELf/75p7KyslS+fHm7/eXLl9fhw4dzPebw4cMOtS9q8jNmd7uCGLMhQ4YoKCgoxy9rRVV+x+z06dPy9vaWu7u7OnTooA8//FCtW7e+1eUWCvkZs/Xr12vOnDmaNWvW7Six0MnPmFWtWlVz587Vt99+q88//1zZ2dlq3Lixfvvtt9tRstPlZ8z++9//6uuvv1ZWVpaWLVum6OhoTZ48We+8887tKDlP3JxdAADg5owfP16xsbFas2bNXXlzkSN8fHyUkJCgjIwMxcXF6fXXX1elSpXUvHlzZ5dW6Jw5c0bPPPOMZs2apbJlyzq7nDtGo0aN1KhRI/N148aNVb16dc2YMUNjxoxxYmWFV3Z2tsqVK6eZM2fK1dVV9erV0++//6733ntPI0aMcHZ5kgjMgKWyZcvK1dVVR44csdt/5MgRBQQE5HpMQECAQ+2LmvyM2d3uZsZs0qRJGj9+vFavXq06dercyjILlfyOmYuLi6pUqSJJqlu3rpKSkjRu3Li7IjA7OmbJyclKSUlRx44dzX3Z2dmSJDc3N+3Zs0eVK1e+tUU7WUH8/6xYsWIKDw/X/v37b0WJhU5+xiwwMFDFihWTq6urua969eo6fPiwMjMz5e7ufktrzguWZAAW3N3dVa9ePcXFxZn7srOzFRcXZzeDcLVGjRrZtZekVatWXbd9UZOfMbvb5XfMJk6cqDFjxmjFihWqX7/+7Si10Cion7Ps7GxduHDhVpRY6Dg6ZtWqVdPOnTuVkJBgbo899phatGihhIQEBQcH387ynaIgfs6ysrK0c+dOBQYG3qoyC5X8jNlDDz2k/fv3m7+QSdLevXsVGBhYKMKyJB4rB9xIbGys4eHhYcTExBiJiYnG888/b5QsWdJ8TNAzzzxjDB061Gy/YcMGw83NzZg0aZKRlJRkjBgx4q58rJwjY3bhwgUjPj7eiI+PNwIDA41BgwYZ8fHxxr59+5x1Cbedo2M2fvx4w93d3fj666/tHl915swZZ13CbefomI0dO9ZYuXKlkZycbCQmJhqTJk0y3NzcjFmzZjnrEm47R8fsWnfjUzIcHbNRo0YZP/zwg5GcnGxs27bN6NGjh+Hp6Wns3r3bWZdw2zk6ZqmpqYaPj4/x0ksvGXv27DG+++47o1y5csY777zjrEvIgcAM5MGHH35oVKhQwXB3dzcaNGhg/PTTT+Z7zZo1M5599lm79gsXLjTCwsIMd3d3o2bNmsb3339/myt2PkfG7MCBA4akHFuzZs1uf+FO5MiYhYSE5DpmI0aMuP2FO5EjYzZs2DCjSpUqhqenp1GqVCmjUaNGRmxsrBOqdi5H/392tbsxMBuGY2P26quvmm3Lly9vtG/f3ti+fbsTqnYuR3/ONm7caDRs2NDw8PAwKlWqZLz77rvGpUuXbnPV12czDMNw1uw2AAAAUNixhhkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMACgRkAAACwQGAGAAAALBCYAQAAAAsEZgAAAMDC/wP65VIMye3TlgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] From 9fe6f0d5c22e8429394e69a98e976187a9264f45 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 13:07:48 +0100 Subject: [PATCH 091/183] Run formatting --- src/setfit/trainer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 269b7401..0bb91a41 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -208,9 +208,7 @@ def __init__( default_callbacks = DEFAULT_CALLBACKS + get_reporting_integration_callbacks(self.args.report_to) callbacks = default_callbacks if callbacks is None else default_callbacks + callbacks # TODO: Observe optimizer and scheduler by wrapping SentenceTransformer._get_scheduler - self.callback_handler = CallbackHandler( - callbacks, self.model, self.model.model_body.tokenizer, None, None - ) + self.callback_handler = CallbackHandler(callbacks, self.model, self.model.model_body.tokenizer, None, None) self.state = TrainerState() self.control = TrainerControl() self.add_callback(DEFAULT_PROGRESS_CALLBACK if self.args.show_progress_bar else PrinterCallback) From da338ad80d95a7446fa71ccb8fe168a7cdb2db7f Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 14:22:53 +0100 Subject: [PATCH 092/183] Remove unused arguments in tests --- tests/test_training_args.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_training_args.py b/tests/test_training_args.py index 391e7c16..ee7b4f88 100644 --- a/tests/test_training_args.py +++ b/tests/test_training_args.py @@ -19,7 +19,6 @@ def test_raises_error_with_wrong_warmup_proportion(self): def test_batch_sizes(self): batch_size_A = 12 batch_size_B = 4 - batch_size_C = 6 args = TrainingArguments(batch_size=batch_size_A) self.assertEqual(args.batch_size, (batch_size_A, batch_size_A)) @@ -34,7 +33,6 @@ def test_batch_sizes(self): def test_num_epochs(self): num_epochs_A = 12 num_epochs_B = 4 - num_epochs_C = 6 args = TrainingArguments(num_epochs=num_epochs_A) self.assertEqual(args.num_epochs, (num_epochs_A, num_epochs_A)) @@ -49,7 +47,6 @@ def test_num_epochs(self): def test_learning_rates(self): learning_rate_A = 1e-2 learning_rate_B = 1e-3 - learning_rate_C = 1e-4 base = TrainingArguments() From be4c9004772fb0a8ff822c7649b8e866e757d6f5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 14:55:29 +0100 Subject: [PATCH 093/183] Initial documentation for SetFit v1.0.0 Should be built/previewed with 'doc-builder preview setfit docs/source/en' --- README.md | 257 +---------- docs/source/en/_toctree.yml | 44 +- docs/source/en/api/main.mdx | 12 - .../en/conceptual_guides/placeholder.mdx | 3 - .../conceptual_guides/sampling_strategies.mdx | 87 ++++ docs/source/en/conceptual_guides/setfit.mdx | 28 ++ docs/source/en/how_to/absa.mdx | 203 +++++++++ docs/source/en/how_to/callbacks.mdx | 104 +++++ .../source/en/how_to/classification_heads.mdx | 208 +++++++++ .../en/how_to/hyperparameter_optimization.mdx | 376 ++++++++++++++++ .../en/how_to/knowledge_distillation.mdx | 293 +++++++++++++ docs/source/en/how_to/multilabel.mdx | 48 +++ docs/source/en/how_to/overview.mdx | 9 + docs/source/en/how_to/placeholder.mdx | 3 - .../en/how_to/v1.0.0_migration_guide.mdx | 82 ++++ docs/source/en/how_to/zero_shot.mdx | 167 ++++++++ docs/source/en/index.mdx | 10 +- docs/source/en/installation.mdx | 36 +- docs/source/en/quickstart.mdx | 405 +++++++----------- docs/source/en/reference/main.mdx | 43 ++ docs/source/en/{api => reference}/trainer.mdx | 4 + docs/source/en/reference/utility.mdx | 6 + docs/source/en/tutorials/overview.mdx | 8 + docs/source/en/tutorials/placeholder.mdx | 3 - docs/source/en/tutorials/zero_shot.mdx | 327 ++++++++++++++ 25 files changed, 2226 insertions(+), 540 deletions(-) delete mode 100644 docs/source/en/api/main.mdx delete mode 100644 docs/source/en/conceptual_guides/placeholder.mdx create mode 100644 docs/source/en/conceptual_guides/sampling_strategies.mdx create mode 100644 docs/source/en/conceptual_guides/setfit.mdx create mode 100644 docs/source/en/how_to/absa.mdx create mode 100644 docs/source/en/how_to/callbacks.mdx create mode 100644 docs/source/en/how_to/classification_heads.mdx create mode 100644 docs/source/en/how_to/hyperparameter_optimization.mdx create mode 100644 docs/source/en/how_to/knowledge_distillation.mdx create mode 100644 docs/source/en/how_to/multilabel.mdx create mode 100644 docs/source/en/how_to/overview.mdx delete mode 100644 docs/source/en/how_to/placeholder.mdx create mode 100644 docs/source/en/how_to/v1.0.0_migration_guide.mdx create mode 100644 docs/source/en/how_to/zero_shot.mdx create mode 100644 docs/source/en/reference/main.mdx rename docs/source/en/{api => reference}/trainer.mdx (70%) create mode 100644 docs/source/en/reference/utility.mdx create mode 100644 docs/source/en/tutorials/overview.mdx delete mode 100644 docs/source/en/tutorials/placeholder.mdx create mode 100644 docs/source/en/tutorials/zero_shot.mdx diff --git a/README.md b/README.md index 4a8156e6..491a26d0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

- 🤗 Models & Datasets | 📖 Blog | 📃 Paper + 🤗 Models & Datasets | 📕 Documentation | 📖 Blog | 📃 Paper

# SetFit - Efficient Few-shot Learning with Sentence Transformers @@ -87,259 +87,6 @@ model = SetFitModel.from_pretrained("lewtun/my-awesome-setfit-model") preds = model(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) ``` -Here is an end-to-end example using `SetFitHead`: - - -```python -from datasets import load_dataset -from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset - - -# Load a dataset from the Hugging Face Hub -dataset = load_dataset("sst2") - -# Simulate the few-shot regime by sampling 8 examples per class -train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) -eval_dataset = dataset["validation"] -num_classes = 2 - -# Load a SetFit model from Hub -model = SetFitModel.from_pretrained( - "sentence-transformers/paraphrase-mpnet-base-v2", - use_differentiable_head=True, - head_params={"out_features": num_classes}, -) - -args = TrainingArguments( - body_learning_rate=2e-5, - head_learning_rate=1e-2, - batch_size=16, - num_iterations=20, # The number of text pairs to generate for contrastive learning - num_epochs=(1, 25), # For finetuning the embeddings and training the classifier, respectively - l2_weight=0.0, - end_to_end=False, # Don't train the classifier end-to-end, i.e. only train the head -) - -trainer = Trainer( - model=model, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - metric="accuracy", - column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer -) - -# Train and evaluate -trainer.train() -metrics = trainer.evaluate() - -# Push model to the Hub -trainer.push_to_hub("my-awesome-setfit-model") - -# Download from Hub and run inference -model = SetFitModel.from_pretrained("lewtun/my-awesome-setfit-model") -# Run inference -preds = model(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) -``` - -Based on our experiments, `SetFitHead` can achieve similar performance as using a `scikit-learn` head. We use `AdamW` as the optimizer and scale down learning rates by 0.5 every 5 epochs. For more details about the experiments, please check out [here](https://github.com/huggingface/setfit/pull/112#issuecomment-1295773537). We recommend using a large learning rate (e.g. `1e-2`) for `SetFitHead` and a small learning rate (e.g. `1e-5`) for the body in your first attempt. - -### Training on multilabel datasets - -To train SetFit models on multilabel datasets, specify the `multi_target_strategy` argument when loading the pretrained model: - -#### Example using a classification head from `scikit-learn`: - -```python -from setfit import SetFitModel - -model = SetFitModel.from_pretrained( - model_id, - multi_target_strategy="one-vs-rest", -) -``` - -This will initialise a multilabel classification head from `sklearn` - the following options are available for `multi_target_strategy`: - -* `one-vs-rest`: uses a `OneVsRestClassifier` head. -* `multi-output`: uses a `MultiOutputClassifier` head. -* `classifier-chain`: uses a `ClassifierChain` head. - -From here, you can instantiate a `Trainer` using the same example above, and train it as usual. - -#### Example using the differentiable `SetFitHead`: - -```python -from setfit import SetFitModel - -model = SetFitModel.from_pretrained( - model_id, - multi_target_strategy="one-vs-rest" - use_differentiable_head=True, - head_params={"out_features": num_classes}, -) -``` -**Note:** If you use the differentiable `SetFitHead` classifier head, it will automatically use `BCEWithLogitsLoss` for training. The prediction involves a `sigmoid` after which probabilities are rounded to 1 or 0. Furthermore, the `"one-vs-rest"` and `"multi-output"` multi-target strategies are equivalent for the differentiable `SetFitHead`. - -### Zero-shot text classification - -SetFit can also be applied to scenarios where no labels are available. To do so, create a synthetic dataset of training examples: - -```python -from setfit import get_templated_dataset - -candidate_labels = ["negative", "positive"] -train_dataset = get_templated_dataset(candidate_labels=candidate_labels, sample_size=8) -``` - -This will create examples of the form `"This sentence is {}"`, where the `{}` is filled in with one of the candidate labels. From here you can train a SetFit model as usual: - -```python -from setfit import SetFitModel, Trainer - -model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") -trainer = Trainer( - model=model, - train_dataset=train_dataset -) -trainer.train() -``` - -We find this approach typically outperforms the [zero-shot pipeline](https://huggingface.co/docs/transformers/v4.24.0/en/main_classes/pipelines#transformers.ZeroShotClassificationPipeline) in 🤗 Transformers (based on MNLI with BART), while being 5x faster to generate predictions with. - - -### Running hyperparameter search - -`Trainer` provides a `hyperparameter_search()` method that you can use to find good hyperparameters for your data. To use this feature, first install the `optuna` backend: - -```bash -python -m pip install setfit[optuna] -``` - -To use this method, you need to define two functions: - -* `model_init()`: A function that instantiates the model to be used. If provided, each call to `train()` will start from a new instance of the model as given by this function. -* `hp_space()`: A function that defines the hyperparameter search space. - -Here is an example of a `model_init()` function that we'll use to scan over the hyperparameters associated with the classification head in `SetFitModel`: - -```python -from setfit import SetFitModel - -def model_init(params): - params = params or {} - max_iter = params.get("max_iter", 100) - solver = params.get("solver", "liblinear") - params = { - "head_params": { - "max_iter": max_iter, - "solver": solver, - } - } - return SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", **params) -``` - -Similarly, to scan over hyperparameters associated with the SetFit training process, we can define a `hp_space()` function as follows: - -```python -def hp_space(trial): # Training parameters - return { - "learning_rate": trial.suggest_float("learning_rate", 1e-6, 1e-4, log=True), - "num_epochs": trial.suggest_int("num_epochs", 1, 5), - "batch_size": trial.suggest_categorical("batch_size", [4, 8, 16, 32, 64]), - "seed": trial.suggest_int("seed", 1, 40), - "num_iterations": trial.suggest_categorical("num_iterations", [5, 10, 20]), - "max_iter": trial.suggest_int("max_iter", 50, 300), - "solver": trial.suggest_categorical("solver", ["newton-cg", "lbfgs", "liblinear"]), - } -``` - -**Note:** In practice, we found `num_iterations` to be the most important hyperparameter for the contrastive learning process. - -The next step is to instantiate a `Trainer` and call `hyperparameter_search()`: - -```python -from datasets import Dataset -from setfit import Trainer - -dataset = Dataset.from_dict( - {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} -) - -trainer = Trainer( - train_dataset=dataset, - eval_dataset=dataset, - model_init=model_init, - column_mapping={"text_new": "text", "label_new": "label"}, -) -best_run = trainer.hyperparameter_search(direction="maximize", hp_space=hp_space, n_trials=5) -``` - -Finally, you can apply the hyperparameters you found to the trainer, and lock in the optimal model, before training for -a final time. - -```python -trainer.apply_hyperparameters(best_run.hyperparameters, final_model=True) -trainer.train() -``` - -## Compressing a SetFit model with knowledge distillation - -If you have access to unlabeled data, you can use knowledge distillation to compress a trained SetFit model into a smaller version. The result is a model that can run inference much faster, with little to no drop in accuracy. Here's an end-to-end example (see our paper for more details): - -```python -from datasets import load_dataset -from setfit import SetFitModel, Trainer, DistillationTrainer, sample_dataset -from setfit.training_args import TrainingArguments - -# Load a dataset from the Hugging Face Hub -dataset = load_dataset("ag_news") - -# Create a sample few-shot dataset to train the teacher model -train_dataset_teacher = sample_dataset(dataset["train"], label_column="label", num_samples=16) -# Create a dataset of unlabeled examples to train the student -train_dataset_student = dataset["train"].shuffle(seed=0).select(range(500)) -# Dataset for evaluation -eval_dataset = dataset["test"] - -# Load teacher model -teacher_model = SetFitModel.from_pretrained( - "sentence-transformers/paraphrase-mpnet-base-v2" -) - -# Create trainer for teacher model -teacher_trainer = Trainer( - model=teacher_model, - train_dataset=train_dataset_teacher, - eval_dataset=eval_dataset, -) - -# Train teacher model -teacher_trainer.train() -teacher_metrics = teacher_trainer.evaluate() - -# Load small student model -student_model = SetFitModel.from_pretrained("paraphrase-MiniLM-L3-v2") - -args = TrainingArguments( - batch_size=16, - num_iterations=20, - num_epochs=1 -) - -# Create trainer for knowledge distillation -student_trainer = DistillationTrainer( - teacher_model=teacher_model, - student_model=student_model, - args=args, - train_dataset=train_dataset_student, - eval_dataset=eval_dataset, -) - -# Train student with knowledge distillation -student_trainer.train() -student_metrics = student_trainer.evaluate() -``` - ## Reproducing the results from the paper @@ -369,8 +116,6 @@ We use `black` and `isort` to ensure consistent code formatting. After following make style && make quality ``` - - ## Project structure ``` diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index bea05d0b..af85e43b 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -6,21 +6,47 @@ - local: installation title: Installation title: Get started + - sections: - - local: tutorials/placeholder - title: Placeholder + - local: tutorials/overview + title: Overview + - local: tutorials/zero_shot + title: Zero-shot Text Classification title: Tutorials + - sections: - - local: how_to/placeholder - title: Placeholder + - local: how_to/overview + title: Overview + - local: how_to/callbacks + title: Callbacks + - local: how_to/classification_heads + title: Classification Heads + - local: how_to/multilabel + title: Multilabel Text Classification + - local: how_to/zero_shot + title: Zero-shot Text Classification + - local: how_to/hyperparameter_optimization + title: Hyperparameter Optimization + - local: how_to/knowledge_distillation + title: Knowledge Distillation + - local: how_to/absa + title: Aspect Based Sentiment Analysis + - local: how_to/v1.0.0_migration_guide + title: v1.0.0 Migration Guide title: How-to Guides + - sections: - - local: conceptual_guides/placeholder - title: Placeholder + - local: conceptual_guides/setfit + title: SetFit + - local: conceptual_guides/sampling_strategies + title: Sampling Strategies title: Conceptual Guides + - sections: - - local: api/main + - local: reference/main title: Main classes - - local: api/trainer + - local: reference/trainer title: Trainer classes - title: API \ No newline at end of file + - local: reference/utility + title: Utility + title: Reference \ No newline at end of file diff --git a/docs/source/en/api/main.mdx b/docs/source/en/api/main.mdx deleted file mode 100644 index a65b3db4..00000000 --- a/docs/source/en/api/main.mdx +++ /dev/null @@ -1,12 +0,0 @@ - -# SetFitModel - -[[autodoc]] SetFitModel - -# SetFitHead - -[[autodoc]] SetFitHead - -# AbsaModel - -[[autodoc]] AbsaModel \ No newline at end of file diff --git a/docs/source/en/conceptual_guides/placeholder.mdx b/docs/source/en/conceptual_guides/placeholder.mdx deleted file mode 100644 index b79fc271..00000000 --- a/docs/source/en/conceptual_guides/placeholder.mdx +++ /dev/null @@ -1,3 +0,0 @@ - -# Conceptual Guides -Work in Progress! \ No newline at end of file diff --git a/docs/source/en/conceptual_guides/sampling_strategies.mdx b/docs/source/en/conceptual_guides/sampling_strategies.mdx new file mode 100644 index 00000000..817b0a18 --- /dev/null +++ b/docs/source/en/conceptual_guides/sampling_strategies.mdx @@ -0,0 +1,87 @@ + +# SetFit Sampling Strategies + +SetFit supports various contrastive pair sampling strategies in [`TrainingArguments`]. In this conceptual guide, we will learn about the following four sampling strategies: + +1. `"oversampling"` (the default) +2. `"undersampling"` +3. `"unique"` +4. `"num_iterations"` + +Consider first reading the [SetFit conceptual guide](../setfit) for a background on contrastive learning and positive & negative pairs. + +## Running example + +Throughout this conceptual guide, we will use to the following example scenario: + +* 3 classes: "happy", "content", and "sad". +* 20 total samples: 8 "happy", 4 "content", and 8 "sad" samples. + +Considering that a sentence pair of `(X, Y)` and `(Y, X)` result in the same embedding distance/loss, we only want to consider one of those two cases. Furthermore, we don't want pairs where both sentences are the same, e.g. no `(X, X)`. + +The resulting positive and negative pairs can be visualized in a table like below. The `+` and `-` represent positive and negative pairs, respectively. Furthermore, `h-n` represents the n-th "happy" sentence, `c-n` the n-th "content" sentence, and `s-n` the n-th "sad" sentence. Note that the area below the diagonal is not used as `(X, Y)` and `(Y, X)` result in the same embedding distances, and that the diagonal is not used as we are not interested in pairs where both sentences are identical. + +| |h-1|h-2|h-3|h-4|h-5|h-6|h-7|h-8|c-1|c-2|c-3|c-4|s-1|s-2|s-3|s-4|s-5|s-6|s-7|s-8| +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +|h-1| | + | + | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|h-2| | | + | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|h-3| | | | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|h-4| | | | | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|h-5| | | | | | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|h-6| | | | | | | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|h-7| | | | | | | | + | - | - | - | - | - | - | - | - | - | - | - | - | +|h-8| | | | | | | | | - | - | - | - | - | - | - | - | - | - | - | - | +|c-1| | | | | | | | | | + | + | + | - | - | - | - | - | - | - | - | +|c-2| | | | | | | | | | | + | + | - | - | - | - | - | - | - | - | +|c-3| | | | | | | | | | | | + | - | - | - | - | - | - | - | - | +|c-4| | | | | | | | | | | | | - | - | - | - | - | - | - | - | +|s-1| | | | | | | | | | | | | | + | + | + | + | + | + | + | +|s-2| | | | | | | | | | | | | | | + | + | + | + | + | + | +|s-3| | | | | | | | | | | | | | | | + | + | + | + | + | +|s-4| | | | | | | | | | | | | | | | | + | + | + | + | +|s-5| | | | | | | | | | | | | | | | | | + | + | + | +|s-6| | | | | | | | | | | | | | | | | | | + | + | +|s-7| | | | | | | | | | | | | | | | | | | | + | +|s-8| | | | | | | | | | | | | | | | | | | | | + +As shown in the prior table, we have 28 positive pairs for "happy", 6 positive pairs for "content", and another 28 positive pairs for "sad". In total, this is 62 positive pairs. Also, we have 32 negative pairs between "happy" and "content", 64 negative pairs between "happy" and "sad", and 32 negative pairs between "content" and "sad". In total, this is 128 negative pairs. + +## Oversampling + +By default, SetFit applies the oversampling strategy for its contrastive pairs. This strategy samples an equal amount of positive and negative training pairs, oversampling the minority pair type to match that of the majority pair type. As the number of negative pairs is generally larger than the number of positive pairs, this usually involves oversampling the positive pairs. + +In our running example, this would involve oversampling the 62 positive pairs up to 128, resulting in one epoch of 128 + 128 = 256 pairs. In summary: + +* ✅ An equal amount of positive and negative pairs are sampled. +* ✅ Every possible pair is used. +* ❌ There is some data duplication. + +## Undersampling + +Like oversampling, this strategy samples an equal amount of positive and negative training pairs. However, it undersamples the majority pair type to match that of the minority pair type. This usually involves undersampling the negative pairs to match the positive pairs. + +In our running example, this would involve undersampling the 128 negative pairs down to 62, resulting in one epoch of 62 + 62 = 124 pairs. In summary: + +* ✅ An equal amount of positive and negative pairs are sampled. +* ❌ **Not** every possible pair is used. +* ✅ There is **no** data duplication. + +## Unique + +Thirdly, the unique strategy does not sample an equal amount of positive and negative training pairs. Instead, it simply samples all possible pairs exactly once. No form of oversampling or undersampling is used here. + +In our running example, this would involve sampling all negative and positive pairs, resulting in one epoch of 62 + 128 = 190 pairs. In summary: + +* ❌ **Not** an equal amount of positive and negative pairs are sampled. +* ✅ Every possible pair is used. +* ✅ There is **no** data duplication. + +## `num_iterations` + +Lastly, SetFit can still be used with a deprecated sampling strategy involving the `num_iterations` training argument. Unlike the other sampling strategies, this strategy does not involve the number of possible pairs. Instead, it samples `num_iterations` positive pairs and `num_iterations` negative pairs for each training sample. + +In our running example, if we assume `num_iterations=20`, then we would sample 20 positive pairs and 20 negative pairs per training sample. Because there's 20 samples, this involves (20 + 20) * 20 = 800 pairs. Because there are only 190 unique pairs, this certainly involves some data duplication. However, it does not guarantee that every possible pair is used. In summary: + +* ✅ **Not** an equal amount of positive and negative pairs are sampled. +* ❌ Not necessarily every possible pair is used. +* ❌ There is some data duplication. \ No newline at end of file diff --git a/docs/source/en/conceptual_guides/setfit.mdx b/docs/source/en/conceptual_guides/setfit.mdx new file mode 100644 index 00000000..b4f158f7 --- /dev/null +++ b/docs/source/en/conceptual_guides/setfit.mdx @@ -0,0 +1,28 @@ + +# Sentence Transformers Finetuning (SetFit) + +SetFit is a model framework to efficiently train text classification models with surprisingly little training data. For example, with only 8 labeled examples per class on the Customer Reviews (CR) sentiment dataset, SetFit is competitive with fine-tuning RoBERTa Large on the full training set of 3k examples. Furthermore, SetFit is fast to train and run inference with, and can easily support multilingual tasks. + +Every SetFit model consists of two parts: a **sentence transformer** embedding model (the body) and a **classifier** (the head). These two parts are trained in two separate phases: the **embedding finetuning phase** and the **classifier training phase**. This conceptual guide will elaborate on the intuition between these phases, and why SetFit works so well. + +## Embedding finetuning phase + +The first phase has one primary goal: finetune a sentence transformer embedding model to produce useful embeddings for *our* classification task. The [Hugging Face Hub](https://huggingface.co/models?library=sentence-transformers) already has thousands of sentence transformer available, many of which have been trained to very accurately group the embeddings of texts with similar semantic meaning. + +However, models that are good at Semantic Textual Similarity (STS) are not necessarily immediately good at *our* classification task. For example, according to an embedding model, the sentence of 1) `"He biked to work."` will be much more similar to 2) `"He drove his car to work."` than to 3) `"Peter decided to take the bicycle to the beach party!"`. But if our classification task involves classifying texts into transportation modes, then we want our embedding model to place sentences 1 and 3 closely together, and 2 further away. + +To do so, we can finetune the chosen sentence transformer embedding model. The goal here is to nudge the model to use its pretrained knowledge in a different way that better aligns with our classification task, rather than making the completely forget what it has learned. + +For finetuning, SetFit uses **contrastive learning**. This training approach involves creating **positive and negative pairs** of sentences. A sentence pair will be positive if both of the sentences are of the same class, and negative otherwise. For example, in the case of binary "positive"-"negative" sentiment analysis, `("The movie was awesome", "I loved it")` is a positive pair, and `("The movie was awesome", "It was quite disappointing")` is a negative pair. + +During training, the embedding model receives these pairs, and will convert the sentences to embeddings. If the pair is positive, then it will pull on the model weights such that the text embeddings will be more similar, and vice versa for a negative pair. Through this approach, sentences with the same label will be embedded more similarly, and sentences with different labels less similarly. + +Conveniently, this contrastive learning works with pairs rather than individual samples, and we can create plenty of unique pairs from just a few samples. For example, given 8 positive sentences and 8 negative sentences, we can create 28 positive pairs and 64 negative pairs for 92 unique training pairs. This grows exponentially to the number of sentences and classes, and that is why SetFit can train with just a few examples and still correctly finetune the sentence transformer embedding model. However, we should still be wary of overfitting. + +## Classifier training phase + +Once the sentence transformer embedding model has been finetuned for our task at hand, we can start training the classifier. This phase has one primary goal: create a good mapping from the sentence transformer embeddings to the classes. + +Unlike with the first phase, training the classifier is done from scratch and using the labeled samples directly, rather than using pairs. By default, the classifier is a simple **logistic regression** classifier from scikit-learn. First, all training sentences are fed through the now-finetuned sentence transformer embedding model, and then the sentence embeddings and labels are used to fit the logistic regression classifier. The result is a strong and efficient classifier. + +Using these two parts, SetFit models are efficient, performant and easy to train, even on CPU-only devices. \ No newline at end of file diff --git a/docs/source/en/how_to/absa.mdx b/docs/source/en/how_to/absa.mdx new file mode 100644 index 00000000..d558efe2 --- /dev/null +++ b/docs/source/en/how_to/absa.mdx @@ -0,0 +1,203 @@ + +# SetFit for Aspect Based Sentiment Analysis + +SetFitABSA is an efficient framework for few-shot Aspect Based Sentiment Analysis, achieving competitive performance with little training data. It consists of three phases: + +1. Using spaCy to find potential aspect candidates. +2. Using a SetFit model for filtering these aspect candidates. +3. Using a SetFit model for classifying the filtered aspect candidates. + +This guide will show you how to train, predict, save and load these models. + +## Training SetFitABSA + +First of all, we must instantiate a new [`AbsaModel`] via [`AbsaModel.from_pretrained`]. This can be done by providing configuration for each of the three phases for SetFitABSA: + +1. Provide the name or path of a Sentence Transformer model to be used for the **aspect filtering** SetFit model as the first argument. +2. (Optional) Provide the name or path of a Sentence Transformer model to be used for the **polarity classification** SetFit model as the second argument. If not provided, the same Sentence Transformer model as the aspect filtering model is also used for the polarity classification model. +3. (Optional) Provide the spaCy model to use via the `spacy_model` keyword argument. + +For example: + +```py +from setfit import AbsaModel + +model = AbsaModel.from_pretrained( + "sentence-transformers/all-MiniLM-L6-v2", + "sentence-transformers/all-mpnet-base-v2", + spacy_model="en_core_web_sm", +) +``` + +Or a minimal example: + +```py +from setfit import AbsaModel + +model = AbsaModel.from_pretrained("BAAI/bge-small-en-v1.5") +``` + +Then we have to prepare a training/testing set. These datasets must have `"text"`, `"span"`, `"polarity"`, and `"ordinal"` columns: + +* `"text"`: The full sentence or text containing the aspects. For example: `"But the staff was so horrible to us."`. +* `"span"`: An aspect from the full sentence. Can be multiple words. For example: `"staff"`. +* `"polarity"`: The (polarity) label corresponding to the aspect span. For example: `"negative"`. +* `"ordinal"`: If the aspect span occurs multiple times in the text, then this ordinal represents the index of those occurrences. Often this is just 0. For example: `0`. + +Two datasets that already match this format are these datasets of reviews from the SemEval-2014 Task 4: + +* [tomaarsen/setfit-absa-semeval-restaurants](https://huggingface.co/datasets/tomaarsen/setfit-absa-semeval-restaurants) +* [tomaarsen/setfit-absa-semeval-laptops](https://huggingface.co/datasets/tomaarsen/setfit-absa-semeval-laptops) + +```py +# The training/eval dataset must have `text`, `span`, `polarity`, and `ordinal` columns +dataset = load_dataset("tomaarsen/setfit-absa-semeval-laptops", split="train") +train_dataset = dataset.select(range(128)) +eval_dataset = dataset.select(range(128, 256)) +``` + +We can commence training like with normal SetFit, but now using [`AbsaTrainer`] instead. + + + +If you wish, you can specify separate training arguments for the aspect model as the polarity model by using both the `args` and `polarity_args` keyword arguments. + + + +```py +args = TrainingArguments( + output_dir="models", + num_epochs=5, + use_amp=True, + batch_size=128, + evaluation_strategy="steps", + eval_steps=50, + save_steps=50, + load_best_model_at_end=True, +) + +trainer = AbsaTrainer( + model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + callbacks=[EarlyStoppingCallback(early_stopping_patience=5)], +) +trainer.train() +``` +``` +***** Running training ***** + Num examples = 249 + Num epochs = 5 + Total optimization steps = 1245 + Total train batch size = 128 +{'aspect_embedding_loss': 0.2542, 'learning_rate': 1.6e-07, 'epoch': 0.0} +{'aspect_embedding_loss': 0.2437, 'learning_rate': 8.000000000000001e-06, 'epoch': 0.2} +{'eval_aspect_embedding_loss': 0.2511, 'learning_rate': 8.000000000000001e-06, 'epoch': 0.2} +{'aspect_embedding_loss': 0.2209, 'learning_rate': 1.6000000000000003e-05, 'epoch': 0.4} +{'eval_aspect_embedding_loss': 0.2385, 'learning_rate': 1.6000000000000003e-05, 'epoch': 0.4} +{'aspect_embedding_loss': 0.0165, 'learning_rate': 1.955357142857143e-05, 'epoch': 0.6} +{'eval_aspect_embedding_loss': 0.2776, 'learning_rate': 1.955357142857143e-05, 'epoch': 0.6} +{'aspect_embedding_loss': 0.0158, 'learning_rate': 1.8660714285714287e-05, 'epoch': 0.8} +{'eval_aspect_embedding_loss': 0.2848, 'learning_rate': 1.8660714285714287e-05, 'epoch': 0.8} +{'aspect_embedding_loss': 0.0015, 'learning_rate': 1.7767857142857143e-05, 'epoch': 1.0} +{'eval_aspect_embedding_loss': 0.3133, 'learning_rate': 1.7767857142857143e-05, 'epoch': 1.0} +{'aspect_embedding_loss': 0.0012, 'learning_rate': 1.6875e-05, 'epoch': 1.2} +{'eval_aspect_embedding_loss': 0.2966, 'learning_rate': 1.6875e-05, 'epoch': 1.2} +{'aspect_embedding_loss': 0.0009, 'learning_rate': 1.598214285714286e-05, 'epoch': 1.41} +{'eval_aspect_embedding_loss': 0.2996, 'learning_rate': 1.598214285714286e-05, 'epoch': 1.41} + 28%|██████████████████████████████████▎ | 350/1245 [03:40<09:24, 1.59it/s] +Loading best SentenceTransformer model from step 100. +{'train_runtime': 226.7429, 'train_samples_per_second': 702.822, 'train_steps_per_second': 5.491, 'epoch': 1.41} +***** Running training ***** + Num examples = 39 + Num epochs = 5 + Total optimization steps = 195 + Total train batch size = 128 +{'polarity_embedding_loss': 0.2267, 'learning_rate': 1.0000000000000002e-06, 'epoch': 0.03} +{'polarity_embedding_loss': 0.1038, 'learning_rate': 1.6571428571428574e-05, 'epoch': 1.28} +{'eval_polarity_embedding_loss': 0.1946, 'learning_rate': 1.6571428571428574e-05, 'epoch': 1.28} +{'polarity_embedding_loss': 0.0116, 'learning_rate': 1.0857142857142858e-05, 'epoch': 2.56} +{'eval_polarity_embedding_loss': 0.2364, 'learning_rate': 1.0857142857142858e-05, 'epoch': 2.56} +{'polarity_embedding_loss': 0.0059, 'learning_rate': 5.142857142857142e-06, 'epoch': 3.85} +{'eval_polarity_embedding_loss': 0.2401, 'learning_rate': 5.142857142857142e-06, 'epoch': 3.85} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 195/195 [00:54<00:00, 3.58it/s] +Loading best SentenceTransformer model from step 50. +{'train_runtime': 54.4104, 'train_samples_per_second': 458.736, 'train_steps_per_second': 3.584, 'epoch': 5.0} +``` + +Evaluation is also like normal, although you now get results from the aspect and polarity models separately: + +```py +metrics = trainer.evaluate(eval_dataset) +print(metrics) +``` +``` +***** Running evaluation ***** +{'aspect': {'accuracy': 0.7130649876321116}, 'polarity': {'accuracy': 0.7102310231023102}} +``` + + + +Note that the aspect accuracy refers to the accuracy of classifying aspect candidate spans from the spaCy model as a true aspect or not, and the polarity accuracy refers to the accuracy of classifying only the filtered aspect candidate spans to the correct class. + + + +## Saving a SetFitABSA model + +Once trained, we can use familiar [`AbsaTrainer.save_pretrained`]/[`AbsaModel.save_pretrained`] and [`AbsaTrainer.push_to_hub`]/[`AbsaModel.push_to_hub`] methods to save the model. However, unlike normally, saving an [`AbsaModel`] involves saving two separate models: the **aspect** SetFit model and the **polarity** SetFit model. Consequently, we can provide two directories or `repo_id`'s: + +```py +model.save_pretrained("models/setfit-absa-model-aspect", "models/setfit-absa-model-polarity") +# or +model.push_to_hub("tomaarsen/setfit-absa-model-aspect", "tomaarsen/setfit-absa-model-polarity") +``` +However, you can also provide just one directory or `repo_id`, and `-aspect` and `-polarity` will be automatically added. So, the following code is equivalent to the previous snippet: + +```py +model.save_pretrained("models/setfit-absa-model") +# or +model.push_to_hub("tomaarsen/setfit-absa-model") +``` + +## Loading a SetFitABSA model + +Loading a trained [`AbsaModel`] involves calling [`AbsaModel.from_pretrained`] with details for each of the three phases for SetFitABSA: + +1. Provide the name or path of a trained SetFit ABSA model to be used for the **aspect filtering** model as the first argument. +2. Provide the name or path of a trained SetFit ABSA model to be used for the **polarity classification** model as the second argument. +3. (Optional) Provide the spaCy model to use via the `spacy_model` keyword argument. It is recommended to match this with the model used during training. The default is `"en_core_web_lg"`. + +For example: + +```py +from setfit import AbsaModel + +model = AbsaModel.from_pretrained( + "tomaarsen/setfit-absa-restaurants-aspect", + "tomaarsen/setfit-absa-restaurants-polarity", + spacy_model="en_core_web_lg", +) +``` + +We've now successfully loaded the SetFitABSA model from: +* [tomaarsen/setfit-absa-restaurants-aspect](https://huggingface.co/tomaarsen/setfit-absa-restaurants-aspect) +* [tomaarsen/setfit-absa-restaurants-polarity](https://huggingface.co/tomaarsen/setfit-absa-restaurants-polarity) + +## Inference with a SetFitABSA model + +To perform inference with a trained [`AbsaModel`], we can use [`AbsaModel.predict`]: + +```py +preds = model.predict([ + "Best pizza outside of Italy and really tasty.", + "The food variations are great and the prices are absolutely fair.", + "Unfortunately, you have to expect some waiting time and get a note with a waiting number if it should be very full." +]) +print(preds) +# [ +# [{'span': 'pizza', 'polarity': 'positive'}], +# [{'span': 'food variations', 'polarity': 'positive'}, {'span': 'prices', 'polarity': 'positive'}], +# [{'span': 'waiting number', 'polarity': 'negative'}] +# ] +``` diff --git a/docs/source/en/how_to/callbacks.mdx b/docs/source/en/how_to/callbacks.mdx new file mode 100644 index 00000000..6ff557f5 --- /dev/null +++ b/docs/source/en/how_to/callbacks.mdx @@ -0,0 +1,104 @@ + +# Callbacks +SetFit models can be influenced by callbacks, for example for logging or early stopping. + +This guide will show you what they are and how they can be used. + +## Callbacks in SetFit + +Callbacks are objects that customize the behaviour of the training loop in the SetFit [`Trainer`] that can inspect the training loop state (for progress reporting, logging, inspecting embeddings during training) and take decisions (e.g. early stopping). + +In particular, the [`Trainer`] uses a [`TrainerControl`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.TrainerControl) that can be influenced by callbacks to stop training, save models, evaluate, or log, and a [`TrainerState`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.TrainerState) which tracks some training loop metrics during training, such as the number of training steps so far. + +SetFit relies on the Callbacks implemented in `transformers`, as described in the `transformers` documentation [here](https://huggingface.co/docs/transformers/main_classes/callback). + +## Default Callbacks + +SetFit uses the `TrainingArguments.report_to` argument to specify which of the built-in callbacks should be enabled. This argument defaults to `"all"`, meaning that all third-party callbacks from `transformers` that are also installed will be enabled. For example the [`TensorBoardCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.integrations.TensorBoardCallback) or the [`WandbCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.integrations.WandbCallback). + +Beyond that, the [`PrinterCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.PrinterCallback) or [`ProgressCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.ProgressCallback) is always enabled to show the training progress, and [`DefaultFlowCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.DefaultFlowCallback) is also always enabled to properly update the `TrainerControl`. + +## Using Callbacks + +As mentioned, you can use `TrainingArguments.report_to` to specify exactly which callbacks you would like to enable. For example: + +```py +from setfit import TrainingArguments + +args = TrainingArguments( + ..., + report_to="wandb", + ..., +) +# or +args = TrainingArguments( + ..., + report_to=["wandb", "tensorboard"], + ..., +) +``` +You can also use [`Trainer.add_callback`], [`Trainer.pop_callback`] and [`Trainer.remove_callback`] to influence the trainer callbacks, and you can specify callbacks via the [`Trainer`] init, e.g.: + +```py +from setfit import Trainer + +... + +trainer = Trainer( + model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + callbacks=[EarlyStoppingCallback(early_stopping_patience=5)], +) +trainer.train() +``` + +## Custom Callbacks + +SetFit supports custom callbacks in the same way that `transformers` does: by subclassing [`TrainerCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.TrainerCallback). This class implements a lot of `on_...` methods that can be overridden. For example, the following script shows a custom callback that saves plots of the tSNE of the training and evaluation embeddings during training. + +```py +import matplotlib.pyplot as plt +from sklearn.manifold import TSNE + +class EmbeddingPlotCallback(TrainerCallback): + """Simple embedding plotting callback that plots the tSNE of the training and evaluation datasets throughout training.""" + def on_evaluate(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, model: SetFitModel, **kwargs): + train_embeddings = model.encode(train_dataset["text"]) + eval_embeddings = model.encode(eval_dataset["text"]) + + fig, (train_ax, eval_ax) = plt.subplots(ncols=2) + + train_X = TSNE(n_components=2).fit_transform(train_embeddings) + train_ax.scatter(*train_X.T, c=train_dataset["label"], label=train_dataset["label"]) + train_ax.set_title("Training embeddings") + + eval_X = TSNE(n_components=2).fit_transform(eval_embeddings) + eval_ax.scatter(*eval_X.T, c=eval_dataset["label"], label=eval_dataset["label"]) + eval_ax.set_title("Evaluation embeddings") + + fig.suptitle(f"tSNE of training and evaluation embeddings at step {state.global_step} of {state.max_steps}.") + fig.savefig(f"logs/step_{state.global_step}.png") +``` + +with + +```py +trainer = Trainer( + model=model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + callbacks=[EmbeddingPlotCallback()] +) +trainer.train() +``` + +The `on_evaluate` from `EmbeddingPlotCallback` will be triggered on every single evaluation call. In the case of this example, it resulted in the following figures being plotted: + +| Step 20 | Step 40 | +|-------------|-------------| +| ![step_20](https://github.com/huggingface/setfit/assets/37621491/7200d00a-fd48-4038-bcbe-f2d5f1280162) | ![step_40](https://github.com/huggingface/setfit/assets/37621491/be12e3c4-867c-452d-89a0-0677f035516d) | +| **Step 60** | **Step 80** | +| ![step_60](https://github.com/huggingface/setfit/assets/37621491/3a384aa2-51ce-40d7-b02c-a2c986f3aeb4) | ![step_80](https://github.com/huggingface/setfit/assets/37621491/b5aa9835-40cb-4327-9f31-b3ababeca769) | \ No newline at end of file diff --git a/docs/source/en/how_to/classification_heads.mdx b/docs/source/en/how_to/classification_heads.mdx new file mode 100644 index 00000000..cb1ef413 --- /dev/null +++ b/docs/source/en/how_to/classification_heads.mdx @@ -0,0 +1,208 @@ + +# Classification heads + +[[open-in-colab]] + +Any 🤗 SetFit model consists of two parts: a [SentenceTransformer](https://sbert.net/) embedding body and a classification head. + +This guide will show you: +* The built-in logistic regression classification head +* The built-in differentiable classification head +* The requirements for a custom classification head + +## Logistic Regression classification head + +When a new SetFit model is initialized, a [scikit-learn logistic regression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) head is chosen by default. This has been shown to be highly effective when applied on top of a finetuned sentence transformer body, and it remains the recommended classification head. Initializing a new SetFit model with a Logistic Regression head is simple: + +```py +>>> from setfit import SetFitModel + +>>> model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5") +>>> model.model_head +LogisticRegression() +``` + +To initialize the Logistic Regression head (or any other head) with additional parameters, then you can use the `head_params` argument on [`SetFitModel.from_pretrained`]: + +```py +>>> from setfit import SetFitModel + +>>> model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5", head_params={"solver": "liblinear", "max_iter": 300}) +>>> model.model_head +LogisticRegression(max_iter=300, solver='liblinear') +``` + +## Differentiable classification head + +SetFit also provides [`SetFitHead`] as an exclusively `torch` classification head. It uses a linear layer to map the embeddings to the class. It can be used by setting the `use_differentiable_head` argument on [`SetFitModel.from_pretrained`] to `True`: + +```py +>>> from setfit import SetFitModel + +>>> model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5", use_differentiable_head=True) +>>> model.model_head +SetFitHead({'in_features': 384, 'out_features': 2, 'temperature': 1.0, 'bias': True, 'device': 'cuda'}) +``` + +By default, this will assume binary classification. To change that, also set the `out_features` via `head_params` to the number of classes that you are using. + +```py +>>> from setfit import SetFitModel + +>>> model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5", use_differentiable_head=True, head_params={"out_features": 5}) +>>> model.model_head +SetFitHead({'in_features': 384, 'out_features': 5, 'temperature': 1.0, 'bias': True, 'device': 'cuda'}) +``` + + + +Unlike the default Logistic Regression head, the differentiable classification head only supports integer labels in the following range: `[0, num_classes)`. + + + +### Training with a differentiable classification head + +Using the [`SetFitHead`] unlocks some new [`TrainingArguments`] that are not used with a sklearn-based head. Note that training with SetFit consists of two phases behind the scenes: **finetuning embeddings** and **training a classification head**. As a result, some of the training arguments can be tuples, where the two values are used for each of the two phases, respectively. For a lot of these cases, the second value is only used if the classification head is differentiable. For example: + +* **batch_size**: (`Union[int, Tuple[int, int]]`, defaults to `(16, 2)`) - The second value in the tuple determines the batch size when training the differentiable SetFitHead. +* **num_epochs**: (`Union[int, Tuple[int, int]]`, defaults to `(1, 16)`) - The second value in the tuple determines the number of epochs when training the differentiable SetFitHead. In practice, the `num_epochs` is usually larger for training the classification head. There are two reasons for this: + + 1. This training phase does not train with contrastive pairs, so unlike when finetuning the embedding model, you only get one training sample per labeled training text. + 2. This training phase involves training a classifier from scratch, not finetuning an already capable model. We need more training steps for this. +* **end_to_end**: (`bool`, defaults to `False`) - If `True`, train the entire model end-to-end during the classifier training phase. Otherwise, freeze the Sentence Transformer body and only train the head. +* **body_learning_rate**: (`Union[float, Tuple[float, float]]`, defaults to `(2e-5, 1e-5)`) - The second value in the tuple determines the learning rate of the Sentence Transformer body during the classifier training phase. This is only relevant if `end_to_end` is `True`, as otherwise the Sentence Transformer body is frozen when training the classifier. +* **head_learning_rate** (`float`, defaults to `1e-2`) - This value determines the learning rate of the differentiable head during the classifier training phase. It is only used if the differentiable head is used. +* **l2_weight** (`float`, *optional*) - Optional l2 weight for both the model body and head, passed to the `AdamW` optimizer in the classifier training phase only if a differentiable head is used. + +For example, a full training script using a differentiable classification head may look something like this: + +```py +from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset +from datasets import load_dataset + +# Initializing a new SetFit model +model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5", use_differentiable_head=True, head_params={"out_features": 2}) + +# Preparing the dataset +dataset = load_dataset("SetFit/sst2") +train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=32) +test_dataset = dataset["test"] + +# Preparing the training arguments +args = TrainingArguments( + batch_size=(32, 16), + num_epochs=(3, 8), + end_to_end=True, + body_learning_rate=(2e-5, 5e-6), + head_learning_rate=2e-3, + l2_weight=0.01, +) + +# Preparing the trainer +trainer = Trainer( + model=model, + args=args, + train_dataset=train_dataset, +) +trainer.train() +# ***** Running training ***** +# Num examples = 66 +# Num epochs = 3 +# Total optimization steps = 198 +# Total train batch size = 3 +# {'embedding_loss': 0.2204, 'learning_rate': 1.0000000000000002e-06, 'epoch': 0.02} +# {'embedding_loss': 0.0058, 'learning_rate': 1.662921348314607e-05, 'epoch': 0.76} +# {'embedding_loss': 0.0026, 'learning_rate': 1.101123595505618e-05, 'epoch': 1.52} +# {'embedding_loss': 0.0022, 'learning_rate': 5.393258426966292e-06, 'epoch': 2.27} +# {'train_runtime': 36.6756, 'train_samples_per_second': 172.758, 'train_steps_per_second': 5.399, 'epoch': 3.0} +# 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 198/198 [00:30<00:00, 6.45it/s] +# The `max_length` is `None`. Using the maximum acceptable length according to the current model body: 512. +# Epoch: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 8/8 [00:07<00:00, 1.03it/s] + +# Evaluating +metrics = trainer.evaluate(test_dataset) +print(metrics) +# => {'accuracy': 0.8632619439868204} + +# Performing inference +preds = model.predict([ + "It's a charming and often affecting journey.", + "It's slow -- very, very slow.", + "A sometimes tedious film.", +]) +print(preds) +# => tensor([1, 0, 0], device='cuda:0') +``` + +## Custom classification head +Alongside the two built-in options, SetFit allows you to specify a custom classification head. There are two forms of supported heads: a custom **differentiable** head or a custom **non-differentiable** head. Both heads must implement the following two methods: + +### Custom differentiable head +A custom differentiable head must follow these requirements: + +* Must subclass `nn.Module`. +* A `predict` method: `(self, torch.Tensor with shape [num_inputs, embedding_size]) -> torch.Tensor with shape [num_inputs]` - This method classifies the embeddings. The output must integers in the range of `[0, num_classes)`. +* A `predict_proba` method: `(self, torch.Tensor with shape [num_inputs, embedding_size]) -> torch.Tensor with shape [num_inputs, num_classes]` - This method classifies the embeddings into probabilities for each class. For each input, the tensor of size `num_classes` must sum to 1. Applying `torch.argmax(output, dim=-1)` should result in the output for `predict`. +* A `get_loss_fn` method: `(self) -> nn.Module` - Returns an initialized loss function, e.g. `torch.nn.CrossEntropyLoss()`. +* A `forward` method: `(self, Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]` - Given the output from the Sentence Transformer body, i.e. a dictionary of `'input_ids'`, `'token_type_ids'`, `'attention_mask'`, `'token_embeddings'` and `'sentence_embedding'` keys, return a dictionary with a `'logits'` key and a `torch.Tensor` value with shape `[batch_size, num_classes]`. + +### Custom non-differentiable head +A custom non-differentiable head must follow these requirements: + +* A `predict` method: `(self, np.array with shape [num_inputs, embedding_size]) -> np.array with shape [num_inputs]` - This method classifies the embeddings. The output must integers in the range of `[0, num_classes)`. +* A `predict_proba` method: `(self, np.array with shape [num_inputs, embedding_size]) -> np.array with shape [num_inputs, num_classes]` - This method classifies the embeddings into probabilities for each class. For each input, the array of size `num_classes` must sum to 1. Applying `np.argmax(output, dim=-1)` should result in the output for `predict`. +* A `fit` method: `(self, np.array with shape [num_inputs, embedding_size], List[Any]) -> None` - This method must take a `numpy` array of embeddings and a list of corresponding labels. The labels need not be integers per se. + +Many classifiers from sklearn already fit these requirements, such as [`RandomForestClassifier`](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier), [`MLPClassifier`](https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html#sklearn.neural_network.MLPClassifier), [`KNeighborsClassifier`](https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html#sklearn.neighbors.KNeighborsClassifier), etc. + +When initializing a SetFit model using your custom (non-)differentiable classification head, it is recommended to use the regular `__init__` method: + +```py +from setfit import SetFitModel +from sklearn.svm import LinearSVC +from sentence_transformers import SentenceTransformer + +# Initializing a new SetFit model +model_body = SentenceTransformer("BAAI/bge-small-en-v1.5") +model_head = LinearSVC() +model = SetFitModel(model_body, model_head) +``` + +Then, training and inference can commence like normal, e.g.: +```py +from setfit import Trainer, TrainingArguments, sample_dataset +from datasets import load_dataset + +# Preparing the dataset +dataset = load_dataset("SetFit/sst2") +train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=32) +test_dataset = dataset["test"] + +# Preparing the training arguments +args = TrainingArguments( + batch_size=32, + num_epochs=3, +) + +# Preparing the trainer +trainer = Trainer( + model=model, + args=args, + train_dataset=train_dataset, +) +trainer.train() + +# Evaluating +metrics = trainer.evaluate(test_dataset) +print(metrics) +# => {'accuracy': 0.8638110928061504} + +# Performing inference +preds = model.predict([ + "It's a charming and often affecting journey.", + "It's slow -- very, very slow.", + "A sometimes tedious film.", +]) +print(preds) +# => tensor([1, 0, 0], dtype=torch.int32) +``` diff --git a/docs/source/en/how_to/hyperparameter_optimization.mdx b/docs/source/en/how_to/hyperparameter_optimization.mdx new file mode 100644 index 00000000..db687044 --- /dev/null +++ b/docs/source/en/how_to/hyperparameter_optimization.mdx @@ -0,0 +1,376 @@ + +# Hyperparameter Optimization + +SetFit models are often very quick to train, making them very suitable for hyperparameter optimization (HPO) to select the best hyperparameters. + +This guide will show you how to apply HPO for SetFit. + +## Requirements + +To use HPO, first install the `optuna` backend: + +```bash +pip install optuna +``` + +To use this method, you need to define two functions: + +* `model_init()`: A function that instantiates the model to be used. If provided, each call to `train()` will start from a new instance of the model as given by this function. +* `hp_space()`: A function that defines the hyperparameter search space. + +Here is an example of a `model_init()` function that we'll use to scan over the hyperparameters associated with the classification head in `SetFitModel`: + +```python +from setfit import SetFitModel +from typing import Dict, Any + +def model_init(params: Dict[str, Any]) -> SetFitModel: + params = params or {} + max_iter = params.get("max_iter", 100) + solver = params.get("solver", "liblinear") + params = { + "head_params": { + "max_iter": max_iter, + "solver": solver, + } + } + return SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5", **params) +``` + +Then, to scan over hyperparameters associated with the SetFit training process, we can define a `hp_space(trial)` function as follows: + +```python +from optuna import Trial +from typing import Dict, Union + +def hp_space(trial: Trial) -> Dict[str, Union[float, int, str]]: + return { + "body_learning_rate": trial.suggest_float("body_learning_rate", 1e-6, 1e-3, log=True), + "num_epochs": trial.suggest_int("num_epochs", 1, 3), + "batch_size": trial.suggest_categorical("batch_size", [16, 32, 64]), + "seed": trial.suggest_int("seed", 1, 40), + "max_iter": trial.suggest_int("max_iter", 50, 300), + "solver": trial.suggest_categorical("solver", ["newton-cg", "lbfgs", "liblinear"]), + } +``` + + + +In practice, we found `num_epochs`, `max_steps`, and `body_learning_rate` to be the most important hyperparameters for the contrastive learning process. + + + +The next step is to prepare a dataset. + +```py +from datasets import load_dataset +from setfit import Trainer, sample_dataset + +dataset = load_dataset("SetFit/emotion") +train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) +test_dataset = dataset["test"] +``` + +After which we can instantiate a [`Trainer`] and commence HPO via [`Trainer.hyperparameter_search`]. I've split up the logs from each trial into separate codeblocks for readability: + +```py +trainer = Trainer( + train_dataset=train_dataset, + eval_dataset=test_dataset, + model_init=model_init, +) +best_run = trainer.hyperparameter_search(direction="maximize", hp_space=hp_space, n_trials=10) +``` +``` +[I 2023-11-14 20:36:55,736] A new study created in memory with name: no-name-d9c6ec29-c5d8-48a2-8f09-299b1f3740f1 +Trial: {'body_learning_rate': 1.937397586885703e-06, 'num_epochs': 3, 'batch_size': 32, 'seed': 16, 'max_iter': 223, 'solver': 'newton-cg'} +model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference. +***** Running training ***** + Num examples = 60 + Num epochs = 3 + Total optimization steps = 180 + Total train batch size = 32 +{'embedding_loss': 0.26, 'learning_rate': 1.0763319927142795e-07, 'epoch': 0.02} +{'embedding_loss': 0.2069, 'learning_rate': 1.5547017672539594e-06, 'epoch': 0.83} +{'embedding_loss': 0.2145, 'learning_rate': 9.567395490793595e-07, 'epoch': 1.67} +{'embedding_loss': 0.2236, 'learning_rate': 3.587773309047598e-07, 'epoch': 2.5} +{'train_runtime': 36.1299, 'train_samples_per_second': 159.425, 'train_steps_per_second': 4.982, 'epoch': 3.0} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 180/180 [00:29<00:00, 6.02it/s] +***** Running evaluation ***** +[I 2023-11-14 20:37:33,895] Trial 0 finished with value: 0.44 and parameters: {'body_learning_rate': 1.937397586885703e-06, 'num_epochs': 3, 'batch_size': 32, 'seed': 16, 'max_iter': 223, 'solver': 'newton-cg'}. Best is trial 0 with value: 0.44. +``` +``` +Trial: {'body_learning_rate': 0.000946449838705604, 'num_epochs': 2, 'batch_size': 16, 'seed': 8, 'max_iter': 60, 'solver': 'liblinear'} +model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference. +***** Running training ***** + Num examples = 120 + Num epochs = 2 + Total optimization steps = 240 + Total train batch size = 16 +{'embedding_loss': 0.2354, 'learning_rate': 3.943540994606683e-05, 'epoch': 0.01} +{'embedding_loss': 0.2419, 'learning_rate': 0.0008325253210836332, 'epoch': 0.42} +{'embedding_loss': 0.3601, 'learning_rate': 0.0006134397102721508, 'epoch': 0.83} +{'embedding_loss': 0.2694, 'learning_rate': 0.00039435409946066835, 'epoch': 1.25} +{'embedding_loss': 0.2496, 'learning_rate': 0.0001752684886491859, 'epoch': 1.67} +{'train_runtime': 33.5015, 'train_samples_per_second': 114.622, 'train_steps_per_second': 7.164, 'epoch': 2.0} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 240/240 [00:33<00:00, 7.16it/s] +***** Running evaluation ***** +[I 2023-11-14 20:38:09,485] Trial 1 finished with value: 0.207 and parameters: {'body_learning_rate': 0.000946449838705604, 'num_epochs': 2, 'batch_size': 16, 'seed': 8, 'max_iter': 60, 'solver': 'liblinear'}. Best is trial 0 with value: 0.44. +``` +``` +Trial: {'body_learning_rate': 8.050718146495058e-06, 'num_epochs': 1, 'batch_size': 32, 'seed': 20, 'max_iter': 260, 'solver': 'lbfgs'} +model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference. +***** Running training ***** + Num examples = 60 + Num epochs = 1 + Total optimization steps = 60 + Total train batch size = 32 +{'embedding_loss': 0.2499, 'learning_rate': 1.3417863577491763e-06, 'epoch': 0.02} +{'embedding_loss': 0.1714, 'learning_rate': 1.490873730832418e-06, 'epoch': 0.83} +{'train_runtime': 9.5338, 'train_samples_per_second': 201.388, 'train_steps_per_second': 6.293, 'epoch': 1.0} +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:09<00:00, 6.30it/s] +***** Running evaluation ***** +[I 2023-11-14 20:38:21,069] Trial 2 finished with value: 0.436 and parameters: {'body_learning_rate': 8.050718146495058e-06, 'num_epochs': 1, 'batch_size': 32, 'seed': 20, 'max_iter': 260, 'solver': 'lbfgs'}. Best is trial 0 with value: 0.44. +``` +``` +Trial: {'body_learning_rate': 0.000995585414046506, 'num_epochs': 1, 'batch_size': 32, 'seed': 29, 'max_iter': 105, 'solver': 'lbfgs'} +model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference. +***** Running training ***** + Num examples = 60 + Num epochs = 1 + Total optimization steps = 60 + Total train batch size = 32 +{'embedding_loss': 0.2556, 'learning_rate': 0.00016593090234108434, 'epoch': 0.02} +{'embedding_loss': 0.0625, 'learning_rate': 0.0001843676692678715, 'epoch': 0.83} +{'train_runtime': 9.5629, 'train_samples_per_second': 200.776, 'train_steps_per_second': 6.274, 'epoch': 1.0} +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:09<00:00, 6.28it/s] +***** Running evaluation ***** +[I 2023-11-14 20:38:32,890] Trial 3 finished with value: 0.283 and parameters: {'body_learning_rate': 0.000995585414046506, 'num_epochs': 1, 'batch_size': 32, 'seed': 29, 'max_iter': 105, 'solver': 'lbfgs'}. Best is trial 0 with value: 0.44. +``` +``` +Trial: {'body_learning_rate': 8.541092571911196e-06, 'num_epochs': 3, 'batch_size': 32, 'seed': 2, 'max_iter': 223, 'solver': 'newton-cg'} +model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference. +***** Running training ***** + Num examples = 60 + Num epochs = 3 + Total optimization steps = 180 + Total train batch size = 32 +{'embedding_loss': 0.2578, 'learning_rate': 4.745051428839553e-07, 'epoch': 0.02} +{'embedding_loss': 0.1725, 'learning_rate': 6.8539631749904665e-06, 'epoch': 0.83} +{'embedding_loss': 0.1589, 'learning_rate': 4.217823492301825e-06, 'epoch': 1.67} +{'embedding_loss': 0.1153, 'learning_rate': 1.5816838096131844e-06, 'epoch': 2.5} +{'train_runtime': 28.3099, 'train_samples_per_second': 203.462, 'train_steps_per_second': 6.358, 'epoch': 3.0} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 180/180 [00:28<00:00, 6.36it/s] +***** Running evaluation ***** +[I 2023-11-14 20:39:03,196] Trial 4 finished with value: 0.4415 and parameters: {'body_learning_rate': 8.541092571911196e-06, 'num_epochs': 3, 'batch_size': 32, 'seed': 2, 'max_iter': 223, 'solver': 'newton-cg'}. Best is trial 4 with value: 0.4415. +``` +``` +Trial: {'body_learning_rate': 2.3916782417792657e-05, 'num_epochs': 1, 'batch_size': 64, 'seed': 23, 'max_iter': 258, 'solver': 'liblinear'} +model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference. +***** Running training ***** + Num examples = 30 + Num epochs = 1 + Total optimization steps = 30 + Total train batch size = 64 +{'embedding_loss': 0.2478, 'learning_rate': 7.972260805930886e-06, 'epoch': 0.03} +{'train_runtime': 6.4905, 'train_samples_per_second': 295.818, 'train_steps_per_second': 4.622, 'epoch': 1.0} +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 30/30 [00:06<00:00, 4.62it/s] +***** Running evaluation ***** +[I 2023-11-14 20:39:12,024] Trial 5 finished with value: 0.4345 and parameters: {'body_learning_rate': 2.3916782417792657e-05, 'num_epochs': 1, 'batch_size': 64, 'seed': 23, 'max_iter': 258, 'solver': 'liblinear'}. Best is trial 4 with value: 0.4415. +``` +``` +Trial: {'body_learning_rate': 0.00012856431493122938, 'num_epochs': 1, 'batch_size': 32, 'seed': 29, 'max_iter': 97, 'solver': 'liblinear'} +model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference. +***** Running training ***** + Num examples = 60 + Num epochs = 1 + Total optimization steps = 60 + Total train batch size = 32 +{'embedding_loss': 0.2556, 'learning_rate': 2.1427385821871562e-05, 'epoch': 0.02} +{'embedding_loss': 0.023, 'learning_rate': 2.380820646874618e-05, 'epoch': 0.83} +{'train_runtime': 9.2295, 'train_samples_per_second': 208.029, 'train_steps_per_second': 6.501, 'epoch': 1.0} +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:09<00:00, 6.50it/s] +***** Running evaluation ***** +[I 2023-11-14 20:39:23,302] Trial 6 finished with value: 0.4675 and parameters: {'body_learning_rate': 0.00012856431493122938, 'num_epochs': 1, 'batch_size': 32, 'seed': 29, 'max_iter': 97, 'solver': 'liblinear'}. Best is trial 6 with value: 0.4675. +``` +``` +Trial: {'body_learning_rate': 3.839168294105717e-06, 'num_epochs': 3, 'batch_size': 16, 'seed': 16, 'max_iter': 297, 'solver': 'newton-cg'} +***** Running training ***** + Num examples = 120 + Num epochs = 3 + Total optimization steps = 360 + Total train batch size = 16 +{'embedding_loss': 0.2357, 'learning_rate': 1.066435637251588e-07, 'epoch': 0.01} +{'embedding_loss': 0.2268, 'learning_rate': 3.6732783060888037e-06, 'epoch': 0.42} +{'embedding_loss': 0.1308, 'learning_rate': 3.0808140631712545e-06, 'epoch': 0.83} +{'embedding_loss': 0.2032, 'learning_rate': 2.4883498202537057e-06, 'epoch': 1.25} +{'embedding_loss': 0.1617, 'learning_rate': 1.8958855773361567e-06, 'epoch': 1.67} +{'embedding_loss': 0.1363, 'learning_rate': 1.3034213344186077e-06, 'epoch': 2.08} +{'embedding_loss': 0.1559, 'learning_rate': 7.109570915010587e-07, 'epoch': 2.5} +{'embedding_loss': 0.1761, 'learning_rate': 1.1849284858350979e-07, 'epoch': 2.92} +{'train_runtime': 49.8712, 'train_samples_per_second': 115.497, 'train_steps_per_second': 7.219, 'epoch': 3.0} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 360/360 [00:49<00:00, 7.22it/s] +***** Running evaluation ***** +[I 2023-11-14 20:40:15,350] Trial 7 finished with value: 0.442 and parameters: {'body_learning_rate': 3.839168294105717e-06, 'num_epochs': 3, 'batch_size': 16, 'seed': 16, 'max_iter': 297, 'solver': 'newton-cg'}. Best is trial 6 with value: 0.4675. +``` +``` +Trial: {'body_learning_rate': 0.0005575631179396824, 'num_epochs': 1, 'batch_size': 32, 'seed': 31, 'max_iter': 264, 'solver': 'newton-cg'} +***** Running training ***** + Num examples = 60 + Num epochs = 1 + Total optimization steps = 60 + Total train batch size = 32 +{'embedding_loss': 0.2588, 'learning_rate': 9.29271863232804e-05, 'epoch': 0.02} +{'embedding_loss': 0.0025, 'learning_rate': 0.00010325242924808932, 'epoch': 0.83} +{'train_runtime': 9.4608, 'train_samples_per_second': 202.942, 'train_steps_per_second': 6.342, 'epoch': 1.0} +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:09<00:00, 6.34it/s] +***** Running evaluation ***** +[I 2023-11-14 20:40:26,886] Trial 8 finished with value: 0.4785 and parameters: {'body_learning_rate': 0.0005575631179396824, 'num_epochs': 1, 'batch_size': 32, 'seed': 31, 'max_iter': 264, 'solver': 'newton-cg'}. Best is trial 8 with value: 0.4785. +``` +``` +Trial: {'body_learning_rate': 0.00021830594983845785, 'num_epochs': 2, 'batch_size': 16, 'seed': 38, 'max_iter': 267, 'solver': 'lbfgs'} +***** Running training ***** + Num examples = 120 + Num epochs = 2 + Total optimization steps = 240 + Total train batch size = 16 +{'embedding_loss': 0.2356, 'learning_rate': 9.096081243269076e-06, 'epoch': 0.01} +{'embedding_loss': 0.071, 'learning_rate': 0.00019202838180234718, 'epoch': 0.42} +{'embedding_loss': 0.0021, 'learning_rate': 0.000141494597117519, 'epoch': 0.83} +{'embedding_loss': 0.0018, 'learning_rate': 9.096081243269078e-05, 'epoch': 1.25} +{'embedding_loss': 0.0012, 'learning_rate': 4.0427027747862565e-05, 'epoch': 1.67} +{'train_runtime': 32.7462, 'train_samples_per_second': 117.265, 'train_steps_per_second': 7.329, 'epoch': 2.0} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 240/240 [00:32<00:00, 7.33it/s] +***** Running evaluation ***** +[I 2023-11-14 20:41:01,722] Trial 9 finished with value: 0.4615 and parameters: {'body_learning_rate': 0.00021830594983845785, 'num_epochs': 2, 'batch_size': 16, 'seed': 38, 'max_iter': 267, 'solver': 'lbfgs'}. Best is trial 8 with value: 0.4785. +``` +Let's observe the best found hyperparameters: + +```py +print(best_run) +``` +``` +BestRun(run_id='8', objective=0.4785, hyperparameters={'body_learning_rate': 0.0005575631179396824, 'num_epochs': 1, 'batch_size': 32, 'seed': 31, 'max_iter': 264, 'solver': 'newton-cg'}, backend=) +``` + +Finally, you can apply the hyperparameters you found to the trainer, and lock in the optimal model, before training for +a final time. + +```py +trainer.apply_hyperparameters(best_run.hyperparameters, final_model=True) +trainer.train() +``` +``` +***** Running training ***** + Num examples = 60 + Num epochs = 1 + Total optimization steps = 60 + Total train batch size = 32 +{'embedding_loss': 0.2588, 'learning_rate': 9.29271863232804e-05, 'epoch': 0.02} +{'embedding_loss': 0.0025, 'learning_rate': 0.00010325242924808932, 'epoch': 0.83} +{'train_runtime': 9.4331, 'train_samples_per_second': 203.54, 'train_steps_per_second': 6.361, 'epoch': 1.0} +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:09<00:00, 6.36it/s] +``` +For peace of mind, we can evaluate this model once more: + +```py +metrics = trainer.evaluate() +print(metrics) +``` +``` +***** Running evaluation ***** +{'accuracy': 0.4785} +``` +As expected, the accuracy matches that of the best run. + +## Baseline + +As a comparison, let's observe the same metrics for the same setup but with the default training arguments: + +```py +from datasets import load_dataset +from setfit import SetFitModel, Trainer, sample_dataset + +model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5") + +dataset = load_dataset("SetFit/emotion") +train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) +test_dataset = dataset["test"] + +trainer = Trainer( + model=model, + train_dataset=train_dataset, + eval_dataset=test_dataset, +) +trainer.train() + +metrics = trainer.evaluate() +print(metrics) +``` +``` +***** Running training ***** + Num examples = 120 + Num epochs = 1 + Total optimization steps = 120 + Total train batch size = 16 +{'embedding_loss': 0.246, 'learning_rate': 1.6666666666666667e-06, 'epoch': 0.01} +{'embedding_loss': 0.1734, 'learning_rate': 1.2962962962962964e-05, 'epoch': 0.42} +{'embedding_loss': 0.0411, 'learning_rate': 3.7037037037037037e-06, 'epoch': 0.83} +{'train_runtime': 23.8184, 'train_samples_per_second': 80.61, 'train_steps_per_second': 5.038, 'epoch': 1.0} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 120/120 [00:17<00:00, 6.83it/s] +***** Running evaluation ***** +{'accuracy': 0.4235} +``` +42.35% versus 47.85%! Quite a big difference for just a few minutes of hyperparameter searching. + +## End-to-end + +This snippet shows the entire hyperparameter optimization strategy in an end-to-end example: + +```py +from datasets import load_dataset +from setfit import SetFitModel, Trainer, sample_dataset +from optuna import Trial +from typing import Dict, Union, Any + +def model_init(params: Dict[str, Any]) -> SetFitModel: + params = params or {} + max_iter = params.get("max_iter", 100) + solver = params.get("solver", "liblinear") + params = { + "head_params": { + "max_iter": max_iter, + "solver": solver, + } + } + return SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5", **params) + +def hp_space(trial: Trial) -> Dict[str, Union[float, int, str]]: + return { + "body_learning_rate": trial.suggest_float("body_learning_rate", 1e-6, 1e-3, log=True), + "num_epochs": trial.suggest_int("num_epochs", 1, 3), + "batch_size": trial.suggest_categorical("batch_size", [16, 32, 64]), + "seed": trial.suggest_int("seed", 1, 40), + "max_iter": trial.suggest_int("max_iter", 50, 300), + "solver": trial.suggest_categorical("solver", ["newton-cg", "lbfgs", "liblinear"]), + } + +dataset = load_dataset("SetFit/emotion") +train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) +test_dataset = dataset["test"] + +trainer = Trainer( + train_dataset=train_dataset, + eval_dataset=test_dataset, + model_init=model_init, +) +best_run = trainer.hyperparameter_search(direction="maximize", hp_space=hp_space, n_trials=10) +print(best_run) + +trainer.apply_hyperparameters(best_run.hyperparameters, final_model=True) +trainer.train() + +metrics = trainer.evaluate() +print(metrics) +# => {'accuracy': 0.4785} +``` \ No newline at end of file diff --git a/docs/source/en/how_to/knowledge_distillation.mdx b/docs/source/en/how_to/knowledge_distillation.mdx new file mode 100644 index 00000000..031969e0 --- /dev/null +++ b/docs/source/en/how_to/knowledge_distillation.mdx @@ -0,0 +1,293 @@ + +# Knowledge Distillation + +If you have access to unlabeled data, then you can use knowledge distillation to improve the performance of your small SetFit model. The approach involves training a larger model and using unlabeled data to distil its performance into your smaller SetFit model. As a result, your SetFit model will become stronger. + +Additionally, you can also use knowledge distillation to replace your trained SetFit model with a more efficient model at less of a performance decrease. + +This guide will show you how to proceed with knowledge distillation. + +## Data preparation + +Let's consider a scenario with a little bit of labeled training data (e.g. 64 sentences). We will simulate this scenario using the [ag_news](https://huggingface.co/datasets/ag_news) dataset for this guide. + +```py +from datasets import load_dataset +from setfit import sample_dataset + +# Load a dataset from the Hugging Face Hub +dataset = load_dataset("ag_news") + +# Create a sample few-shot dataset to train with +train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=16) +# Dataset({ +# features: ['text', 'label'], +# num_rows: 64 +# }) + +# Dataset for evaluation +eval_dataset = dataset["test"] +# Dataset({ +# features: ['text', 'label'], +# num_rows: 7600 +# }) +``` + +## Baseline model +We can use standard SetFit training approach to prepare a model. + +```py +from setfit import SetFitModel, TrainingArguments, Trainer + +model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-MiniLM-L3-v2") + +args = TrainingArguments( + batch_size=64, + num_epochs=5, +) + +trainer = Trainer( + model=model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, +) +trainer.train() + +metrics = trainer.evaluate() +print(metrics) +``` +``` +***** Running training ***** + Num examples = 48 + Num epochs = 5 + Total optimization steps = 240 + Total train batch size = 64 +{'embedding_loss': 0.4173, 'learning_rate': 8.333333333333333e-07, 'epoch': 0.02} +{'embedding_loss': 0.1756, 'learning_rate': 1.7592592592592595e-05, 'epoch': 1.04} +{'embedding_loss': 0.119, 'learning_rate': 1.2962962962962964e-05, 'epoch': 2.08} +{'embedding_loss': 0.0872, 'learning_rate': 8.333333333333334e-06, 'epoch': 3.12} +{'embedding_loss': 0.0542, 'learning_rate': 3.7037037037037037e-06, 'epoch': 4.17} +{'train_runtime': 26.0837, 'train_samples_per_second': 588.873, 'train_steps_per_second': 9.201, 'epoch': 5.0} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 240/240 [00:20<00:00, 11.97it/s] +***** Running evaluation ***** +{'accuracy': 0.7818421052631579} +``` +This model reaches 78.18% on our dataset. Certainly respectable given the tiny amount of training data, but we can use knowledge distillation to squeeze more performance out of our model. + +## Unlabeled Data Preparation + +Alongside our labeled training data, we may als have a lot of unlabeled training data (e.g. 500 sentences). Let's prepare it: + +```py +# Create a dataset of unlabeled examples to perform knowledge distillation +unlabeled_train_dataset = dataset["train"].shuffle(seed=0).select(range(500)) +unlabeled_train_dataset = unlabeled_train_dataset.remove_columns("label") +# Dataset({ +# features: ['text'], +# num_rows: 500 +# }) +``` + +## Teacher model + +Then, we will prepare a larger trained SetFit model that will act as the teacher to our smaller student model. The strong [`sentence-transformers/paraphrase-mpnet-base-v2`](https://huggingface.co/sentence-transformers/paraphrase-mpnet-base-v2) Sentence Transformer model will be used to initialize the SetFit model. + +```py +from setfit import SetFitModel + +teacher_model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") +``` + +We need to train this model on the labeled dataset first: + +```py +from setfit import TrainingArguments, Trainer + +teacher_args = TrainingArguments( + batch_size=16, + num_epochs=2, +) + +teacher_trainer = Trainer( + model=teacher_model, + args=teacher_args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, +) + +# Train teacher model +teacher_trainer.train() +teacher_metrics = teacher_trainer.evaluate() +print(teacher_metrics) +``` +``` +***** Running training ***** + Num examples = 192 + Num epochs = 2 + Total optimization steps = 384 + Total train batch size = 16 +{'embedding_loss': 0.4093, 'learning_rate': 5.128205128205128e-07, 'epoch': 0.01} +{'embedding_loss': 0.1087, 'learning_rate': 1.9362318840579713e-05, 'epoch': 0.26} +{'embedding_loss': 0.001, 'learning_rate': 1.6463768115942028e-05, 'epoch': 0.52} +{'embedding_loss': 0.0006, 'learning_rate': 1.3565217391304348e-05, 'epoch': 0.78} +{'embedding_loss': 0.0003, 'learning_rate': 1.0666666666666667e-05, 'epoch': 1.04} +{'embedding_loss': 0.0004, 'learning_rate': 7.768115942028987e-06, 'epoch': 1.3} +{'embedding_loss': 0.0002, 'learning_rate': 4.869565217391305e-06, 'epoch': 1.56} +{'embedding_loss': 0.0003, 'learning_rate': 1.9710144927536233e-06, 'epoch': 1.82} +{'train_runtime': 84.3703, 'train_samples_per_second': 72.822, 'train_steps_per_second': 4.551, 'epoch': 2.0} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 384/384 [01:24<00:00, 4.55it/s] +***** Running evaluation ***** +{'accuracy': 0.8378947368421052} +``` +This large teacher model reaches 83.79%, which is quite strong for this little data, and noticeably, stronger than the 78.18% from our smaller (but more efficient) model. + +## Knowledge Distillation + +The performance from the stronger teacher_model can be distilled into the smaller model using the [`DistillationTrainer`]. It accepts a teacher and a student model, as well as an unlabeled dataset. + + + +Note that this trainer uses pairs between sentences as the training samples, so the number of training steps grows exponentially to the number of unlabeled examples. To avoid overfitting, consider setting `max_steps` relatively low. + + + +```py +from setfit import DistillationTrainer + +distillation_args = TrainingArguments( + batch_size=16, + max_steps=500, +) + +distillation_trainer = DistillationTrainer( + teacher_model=teacher_model, + student_model=model, + args=distillation_args, + train_dataset=unlabeled_train_dataset, + eval_dataset=eval_dataset, +) +# Train student with knowledge distillation +distillation_trainer.train() +distillation_metrics = distillation_trainer.evaluate() +print(distillation_metrics) +``` +```py +***** Running training ***** + Num examples = 7829 + Num epochs = 1 + Total optimization steps = 7829 + Total train batch size = 16 +{'embedding_loss': 0.5048, 'learning_rate': 2.554278416347382e-08, 'epoch': 0.0} +{'embedding_loss': 0.4514, 'learning_rate': 1.277139208173691e-06, 'epoch': 0.01} +{'embedding_loss': 0.33, 'learning_rate': 2.554278416347382e-06, 'epoch': 0.01} +{'embedding_loss': 0.1218, 'learning_rate': 3.831417624521073e-06, 'epoch': 0.02} +{'embedding_loss': 0.0213, 'learning_rate': 5.108556832694764e-06, 'epoch': 0.03} +{'embedding_loss': 0.016, 'learning_rate': 6.385696040868455e-06, 'epoch': 0.03} +{'embedding_loss': 0.0054, 'learning_rate': 7.662835249042147e-06, 'epoch': 0.04} +{'embedding_loss': 0.0049, 'learning_rate': 8.939974457215838e-06, 'epoch': 0.04} +{'embedding_loss': 0.002, 'learning_rate': 1.0217113665389528e-05, 'epoch': 0.05} +{'embedding_loss': 0.0019, 'learning_rate': 1.1494252873563218e-05, 'epoch': 0.06} +{'embedding_loss': 0.0012, 'learning_rate': 1.277139208173691e-05, 'epoch': 0.06} +{'train_runtime': 22.2725, 'train_samples_per_second': 359.188, 'train_steps_per_second': 22.449, 'epoch': 0.06} +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 500/500 [00:22<00:00, 22.45it/s] +***** Running evaluation ***** +{'accuracy': 0.8084210526315789} +``` +Using knowledge distillation, we were able to improve our model from 78.18% to 80.84% in a few minutes of training. + +## End-to-end + +This snippet shows the entire knowledge distillation strategy in an end-to-end example: + +```py +from datasets import load_dataset +from setfit import sample_dataset + +# Load a dataset from the Hugging Face Hub +dataset = load_dataset("ag_news") + +# Create a sample few-shot dataset to train with +train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=16) +# Dataset({ +# features: ['text', 'label'], +# num_rows: 64 +# }) + +# Dataset for evaluation +eval_dataset = dataset["test"] +# Dataset({ +# features: ['text', 'label'], +# num_rows: 7600 +# }) + +from setfit import SetFitModel, TrainingArguments, Trainer + +model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-MiniLM-L3-v2") + +args = TrainingArguments( + batch_size=64, + num_epochs=5, +) + +trainer = Trainer( + model=model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, +) +trainer.train() + +metrics = trainer.evaluate() +print(metrics) + +# Create a dataset of unlabeled examples to perform knowledge distillation +unlabeled_train_dataset = dataset["train"].shuffle(seed=0).select(range(500)) +unlabeled_train_dataset = unlabeled_train_dataset.remove_columns("label") +# Dataset({ +# features: ['text'], +# num_rows: 500 +# }) + +from setfit import SetFitModel + +teacher_model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") + +from setfit import TrainingArguments, Trainer + +teacher_args = TrainingArguments( + batch_size=16, + num_epochs=2, +) + +teacher_trainer = Trainer( + model=teacher_model, + args=teacher_args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, +) + +# Train teacher model +teacher_trainer.train() +teacher_metrics = teacher_trainer.evaluate() +print(teacher_metrics) + +from setfit import DistillationTrainer + +distillation_args = TrainingArguments( + batch_size=16, + max_steps=500, +) + +distillation_trainer = DistillationTrainer( + teacher_model=teacher_model, + student_model=model, + args=distillation_args, + train_dataset=unlabeled_train_dataset, + eval_dataset=eval_dataset, +) +# Train student with knowledge distillation +distillation_trainer.train() +distillation_metrics = distillation_trainer.evaluate() +print(distillation_metrics) +``` \ No newline at end of file diff --git a/docs/source/en/how_to/multilabel.mdx b/docs/source/en/how_to/multilabel.mdx new file mode 100644 index 00000000..da09fa66 --- /dev/null +++ b/docs/source/en/how_to/multilabel.mdx @@ -0,0 +1,48 @@ + +# Multilabel Text Classification + +SetFit supports multilabel classification, allowing multiple labels to be assigned to each instance. + + + +Unless each instance must be assigned multiple outputs, you frequently do not need to specify a multi target strategy. + + + +This guide will show you how to train and use multilabel SetFit models. + +## Multilabel strategies + +SetFit will initialise a multilabel classification head from `sklearn` - the following options are available for `multi_target_strategy`: + +* `"one-vs-rest"`: uses a [`OneVsRestClassifier`](https://scikit-learn.org/stable/modules/generated/sklearn.multiclass.OneVsRestClassifier.html) head. +* `"multi-output"`: uses a [`MultiOutputClassifier`](https://scikit-learn.org/stable/modules/generated/sklearn.multioutput.MultiOutputClassifier.html) head. +* `"classifier-chain"`: uses a [`ClassifierChain`](https://scikit-learn.org/stable/modules/generated/sklearn.multioutput.ClassifierChain.html) head. + +See the [scikit-learn documentation for multiclass and multioutput classification](https://scikit-learn.org/stable/modules/multiclass.html#multiclass-classification) for more details. + +## Initializing SetFit models with multilabel strategies + +Using the default LogisticRegression head, we can apply multi target strategies like so: + +```py +from setfit import SetFitModel + +model = SetFitModel.from_pretrained( + model_id, # e.g. "BAAI/bge-small-en-v1.5" + multi_target_strategy="multi-output", +) +``` + +With a differentiable head it looks like so: + +```py +from setfit import SetFitModel + +model = SetFitModel.from_pretrained( + model_id, # e.g. "BAAI/bge-small-en-v1.5" + multi_target_strategy="one-vs-rest" + use_differentiable_head=True, + head_params={"out_features": num_classes}, +) +``` \ No newline at end of file diff --git a/docs/source/en/how_to/overview.mdx b/docs/source/en/how_to/overview.mdx new file mode 100644 index 00000000..6bf99847 --- /dev/null +++ b/docs/source/en/how_to/overview.mdx @@ -0,0 +1,9 @@ + +# Overview + +Welcome to the SetFit How-to Guides! The how-to guides offer a more comprehensive overview of all the tools 🤗 SetFit offers and how to use them. +These guides are designed to be concise and code-heavy, written in "show, don't tell" style. For example, using these guides you may learn how to perform hyperparameter optimization, knowledge distillation, apply callbacks, etc. + +Most how-to guides end with an "end to end" script showing all code from the guide for easy adaptation into your own code. + +For simpler documentation explaining SetFit functionality from start to finish, consider visiting the [Tutorials](../tutorials/overview) section or the [quickstart](../quickstart). diff --git a/docs/source/en/how_to/placeholder.mdx b/docs/source/en/how_to/placeholder.mdx deleted file mode 100644 index 219da7be..00000000 --- a/docs/source/en/how_to/placeholder.mdx +++ /dev/null @@ -1,3 +0,0 @@ - -# How-to Guide -Work in Progress! \ No newline at end of file diff --git a/docs/source/en/how_to/v1.0.0_migration_guide.mdx b/docs/source/en/how_to/v1.0.0_migration_guide.mdx new file mode 100644 index 00000000..a4d44df1 --- /dev/null +++ b/docs/source/en/how_to/v1.0.0_migration_guide.mdx @@ -0,0 +1,82 @@ + +# SetFit v1.0.0 Migration Guide + +To update your code to work with v1.0.0, the following changes must be made: + +## General Migration Guide + +1. `keep_body_frozen` from `SetFitModel.unfreeze` has been deprecated, simply either pass `"head"`, `"body"` or no arguments to unfreeze both. +2. `SupConLoss` has been moved from `setfit.modeling` to `setfit.losses`. If you are importing it using `from setfit.modeling import SupConLoss`, then import it like `from setfit import SupConLoss` now instead. + +## Training Migration Guide + +1. Replace all uses of `SetFitTrainer` with [`Trainer`], and all uses of `DistillationSetFitTrainer` with [`DistillationTrainer`]. +2. Remove `num_iterations`, `num_epochs`, `learning_rate`, `batch_size`, `seed`, `use_amp`, `warmup_proportion`, `distance_metric`, `margin`, `samples_per_label` and `loss_class` from a `Trainer` initialisation, and move them to a `TrainerArguments` initialisation instead. This instance should then be passed to the trainer via the `args` argument. + + * `num_iterations` has been deprecated, the number of training steps should now be controlled exclusively via `num_epochs`, `max_steps` or [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback). + * `learning_rate` has been split up into `body_learning_rate` and `head_learning_rate`. + * `loss_class` has been renamed to `loss`. + +3. Stop providing training arguments like `num_epochs` directly to `Trainer.train`: pass a `TrainingArguments` instance instead. +4. Refactor multiple `trainer.train()`, `trainer.freeze()` and `trainer.unfreeze()` calls that were previously necessary to train the differentiable head into just one `trainer.train()` call by setting `batch_size` and `num_epochs` on the `TrainingArguments` dataclass with Tuples. The first value is for training the embeddings, and the second is for training the classifier. + +## Hard deprecations + +* `SetFitBaseModel`, `SKLearnWrapper` and `SetFitPipeline` have been removed. These can no longer be used starting from v1.0.0. + +## New functionality + +This list contains new functionality that can be used starting from v1.0.0. + +* [`SetFitModel.encode`] has been introduce to convert input sentences to embeddings using the `SentenceTransformer` body. +* [`SetFitModel.device`] has been introduced to determine the device of the model. +* [`AbsaTrainer`] and [`AbsaModel`] have been introduced for applying SetFit for Aspect Based Sentiment Analysis. +* [`Trainer`] now supports a `callbacks` argument for a list of [`transformers` `TrainerCallback` instances](https://huggingface.co/docs/transformers/main/en/main_classes/callback). + * By default, all installed callbacks integrated with `transformers` are supported, including [`TensorBoardCallback`](https://huggingface.co/docs/transformers/main/en/main_classes/callback#transformers.integrations.TensorBoardCallback), [`WandbCallback`](https://huggingface.co/docs/transformers/main/en/main_classes/callback#transformers.integrations.WandbCallback) to log training logs to TensorBoard and W&B, respectively. + * The [`Trainer`] will now print `embedding_loss` in the terminal, as well as `eval_embedding_loss` if `evaluation_strategy` is set to `"epoch"` or `"steps"` in [`TrainingArguments`]. +* [`Trainer.evaluate`] now works with string labels. +* An updated contrastive pair sampler increases the variety of training pairs. +* [`TrainingArguments`] supports various new arguments: + * `output_dir`: The output directory where the model predictions and checkpoints will be written. + * `max_steps`: If set to a positive number, the total number of training steps to perform. Overrides num_epochs. The training may stop before reaching the set number of steps when all data is exhausted. + * `sampling_strategy`: The sampling strategy of how to draw pairs in training. Possible values are: + + * `"oversampling"`: Draws even number of positive/negative sentence pairs until every sentence pair has been drawn. + * `"undersampling"`: Draws the minimum number of positive/negative sentence pairs until every sentence pair in the minority class has been drawn. + * `"unique"`: Draws every sentence pair combination (likely resulting in unbalanced number of positive/negative sentence pairs). + + The default is set to `"oversampling"`, ensuring all sentence pairs are drawn at least once. Alternatively, setting `num_iterations` will override this argument and determine the number of generated sentence pairs. + * `report_to`: The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`, `"comet_ml"`, `"mlflow"`, `"neptune"`, `"tensorboard"`,`"clearml"` and `"wandb"`. Use `"all"` to report to all integrations installed, `"none"` for no integrations. + * `run_name`: A descriptor for the run. Typically used for [wandb](https://www.wandb.com/) and [mlflow](https://www.mlflow.org/) logging. + * `logging_strategy`: The logging strategy to adopt during training. Possible values are: + + - `"no"`: No logging is done during training. + - `"epoch"`: Logging is done at the end of each epoch. + - `"steps"`: Logging is done every `logging_steps`. + + * `logging_first_step`: Whether to log and evaluate the first `global_step` or not. + * `logging_steps`: Number of update steps between two logs if `logging_strategy="steps"`. + * `evaluation_strategy`: The evaluation strategy to adopt during training. Possible values are: + + - `"no"`: No evaluation is done during training. + - `"steps"`: Evaluation is done (and logged) every `eval_steps`. + - `"epoch"`: Evaluation is done at the end of each epoch. + + * `eval_steps`: Number of update steps between two evaluations if `evaluation_strategy="steps"`. Will default to the same as `logging_steps` if not set. + * `eval_delay`: Number of epochs or steps to wait for before the first evaluation can be performed, depending on the `evaluation_strategy`. + * `save_strategy`: The checkpoint save strategy to adopt during training. Possible values are: + + - `"no"`: No save is done during training. + - `"epoch"`: Save is done at the end of each epoch. + - `"steps"`: Save is done every `save_steps`. + + * `save_steps`: Number of updates steps before two checkpoint saves if `save_strategy="steps"`. + * `save_total_limit`: If a value is passed, will limit the total amount of checkpoints. Deletes the older checkpoints in `output_dir`. Note, the best model is always preserved if the `evaluation_strategy` is not `"no"`. + * `load_best_model_at_end`: Whether or not to load the best model found during training at the end of training. + + + + When set to `True`, the parameters `save_strategy` needs to be the same as `evaluation_strategy`, and in + the case it is "steps", `save_steps` must be a round multiple of `eval_steps`. + + \ No newline at end of file diff --git a/docs/source/en/how_to/zero_shot.mdx b/docs/source/en/how_to/zero_shot.mdx new file mode 100644 index 00000000..607a4a79 --- /dev/null +++ b/docs/source/en/how_to/zero_shot.mdx @@ -0,0 +1,167 @@ + +# Zero-shot Text Classification + +[[open-in-colab]] + +Your class names are likely already good descriptors of the text that you're looking to classify. With 🤗 SetFit, you can use these class names with strong pretrained Sentence Transformer models to get a strong baseline model without any training samples. + +This guide will show you how to perform zero-shot text classification. + +## Testing dataset + +We'll use the [dair-ai/emotion](https://huggingface.co/datasets/dair-ai/emotion) dataset to test the performance of our zero-shot model. + +```py +from datasets import load_dataset + +test_dataset = load_dataset("dair-ai/emotion", "split", split="test") +``` + +This dataset stores the class names within the dataset `Features`, so we'll extract the classes like so: +```py +classes = test_dataset.features["label"].names +# => ['sadness', 'joy', 'love', 'anger', 'fear', 'surprise'] +``` +Otherwise, we could manually set the list of classes. + +## Synthetic dataset + +Then, we can use [`get_templated_dataset`] to synthetically generate a dummy dataset given these class names. + +```py +from setfit import get_templated_dataset + +train_dataset = get_templated_dataset() +``` +```py +print(train_dataset) +# => Dataset({ +# features: ['text', 'label'], +# num_rows: 48 +# }) +print(train_dataset[0]) +# {'text': 'This sentence is sadness', 'label': 0} +``` + +## Training + +We can use this dataset to train a SetFit model just like normal: + +```py +from setfit import SetFitModel, Trainer, TrainingArguments + +model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5") + +args = TrainingArguments( + batch_size=32, + num_epochs=1, +) + +trainer = Trainer( + model=model, + args=args, + train_dataset=train_dataset, + eval_dataset=test_dataset, +) +trainer.train() +``` +``` +***** Running training ***** + Num examples = 60 + Num epochs = 1 + Total optimization steps = 60 + Total train batch size = 32 +{'embedding_loss': 0.2628, 'learning_rate': 3.3333333333333333e-06, 'epoch': 0.02} +{'embedding_loss': 0.0222, 'learning_rate': 3.7037037037037037e-06, 'epoch': 0.83} +{'train_runtime': 15.4717, 'train_samples_per_second': 124.098, 'train_steps_per_second': 3.878, 'epoch': 1.0} +100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:09<00:00, 6.35it/s] +``` + +Once trained, we can evaluate the model: + +```py +metrics = trainer.evaluate() +print(metrics) +``` +``` +***** Running evaluation ***** +{'accuracy': 0.591} +``` + +And run predictions: + +```py +preds = model.predict([ + "i am just feeling cranky and blue", + "i feel incredibly lucky just to be able to talk to her", + "you're pissing me off right now", + "i definitely have thalassophobia, don't get me near water like that", + "i did not see that coming at all", +]) +print([classes[idx] for idx in preds]) +``` +```py +['sadness', 'joy', 'anger', 'fear', 'surprise'] +``` + +These predictions all look right! + +## Baseline + +To show that the zero-shot performance of SetFit works well, we'll compare it against a zero-shot classification model from `transformers`. + +```py +from transformers import pipeline +from datasets import load_dataset +import evaluate + +# Prepare the testing dataset +test_dataset = load_dataset("dair-ai/emotion", "split", split="test") +classes = test_dataset.features["label"].names + +# Set up the zero-shot classification pipeline from transformers +# Uses 'facebook/bart-large-mnli' by default +pipe = pipeline("zero-shot-classification", device=0) +zeroshot_preds = pipe(test_dataset["text"], batch_size=16, candidate_labels=classes) +preds = [classes.index(pred["labels"][0]) for pred in zeroshot_preds] + +# Compute the accuracy +metric = evaluate.load("accuracy") +transformers_accuracy = metric.compute(predictions=preds, references=test_dataset["label"]) +print(transformers_accuracy) +``` +```py +{'accuracy': 0.3765} +``` + +With its 59.1% accuracy, the 0-shot SetFit heavily outperforms the recommended zero-shot model by `transformers`. + +### Prediction latency + +Beyond getting higher accuracies, SetFit is much faster too. Let's compute the latency of SetFit with `BAAI/bge-small-en-v1.5` versus the latency of `transformers` with `facebook/bart-large-mnli`. Both tests were performed on a GPU. + +```py +import time + +start_t = time.time() +pipe(test_dataset["text"], batch_size=32, candidate_labels=classes) +delta_t = time.time() - start_t +print(f"`transformers` with `facebook/bart-large-mnli` latency: {delta_t / len(test_dataset['text']) * 1000:.4f}ms per sentence") +``` +``` +`transformers` with `facebook/bart-large-mnli` latency: 31.1765ms per sentence +``` + +```py +import time + +start_t = time.time() +model.predict(test_dataset["text"]) +delta_t = time.time() - start_t +print(f"SetFit with `BAAI/bge-small-en-v1.5` latency: {delta_t / len(test_dataset['text']) * 1000:.4f}ms per sentence") +``` +``` +SetFit with `BAAI/bge-small-en-v1.5` latency: 0.4600ms per sentence +``` + +So, SetFit with `BAAI/bge-small-en-v1.5` is 67x faster than `transformers` with `facebook/bart-large-mnli`, alongside being more accurate. diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 35eb5ae3..61253b47 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -11,24 +11,24 @@ Compared to other few-shot learning methods, SetFit has several unique features: * 🗣 **No prompts or verbalisers:** Current techniques for few-shot fine-tuning require handcrafted prompts or verbalisers to convert examples into a format that's suitable for the underlying language model. SetFit dispenses with prompts altogether by generating rich embeddings directly from text examples. -* 🏎 **Fast to train:** SetFit doesn't require large-scale models like T0 or GPT-3 to achieve high accuracy. As a result, it is typically an order of magnitude (or more) faster to train and run inference with. +* 🏎 **Fast to train:** SetFit doesn't require large-scale models like T0, Llama or GPT-4 to achieve high accuracy. As a result, it is typically an order of magnitude (or more) faster to train and run inference with. * 🌎 **Multilingual support**: SetFit can be used with any [Sentence Transformer](https://huggingface.co/models?library=sentence-transformers&sort=downloads) on the Hub, which means you can classify text in multiple languages by simply fine-tuning a multilingual checkpoint.
-
Tutorials

Learn the basics and become familiar with loading pretrained Sentence Transformers and fine-tuning them on data. Start here if you are using 🤗 SetFit for the first time!

 -
How-to guides

Practical guides to help you achieve a specific goal. Take a look at these guides to learn how to use 🤗 SetFit to solve real-world problems.

 -
Conceptual guides

High-level explanations for building a better understanding about important topics such as few-shot and contrastive learning.

 -
Reference

Technical descriptions of how 🤗 SetFit classes and methods work.

 diff --git a/docs/source/en/installation.mdx b/docs/source/en/installation.mdx index 140163fa..8b3fa13d 100644 --- a/docs/source/en/installation.mdx +++ b/docs/source/en/installation.mdx @@ -1,14 +1,42 @@ # Installation -Download and install `setfit` by running: +Before you start, you'll need to setup your environment and install the appropriate packages. 🤗 SetFit is tested on **Python 3.7+**. + +## pip + +The most straightforward way to install 🤗 Datasets is with pip: + +```bash +pip install setfit +``` + +If you have a CUDA-capable graphics card, then it is recommended to [install `torch` with CUDA support](https://pytorch.org/get-started/locally) to train and performing inference much more quickly: + +```bash +pip install torch --index-url https://download.pytorch.org/whl/cu118 +``` + +## Installing from source + +Building 🤗 SetFit from source lets you make changes to the code base. To install from the source, clone the repository and install 🤗 SetFit in [editable mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html) with the following commands: + +```bash +git clone https://github.com/huggingface/setfit.git +cd setfit +pip install -e . +``` + +If you just want the bleeding-edge version without making any changes of your own, then install from source by running: ```bash -python -m pip install setfit +pip install git+https://github.com/huggingface/setfit.git ``` -If you want the bleeding-edge version, install from source by running: +## Conda + +If conda is your package management system of choice, then you can install 🤗 SetFit like so: ```bash -python -m pip install git+https://github.com/huggingface/setfit.git +conda install -c conda-forge setfit ``` \ No newline at end of file diff --git a/docs/source/en/quickstart.mdx b/docs/source/en/quickstart.mdx index 9e46933b..476971f7 100644 --- a/docs/source/en/quickstart.mdx +++ b/docs/source/en/quickstart.mdx @@ -1,313 +1,228 @@ # Quickstart -## Usage +[[open-in-colab]] -The examples below provide a quick overview on the various features supported in `setfit`. For more examples, check out the [`notebooks`](https://github.com/huggingface/setfit/tree/main/notebooks) folder. +This quickstart is intended for developers who are ready to dive into the code and see an example of how to train and use 🤗 SetFit models. We recommend starting with this quickstart, and then proceeding to the [tutorials](./tutorials/overview) or [how-to guides](./how_to/overview) for additional material. Additionally, the [conceptual guides](./conceptual_guides/setfit) help explain exactly how SetFit works. +Start by installing 🤗 SetFit: -### Training a SetFit model - -`setfit` is integrated with the [Hugging Face Hub](https://huggingface.co/) and provides two main classes: - -* `SetFitModel`: a wrapper that combines a pretrained body from `sentence_transformers` and a classification head from either [`scikit-learn`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) or [`SetFitHead`](https://github.com/huggingface/setfit/blob/main/src/setfit/modeling.py) (a differentiable head built upon `PyTorch` with similar APIs to `sentence_transformers`). -* `Trainer`: a helper class that wraps the fine-tuning process of SetFit. - -Here is an end-to-end example using a classification head from `scikit-learn`: - - -```python -from datasets import load_dataset -from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset - - -# Load a dataset from the Hugging Face Hub -dataset = load_dataset("sst2") - -# Simulate the few-shot regime by sampling 8 examples per class -train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) -eval_dataset = dataset["validation"] - -# Load a SetFit model from Hub -model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") - -args = TrainingArguments( - batch_size=16, - num_iterations=20, # The number of text pairs to generate for contrastive learning - num_epochs=1 # The number of epochs to use for contrastive learning -) - -trainer = Trainer( - model=model, - args=args, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - metric="accuracy", - column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer -) - -# Train and evaluate -trainer.train() -metrics = trainer.evaluate() +```bash +pip install setfit +``` -# Push model to the Hub -trainer.push_to_hub("my-awesome-setfit-model") +If you have a CUDA-capable graphics card, then it is recommended to [install `torch` with CUDA support](https://pytorch.org/get-started/locally) to train and performing inference much more quickly: -# Download from Hub -model = SetFitModel.from_pretrained("lewtun/my-awesome-setfit-model") -# Run inference -preds = model(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) +```bash +pip install torch --index-url https://download.pytorch.org/whl/cu118 ``` -Here is an end-to-end example using `SetFitHead`: +## SetFit +SetFit is an efficient framework to train low-latency text classification models using little training data. In this Quickstart, you'll learn how to train a SetFit model, how to perform inference with it, and how to save it to the Hugging Face Hub. -```python -from datasets import load_dataset -from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset +### Training +In this section, you'll load a [Sentence Transformer model](https://huggingface.co/models?library=sentence-transformers) and further finetune it for classifying movie reviews as positive or negative. To train a model, we will need to prepare the following three: 1) a **model**, 2) a **dataset**, and 3) **training arguments**. -# Load a dataset from the Hugging Face Hub -dataset = load_dataset("sst2") -# Simulate the few-shot regime by sampling 8 examples per class -train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) -eval_dataset = dataset["validation"] -num_classes = 2 - -# Load a SetFit model from Hub -model = SetFitModel.from_pretrained( - "sentence-transformers/paraphrase-mpnet-base-v2", - use_differentiable_head=True, - head_params={"out_features": num_classes}, -) +**1**. Initialize a SetFit model using a Sentence Transformer model of our choice. Consider using the [MTEB Leaderboard](https://huggingface.co/spaces/mteb/leaderboard) to guide your decision on which Sentence Transformer model to choose. We will use [BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5), a small but performant model. -args = TrainingArguments( - body_learning_rate=2e-5, - head_learning_rate=1e-2, - batch_size=16, - num_iterations=20, # The number of text pairs to generate for contrastive learning - num_epochs=(1, 25), # For finetuning the embeddings and training the classifier, respectively - l2_weight=0.0, - end_to_end=False, # Don't train the classifier end-to-end, i.e. only train the head -) +```py +>>> from setfit import SetFitModel -trainer = Trainer( - model=model, - train_dataset=train_dataset, - eval_dataset=eval_dataset, - metric="accuracy", - column_mapping={"sentence": "text", "label": "label"} # Map dataset columns to text/label expected by trainer -) - -# Train and evaluate -trainer.train() -metrics = trainer.evaluate() - -# Push model to the Hub -trainer.push_to_hub("my-awesome-setfit-model") - -# Download from Hub and run inference -model = SetFitModel.from_pretrained("lewtun/my-awesome-setfit-model") -# Run inference -preds = model(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) +>>> model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5") ``` -Based on our experiments, `SetFitHead` can achieve similar performance as using a `scikit-learn` head. We use `AdamW` as the optimizer and scale down learning rates by 0.5 every 5 epochs. For more details about the experiments, please check out [here](https://github.com/huggingface/setfit/pull/112#issuecomment-1295773537). We recommend using a large learning rate (e.g. `1e-2`) for `SetFitHead` and a small learning rate (e.g. `1e-5`) for the body in your first attempt. - -### Training on multilabel datasets - -To train SetFit models on multilabel datasets, specify the `multi_target_strategy` argument when loading the pretrained model: +**2a**. Next, load both the "train" and "test" splits of the [SetFit/sst2](https://huggingface.co/datasets/sst2) dataset. Note that the dataset has `"text"` and `"label"` columns: this is exactly the format that 🤗 SetFit expects. If your dataset has different columns, then you can use the column_mapping argument of the [`Trainer`] in step 4 to map the column names to `"text"` and `"label"`. + +```py +>>> from datasets import load_dataset + +>>> dataset = load_dataset("SetFit/sst2") +>>> dataset +DatasetDict({ + train: Dataset({ + features: ['text', 'label', 'label_text'], + num_rows: 6920 + }) + test: Dataset({ + features: ['text', 'label', 'label_text'], + num_rows: 1821 + }) + validation: Dataset({ + features: ['text', 'label', 'label_text'], + num_rows: 872 + }) +}) +``` -#### Example using a classification head from `scikit-learn`: +**2b**. In real world scenarios it is very uncommon to have ~7.000 high quality labeled training samples, so we will heavily shrink the training dataset to give a better idea of how 🤗 SetFit would work in real settings. To be specific, the `sample_dataset` function will sample only 8 samples for each class. The testing set is left unaffected for better evaluation. -```python -from setfit import SetFitModel +```py +>>> from setfit import sample_dataset -model = SetFitModel.from_pretrained( - model_id, - multi_target_strategy="one-vs-rest", -) +>>> train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) +>>> train_dataset +Dataset({ + features: ['text', 'label', 'label_text'], + num_rows: 16 +}) ``` -This will initialise a multilabel classification head from `sklearn` - the following options are available for `multi_target_strategy`: - -* `one-vs-rest`: uses a `OneVsRestClassifier` head. -* `multi-output`: uses a `MultiOutputClassifier` head. -* `classifier-chain`: uses a `ClassifierChain` head. +```py +>>> test_dataset = dataset["test"] +>>> test_dataset +Dataset({ + features: ['text', 'label', 'label_text'], + num_rows: 1821 +}) +``` -From here, you can instantiate a `Trainer` using the same example above, and train it as usual. +**3**. Prepare the [`TrainingArguments`] for training. Note that training with 🤗 SetFit consists of two phases behind the scenes: **finetuning embeddings** and **training a classification head**. As a result, some of the training arguments can be tuples, where the two values are used for each of the two phases, respectively. -#### Example using the differentiable `SetFitHead`: +The `num_epochs` and `max_steps` arguments are frequently used to increase and decrease the number of total training steps. Consider that with SetFit, better performance is reached with **more data, not more training**! Don't be afraid to train for less than 1 epoch if you have a lot of data. -```python -from setfit import SetFitModel +```py +>>> from setfit import TrainingArguments -model = SetFitModel.from_pretrained( - model_id, - multi_target_strategy="one-vs-rest" - use_differentiable_head=True, - head_params={"out_features": num_classes}, -) +>>> args = TrainingArguments( +... batch_size=32, +... num_epochs=10, +... ) ``` -**Note:** If you use the differentiable `SetFitHead` classifier head, it will automatically use `BCEWithLogitsLoss` for training. The prediction involves a `sigmoid` after which probabilities are rounded to 1 or 0. Furthermore, the `"one-vs-rest"` and `"multi-output"` multi-target strategies are equivalent for the differentiable `SetFitHead`. - -### Zero-shot text classification -SetFit can also be applied to scenarios where no labels are available. To do so, create a synthetic dataset of training examples: +**4**. Initialize the [`Trainer`] and perform training. -```python -from setfit import get_templated_dataset +```py +>>> from setfit import Trainer -candidate_labels = ["negative", "positive"] -train_dataset = get_templated_dataset(candidate_labels=candidate_labels, sample_size=8) +>>> trainer = Trainer( +... model=model, +... args=args, +... train_dataset=train_dataset, +... ) ``` -This will create examples of the form `"This sentence is {}"`, where the `{}` is filled in with one of the candidate labels. From here you can train a SetFit model as usual: +```py +>>> trainer.train() +***** Running training ***** + Num examples = 5 + Num epochs = 10 + Total optimization steps = 50 + Total train batch size = 32 +{'embedding_loss': 0.2077, 'learning_rate': 4.000000000000001e-06, 'epoch': 0.2} +{'embedding_loss': 0.0097, 'learning_rate': 0.0, 'epoch': 10.0} +{'train_runtime': 14.705, 'train_samples_per_second': 108.807, 'train_steps_per_second': 3.4, 'epoch': 10.0} +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:08<00:00, 5.70it/s] +``` -```python -from setfit import SetFitModel, Trainer +**5**. Perform evaluation using the provided testing dataset. -model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") -trainer = Trainer( - model=model, - train_dataset=train_dataset -) -trainer.train() +```py +>>> trainer.evaluate(test_dataset) +***** Running evaluation ***** +{'accuracy': 0.8511806699615596} ``` -We find this approach typically outperforms the [zero-shot pipeline](https://huggingface.co/docs/transformers/v4.24.0/en/main_classes/pipelines#transformers.ZeroShotClassificationPipeline) in 🤗 Transformers (based on MNLI with BART), while being 5x faster to generate predictions with. +Feel free to experiment with increasing the number of samples per class to observe the improvements in accuracy. As a challenge, you can play with the samples per class, learning rate, number of epochs, maximum number of steps, and the base Sentence Transformer model to try and improve the accuracy over 90% using very little data. +### Saving a 🤗 SetFit model -### Running hyperparameter search +After training, you can save a 🤗 SetFit model to your local filesystem or to the Hugging Face Hub. Save a model to a local directory using [`SetFitModel.save_pretrained`] by providing a `save_directory`: -`Trainer` provides a `hyperparameter_search()` method that you can use to find good hyperparameters for your data. To use this feature, first install the `optuna` backend: - -```bash -python -m pip install setfit[optuna] +```py +>>> model.save_pretrained("setfit-8-shot-sst2") ``` -To use this method, you need to define two functions: +Alternatively, push a model to the Hugging Face Hub using [`SetFitModel.push_to_hub`] by providing a `repo_id`: -* `model_init()`: A function that instantiates the model to be used. If provided, each call to `train()` will start from a new instance of the model as given by this function. -* `hp_space()`: A function that defines the hyperparameter search space. +```py +>>> model.push_to_hub("tomaarsen/setfit-8-shot-sst2") +``` -Here is an example of a `model_init()` function that we'll use to scan over the hyperparameters associated with the classification head in `SetFitModel`: +### Loading a 🤗 SetFit model -```python -from setfit import SetFitModel +A 🤗 SetFit model can be loaded using [`SetFitModel.from_pretrained`] by providing 1) a `repo_id` from the Hugging Face Hub or 2) a path to a local directory: -def model_init(params): - params = params or {} - max_iter = params.get("max_iter", 100) - solver = params.get("solver", "liblinear") - params = { - "head_params": { - "max_iter": max_iter, - "solver": solver, - } - } - return SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", **params) -``` +```py +>>> model = SetFitModel.from_pretrained("tomaarsen/setfit-8-shot-sst2") # Load from the Hugging Face Hub -Similarly, to scan over hyperparameters associated with the SetFit training process, we can define a `hp_space()` function as follows: - -```python -def hp_space(trial): # Training parameters - return { - "learning_rate": trial.suggest_float("learning_rate", 1e-6, 1e-4, log=True), - "num_epochs": trial.suggest_int("num_epochs", 1, 5), - "batch_size": trial.suggest_categorical("batch_size", [4, 8, 16, 32, 64]), - "seed": trial.suggest_int("seed", 1, 40), - "num_iterations": trial.suggest_categorical("num_iterations", [5, 10, 20]), - "max_iter": trial.suggest_int("max_iter", 50, 300), - "solver": trial.suggest_categorical("solver", ["newton-cg", "lbfgs", "liblinear"]), - } +>>> model = SetFitModel.from_pretrained("setfit-8-shot-sst2") # Load from a local directory ``` -**Note:** In practice, we found `num_iterations` to be the most important hyperparameter for the contrastive learning process. +### Inference -The next step is to instantiate a `Trainer` and call `hyperparameter_search()`: +Once a 🤗 SetFit model has been trained, then it can be used for inference to classify reviews using [`SetFitModel.predict`] or [`SetFitModel.__call__`]: -```python -from datasets import Dataset -from setfit import Trainer - -dataset = Dataset.from_dict( - {"text_new": ["a", "b", "c"], "label_new": [0, 1, 2], "extra_column": ["d", "e", "f"]} -) - -trainer = Trainer( - train_dataset=dataset, - eval_dataset=dataset, - model_init=model_init, - column_mapping={"text_new": "text", "label_new": "label"}, -) -best_run = trainer.hyperparameter_search(direction="maximize", hp_space=hp_space, n_trials=5) +```py +>>> preds = model.predict([ +... "It's a charming and often affecting journey.", +... "It's slow -- very, very slow.", +... "A sometimes tedious film.", +... ]) +>>> preds +tensor([1, 0, 0], dtype=torch.int32) ``` +These predictions match the format of the input labels. If we had used string labels, then the outputs would have been `['positive' 'negative' 'negative']`. -Finally, you can apply the hyperparameters you found to the trainer, and lock in the optimal model, before training for -a final time. +## What's next? -```python -trainer.apply_hyperparameters(best_run.hyperparameters, final_model=True) -trainer.train() -``` +You've completed the 🤗 SetFit quickstart! You can train, save, load and perform inference with 🤗 SetFit models! -## Compressing a SetFit model with knowledge distillation +For your next steps, take a look at our [How-to guides](./how_to/overview) and learn how to do more specific things like hyperparameter search, knowledge distillation, or zero-shot text classification. If you're interested in learning more about how 🤗 SetFit works, grab a cup of coffee and read our [Conceptual Guides](./conceptual_guides/setfit)! -If you have access to unlabeled data, you can use knowledge distillation to compress a trained SetFit model into a smaller version. The result is a model that can run inference much faster, with little to no drop in accuracy. Here's an end-to-end example (see our paper for more details): +## End-to-end -```python -from datasets import load_dataset -from setfit import SetFitModel, Trainer, DistillationTrainer, sample_dataset -from setfit.training_args import TrainingArguments - -# Load a dataset from the Hugging Face Hub -dataset = load_dataset("ag_news") - -# Create a sample few-shot dataset to train the teacher model -train_dataset_teacher = sample_dataset(dataset["train"], label_column="label", num_samples=16) -# Create a dataset of unlabeled examples to train the student -train_dataset_student = dataset["train"].shuffle(seed=0).select(range(500)) -# Dataset for evaluation -eval_dataset = dataset["test"] - -# Load teacher model -teacher_model = SetFitModel.from_pretrained( - "sentence-transformers/paraphrase-mpnet-base-v2" -) +This snippet shows the entire quickstart in an end-to-end example: -# Create trainer for teacher model -teacher_trainer = Trainer( - model=teacher_model, - train_dataset=train_dataset_teacher, - eval_dataset=eval_dataset, -) +```py +from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset +from datasets import load_dataset -# Train teacher model -teacher_trainer.train() -teacher_metrics = teacher_trainer.evaluate() +# Initializing a new SetFit model +model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5") -# Load small student model -student_model = SetFitModel.from_pretrained("paraphrase-MiniLM-L3-v2") +# Preparing the dataset +dataset = load_dataset("SetFit/sst2") +train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) +test_dataset = dataset["test"] +# Preparing the training arguments args = TrainingArguments( - batch_size=16, - num_iterations=20, - num_epochs=1 + batch_size=32, + num_epochs=10, ) -# Create trainer for knowledge distillation -student_trainer = DistillationTrainer( - teacher_model=teacher_model, - student_model=student_model, +# Preparing the trainer +trainer = Trainer( + model=model, args=args, - train_dataset=train_dataset_student, - eval_dataset=eval_dataset, + train_dataset=train_dataset, ) +trainer.train() -# Train student with knowledge distillation -student_trainer.train() -student_metrics = student_trainer.evaluate() -``` \ No newline at end of file +# Evaluating +metrics = trainer.evaluate(test_dataset) +print(metrics) +# => {'accuracy': 0.8511806699615596} + +# Saving the trained model +model.save_pretrained("setfit-8-shot-sst2") +# or +model.push_to_hub("tomaarsen/setfit-8-shot-sst2") + +# Loading a trained model +model = SetFitModel.from_pretrained("tomaarsen/setfit-8-shot-sst2") # Load from the Hugging Face Hub +# or +model = SetFitModel.from_pretrained("setfit-8-shot-sst2") # Load from a local directory + +# Performing inference +preds = model.predict([ + "It's a charming and often affecting journey.", + "It's slow -- very, very slow.", + "A sometimes tedious film.", +]) +print(preds) +# => tensor([1, 0, 0], dtype=torch.int32) +``` diff --git a/docs/source/en/reference/main.mdx b/docs/source/en/reference/main.mdx new file mode 100644 index 00000000..a533ad56 --- /dev/null +++ b/docs/source/en/reference/main.mdx @@ -0,0 +1,43 @@ + +# SetFitModel + +[[autodoc]] SetFitModel + - all + - __call__ + +# SetFitHead + +[[autodoc]] SetFitHead + +# AbsaModel + +[[autodoc]] AbsaModel + - __call__ + - device + - from_pretrained + - predict + - push_to_hub + - to + - save_pretrained + +## AspectModel + +[[autodoc]] AspectModel + - __call__ + - device + - from_pretrained + - predict + - push_to_hub + - save_pretrained + - to + +## PolarityModel + +[[autodoc]] PolarityModel + - __call__ + - device + - from_pretrained + - predict + - push_to_hub + - save_pretrained + - to diff --git a/docs/source/en/api/trainer.mdx b/docs/source/en/reference/trainer.mdx similarity index 70% rename from docs/source/en/api/trainer.mdx rename to docs/source/en/reference/trainer.mdx index 3e3d39d1..797418c2 100644 --- a/docs/source/en/api/trainer.mdx +++ b/docs/source/en/reference/trainer.mdx @@ -1,4 +1,8 @@ +# TrainingArguments + +[[autodoc]] TrainingArguments + # Trainer [[autodoc]] Trainer diff --git a/docs/source/en/reference/utility.mdx b/docs/source/en/reference/utility.mdx new file mode 100644 index 00000000..4c56df25 --- /dev/null +++ b/docs/source/en/reference/utility.mdx @@ -0,0 +1,6 @@ + +# Utility + +[[autodoc]] get_templated_dataset + +[[autodoc]] sample_dataset \ No newline at end of file diff --git a/docs/source/en/tutorials/overview.mdx b/docs/source/en/tutorials/overview.mdx new file mode 100644 index 00000000..52010122 --- /dev/null +++ b/docs/source/en/tutorials/overview.mdx @@ -0,0 +1,8 @@ + +# Overview + +Welcome to the SetFit tutorials! These tutorials are designed to walk you through particular applications. For example, we'll delve into topics such as zero-shot text classification, where you'll learn how to use SetFit without any predefined labels or examples during training. See also the [SetFit Notebooks](https://github.com/huggingface/setfit/tree/main/notebooks) for more applications, such as hyperparameter searching and ONNX, though some might be outdated. + +For more concise guides on how to configure SetFit or use it for specific forms of text classification, see the [How-to Guides](../how_to/overview) section. + +If you have any questions about SetFit, feel free to open an [issue](https://github.com/huggingface/setfit/issues). diff --git a/docs/source/en/tutorials/placeholder.mdx b/docs/source/en/tutorials/placeholder.mdx deleted file mode 100644 index f68bd40a..00000000 --- a/docs/source/en/tutorials/placeholder.mdx +++ /dev/null @@ -1,3 +0,0 @@ - -# Tutorial -Work in Progress! \ No newline at end of file diff --git a/docs/source/en/tutorials/zero_shot.mdx b/docs/source/en/tutorials/zero_shot.mdx new file mode 100644 index 00000000..28bb0fcd --- /dev/null +++ b/docs/source/en/tutorials/zero_shot.mdx @@ -0,0 +1,327 @@ +# Zero-shot Text Classification + +[[open-in-colab]] + +Although SetFit was designed for few-shot learning, the method can also be applied in scenarios where no labeled data is available. The main trick is to create _synthetic examples_ that resemble the classification task, and then train a SetFit model on them. + +Remarkably, this simple technique typically outperforms the zero-shot pipeline in 🤗 Transformers, and can generate predictions by a factor of 5x (or more) faster! + +In this tutorial, we'll explore how: + +* SetFit can be applied for zero-shot classification +* Adding synthetic examples can also provide a performance boost to few-shot classification. + +## Setup + +If you're running this Notebook on Colab or some other cloud platform, you will need to install the `setfit` library. Uncomment the following cell and run it: + +```py +# %pip install setfit matplotlib +``` + +To benchmark the performance of the "zero-shot" method, we'll use the following dataset and pretrained model: + +```py +dataset_id = "emotion" +model_id = "sentence-transformers/paraphrase-mpnet-base-v2" +``` + +Next, we'll download the reference dataset from the Hugging Face Hub: + +```py +from datasets import load_dataset + +reference_dataset = load_dataset(dataset_id) +reference_dataset +``` +```py +DatasetDict({ + train: Dataset({ + features: ['text', 'label'], + num_rows: 16000 + }) + validation: Dataset({ + features: ['text', 'label'], + num_rows: 2000 + }) + test: Dataset({ + features: ['text', 'label'], + num_rows: 2000 + }) +}) +``` + +Now that we're set up, let's create some synthetic data to train on! + +## Creating a synthetic dataset + +The first thing we need to do is create a dataset of synthetic examples. In `setfit`, we can do this by applying the `get_templated_dataset()` function to a dummy dataset. This function expects a few main things: + +* A list of candidate labels to classify with. We'll use the labels from the reference dataset here, but this could be anything that's relevant to the task and dataset at hand. +* A template to generate examples with. By default, it is `"This sentence is {}"`, where the `{}` will be filled by one of the candidate labels +* A sample size $N$, which will create $N$ synthetic examples per class. We find $N=8$ usually works best. + +Armed with this information, let's first extract some candidate labels from the dataset: + +```py +# Extract ClassLabel feature from "label" column +label_features = reference_dataset["train"].features["label"] +# Label names to classify with +candidate_labels = label_features.names +candidate_labels +``` +``` +['sadness', 'joy', 'love', 'anger', 'fear', 'surprise'] +``` + + + +Some datasets on the Hugging Face Hub don't have a `ClassLabel` feature for the label column. In these cases, you should compute the candidate labels manually by first computing the id2label mapping as follows: + + + +```py +def get_id2label(dataset): + # The column with the label names + label_names = dataset.unique("label_text") + # The column with the label IDs + label_ids = dataset.unique("label") + id2label = dict(zip(label_ids, label_names)) + # Sort by label ID + return {key: val for key, val in sorted(id2label.items(), key = lambda x: x[0])} + +id2label = get_id2label(reference_dataset["train"]) +candidate_labels = list(id2label.values()) +``` + +Now that we have the labels, it's a simple matter to create synthetic examples: + +```py +from datasets import Dataset +from setfit import get_templated_dataset + +# A dummy dataset to fill with synthetic examples +dummy_dataset = Dataset.from_dict({}) +train_dataset = get_templated_dataset(dummy_dataset, candidate_labels=candidate_labels, sample_size=8) +train_dataset +``` +``` +Dataset({ + features: ['text', 'label'], + num_rows: 48 +}) +``` + + + +You might find you can get better performance by tweaking the `template` argument from the default of `"The sentence is {}"` to variants like `"This sentence is {}"` or `"This example is {}"`. + + + + +Since our dataset has 6 classes and we chose a sample size of 8, our synthetic dataset contains $6\times 8=48$ examples. If we take a look at a few of the examples: + +```py +train_dataset.shuffle()[:3] +``` +``` +{'text': ['This sentence is love', + 'This sentence is fear', + 'This sentence is joy'], + 'label': [2, 4, 1]} +``` + +We can see that each input takes the form of the template and has a corresponding label associated with it. + +Let's not train a SetFit model on these examples! + +## Fine-tuning the model + +To train a SetFit model, the first thing to do is download a pretrained checkpoint from the Hub. We can do so by using the [`SetFitModel.from_pretrained`] method: + +```py +from setfit import SetFitModel + +model = SetFitModel.from_pretrained(model_id) +``` + +Here, we've downloaded a pretrained Sentence Transformer from the Hub and added a logistic classification head to the create the SetFit model. As indicated in the message, we need to train this model on some labeled examples. We can do so by using the [`Trainer`] class as follows: + +```py +from setfit import Trainer + +trainer = Trainer( + model=model, + train_dataset=train_dataset, + eval_dataset=reference_dataset["test"] +) +``` + +Now that we've created a trainer, we can train it! While we're at it, let's time how long it takes to train and evaluate the model: + +```py +%%time +trainer.train() +zeroshot_metrics = trainer.evaluate() +zeroshot_metrics +``` +```py +***** Running training ***** + Num examples = 1920 + Num epochs = 1 + Total optimization steps = 120 + Total train batch size = 16 +***** Running evaluation ***** +{'accuracy': 0.5345} +``` +``` +CPU times: user 12.9 s, sys: 2.37 s, total: 15.2 s +Wall time: 11 s +``` + +Great, now that we have a reference score let's compare against the zero-shot pipeline from 🤗 Transformers. + +## Comparing against the zero-shot pipeline from 🤗 Transformers +🤗 Transformers provides a zero-shot pipeline that frames text classification as a natural language inference task. Let's load the pipeline and place it on the GPU for fast inference: + +```py +from transformers import pipeline + +pipe = pipeline("zero-shot-classification", device=0) +``` + +Now that we have the model, let's generate some predictions. We'll use the same candidate labels as we did with SetFit and increase the batch size for to speed things up: + +```py +%%time +zeroshot_preds = pipe(reference_dataset["test"]["text"], batch_size=16, candidate_labels=candidate_labels) +``` +``` +CPU times: user 1min 10s, sys: 166 ms, total: 1min 11s +Wall time: 53.1 s +``` + +Note that this took almost 5x longer to generate predictions than SetFit! OK, so how well does it perform? Since each prediction is a dictionary of label names ranked by score: + +```py +zeroshot_preds[0] +``` +```py +{'sequence': 'im feeling rather rotten so im not very ambitious right now', + 'labels': ['sadness', 'anger', 'surprise', 'fear', 'joy', 'love'], + 'scores': [0.7367985844612122, + 0.10041674226522446, + 0.09770156443119049, + 0.05880110710859299, + 0.004266355652362108, + 0.0020156768150627613]} +``` + +We can use the `str2int()` function from the `label` column to convert them to integers. + +```py +preds = [label_features.str2int(pred["labels"][0]) for pred in zeroshot_preds] +``` + +**Note:** As noted earlier, if you're using a dataset that doesn't have a `ClassLabel` feature for the label column, you'll need to compute the label mapping manually with something like: + +```py +id2label = get_id2label(reference_dataset["train"]) +label2id = {v:k for k,v in id2label.items()} +preds = [label2id[pred["labels"][0]] for pred in zeroshot_preds] +``` + +The last step is to compute accuracy using 🤗 Evaluate: + +```py +import evaluate + +metric = evaluate.load("accuracy") +transformers_metrics = metric.compute(predictions=preds, references=reference_dataset["test"]["label"]) +transformers_metrics +``` +```py +{'accuracy': 0.3765} +``` + +Compared to SetFit, this approach performs significantly worse. Let's wrap up our analysis by combining synthetic examples with a few labeled ones. + +## Augmenting labeled data with synthetic examples + +If you have a few labeled examples, adding synthetic data can often boost performance. To simulate this, let's first sample 8 labeled examples from our reference dataset: + +```py +from setfit import sample_dataset + +train_dataset = sample_dataset(reference_dataset["train"]) +train_dataset +``` +```py +Dataset({ + features: ['text', 'label'], + num_rows: 48 +}) +``` + +To warm up, we'll train a SetFit model on these true labels: +```py +model = SetFitModel.from_pretrained(model_id) + +trainer = Trainer( + model=model, + train_dataset=train_dataset, + eval_dataset=reference_dataset["test"] +) +trainer.train() +fewshot_metrics = trainer.evaluate() +fewshot_metrics +``` +```py +{'accuracy': 0.4705} +``` + +Note that for this particular dataset, the performance with true labels is _worse_ than training on synthetic examples! In our experiments, we found that the difference depends strongly on the dataset in question. Since SetFit models are fast to train, you can always try both approaches and pick the best one. + +In any case, let's now add some synthetic examples to our training set: + +```py +augmented_dataset = get_templated_dataset(train_dataset, candidate_labels=candidate_labels, sample_size=8) +augmented_dataset +``` +```py +Dataset({ + features: ['text', 'label'], + num_rows: 96 +}) +``` + +As before, we can train and evaluate SetFit with the augmented dataset: + +```py +model = SetFitModel.from_pretrained(model_id) + +trainer = Trainer( + model=model, + train_dataset=augmented_dataset, + eval_dataset=reference_dataset["test"] +) +trainer.train() +augmented_metrics = trainer.evaluate() +augmented_metrics +``` +``` +{'accuracy': 0.613} +``` + +Great, this has given us a significant boost in performance and given us a few percentage points over the purely synthetic example. + +Let's plot the final results for comparison: + +```py +import pandas as pd + +df = pd.DataFrame.from_dict({"Method":["Transformers (zero-shot)", "SetFit (zero-shot)", "SetFit (augmented)"], "Accuracy": [transformers_metrics["accuracy"], zeroshot_metrics["accuracy"], augmented_metrics["accuracy"]]}) +df.plot(kind="barh", x="Method"); +``` + +![setfit_zero_shot_results](https://github.com/huggingface/setfit/assets/37621491/b02d3e62-d51c-4506-91f6-2fe9b7ef554d) \ No newline at end of file From fb42dd750c4c218b893090f9362e61d1b7ee1940 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 15:13:12 +0100 Subject: [PATCH 094/183] Update the documentation related workflows As per internal documentation --- .github/workflows/build_documentation.yml | 3 +++ ...tation.yml => delete_doc_comment_trigger.yml} | 7 +++---- .github/workflows/upload_pr_documentation.yml | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) rename .github/workflows/{delete_pr_documentation.yml => delete_doc_comment_trigger.yml} (54%) create mode 100644 .github/workflows/upload_pr_documentation.yml diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index 70f46f86..cfe3dc17 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -14,3 +14,6 @@ jobs: commit_sha: ${{ github.sha }} package: setfit languages: en + secrets: + token: ${{ secrets.HUGGINGFACE_PUSH }} + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} \ No newline at end of file diff --git a/.github/workflows/delete_pr_documentation.yml b/.github/workflows/delete_doc_comment_trigger.yml similarity index 54% rename from .github/workflows/delete_pr_documentation.yml rename to .github/workflows/delete_doc_comment_trigger.yml index e925fcfe..5e39e253 100644 --- a/.github/workflows/delete_pr_documentation.yml +++ b/.github/workflows/delete_doc_comment_trigger.yml @@ -1,4 +1,4 @@ -name: Delete dev documentation +name: Delete doc comment trigger on: pull_request: @@ -7,7 +7,6 @@ on: jobs: delete: - uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main + uses: huggingface/doc-builder/.github/workflows/delete_doc_comment_trigger.yml@main with: - pr_number: ${{ github.event.number }} - package: setfit \ No newline at end of file + pr_number: ${{ github.event.number }} \ No newline at end of file diff --git a/.github/workflows/upload_pr_documentation.yml b/.github/workflows/upload_pr_documentation.yml new file mode 100644 index 00000000..4f79605b --- /dev/null +++ b/.github/workflows/upload_pr_documentation.yml @@ -0,0 +1,16 @@ +name: Upload PR Documentation + +on: + workflow_run: + workflows: ["Build PR Documentation"] + types: + - completed + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main + with: + package_name: setfit + secrets: + hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} + comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }} \ No newline at end of file From bfe6ef6c214d8434db264d9041c914b5be0cf445 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 17:41:55 +0100 Subject: [PATCH 095/183] Add figure to zero-shot how-to guide --- docs/source/en/how_to/zero_shot.mdx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/en/how_to/zero_shot.mdx b/docs/source/en/how_to/zero_shot.mdx index 607a4a79..95b45b2d 100644 --- a/docs/source/en/how_to/zero_shot.mdx +++ b/docs/source/en/how_to/zero_shot.mdx @@ -136,7 +136,7 @@ print(transformers_accuracy) With its 59.1% accuracy, the 0-shot SetFit heavily outperforms the recommended zero-shot model by `transformers`. -### Prediction latency +## Prediction latency Beyond getting higher accuracies, SetFit is much faster too. Let's compute the latency of SetFit with `BAAI/bge-small-en-v1.5` versus the latency of `transformers` with `facebook/bart-large-mnli`. Both tests were performed on a GPU. @@ -164,4 +164,6 @@ print(f"SetFit with `BAAI/bge-small-en-v1.5` latency: {delta_t / len(test_datase SetFit with `BAAI/bge-small-en-v1.5` latency: 0.4600ms per sentence ``` -So, SetFit with `BAAI/bge-small-en-v1.5` is 67x faster than `transformers` with `facebook/bart-large-mnli`, alongside being more accurate. +So, SetFit with `BAAI/bge-small-en-v1.5` is 67x faster than `transformers` with `facebook/bart-large-mnli`, alongside being more accurate: + +![zero_shot_transformers_vs_setfit](https://github.com/huggingface/setfit/assets/37621491/33f574d9-c51b-4e02-8d98-6e04e18427ef) From 773b860a3ab1fc8cdddb01e6bc923dc18f8af8a5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 18:04:44 +0100 Subject: [PATCH 096/183] Add docs notebook building support --- .github/workflows/build_documentation.yml | 1 + docs/source/_config.py | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 docs/source/_config.py diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index cfe3dc17..d66ee134 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -13,6 +13,7 @@ jobs: with: commit_sha: ${{ github.sha }} package: setfit + notebook_folder: setfit_doc languages: en secrets: token: ${{ secrets.HUGGINGFACE_PUSH }} diff --git a/docs/source/_config.py b/docs/source/_config.py new file mode 100644 index 00000000..2f4f5c51 --- /dev/null +++ b/docs/source/_config.py @@ -0,0 +1,9 @@ +# docstyle-ignore +INSTALL_CONTENT = """ +# SetFit installation +! pip install setfit +# To install from source instead of the last release, comment the command above and uncomment the following one. +# ! pip install git+https://github.com/huggingface/setfit.git +""" + +notebook_first_cells = [{"type": "code", "content": INSTALL_CONTENT}] \ No newline at end of file From 883889c132910002553fdcdd8bafb0019f7a52c3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 21 Nov 2023 18:14:53 +0100 Subject: [PATCH 097/183] Update broken, redirecting links --- docs/README.md | 34 +++---------------- .../en/how_to/v1.0.0_migration_guide.mdx | 2 +- docs/source/en/installation.mdx | 2 +- docs/source/en/quickstart.mdx | 2 +- 4 files changed, 8 insertions(+), 32 deletions(-) diff --git a/docs/README.md b/docs/README.md index befedf88..0178011d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -78,7 +78,7 @@ The `preview` command only works with existing doc files. When you add a complet Accepted files are Markdown (.md or .mdx). Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting -the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/setfit/blob/main/docs/source/_toctree.yml) file. +the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/setfit/blob/main/docs/source/en/_toctree.yml) file. ## Renaming section headers and moving sections @@ -103,7 +103,7 @@ Sections that were moved: Use the relative style to link to the new file so that the versioned docs continue to work. -For an example of a rich moved section set please see the very end of [the Trainer doc](https://github.com/huggingface/transformers/blob/main/docs/source/en/main_classes/trainer.mdx). +For an example of a rich moved section set please see the very end of [the Trainer doc](https://github.com/huggingface/transformers/blob/main/docs/source/en/main_classes/trainer.md). ## Writing Documentation - Specification @@ -123,34 +123,10 @@ Make sure to put your new file under the proper section. It's unlikely to go in depending on the intended targets (beginners, more advanced users, or researchers) it should go in sections two, three, or four. -### Translating -When translating, refer to the guide at [./TRANSLATING.md](https://github.com/huggingface/setfit/blob/main/docs/TRANSLATING.md). +### Autodoc - -### Adding a new model - -When adding a new model: - -- Create a file `xxx.mdx` or under `./source/model_doc` (don't hesitate to copy an existing file as template). -- Link that file in `./source/_toctree.yml`. -- Write a short overview of the model: - - Overview with paper & authors - - Paper abstract - - Tips and tricks and how to use it best -- Add the classes that should be linked in the model. This generally includes the configuration, the tokenizer, and - every model of that class (the base model, alongside models with additional heads), both in PyTorch and TensorFlow. - The order is generally: - - Configuration, - - Tokenizer - - PyTorch base model - - PyTorch head models - - TensorFlow base model - - TensorFlow head models - - Flax base model - - Flax head models - -These classes should be added using our Markdown syntax. Usually as follows: +The following are some examples of `[[autodoc]]` for documentation building. ``` ## XXXConfig diff --git a/docs/source/en/how_to/v1.0.0_migration_guide.mdx b/docs/source/en/how_to/v1.0.0_migration_guide.mdx index a4d44df1..701af807 100644 --- a/docs/source/en/how_to/v1.0.0_migration_guide.mdx +++ b/docs/source/en/how_to/v1.0.0_migration_guide.mdx @@ -47,7 +47,7 @@ This list contains new functionality that can be used starting from v1.0.0. The default is set to `"oversampling"`, ensuring all sentence pairs are drawn at least once. Alternatively, setting `num_iterations` will override this argument and determine the number of generated sentence pairs. * `report_to`: The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`, `"comet_ml"`, `"mlflow"`, `"neptune"`, `"tensorboard"`,`"clearml"` and `"wandb"`. Use `"all"` to report to all integrations installed, `"none"` for no integrations. - * `run_name`: A descriptor for the run. Typically used for [wandb](https://www.wandb.com/) and [mlflow](https://www.mlflow.org/) logging. + * `run_name`: A descriptor for the run. Typically used for [wandb](https://wandb.ai/) and [mlflow](https://www.mlflow.org/) logging. * `logging_strategy`: The logging strategy to adopt during training. Possible values are: - `"no"`: No logging is done during training. diff --git a/docs/source/en/installation.mdx b/docs/source/en/installation.mdx index 8b3fa13d..f792501c 100644 --- a/docs/source/en/installation.mdx +++ b/docs/source/en/installation.mdx @@ -11,7 +11,7 @@ The most straightforward way to install 🤗 Datasets is with pip: pip install setfit ``` -If you have a CUDA-capable graphics card, then it is recommended to [install `torch` with CUDA support](https://pytorch.org/get-started/locally) to train and performing inference much more quickly: +If you have a CUDA-capable graphics card, then it is recommended to [install `torch` with CUDA support](https://pytorch.org/get-started/locally/) to train and performing inference much more quickly: ```bash pip install torch --index-url https://download.pytorch.org/whl/cu118 diff --git a/docs/source/en/quickstart.mdx b/docs/source/en/quickstart.mdx index 476971f7..4845055d 100644 --- a/docs/source/en/quickstart.mdx +++ b/docs/source/en/quickstart.mdx @@ -11,7 +11,7 @@ Start by installing 🤗 SetFit: pip install setfit ``` -If you have a CUDA-capable graphics card, then it is recommended to [install `torch` with CUDA support](https://pytorch.org/get-started/locally) to train and performing inference much more quickly: +If you have a CUDA-capable graphics card, then it is recommended to [install `torch` with CUDA support](https://pytorch.org/get-started/locally/) to train and performing inference much more quickly: ```bash pip install torch --index-url https://download.pytorch.org/whl/cu118 From b4e5db00753fe9ee2607abe5d253a9c7dab586f8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 22 Nov 2023 11:49:20 +0100 Subject: [PATCH 098/183] polarity -> label --- docs/source/en/how_to/absa.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/en/how_to/absa.mdx b/docs/source/en/how_to/absa.mdx index d558efe2..5d958931 100644 --- a/docs/source/en/how_to/absa.mdx +++ b/docs/source/en/how_to/absa.mdx @@ -37,11 +37,11 @@ from setfit import AbsaModel model = AbsaModel.from_pretrained("BAAI/bge-small-en-v1.5") ``` -Then we have to prepare a training/testing set. These datasets must have `"text"`, `"span"`, `"polarity"`, and `"ordinal"` columns: +Then we have to prepare a training/testing set. These datasets must have `"text"`, `"span"`, `"label"`, and `"ordinal"` columns: * `"text"`: The full sentence or text containing the aspects. For example: `"But the staff was so horrible to us."`. * `"span"`: An aspect from the full sentence. Can be multiple words. For example: `"staff"`. -* `"polarity"`: The (polarity) label corresponding to the aspect span. For example: `"negative"`. +* `"label"`: The (polarity) label corresponding to the aspect span. For example: `"negative"`. * `"ordinal"`: If the aspect span occurs multiple times in the text, then this ordinal represents the index of those occurrences. Often this is just 0. For example: `0`. Two datasets that already match this format are these datasets of reviews from the SemEval-2014 Task 4: @@ -50,7 +50,7 @@ Two datasets that already match this format are these datasets of reviews from t * [tomaarsen/setfit-absa-semeval-laptops](https://huggingface.co/datasets/tomaarsen/setfit-absa-semeval-laptops) ```py -# The training/eval dataset must have `text`, `span`, `polarity`, and `ordinal` columns +# The training/eval dataset must have `text`, `span`, `label`, and `ordinal` columns dataset = load_dataset("tomaarsen/setfit-absa-semeval-laptops", split="train") train_dataset = dataset.select(range(128)) eval_dataset = dataset.select(range(128, 256)) From dbd707baad19d8e237091cc612b08c14563e751a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 22 Nov 2023 11:52:02 +0100 Subject: [PATCH 099/183] Mention extra download requirements for ABSA --- docs/source/en/how_to/absa.mdx | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/source/en/how_to/absa.mdx b/docs/source/en/how_to/absa.mdx index 5d958931..20695d99 100644 --- a/docs/source/en/how_to/absa.mdx +++ b/docs/source/en/how_to/absa.mdx @@ -9,6 +9,23 @@ SetFitABSA is an efficient framework for few-shot Aspect Based Sentiment Analysi This guide will show you how to train, predict, save and load these models. +## Getting Started + +First of all, SetFitABSA also requires spaCy to be installed, so we must install it: + +``` +!pip install "setfit[absa]" +# or +# !pip install spacy +``` + +Then, we must download the spaCy model that we intend on using. By default, SetFitABSA uses `en_core_web_lg`, but `en_core_web_sm` and `en_core_web_md` are also good options. + +``` +!spacy download en_core_web_lg +!spacy download en_core_web_sm +``` + ## Training SetFitABSA First of all, we must instantiate a new [`AbsaModel`] via [`AbsaModel.from_pretrained`]. This can be done by providing configuration for each of the three phases for SetFitABSA: From 0d32dd1e05d8d55e2976bcad3238e6817e75f3c2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 13:00:37 +0100 Subject: [PATCH 100/183] Implement 'batch_size' on model.predict --- docs/source/en/how_to/batch_sizes.mdx | 21 +++++++++++++++++++ src/setfit/modeling.py | 29 ++++++++++++++++++++------- 2 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 docs/source/en/how_to/batch_sizes.mdx diff --git a/docs/source/en/how_to/batch_sizes.mdx b/docs/source/en/how_to/batch_sizes.mdx new file mode 100644 index 00000000..781e7a98 --- /dev/null +++ b/docs/source/en/how_to/batch_sizes.mdx @@ -0,0 +1,21 @@ + +# Batch sizes +In this how-to guide we will explore the effects of increasing the batch sizes in [`SetFitModel.predict`]. + +## What are they? +When processing on GPUs, often times not all data fits on the GPU its VRAM at once. As a result, the data gets split up into **batches** of some often pre-determined batch size. This is done both during training and during inference. In both scenarios, increasing the batch size often has notable consequences to processing efficiency and VRAM memory usage, as transferring data to and from the GPU can be relatively slow. + +For inference, it is often recommended to set the batch size high to get notably quicker processing speeds. + +## In SetFit +The batch size for inference in SetFit is set to 32, but it can be affected by passing a `batch_size` argument to [`SetFitModel.predict`]. For example, on a RTX 3090 with a SetFit model based on the [paraphrase-mpnet-base-v2](https://huggingface.co/sentence-transformers/paraphrase-mpnet-base-v2) Sentence Transformer, the following throughputs are reached: + +![setfit_speed_per_batch_size](https://github.com/huggingface/setfit/assets/37621491/c01d391b-aeba-4a4b-83f8-b09970a0d6e6) + + + +Each sentence consists of 11 words in this experiment. + + + +The default batch size of 32 does not result in the highest possible throughput on this hardware. Consider experimenting with the batch size to reach your highest possible throughput. \ No newline at end of file diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 793b2c72..29ba9e61 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -432,11 +432,15 @@ def _freeze_or_not(self, model: nn.Module, to_freeze: bool) -> None: for param in model.parameters(): param.requires_grad = not to_freeze - def encode(self, inputs: List[str], show_progress_bar: Optional[bool] = None) -> Union[torch.Tensor, np.ndarray]: + def encode( + self, inputs: List[str], batch_size: int = 32, show_progress_bar: Optional[bool] = None + ) -> Union[torch.Tensor, np.ndarray]: """Convert input sentences to embeddings using the `SentenceTransformer` body. Args: inputs (`List[str]`): The input sentences to embed. + batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. + Higher often means faster processing but higher memory usage. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. Returns: @@ -445,6 +449,7 @@ def encode(self, inputs: List[str], show_progress_bar: Optional[bool] = None) -> """ return self.model_body.encode( inputs, + batch_size=batch_size, normalize_embeddings=self.normalize_embeddings, convert_to_tensor=self.has_differentiable_head, show_progress_bar=show_progress_bar, @@ -472,12 +477,14 @@ def _output_type_conversion( return outputs def predict( - self, inputs: List[str], as_numpy: bool = False, show_progress_bar: Optional[bool] = None + self, inputs: List[str], batch_size: int = 32, as_numpy: bool = False, show_progress_bar: Optional[bool] = None ) -> Union[torch.Tensor, np.ndarray]: """Predict the various classes. Args: inputs (`List[str]`): The input sentences to predict classes for. + batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. + Higher often means faster processing but higher memory usage. as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. @@ -490,17 +497,19 @@ def predict( `Union[torch.Tensor, np.ndarray]`: A vector with equal length to the inputs, denoting to which class each input is predicted to belong. """ - embeddings = self.encode(inputs, show_progress_bar=show_progress_bar) + embeddings = self.encode(inputs, batch_size=batch_size, show_progress_bar=show_progress_bar) outputs = self.model_head.predict(embeddings) return self._output_type_conversion(outputs, as_numpy=as_numpy) def predict_proba( - self, inputs: List[str], as_numpy: bool = False, show_progress_bar: Optional[bool] = None + self, inputs: List[str], batch_size: int = 32, as_numpy: bool = False, show_progress_bar: Optional[bool] = None ) -> Union[torch.Tensor, np.ndarray]: """Predict the probabilities of the various classes. Args: inputs (`List[str]`): The input sentences to predict class probabilities for. + batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. + Higher often means faster processing but higher memory usage. as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. @@ -515,7 +524,7 @@ def predict_proba( `Union[torch.Tensor, np.ndarray]`: A matrix with shape [INPUT_LENGTH, NUM_CLASSES] denoting probabilities of predicting an input as a class. """ - embeddings = self.encode(inputs, show_progress_bar=show_progress_bar) + embeddings = self.encode(inputs, batch_size=batch_size, show_progress_bar=show_progress_bar) outputs = self.model_head.predict_proba(embeddings) return self._output_type_conversion(outputs, as_numpy=as_numpy) @@ -574,11 +583,17 @@ def create_model_card(self, path: str, model_name: Optional[str] = "SetFit Model with open(os.path.join(path, "README.md"), "w", encoding="utf-8") as f: f.write(model_card_content) - def __call__(self, inputs: List[str]) -> torch.Tensor: + def __call__( + self, inputs: List[str], batch_size: int = 32, as_numpy: bool = False, show_progress_bar: Optional[bool] = None + ) -> Union[torch.Tensor, np.ndarray]: """Predict the various classes. Args: inputs (`List[str]`): The input sentences to predict classes for. + batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. + Higher often means faster processing but higher memory usage. + as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. + show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. Example: >>> model = SetFitModel.from_pretrained(...) @@ -589,7 +604,7 @@ def __call__(self, inputs: List[str]) -> torch.Tensor: `torch.Tensor`: A vector with equal length to the inputs, denoting to which class each input is predicted to belong. """ - return self.predict(inputs) + return self.predict(inputs, batch_size=batch_size, as_numpy=as_numpy, show_progress_bar=show_progress_bar) def _save_pretrained(self, save_directory: Union[Path, str]) -> None: save_directory = str(save_directory) From 392cf0dd3be2842499f87c7962eb1d78b02a9134 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 13:08:01 +0100 Subject: [PATCH 101/183] Add batch sizes to toctree --- docs/source/en/_toctree.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index af85e43b..5f0367c6 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -29,6 +29,8 @@ title: Hyperparameter Optimization - local: how_to/knowledge_distillation title: Knowledge Distillation + - local: how_to/batch_sizes + title: Batch Sizes - local: how_to/absa title: Aspect Based Sentiment Analysis - local: how_to/v1.0.0_migration_guide From 17d6513061404546009565957874a62756599c59 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 13:23:28 +0100 Subject: [PATCH 102/183] Save model head on CPU And move models to the right head after loading Co-authored-by: karter-liner <88304414+karter-liner@users.noreply.github.com> --- src/setfit/modeling.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 29ba9e61..bcf0ddc5 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -610,7 +610,12 @@ def _save_pretrained(self, save_directory: Union[Path, str]) -> None: save_directory = str(save_directory) self.model_body.save(path=save_directory, create_model_card=False) self.create_model_card(path=save_directory, model_name=save_directory) + # Move the head to the CPU before saving + if self.has_differentiable_head: + self.model_head.to("cpu") joblib.dump(self.model_head, str(Path(save_directory) / MODEL_HEAD_NAME)) + if self.has_differentiable_head: + self.model_head.to(self.device) @classmethod @validate_hf_hub_args @@ -665,6 +670,8 @@ def _from_pretrained( if model_head_file is not None: model_head = joblib.load(model_head_file) + if isinstance(model_head, torch.Module): + model_head.to(target_device) else: head_params = model_kwargs.pop("head_params", {}) if use_differentiable_head: From dca6fd0959d3410f40ed8607758d9733ce6c7f28 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 13:32:00 +0100 Subject: [PATCH 103/183] torch.Module -> torch.nn.Module --- src/setfit/modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index bcf0ddc5..e1d85809 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -670,7 +670,7 @@ def _from_pretrained( if model_head_file is not None: model_head = joblib.load(model_head_file) - if isinstance(model_head, torch.Module): + if isinstance(model_head, torch.nn.Module): model_head.to(target_device) else: head_params = model_kwargs.pop("head_params", {}) From b5a63614de0406e409fa36a44ca7bac13a66db8d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 14:28:45 +0100 Subject: [PATCH 104/183] Add new top-level header to docs reference This might help with the sidebar on the docs --- docs/source/en/reference/main.mdx | 12 +++++++----- docs/source/en/reference/trainer.mdx | 10 ++++++---- docs/source/en/reference/utility.mdx | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/source/en/reference/main.mdx b/docs/source/en/reference/main.mdx index a533ad56..0b62e376 100644 --- a/docs/source/en/reference/main.mdx +++ b/docs/source/en/reference/main.mdx @@ -1,15 +1,17 @@ -# SetFitModel +# Main Classes + +## SetFitModel [[autodoc]] SetFitModel - all - __call__ -# SetFitHead +## SetFitHead [[autodoc]] SetFitHead -# AbsaModel +## AbsaModel [[autodoc]] AbsaModel - __call__ @@ -20,7 +22,7 @@ - to - save_pretrained -## AspectModel +### AspectModel [[autodoc]] AspectModel - __call__ @@ -31,7 +33,7 @@ - save_pretrained - to -## PolarityModel +### PolarityModel [[autodoc]] PolarityModel - __call__ diff --git a/docs/source/en/reference/trainer.mdx b/docs/source/en/reference/trainer.mdx index 797418c2..77230e85 100644 --- a/docs/source/en/reference/trainer.mdx +++ b/docs/source/en/reference/trainer.mdx @@ -1,16 +1,18 @@ -# TrainingArguments +# Trainer Classes + +## TrainingArguments [[autodoc]] TrainingArguments -# Trainer +## Trainer [[autodoc]] Trainer -# DistillationTrainer +## DistillationTrainer [[autodoc]] DistillationTrainer -# AbsaTrainer +## AbsaTrainer [[autodoc]] AbsaTrainer \ No newline at end of file diff --git a/docs/source/en/reference/utility.mdx b/docs/source/en/reference/utility.mdx index 4c56df25..d4741acf 100644 --- a/docs/source/en/reference/utility.mdx +++ b/docs/source/en/reference/utility.mdx @@ -1,5 +1,5 @@ -# Utility +# Utility Functions [[autodoc]] get_templated_dataset From 6ca989e734ec566feca5904e97ae98a05da5b7ce Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 14:30:20 +0100 Subject: [PATCH 105/183] Update docs about return value of metric function --- src/setfit/trainer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 0bb91a41..c97b143f 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -151,8 +151,8 @@ class Trainer(ColumnMappingMixin): function when a `trial` is passed. metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): The metric to use for evaluation. If a string is provided, we treat it as the metric - name and load it with default settings. - If a callable is provided, it must take two arguments (`y_pred`, `y_test`). + name and load it with default settings. If a callable is provided, it must take two arguments + (`y_pred`, `y_test`) and return a dictionary with metric keys to values. metric_kwargs (`Dict[str, Any]`, *optional*): Keyword arguments passed to the evaluation function if `metric` is an evaluation string like "f1". For example useful for providing an averaging strategy for computing f1 in a multi-label setting. From 93c52dd5175bcef8ad69ee1288434598156adff2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 15:16:39 +0100 Subject: [PATCH 106/183] Add "use_auth_token" to migration guide --- docs/source/en/how_to/v1.0.0_migration_guide.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/en/how_to/v1.0.0_migration_guide.mdx b/docs/source/en/how_to/v1.0.0_migration_guide.mdx index 701af807..5051f930 100644 --- a/docs/source/en/how_to/v1.0.0_migration_guide.mdx +++ b/docs/source/en/how_to/v1.0.0_migration_guide.mdx @@ -7,6 +7,7 @@ To update your code to work with v1.0.0, the following changes must be made: 1. `keep_body_frozen` from `SetFitModel.unfreeze` has been deprecated, simply either pass `"head"`, `"body"` or no arguments to unfreeze both. 2. `SupConLoss` has been moved from `setfit.modeling` to `setfit.losses`. If you are importing it using `from setfit.modeling import SupConLoss`, then import it like `from setfit import SupConLoss` now instead. +3. `use_auth_token` has been renamed to `token` in [`SetFitModel.from_pretrained`]. `use_auth_token` will keep working until the next major version, but with a warning. ## Training Migration Guide From 44daad46050f7237c26eea383ae62281ae9f9801 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 15:57:54 +0100 Subject: [PATCH 107/183] Allow 'device' on SetFitModel.from_pretrained() --- src/setfit/modeling.py | 11 ++++++----- tests/test_modeling.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index e1d85809..ec0d7a3c 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -632,11 +632,12 @@ def _from_pretrained( multi_target_strategy: Optional[str] = None, use_differentiable_head: bool = False, normalize_embeddings: bool = False, + device: Optional[Union[torch.device, str]] = None, **model_kwargs, ) -> "SetFitModel": - model_body = SentenceTransformer(model_id, cache_folder=cache_dir, use_auth_token=token) - target_device = model_body._target_device - model_body.to(target_device) # put `model_body` on the target device + model_body = SentenceTransformer(model_id, cache_folder=cache_dir, use_auth_token=token, device=device) + device = model_body._target_device + model_body.to(device) # put `model_body` on the target device if os.path.isdir(model_id): if MODEL_HEAD_NAME in os.listdir(model_id): @@ -671,7 +672,7 @@ def _from_pretrained( if model_head_file is not None: model_head = joblib.load(model_head_file) if isinstance(model_head, torch.nn.Module): - model_head.to(target_device) + model_head.to(device) else: head_params = model_kwargs.pop("head_params", {}) if use_differentiable_head: @@ -689,7 +690,7 @@ def _from_pretrained( # - follow the `model_body`, put `model_head` on the target device base_head_params = { "in_features": model_body.get_sentence_embedding_dimension(), - "device": target_device, + "device": device, "multitarget": use_multitarget, } model_head = SetFitHead(**{**head_params, **base_head_params}) diff --git a/tests/test_modeling.py b/tests/test_modeling.py index a5e279f6..71c683a1 100644 --- a/tests/test_modeling.py +++ b/tests/test_modeling.py @@ -246,3 +246,13 @@ def test_to_sentence_transformer_device_reset(use_differentiable_head): model.model_body.encode("This is a test sample to encode") assert model.model_body.device == torch.device("cpu") + + +@torch_cuda_available +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_load_model_on_device(device): + model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", device=device) + assert model.device.type == device + assert model.model_body.device.type == device + + model.model_body.encode("This is a test sample to encode") From 6f06204c8409ba105990974251e37efeb8a9bd7c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 16:06:02 +0100 Subject: [PATCH 108/183] Add tests for SetFitABSA as well --- tests/span/test_modeling.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/span/test_modeling.py b/tests/span/test_modeling.py index 0bc3ccb8..81675aba 100644 --- a/tests/span/test_modeling.py +++ b/tests/span/test_modeling.py @@ -7,6 +7,7 @@ from setfit import AbsaModel from setfit.span.aspect_extractor import AspectExtractor from setfit.span.modeling import AspectModel, PolarityModel +from tests.test_modeling import torch_cuda_available def test_loading(): @@ -84,3 +85,12 @@ def test_to(absa_model: AbsaModel) -> None: assert absa_model.device.type == "cpu" assert absa_model.aspect_model.device.type == "cpu" assert absa_model.polarity_model.device.type == "cpu" + + +@torch_cuda_available +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +def test_load_model_on_device(device): + model = AbsaModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2", device=device) + assert model.device.type == device + assert model.polarity_model.device.type == device + assert model.aspect_model.device.type == device From b8da4a3b3e0294ca8e1c45bd8e20e965d3c0d3d7 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 24 Nov 2023 18:56:25 +0100 Subject: [PATCH 109/183] Update which trainer methods are documented --- docs/source/en/reference/trainer.mdx | 26 ++++++++++++++++++++++++++ src/setfit/training_args.py | 26 ++++++++++++++++++-------- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/docs/source/en/reference/trainer.mdx b/docs/source/en/reference/trainer.mdx index 77230e85..5b23f545 100644 --- a/docs/source/en/reference/trainer.mdx +++ b/docs/source/en/reference/trainer.mdx @@ -4,14 +4,40 @@ ## TrainingArguments [[autodoc]] TrainingArguments + - to_dict + - from_dict + - copy + - update ## Trainer [[autodoc]] Trainer + - add_callback + - apply_hyperparameters + - evaluate + - hyperparameter_search + - log + - pop_callback + - push_to_hub + - remove_callback + - train + - train_classifier + - train_embeddings ## DistillationTrainer [[autodoc]] DistillationTrainer + - add_callback + - apply_hyperparameters + - evaluate + - hyperparameter_search + - log + - pop_callback + - push_to_hub + - remove_callback + - train + - train_classifier + - train_embeddings ## AbsaTrainer diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index ce667b42..2b8d2b25 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -305,32 +305,42 @@ def body_classifier_learning_rate(self) -> float: return self.body_learning_rate[1] def to_dict(self) -> Dict[str, Any]: - # filter out fields that are defined as field(init=False) + """Convert this instance to a dictionary. + + Returns: + `Dict[str, Any]`: The dictionary variant of this dataclass. + """ return {field.name: getattr(self, field.name) for field in fields(self) if field.init} @classmethod def from_dict(cls, arguments: Dict[str, Any], ignore_extra: bool = False) -> TrainingArguments: + """Initialize a TrainingArguments instance from a dictionary. + + Args: + arguments (`Dict[str, Any]`): A dictionary of arguments. + ignore_extra (`bool`, *optional*): Whether to ignore arguments that do not occur in the + TrainingArguments __init__ signature. Defaults to False. + + Returns: + `TrainingArguments`: The instantiated TrainingArguments instance. + """ if ignore_extra: return cls(**{key: value for key, value in arguments.items() if key in inspect.signature(cls).parameters}) return cls(**arguments) def copy(self) -> TrainingArguments: + """Create a shallow copy of this TrainingArguments instance.""" return copy(self) def update(self, arguments: Dict[str, Any], ignore_extra: bool = False) -> TrainingArguments: return TrainingArguments.from_dict({**self.to_dict(), **arguments}, ignore_extra=ignore_extra) def to_json_string(self): - """ - Serializes this instance to a JSON string. - """ - # TODO: This needs to be improved + # Serializes this instance to a JSON string. return json.dumps({key: str(value) for key, value in self.to_dict().items()}, indent=2) def to_sanitized_dict(self) -> Dict[str, Any]: - """ - Sanitized serialization to use with TensorBoard’s hparams - """ + # Sanitized serialization to use with TensorBoard’s hparams d = self.to_dict() d = {**d, **{"train_batch_size": self.embedding_batch_size, "eval_batch_size": self.embedding_batch_size}} From 639750fe5313bcdcc6255621355eb44c4df31e37 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 27 Nov 2023 10:06:59 +0100 Subject: [PATCH 110/183] Link to the Hub in d ocstring --- src/setfit/modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index ec0d7a3c..2c7a0fce 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -242,7 +242,7 @@ def __repr__(self) -> str: @dataclass class SetFitModel(PyTorchModelHubMixin): - """A SetFit model with integration to the Hugging Face Hub.""" + """A SetFit model with integration to the [Hugging Face Hub](https://huggingface.co).""" model_body: Optional[SentenceTransformer] = (None,) model_head: Optional[Union[SetFitHead, LogisticRegression]] = None From 9ffc262936bbf664218be5c4b4c0027f854ef4b0 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 27 Nov 2023 10:10:17 +0100 Subject: [PATCH 111/183] Add scikit-learn API version of SetFit to related work --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 491a26d0..34aa0a4a 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ make style && make quality ## Related work +* [https://github.com/pmbaumgartner/setfit](https://github.com/pmbaumgartner/setfit) - A scikit-learn API version of SetFit. * [jxpress/setfit-pytorch-lightning](https://github.com/jxpress/setfit-pytorch-lightning) - A PyTorch Lightning implementation of SetFit. * [davidberenstein1957/spacy-setfit](https://github.com/davidberenstein1957/spacy-setfit) - An easy and intuitive approach to use SetFit in combination with spaCy. From 2ef61bb1ff55781d966bde032715260224006878 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 27 Nov 2023 10:21:38 +0100 Subject: [PATCH 112/183] Batch Sizes + "for Inference" --- docs/source/en/_toctree.yml | 2 +- docs/source/en/how_to/batch_sizes.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 5f0367c6..11e9df0f 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -30,7 +30,7 @@ - local: how_to/knowledge_distillation title: Knowledge Distillation - local: how_to/batch_sizes - title: Batch Sizes + title: Batch Sizes for Inference - local: how_to/absa title: Aspect Based Sentiment Analysis - local: how_to/v1.0.0_migration_guide diff --git a/docs/source/en/how_to/batch_sizes.mdx b/docs/source/en/how_to/batch_sizes.mdx index 781e7a98..5f4d71c0 100644 --- a/docs/source/en/how_to/batch_sizes.mdx +++ b/docs/source/en/how_to/batch_sizes.mdx @@ -1,5 +1,5 @@ -# Batch sizes +# Batch sizes for Inference In this how-to guide we will explore the effects of increasing the batch sizes in [`SetFitModel.predict`]. ## What are they? From b8b841768ebb20316832f5e0425909ad6fec6296 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 27 Nov 2023 10:47:18 +0100 Subject: [PATCH 113/183] Make first column bold in Sampling Strategy table --- .../conceptual_guides/sampling_strategies.mdx | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/source/en/conceptual_guides/sampling_strategies.mdx b/docs/source/en/conceptual_guides/sampling_strategies.mdx index 817b0a18..e076138f 100644 --- a/docs/source/en/conceptual_guides/sampling_strategies.mdx +++ b/docs/source/en/conceptual_guides/sampling_strategies.mdx @@ -21,28 +21,28 @@ Considering that a sentence pair of `(X, Y)` and `(Y, X)` result in the same emb The resulting positive and negative pairs can be visualized in a table like below. The `+` and `-` represent positive and negative pairs, respectively. Furthermore, `h-n` represents the n-th "happy" sentence, `c-n` the n-th "content" sentence, and `s-n` the n-th "sad" sentence. Note that the area below the diagonal is not used as `(X, Y)` and `(Y, X)` result in the same embedding distances, and that the diagonal is not used as we are not interested in pairs where both sentences are identical. -| |h-1|h-2|h-3|h-4|h-5|h-6|h-7|h-8|c-1|c-2|c-3|c-4|s-1|s-2|s-3|s-4|s-5|s-6|s-7|s-8| -|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| -|h-1| | + | + | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | -|h-2| | | + | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | -|h-3| | | | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | -|h-4| | | | | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | -|h-5| | | | | | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | -|h-6| | | | | | | + | + | - | - | - | - | - | - | - | - | - | - | - | - | -|h-7| | | | | | | | + | - | - | - | - | - | - | - | - | - | - | - | - | -|h-8| | | | | | | | | - | - | - | - | - | - | - | - | - | - | - | - | -|c-1| | | | | | | | | | + | + | + | - | - | - | - | - | - | - | - | -|c-2| | | | | | | | | | | + | + | - | - | - | - | - | - | - | - | -|c-3| | | | | | | | | | | | + | - | - | - | - | - | - | - | - | -|c-4| | | | | | | | | | | | | - | - | - | - | - | - | - | - | -|s-1| | | | | | | | | | | | | | + | + | + | + | + | + | + | -|s-2| | | | | | | | | | | | | | | + | + | + | + | + | + | -|s-3| | | | | | | | | | | | | | | | + | + | + | + | + | -|s-4| | | | | | | | | | | | | | | | | + | + | + | + | -|s-5| | | | | | | | | | | | | | | | | | + | + | + | -|s-6| | | | | | | | | | | | | | | | | | | + | + | -|s-7| | | | | | | | | | | | | | | | | | | | + | -|s-8| | | | | | | | | | | | | | | | | | | | | +| |h-1|h-2|h-3|h-4|h-5|h-6|h-7|h-8|c-1|c-2|c-3|c-4|s-1|s-2|s-3|s-4|s-5|s-6|s-7|s-8| +|-------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +|**h-1**| | + | + | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|**h-2**| | | + | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|**h-3**| | | | + | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|**h-4**| | | | | + | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|**h-5**| | | | | | + | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|**h-6**| | | | | | | + | + | - | - | - | - | - | - | - | - | - | - | - | - | +|**h-7**| | | | | | | | + | - | - | - | - | - | - | - | - | - | - | - | - | +|**h-8**| | | | | | | | | - | - | - | - | - | - | - | - | - | - | - | - | +|**c-1**| | | | | | | | | | + | + | + | - | - | - | - | - | - | - | - | +|**c-2**| | | | | | | | | | | + | + | - | - | - | - | - | - | - | - | +|**c-3**| | | | | | | | | | | | + | - | - | - | - | - | - | - | - | +|**c-4**| | | | | | | | | | | | | - | - | - | - | - | - | - | - | +|**s-1**| | | | | | | | | | | | | | + | + | + | + | + | + | + | +|**s-2**| | | | | | | | | | | | | | | + | + | + | + | + | + | +|**s-3**| | | | | | | | | | | | | | | | + | + | + | + | + | +|**s-4**| | | | | | | | | | | | | | | | | + | + | + | + | +|**s-5**| | | | | | | | | | | | | | | | | | + | + | + | +|**s-6**| | | | | | | | | | | | | | | | | | | + | + | +|**s-7**| | | | | | | | | | | | | | | | | | | | + | +|**s-8**| | | | | | | | | | | | | | | | | | | | | As shown in the prior table, we have 28 positive pairs for "happy", 6 positive pairs for "content", and another 28 positive pairs for "sad". In total, this is 62 positive pairs. Also, we have 32 negative pairs between "happy" and "content", 64 negative pairs between "happy" and "sad", and 32 negative pairs between "content" and "sad". In total, this is 128 negative pairs. From a2fa84f881b69da37500a86ba0ef88a2901200f9 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 27 Nov 2023 10:55:37 +0100 Subject: [PATCH 114/183] Remove comment about Google Colab with Python 3.7 --- src/setfit/modeling.py | 2 +- src/setfit/trainer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 2c7a0fce..c2caa7f0 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Tuple, Union -# Google Colab runs on Python 3.7, so we need this to be compatible +# For Python 3.7 compatibility try: from typing import Literal except ImportError: diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index c97b143f..723aeaa9 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -45,7 +45,7 @@ from .utils import BestRun, default_hp_space_optuna -# Google Colab runs on Python 3.7, so we need this to be compatible +# For Python 3.7 compatibility try: from typing import Literal except ImportError: From e2cf782151a0dd4bef93fe676a015e8e7841463e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 27 Nov 2023 14:19:56 +0100 Subject: [PATCH 115/183] Rename file, remove distilBERT, fix typos --- ..._small_SST_2_setfit_Optimum_TensorRT.ipynb | 12652 ---------------- notebooks/setfit-onnx-optimum.ipynb | 774 + 2 files changed, 774 insertions(+), 12652 deletions(-) delete mode 100644 notebooks/bge_small_SST_2_setfit_Optimum_TensorRT.ipynb create mode 100644 notebooks/setfit-onnx-optimum.ipynb diff --git a/notebooks/bge_small_SST_2_setfit_Optimum_TensorRT.ipynb b/notebooks/bge_small_SST_2_setfit_Optimum_TensorRT.ipynb deleted file mode 100644 index 12722a26..00000000 --- a/notebooks/bge_small_SST_2_setfit_Optimum_TensorRT.ipynb +++ /dev/null @@ -1,12652 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7a40382d-295b-45e5-b694-52331d61e657", - "metadata": { - "id": "7a40382d-295b-45e5-b694-52331d61e657" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "markdown", - "id": "76571396-8f54-40ed-9e81-6c7531e6eaee", - "metadata": { - "id": "76571396-8f54-40ed-9e81-6c7531e6eaee" - }, - "source": [ - "# Efficently run SetFit Models with Optimum" - ] - }, - { - "cell_type": "markdown", - "id": "24fd5853-812f-45a4-8a7b-0a0c9a60d0a2", - "metadata": { - "id": "24fd5853-812f-45a4-8a7b-0a0c9a60d0a2" - }, - "source": [ - "[SetFit](https://github.com/huggingface/setfit) is a technique for few-shot text classification that uses contrastive learning to fine-tune Sentence Transformers in domains where little to no labeled data is available. It achieves comparable performance to existing state-of-the-art methods based on large language models, yet requires no prompts and is efficient to train (typically a few seconds on a GPU to minutes on a CPU).\n", - "\n", - "In this notebook you'll learn how to further compress SetFit models for faster inference & deployment on GPU using Optimum Onnx." - ] - }, - { - "cell_type": "markdown", - "id": "a3b30b35-7875-498f-a771-068132f4084f", - "metadata": { - "tags": [], - "id": "a3b30b35-7875-498f-a771-068132f4084f" - }, - "source": [ - "## 1. Setup development environment" - ] - }, - { - "cell_type": "markdown", - "id": "dc40c7af-1f4f-4324-847c-dc9b7797b60c", - "metadata": { - "id": "dc40c7af-1f4f-4324-847c-dc9b7797b60c" - }, - "source": [ - "Our first step is to install SetFit. Running the following cell will install all the required packages for us." - ] - }, - { - "cell_type": "code", - "source": [ - "!pip install setfit" - ], - "metadata": { - "id": "Cu9et-iSaU0i", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "3c1bd043-edf4-443d-9a44-5dc480656943" - }, - "id": "Cu9et-iSaU0i", - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Collecting setfit\n", - " Downloading setfit-0.7.0-py3-none-any.whl (45 kB)\n", - "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/45.9 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.9/45.9 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting datasets>=2.3.0 (from setfit)\n", - " Downloading datasets-2.14.6-py3-none-any.whl (493 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m493.7/493.7 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting sentence-transformers>=2.2.1 (from setfit)\n", - " Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - "Collecting evaluate>=0.3.0 (from setfit)\n", - " Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (1.23.5)\n", - "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (9.0.0)\n", - "Collecting dill<0.3.8,>=0.3.0 (from datasets>=2.3.0->setfit)\n", - " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (1.5.3)\n", - "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (2.31.0)\n", - "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (4.66.1)\n", - "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (3.4.1)\n", - "Collecting multiprocess (from datasets>=2.3.0->setfit)\n", - " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (2023.6.0)\n", - "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (3.8.6)\n", - "Collecting huggingface-hub<1.0.0,>=0.14.0 (from datasets>=2.3.0->setfit)\n", - " Downloading huggingface_hub-0.19.0-py3-none-any.whl (311 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m311.2/311.2 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (23.2)\n", - "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.3.0->setfit) (6.0.1)\n", - "Collecting responses<0.19 (from evaluate>=0.3.0->setfit)\n", - " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", - "Collecting transformers<5.0.0,>=4.6.0 (from sentence-transformers>=2.2.1->setfit)\n", - " Downloading transformers-4.35.0-py3-none-any.whl (7.9 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.9/7.9 MB\u001b[0m \u001b[31m47.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (2.1.0+cu118)\n", - "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (0.16.0+cu118)\n", - "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (1.2.2)\n", - "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (1.11.3)\n", - "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.1->setfit) (3.8.1)\n", - "Collecting sentencepiece (from sentence-transformers>=2.2.1->setfit)\n", - " Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m74.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (23.1.0)\n", - "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (3.3.2)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (6.0.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (4.0.3)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (1.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.3.0->setfit) (1.3.1)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets>=2.3.0->setfit) (3.13.1)\n", - "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets>=2.3.0->setfit) (4.5.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.3.0->setfit) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.3.0->setfit) (2.0.7)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.3.0->setfit) (2023.7.22)\n", - "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (1.12)\n", - "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (3.1.2)\n", - "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (2.1.0)\n", - "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.1->setfit) (2023.6.3)\n", - "Collecting tokenizers<0.15,>=0.14 (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.1->setfit)\n", - " Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m98.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers>=2.2.1->setfit)\n", - " Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m81.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers>=2.2.1->setfit) (8.1.7)\n", - "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers>=2.2.1->setfit) (1.3.2)\n", - "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.3.0->setfit) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.3.0->setfit) (2023.3.post1)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers>=2.2.1->setfit) (3.2.0)\n", - "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence-transformers>=2.2.1->setfit) (9.4.0)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets>=2.3.0->setfit) (1.16.0)\n", - "Collecting huggingface-hub<1.0.0,>=0.14.0 (from datasets>=2.3.0->setfit)\n", - " Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m36.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (2.1.3)\n", - "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.6.0->sentence-transformers>=2.2.1->setfit) (1.3.0)\n", - "Building wheels for collected packages: sentence-transformers\n", - " Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125923 sha256=162630274e9f88976ff1d7790162c5e378a11580d7977416bcefdcbacb07ac7a\n", - " Stored in directory: /root/.cache/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n", - "Successfully built sentence-transformers\n", - "Installing collected packages: sentencepiece, safetensors, dill, responses, multiprocess, huggingface-hub, tokenizers, transformers, datasets, sentence-transformers, evaluate, setfit\n", - "Successfully installed datasets-2.14.6 dill-0.3.7 evaluate-0.4.1 huggingface-hub-0.17.3 multiprocess-0.70.15 responses-0.18.0 safetensors-0.4.0 sentence-transformers-2.2.2 sentencepiece-0.1.99 setfit-0.7.0 tokenizers-0.14.1 transformers-4.35.0\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "id": "1832e3f2-09e7-44b5-a438-87ea658d49ea", - "metadata": { - "id": "1832e3f2-09e7-44b5-a438-87ea658d49ea" - }, - "source": [ - "While we're at it, let's turn off some of the warnings from the 🤗 Datasets library and the tokenizers:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b67e17b0-743a-47ba-9d55-9baea453a0d8", - "metadata": { - "id": "b67e17b0-743a-47ba-9d55-9baea453a0d8", - "outputId": "4a68c21f-eb4e-4652-a1d0-bcf1eb8b9682", - "colab": { - "base_uri": "https://localhost:8080/" - } - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "env: TOKENIZERS_PARALLELISM=false\n" - ] - } - ], - "source": [ - "import datasets\n", - "\n", - "datasets.logging.set_verbosity_error()\n", - "\n", - "%env TOKENIZERS_PARALLELISM=false" - ] - }, - { - "cell_type": "markdown", - "id": "c442bab8-3ae1-4d1d-9ebe-723d7a829bae", - "metadata": { - "id": "c442bab8-3ae1-4d1d-9ebe-723d7a829bae" - }, - "source": [ - "To be able to share your model with the community, there are a few more steps to follow.\n", - "\n", - "First, you have to store your authentication token from the Hugging Face Hub (sign up here if you haven't already!). To do so, execute the following cell and input an **access token with write permissions** associated with your account:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "004da15f-6c49-4e7a-b1f6-b285e22eb6cb", - "metadata": { - "id": "004da15f-6c49-4e7a-b1f6-b285e22eb6cb", - "outputId": "3a5ea685-01e8-495b-a7e3-dc0ce8f87072", - "colab": { - "base_uri": "https://localhost:8080/", - "referenced_widgets": [ - "c22db575e86b45229ab97eb5f5193fe1", - "1da311f97a57441988a622d5b019fa5b", - "458a3ee5ff8d44fca9ca6f4888d79174", - "09d3ebe126e2484d8cc50653f7d8f8fb", - "f867a0c844b547e8bd1d294245c4fa13", - "1993399b93224ec6865eaa01b6a73d8b", - "b35a241e895c41ca8be0f334615df0a3", - "99ef6333ea314434a1dd3ef05fa0e42b", - "ac1617fabea24f8a912e7cb6bb9abca4", - "108c04b373e243218924ab3b101e8908", - "16019f898e814155ae1c27ab2475565a", - "d1df847b4fbe4ab992c64d5d2262cbcf", - "eeb36e1cf46b46328725481ce4b9a23b", - "58c9a6969edf4b5ab636c0fdcb9b5b5b", - "abe2bd79c6574e5aafa865bd18f14221", - "317f736683394e5cb974117fd8f5e7fe", - "a34b80886d56489cb82c36ba1415841c" - ] - } - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "VBox(children=(HTML(value='
:30: MatplotlibDeprecationWarning: The legendHandles attribute was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use legend_handles instead.\n", - " for handle in legend.legendHandles:\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/0ElEQVR4nO3deXgV5f3//9fJShKykIRsNIQQIOyyytYilWhAioCogLQsQUBMRVSkxYpKwVKou/YL0tKwaBDpR1RsBQkaFkVEBKKCASISEAItkhwCSchy//7wx6lHAibkhJMMz8d1zXUxM/fc8z6ZxPNy5p4ZmzHGCAAAwKI83F0AAABAbSLsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAASyPsAAAAS3Nr2Dlz5oymTZumuLg4+fn5qXfv3tqxY4dj/bhx42Sz2ZymAQMGuLFiAABQ33i5c+d33323vvjiC61YsUIxMTF65ZVXlJSUpL1796pJkyaSpAEDBigtLc2xja+vr7vKBQAA9ZDNXS8CLSoqUmBgoN566y0NGjTIsbxr164aOHCg5s6dq3Hjxik/P19vvvmmO0oEAAAW4LYzO2VlZSovL1eDBg2clvv5+Wnr1q2O+czMTEVERKhRo0a68cYbNXfuXIWFhV2y35KSEpWUlDjmKyoq9N133yksLEw2m831HwQAALicMUZnzpxRTEyMPDxqOOrGuFGvXr3MDTfcYL799ltTVlZmVqxYYTw8PEyrVq2MMcasXLnSvPXWWyYrK8usWbPGtGnTxnTv3t2UlZVdss/HH3/cSGJiYmJiYmKywHTkyJEa5w23XcaSpJycHKWkpGjz5s3y9PRUly5d1KpVK+3cuVP79u27qP3XX3+thIQEZWRkqH///pX2+eMzOwUFBWratKmOHDmioKCgWvssAADAdex2u2JjY5Wfn6/g4OAa9eXWAcoJCQnatGmTzp49K7vdrujoaI0YMULNmzevtH3z5s0VHh6ugwcPXjLs+Pr6VjqIOSgoiLADAEA944ohKHXiOTsBAQGKjo7W6dOntX79eg0ZMqTSdkePHtWpU6cUHR19lSsEAAD1lVvP7Kxfv17GGCUmJurgwYN6+OGH1bp1a40fP16FhYWaPXu2hg8frqioKOXk5GjGjBlq0aKFkpOT3Vk2AACoR9x6ZqegoECpqalq3bq1xowZo5///Odav369vL295enpqaysLN16661q1aqVJkyYoK5du2rLli08awcAAFSZWwcoXw12u13BwcEqKChgzA4ASysvL1dpaam7ywCq5MKJjUtx5fe3Wy9jAQBqzhijvLw85efnu7sUoFpCQkIUFRVV68/BI+wAQD13IehERETI39+fB6iizjPG6Ny5czp58qQk1fqNR4QdAKjHysvLHUHnck+XB+oaPz8/SdLJkycVERFx2UtaNVUnbj0HAFyZC2N0/P393VwJUH0Xfm9re6wZYQcALIBLV6iPrtbvLWEHAABYGmEHAOAW/fr107Rp09xdxlWzdOlShYSEOOafeOIJderUyW311IZmzZrpueeec3cZFyHsAABQB33zzTey2WyOKSwsTDfffLN27dpVpe2bNWvmtP2Pp3HjxtXuB6hDuBsLAIA6LCMjQ+3atdPRo0c1depUDRw4UF999ZXTWaLK7NixQ+Xl5ZKkjz76SMOHD1d2drbjAX0X7oaqqtLSUnl7e1/RZ3A3zuwAABxOnz2vQ/89q9Nnz1+V/ZWVlem3v/2tgoODFR4erlmzZumHD/Y/fvy4Bg0aJD8/P8XHxys9Pf2iSyX5+fm6++671bhxYwUFBenGG2/Unj17LrvfzMxMXX/99QoICFBISIj69Omjw4cPS/rf5aV//OMfatq0qRo2bKh7771X5eXlWrBggaKiohQREaEnn3zSqc9nnnlGHTp0UEBAgGJjY3XvvfeqsLCwxj+jsLAwRUVFqVu3bnrqqad04sQJbd++XX/84x/Vvn37i9p36tRJs2bNUuPGjRUVFaWoqCiFhoZKkiIiIhzL0tPTlZCQIB8fHyUmJmrFihVO/dhsNi1cuFC33nqrAgICHJ937dq16t69uxo0aKDw8HANGzbMabtz584pJSVFgYGBatq0qRYvXlzjn0FNEXYAACouLdc/dx7R/HVf6dkN2Zq/7iv9c+cRFZeW1+p+ly1bJi8vL33yySd6/vnn9cwzz+jvf/+7Y/2YMWN07NgxZWZm6v/+7/+0ePFix4PoLrjjjjt08uRJvfvuu9q5c6e6dOmi/v3767vvvqt0n2VlZRo6dKhuuOEGZWVladu2bZo0aZLTnUE5OTl69913tW7dOq1cuVJLlizRoEGDdPToUW3atEnz58/Xo48+qu3btzu28fDw0AsvvKAvv/xSy5Yt0/vvv68ZM2a49Od14WzM+fPnlZKSon379mnHjh2O9bt27VJWVpbGjx9/2X7WrFmj+++/Xw899JC++OILTZ48WePHj9cHH3zg1O6JJ57QsGHD9PnnnyslJUX/+te/NGzYMN1yyy3atWuXNm7cqOuvv95pm6efflrdunXTrl27dO+992rKlCnKzs520U/gChmLKygoMJJMQUGBu0sBAJcrKioye/fuNUVFRTXqZ/WnuWbS8h1m5v9lmXn/3mtm/l+WmbR8h1n9aa6LKr3YDTfcYNq0aWMqKiocy373u9+ZNm3aGGOM2bdvn5FkduzY4Vh/4MABI8k8++yzxhhjtmzZYoKCgkxxcbFT3wkJCebll1+udL+nTp0ykkxmZmal6x9//HHj7+9v7Ha7Y1lycrJp1qyZKS8vdyxLTEw08+bNu+TnW716tQkLC3PMp6WlmeDgYKf9XHfddZfc/tChQ0aS2bVrlzHGmNOnT5thw4aZhg0bmry8PGOMMQMHDjRTpkxxbHPfffeZfv36XdTXBx98YCSZ06dPG2OM6d27t5k4caJTmzvuuMPccsstjnlJZtq0aU5tevXqZUaPHn3JmuPi4syvf/1rx3xFRYWJiIgwCxcurLT95X5/Xfn9zZkdALjGnT57Xp9+c1phAb5qHOgrXy9PNQ70VViAr3Z+c7pWL2n17NnT6YxKr169dODAAZWXlys7O1teXl7q0qWLY32LFi3UqFEjx/yePXtUWFiosLAwNWzY0DEdOnRIOTk5ys3NdVr+pz/9SaGhoRo3bpySk5M1ePBgPf/88zp+/LhTXc2aNVNgYKBjPjIyUm3btpWHh4fTsh+eZcrIyFD//v3VpEkTBQYG6je/+Y1OnTqlc+fO1ehn1Lt3bzVs2FCNGjXSnj17tGrVKkVGRkqSJk6cqJUrV6q4uFjnz59Xenq6UlJSfrLPffv2qU+fPk7L+vTpo3379jkt69atm9P87t271b9//8v23bFjR8e/bTaboqKiLjobd7UxQBkArnH5RaU6d75MMSHOA1aD/Lx0LL9I+UWlahTg46bqLq+wsFDR0dHKzMy8aF1ISIhCQkK0e/dux7ILY1fS0tI0depUrVu3TqtWrdKjjz6qDRs2qGfPnpJ00UBcm81W6bKKigpJ39859atf/UpTpkzRk08+qdDQUG3dulUTJkzQ+fPna/SE61WrVqlt27YKCwu7aFDy4MGD5evrqzVr1sjHx0elpaW6/fbbr3hfPxYQEOA0X5VBzZf7ObkLYQcArnEhft7y9/GSvahMjQP/934ie1GZAny8FOJXe3fg/HDMiyR9/PHHatmypTw9PZWYmKiysjLt2rVLXbt2lSQdPHhQp0+fdrTv0qWL8vLy5OXlpWbNmlW6jxYtWlS6vHPnzurcubNmzpypXr16KT093RF2qmvnzp2qqKjQ008/7Tj78/rrr19RXz8WGxurhISEStd5eXlp7NixSktLk4+Pj0aOHFmlQNKmTRt9+OGHGjt2rGPZhx9+qLZt2152u44dO2rjxo0/OSaoriHsAMA1rlGAj7o1a6QNe09I+v6Mjr2oTKfOluimtpG1elYnNzdXDz74oCZPnqzPPvtML774op5++mlJUuvWrZWUlKRJkyZp4cKF8vb21kMPPSQ/Pz/Hpa+kpCT16tVLQ4cO1YIFC9SqVSsdO3bMMZD2x5dhJOnQoUNavHixbr31VsXExCg7O1sHDhzQmDFjrvhztGjRQqWlpXrxxRc1ePBgffjhh1q0aNEV91cdd999t9q0aSPp+8BSFQ8//LDuvPNOde7cWUlJSVq7dq3eeOMNZWRkXHa7xx9/XP3791dCQoJGjhypsrIy/fvf/9bvfve7Gn+O2sSYHQCAftUxRje1jZQxRsfyi2SM0U1tI/WrjjG1ut8xY8aoqKhI119/vVJTU3X//fdr0qRJjvXLly9XZGSk+vbtq2HDhmnixIkKDAxUgwYNJH1/ieTf//63+vbtq/Hjx6tVq1YaOXKkDh8+7BjX8mP+/v766quvNHz4cLVq1UqTJk1SamqqJk+efMWf47rrrtMzzzyj+fPnq3379nr11Vc1b968K+6vOlq2bKnevXurdevW6tGjR5W2GTp0qJ5//nk99dRTateunV5++WWlpaWpX79+l92uX79+Wr16td5++2116tRJN954oz755BMXfIraZTPmBw80sCC73a7g4GAVFBQ4HqQEAFZRXFysQ4cOKT4+3hEAauL02fPKLypViJ93nRync/ToUcXGxjoGA0Myxqhly5a699579eCDD7q7nGq53O+vK7+/uYwFAHBoFOBTp0LO+++/r8LCQnXo0EHHjx/XjBkz1KxZM/Xt29fdpdUJ//nPf/Taa68pLy+v3o2juZoIOwCAOqu0tFSPPPKIvv76awUGBqp379569dVX6+1rC1wtIiJC4eHhWrx4sdMt+XBG2AEA1FnJyclKTk52dxl1lsVHorgMA5QBAIClEXYAwAL4P3zUR1fr95awAwD12IWxKzV9JQHgDhd+b2t7DBZjdgCgHvP09FRISIjj3UP+/v5O75oC6iJjjM6dO6eTJ08qJCREnp6eP71RDRB2AKCei4qKkiS3v2wRqK6QkBDH729tIuwAQD1ns9kUHR2tiIgIlZaWurscoEq8vb1r/YzOBYQdALAIT0/Pq/blAdQnDFAGAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBAACWRtgBcM2qqODlmcC1gIcKArhmlJZXKDvvjHYePq3c787qfFmFfLw81DQ0QF3jGikxKlDenvw/IGA1hB0AlmeM0e4j+Xpvb56OfFekigqjwAbe8vSwqbC4XNu/PqUd33yn2FA/3dw2Sp2bNnJ3yQBciLADwNKMMdp68L9a89m3Kq2oUJMQPzXwdn6lQlRwAxWXluvo6SKt2HZYhSVl+kXLxm6qGICrcb4WgKXtPpKvNZ99Ky9Pm5qHN7wo6FzQwNtTzcMbysvTpjW7vtXuI/lXt1AAtYawA8CySssr9N7ePJVWVCg62K9K20QH+32/3Zd5Ki2vqOUKAVwNhB0AlpWdd0ZHvitSTBWDzgUxwX7K/e6csvPO1FJlAK4mwg4Ay9p5+LQqKswlL11dSgNvT1VUGO08fLqWKgNwNRF2AFhW7ndnFdjA+4q2bdjAS7nfnXNxRQDcgbADwJIqKozOl1XI08N2Rdt7eth0vqycBw8CFkDYAWBJHh42+Xh5qPwKw0p5hZGPl6c8rjAsAag7CDsALKtpaIDOFJde0baFxWVqGurv4ooAuINbw86ZM2c0bdo0xcXFyc/PT71799aOHTsc640xeuyxxxQdHS0/Pz8lJSXpwIEDbqwYQH3SNa6RPDxsKi4tr9Z2xaXl8vCwqWscT1IGrMCtYefuu+/Whg0btGLFCn3++ee6+eablZSUpG+//VaStGDBAr3wwgtatGiRtm/froCAACUnJ6u4uNidZQOoJxKjAhUb6qdjBUXV2u5YQZGahvorMSqwlioDcDXZjDFuGX1XVFSkwMBAvfXWWxo0aJBjedeuXTVw4EDNmTNHMTExeuihhzR9+nRJUkFBgSIjI7V06VKNHDmySvux2+0KDg5WQUGBgoKCauWzAKi7duWe1opth+XlaavSgwWPFxSprMJoTK9m6hQbUvsFAqiUK7+/3XZmp6ysTOXl5WrQoIHTcj8/P23dulWHDh1SXl6ekpKSHOuCg4PVo0cPbdu27ZL9lpSUyG63O00Arl2dYkM0rEsTlZUbff3fwkte0iouLdfX/y1UWYXRsM5NCDqAhbjtRaCBgYHq1auX5syZozZt2igyMlIrV67Utm3b1KJFC+Xl5UmSIiMjnbaLjIx0rKvMvHnzNHv27FqtHUD9YbPZ9IuWjRXYwFvvfZmn3O/OqaLCqGEDL3l62FReYVRYXCYPD5uahvrr5nZRBB3AYtz61vMVK1YoJSVFTZo0kaenp7p06aJRo0Zp586dV9znzJkz9eCDDzrm7Xa7YmNjXVEugHqsU2yI2sUEKTvvjHYePq3c787pfFm5/H281C4mWF3jGikxKlDentykCliNW8NOQkKCNm3apLNnz8putys6OlojRoxQ8+bNFRUVJUk6ceKEoqOjHducOHFCnTp1umSfvr6+8vX1re3SAdRD3p4eat8kWO2bBEv6/sGDPEcHsL468b8wAQEBio6O1unTp7V+/XoNGTJE8fHxioqK0saNGx3t7Ha7tm/frl69ermxWgBWQdABrg1uPbOzfv16GWOUmJiogwcP6uGHH1br1q01fvx42Ww2TZs2TXPnzlXLli0VHx+vWbNmKSYmRkOHDnVn2QAAoB5xa9gpKCjQzJkzdfToUYWGhmr48OF68skn5e39/Yv7ZsyYobNnz2rSpEnKz8/Xz3/+c61bt+6iO7gAAAAuxW3P2blaeM4OAAD1jyWeswMAAHA1EHYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAIClEXYAAICluTXslJeXa9asWYqPj5efn58SEhI0Z84cGWMcbcaNGyebzeY0DRgwwI1VAwCA+sTLnTufP3++Fi5cqGXLlqldu3b69NNPNX78eAUHB2vq1KmOdgMGDFBaWppj3tfX1x3lAgCAesitYeejjz7SkCFDNGjQIElSs2bNtHLlSn3yySdO7Xx9fRUVFeWOEgEAQD3n1stYvXv31saNG7V//35J0p49e7R161YNHDjQqV1mZqYiIiKUmJioKVOm6NSpU+4oFwAA1ENuPbPz+9//Xna7Xa1bt5anp6fKy8v15JNPavTo0Y42AwYM0G233ab4+Hjl5OTokUce0cCBA7Vt2zZ5enpe1GdJSYlKSkoc83a7/ap8FgAAUDe5Ney8/vrrevXVV5Wenq527dpp9+7dmjZtmmJiYjR27FhJ0siRIx3tO3TooI4dOyohIUGZmZnq37//RX3OmzdPs2fPvmqfAQAA1G0288Nbn66y2NhY/f73v1dqaqpj2dy5c/XKK6/oq6++uuR2jRs31ty5czV58uSL1lV2Zic2NlYFBQUKCgpy7QcAAAC1wm63Kzg42CXf3249s3Pu3Dl5eDgPG/L09FRFRcUltzl69KhOnTql6OjoStf7+vpytxYAAHBwa9gZPHiwnnzySTVt2lTt2rXTrl279MwzzyglJUWSVFhYqNmzZ2v48OGKiopSTk6OZsyYoRYtWig5OdmdpQMAgHrCrZexzpw5o1mzZmnNmjU6efKkYmJiNGrUKD322GPy8fFRUVGRhg4dql27dik/P18xMTG6+eabNWfOHEVGRlZpH648DQYAAK4OV35/uzXsXA2EHQAA6h9Xfn/zbiwAAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBphB0AAGBpXtVpXFFRoU2bNmnLli06fPiwzp07p8aNG6tz585KSkpSbGxsbdUJAABwRap0ZqeoqEhz585VbGysbrnlFr377rvKz8+Xp6enDh48qMcff1zx8fG65ZZb9PHHH9d2zQAAAFVWpTM7rVq1Uq9evfS3v/1NN910k7y9vS9qc/jwYaWnp2vkyJH6wx/+oIkTJ7q8WAAAgOqyGWPMTzXat2+f2rRpU6UOS0tLlZubq4SEhBoX5wp2u13BwcEqKChQUFCQu8sBAABV4Mrv7ypdxqpq0JEkb2/vOhN0AAAAqjVA+YfKysr08ssvKzMzU+Xl5erTp49SU1PVoEEDV9YHAABQI1ccdqZOnar9+/frtttuU2lpqZYvX65PP/1UK1eudGV9AAAANVLlsLNmzRoNGzbMMf/ee+8pOztbnp6ekqTk5GT17NnT9RUCAADUQJUfKviPf/xDQ4cO1bFjxyRJXbp00T333KN169Zp7dq1mjFjhrp3715rhQIAAFyJKoedtWvXatSoUerXr59efPFFLV68WEFBQfrDH/6gWbNmKTY2Vunp6bVZKwAAQLVV6dbzH8rPz9eMGTO0Z88eLVq0SJ07d66t2lyCW88BAKh/rvqt5z8UEhKixYsX6y9/+YvGjBmjhx9+WMXFxTUqAgAAoLZUOezk5ubqzjvvVIcOHTR69Gi1bNlSO3fulL+/v6677jq9++67tVknAADAFanyZax+/fopKipK48aN0/r165WTk6O3335b0vdPWJ48ebKioqL0+uuv12rB1cVlLAAA6h9Xfn9X+dbzTz/9VHv27FFCQoKSk5MVHx/vWNemTRtt3rxZixcvrlExAAAArlblsNO1a1c99thjGjt2rDIyMtShQ4eL2kyaNMmlxQEAANRUlcfsLF++XCUlJXrggQf07bff6uWXX67NugAAAFyiymd24uLi9M9//rM2awEAAHC5Kp3ZOXv2bLU6rW57AACA2lKlsNOiRQv9+c9/1vHjxy/ZxhijDRs2aODAgXrhhRdcViAAAEBNVOkyVmZmph555BE98cQTuu6669StWzfFxMSoQYMGOn36tPbu3att27bJy8tLM2fO1OTJk2u7bgAAgCqp1usicnNztXr1am3ZskWHDx9WUVGRwsPD1blzZyUnJ2vgwIGOt6DXFTxnBwCA+seV39/VfjdWfUPYAQCg/nHru7EAAADqE8IOAACwNMIOAACwNMIOAACwNMIOAACwtGqHnWbNmumPf/yjcnNza7zz8vJyzZo1S/Hx8fLz81NCQoLmzJmjH94gZozRY489pujoaPn5+SkpKUkHDhyo8b4BAMC1odphZ9q0aXrjjTfUvHlz3XTTTXrttddUUlJyRTufP3++Fi5cqJdeekn79u3T/PnztWDBAr344ouONgsWLNALL7ygRYsWafv27QoICFBycrKKi4uvaJ8AAODacsXP2fnss8+0dOlSrVy5UuXl5brrrruUkpKiLl26VLmPX/3qV4qMjNSSJUscy4YPHy4/Pz+98sorMsYoJiZGDz30kKZPny5JKigoUGRkpJYuXaqRI0f+5D54zg4AAPVPnXjOTpcuXfTCCy/o2LFjevzxx/X3v/9d3bt3V6dOnfSPf/xDVclQvXv31saNG7V//35J0p49e7R161YNHDhQknTo0CHl5eUpKSnJsU1wcLB69Oihbdu2VdpnSUmJ7Ha70wQAAK5dVXo3VmVKS0u1Zs0apaWlacOGDerZs6cmTJigo0eP6pFHHlFGRobS09Mv28fvf/972e12tW7dWp6eniovL9eTTz6p0aNHS5Ly8vIkSZGRkU7bRUZGOtb92Lx58zR79uwr/VgAAMBiqh12PvvsM6WlpWnlypXy8PDQmDFj9Oyzz6p169aONsOGDVP37t1/sq/XX39dr776qtLT09WuXTvt3r1b06ZNU0xMjMaOHVvd0iRJM2fO1IMPPuiYt9vtio2NvaK+AABA/VftsNO9e3fddNNNWrhwoYYOHSpvb++L2sTHx1dpPM3DDz+s3//+9462HTp00OHDhzVv3jyNHTtWUVFRkqQTJ04oOjrasd2JEyfUqVOnSvv09fWVr69vdT8WAACwqGqHna+//lpxcXGXbRMQEKC0tLSf7OvcuXPy8HAeNuTp6amKigpJ34emqKgobdy40RFu7Ha7tm/frilTplS3dAAAcA2qdtg5efKk8vLy1KNHD6fl27dvl6enp7p161blvgYPHqwnn3xSTZs2Vbt27bRr1y4988wzSklJkSTZbDZNmzZNc+fOVcuWLRUfH69Zs2YpJiZGQ4cOrW7pAADgGlTtu7FSU1N15MiRi5Z/++23Sk1NrVZfL774om6//Xbde++9atOmjaZPn67Jkydrzpw5jjYzZszQfffdp0mTJql79+4qLCzUunXr1KBBg+qWDgAArkHVfs5Ow4YNlZWVpebNmzstP3TokDp27KgzZ864tMCa4jk7AADUP259zo6vr69OnDhx0fLjx4/Ly+uK72QHAACoFdUOOzfffLNmzpypgoICx7L8/Hw98sgjuummm1xaHAAAQE1V+1TMU089pb59+youLk6dO3eWJO3evVuRkZFasWKFywsEAACoiWqHnSZNmigrK0uvvvqq9uzZIz8/P40fP16jRo2q9Jk7AAAA7nRFg2wCAgI0adIkV9cCAADgclc8onjv3r3Kzc3V+fPnnZbfeuutNS4KAADAVa7oCcrDhg3T559/LpvN5ni7uc1mkySVl5e7tkIAAIAaqPbdWPfff7/i4+N18uRJ+fv768svv9TmzZvVrVs3ZWZm1kKJAAAAV67aZ3a2bdum999/X+Hh4fLw8JCHh4d+/vOfa968eZo6dap27dpVG3UCAABckWqf2SkvL1dgYKAkKTw8XMeOHZMkxcXFKTs727XVAQAA1FC1z+y0b99ee/bsUXx8vHr06KEFCxbIx8dHixcvvugVEgAAAO5W7bDz6KOP6uzZs5KkP/7xj/rVr36lX/ziFwoLC9OqVatcXiAAAEBNVPtFoJX57rvv1KhRI8cdWXUJLwIFAKD+cduLQEtLS+Xl5aUvvvjCaXloaGidDDoAAADVCjve3t5q2rQpz9IBAAD1RrXvxvrDH/6gRx55RN99911t1AMAAOBS1R6g/NJLL+ngwYOKiYlRXFycAgICnNZ/9tlnLisOAACgpqoddoYOHVoLZQAAANQOl9yNVZdxNxYAAPWP2+7GAgAAqG+qfRnLw8PjsreZc6cWAACoS6oddtasWeM0X1paql27dmnZsmWaPXu2ywoDAABwBZeN2UlPT9eqVav01ltvuaI7l2HMDgAA9U+dHLPTs2dPbdy40VXdAQAAuIRLwk5RUZFeeOEFNWnSxBXdAQAAuEy1x+z8+IWfxhidOXNG/v7+euWVV1xaHAAAQE1VO+w8++yzTmHHw8NDjRs3Vo8ePdSoUSOXFgcAAFBT1Q4748aNq4UyAAAAake1x+ykpaVp9erVFy1fvXq1li1b5pKiAAAAXKXaYWfevHkKDw+/aHlERIT+9Kc/uaQoAAAAV6l22MnNzVV8fPxFy+Pi4pSbm+uSogAAAFyl2mEnIiJCWVlZFy3fs2ePwsLCXFIUAACAq1Q77IwaNUpTp07VBx98oPLycpWXl+v999/X/fffr5EjR9ZGjQAAAFes2ndjzZkzR99884369+8vL6/vN6+oqNCYMWMYswMAAOqcK3431oEDB7R79275+fmpQ4cOiouLc3VtLsG7sQAAqH9c+f1d7TM7F7Rs2VItW7as0c4BAABqW7XH7AwfPlzz58+/aPmCBQt0xx13uKQoAAAAV6l22Nm8ebNuueWWi5YPHDhQmzdvdklRAAAArlLtsFNYWCgfH5+Llnt7e8tut7ukKAAAAFepdtjp0KGDVq1addHy1157TW3btnVJUQAAAK5S7QHKs2bN0m233aacnBzdeOONkqSNGzdq5cqVlb4zCwAAwJ2qHXYGDx6sN998U3/605/0z3/+U35+furYsaMyMjJ0ww031EaNAAAAV+yKn7NTmS+++ELt27d3VXcuwXN2AACof1z5/V3tMTs/dubMGS1evFjXX3+9rrvuupp2BwAA4FJXHHY2b96sMWPGKDo6Wk899ZRuvPFGffzxx66sDQAAoMaqNWYnLy9PS5cu1ZIlS2S323XnnXeqpKREb775JndiAQCAOqnKZ3YGDx6sxMREZWVl6bnnntOxY8f04osv1mZtAAAANVblsPPuu+9qwoQJmj17tgYNGiRPT88a77xZs2ay2WwXTampqZKkfv36XbTunnvuqfF+AQDAtaPKYWfr1q06c+aMunbtqh49euill17Sf//73xrtfMeOHTp+/Lhj2rBhgyQ5vWNr4sSJTm0WLFhQo30CAIBrS5XDTs+ePfW3v/1Nx48f1+TJk/Xaa68pJiZGFRUV2rBhg86cOVPtnTdu3FhRUVGO6Z133lFCQoLT83r8/f2d2nD7OAAAqI5q340VEBCglJQUbd26VZ9//rkeeugh/fnPf1ZERIRuvfXWKy7k/PnzeuWVV5SSkiKbzeZY/uqrryo8PFzt27fXzJkzde7cucv2U1JSIrvd7jQBAIBrV42es5OYmKgFCxbo6NGjWrlyZY0KefPNN5Wfn69x48Y5lt1111165ZVX9MEHH2jmzJlasWKFfv3rX1+2n3nz5ik4ONgxxcbG1qguAABQv7n0Cco1kZycLB8fH61du/aSbd5//331799fBw8eVEJCQqVtSkpKVFJS4pi32+2KjY3lCcoAANQjrnyCcrXfjVUbDh8+rIyMDL3xxhuXbdejRw9JumzY8fX1la+vr8trBAAA9VONXxfhCmlpaYqIiNCgQYMu22737t2SpOjo6KtQFQAAsAK3n9mpqKhQWlqaxo4dKy+v/5WTk5Oj9PR03XLLLQoLC1NWVpYeeOAB9e3bVx07dnRjxQAAoD5xe9jJyMhQbm6uUlJSnJb7+PgoIyNDzz33nM6ePavY2FgNHz5cjz76qJsqBQAA9VGdGaBcW1w5wAkAAFwdrvz+rhNjdgAAAGoLYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFgaYQcAAFiaW8NOs2bNZLPZLppSU1MlScXFxUpNTVVYWJgaNmyo4cOH68SJE+4sGQAA1DNuDTs7duzQ8ePHHdOGDRskSXfccYck6YEHHtDatWu1evVqbdq0SceOHdNtt93mzpIBAEA9YzPGGHcXccG0adP0zjvv6MCBA7Lb7WrcuLHS09N1++23S5K++uortWnTRtu2bVPPnj2r1KfdbldwcLAKCgoUFBRUm+UDAAAXceX3d50Zs3P+/Hm98sorSklJkc1m086dO1VaWqqkpCRHm9atW6tp06batm3bJfspKSmR3W53mgAAwLWrzoSdN998U/n5+Ro3bpwkKS8vTz4+PgoJCXFqFxkZqby8vEv2M2/ePAUHBzum2NjYWqwaAADUdXUm7CxZskQDBw5UTExMjfqZOXOmCgoKHNORI0dcVCEAAKiPvNxdgCQdPnxYGRkZeuONNxzLoqKidP78eeXn5zud3Tlx4oSioqIu2Zevr698fX1rs1wAAFCP1IkzO2lpaYqIiNCgQYMcy7p27Spvb29t3LjRsSw7O1u5ubnq1auXO8oEAAD1kNvP7FRUVCgtLU1jx46Vl9f/ygkODtaECRP04IMPKjQ0VEFBQbrvvvvUq1evKt+JBQAA4Pawk5GRodzcXKWkpFy07tlnn5WHh4eGDx+ukpISJScn6//9v//nhioBAEB9Vaees1MbeM4OAAD1jyWfswMAAFAbCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDSCDsAAMDS3B52vv32W/36179WWFiY/Pz81KFDB3366aeO9ePGjZPNZnOaBgwY4MaKAQBAfeLlzp2fPn1affr00S9/+Uu9++67aty4sQ4cOKBGjRo5tRswYIDS0tIc876+vle7VAAAUE+5NezMnz9fsbGxTkEmPj7+ona+vr6Kioq6mqUBAACLcOtlrLffflvdunXTHXfcoYiICHXu3Fl/+9vfLmqXmZmpiIgIJSYmasqUKTp16tQl+ywpKZHdbneaAADAtcutYefrr7/WwoUL1bJlS61fv15TpkzR1KlTtWzZMkebAQMGaPny5dq4caPmz5+vTZs2aeDAgSovL6+0z3nz5ik4ONgxxcbGXq2PAwAA6iCbMca4a+c+Pj7q1q2bPvroI8eyqVOnaseOHdq2bVul23z99ddKSEhQRkaG+vfvf9H6kpISlZSUOObtdrtiY2NVUFCgoKAg138IAADgcna7XcHBwS75/nbrmZ3o6Gi1bdvWaVmbNm2Um5t7yW2aN2+u8PBwHTx4sNL1vr6+CgoKcpoAAMC1y61hp0+fPsrOznZatn//fsXFxV1ym6NHj+rUqVOKjo6u7fIAAIAFuDXsPPDAA/r444/1pz/9SQcPHlR6eroWL16s1NRUSVJhYaEefvhhffzxx/rmm2+0ceNGDRkyRC1atFBycrI7SwcAAPWEW8NO9+7dtWbNGq1cuVLt27fXnDlz9Nxzz2n06NGSJE9PT2VlZenWW29Vq1atNGHCBHXt2lVbtmzhWTsAAKBK3DpA+Wpw5QAnAABwdVhmgDIAAEBtI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABLI+wAAABL83J3AbXNGCNJstvtbq4EAABU1YXv7Qvf4zVh+bBz5swZSVJsbKybKwEAANV16tQpBQcH16gPm3FFZKrDKioqdOzYMQUGBspms7m7nHrNbrcrNjZWR44cUVBQkLvLwQ9wbOoujk3dxvGpuwoKCtS0aVOdPn1aISEhNerL8md2PDw89LOf/czdZVhKUFAQ/1Goozg2dRfHpm7j+NRdHh41H17MAGUAAGBphB0AAGBphB1Uma+vrx5//HH5+vq6uxT8CMem7uLY1G0cn7rLlcfG8gOUAQDAtY0zOwAAwNIIOwAAwNIIOwAAwNIIOwAAwNIIO7isJ554QjabzWlq3bq1u8u6Zm3evFmDBw9WTEyMbDab3nzzTaf1xhg99thjio6Olp+fn5KSknTgwAH3FHuN+aljM27cuIv+lgYMGOCeYq8x8+bNU/fu3RUYGKiIiAgNHTpU2dnZTm2Ki4uVmpqqsLAwNWzYUMOHD9eJEyfcVPG1oyrHpl+/fhf97dxzzz3V2g9hBz+pXbt2On78uGPaunWru0u6Zp09e1bXXXed/vrXv1a6fsGCBXrhhRe0aNEibd++XQEBAUpOTlZxcfFVrvTa81PHRpIGDBjg9Le0cuXKq1jhtWvTpk1KTU3Vxx9/rA0bNqi0tFQ333yzzp4962jzwAMPaO3atVq9erU2bdqkY8eO6bbbbnNj1deGqhwbSZo4caLT386CBQuqtyMDXMbjjz9urrvuOneXgUpIMmvWrHHMV1RUmKioKPOXv/zFsSw/P9/4+vqalStXuqHCa9ePj40xxowdO9YMGTLELfXA2cmTJ40ks2nTJmPM938n3t7eZvXq1Y42+/btM5LMtm3b3FXmNenHx8YYY2644QZz//3316hfzuzgJx04cEAxMTFq3ry5Ro8erdzcXHeXhEocOnRIeXl5SkpKciwLDg5Wjx49tG3bNjdWhgsyMzMVERGhxMRETZkyRadOnXJ3SdekgoICSVJoaKgkaefOnSotLXX622ndurWaNm3K385V9uNjc8Grr76q8PBwtW/fXjNnztS5c+eq1a/lXwSKmunRo4eWLl2qxMREHT9+XLNnz9YvfvELffHFFwoMDHR3efiBvLw8SVJkZKTT8sjISMc6uM+AAQN02223KT4+Xjk5OXrkkUc0cOBAbdu2TZ6enu4u75pRUVGhadOmqU+fPmrfvr2k7/92fHx8LnqzNn87V1dlx0aS7rrrLsXFxSkmJkZZWVn63e9+p+zsbL3xxhtV7puwg8saOHCg498dO3ZUjx49FBcXp9dff10TJkxwY2VA/TJy5EjHvzt06KCOHTsqISFBmZmZ6t+/vxsru7akpqbqiy++YOxhHXSpYzNp0iTHvzt06KDo6Gj1799fOTk5SkhIqFLfXMZCtYSEhKhVq1Y6ePCgu0vBj0RFRUnSRXeQnDhxwrEOdUfz5s0VHh7O39JV9Nvf/lbvvPOOPvjgA/3sZz9zLI+KitL58+eVn5/v1J6/navnUsemMj169JCkav3tEHZQLYWFhcrJyVF0dLS7S8GPxMfHKyoqShs3bnQss9vt2r59u3r16uXGylCZo0eP6tSpU/wtXQXGGP32t7/VmjVr9P777ys+Pt5pfdeuXeXt7e30t5Odna3c3Fz+dmrZTx2byuzevVuSqvW3w2UsXNb06dM1ePBgxcXF6dixY3r88cfl6empUaNGubu0a1JhYaHT/80cOnRIu3fvVmhoqJo2bapp06Zp7ty5atmypeLj4zVr1izFxMRo6NCh7iv6GnG5YxMaGqrZs2dr+PDhioqKUk5OjmbMmKEWLVooOTnZjVVfG1JTU5Wenq633npLgYGBjnE4wcHB8vPzU3BwsCZMmKAHH3xQoaGhCgoK0n333adevXqpZ8+ebq7e2n7q2OTk5Cg9PV233HKLwsLClJWVpQceeEB9+/ZVx44dq76jGt3LBcsbMWKEiY6ONj4+PqZJkyZmxIgR5uDBg+4u65r1wQcfGEkXTWPHjjXGfH/7+axZs0xkZKTx9fU1/fv3N9nZ2e4t+hpxuWNz7tw5c/PNN5vGjRsbb29vExcXZyZOnGjy8vLcXfY1obLjIsmkpaU52hQVFZl7773XNGrUyPj7+5thw4aZ48ePu6/oa8RPHZvc3FzTt29fExoaanx9fU2LFi3Mww8/bAoKCqq1H9v/vzMAAABLYswOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOAACwNMIOgDrlm2++kc1mczwS3orOnz+vFi1a6KOPPqq1fSxatEiDBw+utf6B+oSwA9QD27Ztk6enpwYNGuTuUuqkcePG1atXYixatEjx8fHq3bt3re0jJSVFn332mbZs2VJr+wDqC8IOUA8sWbJE9913nzZv3qxjx47V6r6MMSorK6vVfVzLjDF66aWXNGHChFrdj4+Pj+666y698MILtbofoD4g7AB1XGFhoVatWqUpU6Zo0KBBWrp0qWPdXXfdpREjRji1Ly0tVXh4uJYvXy5Jqqio0Lx58xQfHy8/Pz9dd911+uc//+lon5mZKZvNpnfffVddu3aVr6+vtm7dqpycHA0ZMkSRkZFq2LChunfvroyMDKd9HT9+XIMGDZKfn5/i4+OVnp6uZs2a6bnnnnO0yc/P1913363GjRsrKChIN954o/bs2VPlz19eXq4JEyY46k9MTNTzzz/vWP/EE09o2bJleuutt2Sz2WSz2ZSZmSlJOnLkiO68806FhIQoNDRUQ4YM0TfffOPY9sIZoaeeekrR0dEKCwtTamqqSktLHW1KSkr0u9/9TrGxsfL19VWLFi20ZMkSGWPUokULPfXUU0717t69WzabzemloD+0c+dO5eTkOJ2lu3Dp7vXXX9cvfvEL+fn5qXv37tq/f7927Nihbt26qWHDhho4cKD+85//OB2766+/XgEBAQoJCVGfPn10+PBhx/rBgwfr7bffVlFRUZV/3oAlufaVXgBcbcmSJaZbt27GGGPWrl1rEhISTEVFhTHGmHfeecf4+fmZM2fOONqvXbvW+Pn5GbvdbowxZu7cuaZ169Zm3bp1Jicnx6SlpRlfX1+TmZlpjPnfCyw7duxo3nvvPXPw4EFz6tQps3v3brNo0SLz+eefm/3795tHH33UNGjQwBw+fNixr6SkJNOpUyfz8ccfm507d5obbrjB+Pn5mWeffdapzeDBg82OHTvM/v37zUMPPWTCwsLMqVOnKv28hw4dMpLMrl27jDHGnD9/3jz22GNmx44d5uuvvzavvPKK8ff3N6tWrTLGGHPmzBlz5513mgEDBpjjx4+b48ePm5KSEnP+/HnTpk0bk5KSYrKysszevXvNXXfdZRITE01JSYkxxpixY8eaoKAgc88995h9+/aZtWvXGn9/f7N48WJHPXfeeaeJjY01b7zxhsnJyTEZGRnmtddeM8YY8+STT5q2bds61T916lTTt2/fSx7PZ555xrRu3brSz3zhOO3du9f07NnTdO3a1fTr189s3brVfPbZZ6ZFixbmnnvuMcYYU1paaoKDg8306dPNwYMHzd69e83SpUudjs/Zs2eNh4eH+eCDDy5ZD3AtIOwAdVzv3r3Nc889Z4z5/gsuPDzc8eV1YX758uWO9qNGjTIjRowwxhhTXFxs/P39zUcffeTU54QJE8yoUaOMMf8LO2+++eZP1tKuXTvz4osvGmOM2bdvn5FkduzY4Vh/4MABI8kRdrZs2WKCgoJMcXGxUz8JCQnm5ZdfrnQfPw47lUlNTTXDhw93zI8dO9YMGTLEqc2KFStMYmKiIxgaY0xJSYnx8/Mz69evd2wXFxdnysrKHG3uuOMOx88vOzvbSDIbNmyotI5vv/3WeHp6mu3btxtjvg9m4eHhZunSpZes/f777zc33nhjpZ/573//u2PZypUrjSSzceNGx7J58+aZxMREY4wxp06dMpIcofVSGjVqdNl6gGsBl7GAOiw7O1uffPKJRo0aJUny8vLSiBEjtGTJEsf8nXfeqVdffVWSdPbsWb311lsaPXq0JOngwYM6d+6cbrrpJjVs2NAxLV++XDk5OU776tatm9N8YWGhpk+frjZt2igkJEQNGzbUvn37lJub66jNy8tLXbp0cWzTokULNWrUyDG/Z88eFRYWKiwszGn/hw4dumj/l/PXv/5VXbt2VePGjdWwYUMtXrzYUcel7NmzRwcPHlRgYKBjv6GhoSouLnbad7t27eTp6emYj46O1smTJyV9f0nK09NTN9xwQ6X7iImJ0aBBg/SPf/xDkrR27VqVlJTojjvuuGRdRUVFatCgQaXrOnbs6Ph3ZGSkJKlDhw5Oyy7UFhoaqnHjxik5OVmDBw/W888/r+PHj1/Up5+fn86dO3fJeoBrgZe7CwBwaUuWLFFZWZliYmIcy4wx8vX11UsvvaTg4GCNHj1aN9xwg06ePKkNGzbIz89PAwYMkPR9YJGkf/3rX2rSpIlT376+vk7zAQEBTvPTp0/Xhg0b9NRTT6lFixby8/PT7bffrvPnz1e5/sLCQkVHRzvG0PxQSEhIlfp47bXXNH36dD399NPq1auXAgMD9Ze//EXbt2//yX137drVEQR/qHHjxo5/e3t7O62z2WyqqKiQ9H1Q+Cl33323fvOb3+jZZ59VWlqaRowYIX9//0u2Dw8P1+eff17puh/WYrPZKl12oTZJSktL09SpU7Vu3TqtWrVKjz76qDZs2KCePXs62nz33XdOnxe4FhF2gDqqrKxMy5cv19NPP62bb77Zad3QoUO1cuVK3XPPPerdu7diY2O1atUqvfvuu7rjjjscX5Bt27aVr6+vcnNzL3l24lI+/PBDjRs3TsOGDZP0fXj44eDexMRElZWVadeuXeratauk788knT592tGmS5cuysvLk5eXl5o1a3YFP4Xv6+jdu7fuvfdex7IfnxXy8fFReXm507IuXbpo1apVioiIUFBQ0BXtu0OHDqqoqNCmTZuUlJRUaZtbbrlFAQEBWrhwodatW6fNmzdfts/OnTtr4cKFMsY4Ak1NdO7cWZ07d9bMmTPVq1cvpaenO8JOTk6OiouL1blz5xrvB6jPuIwF1FHvvPOOTp8+rQkTJqh9+/ZO0/Dhwx2XsqTv78patGiRNmzY4LiEJUmBgYGaPn26HnjgAS1btkw5OTn67LPP9OKLL2rZsmWX3X/Lli31xhtvaPfu3dqzZ4/uuusup7MKrVu3VlJSkiZNmqRPPvlEu3bt0qRJk+Tn5+f4Ek9KSlKvXr00dOhQvffee/rmm2/00Ucf6Q9/+IM+/fTTKv0cWrZsqU8//VTr16/X/v37NWvWLO3YscOpTbNmzZSVlaXs7Gz997//VWlpqUaPHq3w8HANGTJEW7Zs0aFDh5SZmampU6fq6NGjVdp3s2bNNHbsWKWkpOjNN9909PH666872nh6emrcuHGaOXOmWrZsqV69el22z1/+8pcqLCzUl19+WaUaLuXQoUOaOXOmtm3bpsOHD+u9997TgQMH1KZNG0ebLVu2qHnz5kpISKjRvoD6jrAD1FFLlixRUlKSgoODL1o3fPhwffrpp8rKypIkjR49Wnv37lWTJk3Up08fp7Zz5szRrFmzNG/ePLVp00YDBgzQv/71L8XHx192/88884waNWqk3r17a/DgwUpOTnYanyNJy5cvV2RkpPr27athw4Zp4sSJCgwMdIxJsdls+ve//62+fftq/PjxatWqlUaOHKnDhw87xqT8lMmTJ+u2227TiBEj1KNHD506dcrpLI8kTZw4UYmJierWrZsaN26sDz/8UP7+/tq8ebOaNm2q2267TW3atNGECRNUXFxcrTM9Cxcu1O233657771XrVu31sSJE3X27FmnNhMmTND58+c1fvz4n+wvLCxMw4YNq/TyWnX4+/vrq6++0vDhw9WqVStNmjRJqampmjx5sqPNypUrNXHixBrtB7ACmzHGuLsIANZw9OhRxcbGKiMjQ/3793d3OVfNli1b1L9/fx05cqRKIS4rK0s33XSTcnJy1LBhw1qp6csvv9SNN96o/fv3VxqYgWsJYQfAFXv//fdVWFioDh066Pjx45oxY4a+/fZb7d+//6KBv1ZUUlKi//znPxo7dqyioqKqdbZm6dKl6tq1q9PdVq6UkZGh8vJyJScn10r/QH1C2AFwxdavX6+HHnpIX3/9tQIDA9W7d28999xziouLc3dpV8XSpUs1YcIEderUSW+//fZFd7wBqBsIOwAAwNIYoAwAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACyNsAMAACzt/wM5YFXqI8XqcgAAAABJRU5ErkJggg==\n" - }, - "metadata": {} - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "import pandas as pd\n", - "\n", - "\n", - "def plot_metrics(perf_metrics, current_optim_type):\n", - " df = pd.DataFrame.from_dict(perf_metrics, orient=\"index\")\n", - "\n", - " for idx in df.index:\n", - " df_opt = df.loc[idx]\n", - " # Add a dashed circle around the current optimization type\n", - " if idx == current_optim_type:\n", - " plt.scatter(\n", - " df_opt[\"time_avg_ms\"],\n", - " df_opt[\"accuracy\"] * 100,\n", - " alpha=0.5,\n", - " s=df_opt[\"size_mb\"],\n", - " label=idx,\n", - " marker=\"$\\u25CC$\",\n", - " )\n", - " else:\n", - " plt.scatter(\n", - " df_opt[\"time_avg_ms\"],\n", - " df_opt[\"accuracy\"] * 100,\n", - " s=df_opt[\"size_mb\"],\n", - " label=idx,\n", - " alpha=0.5,\n", - " )\n", - "\n", - " legend = plt.legend(bbox_to_anchor=(1, 1))\n", - " for handle in legend.legendHandles:\n", - " handle.set_sizes([20])\n", - "\n", - " plt.ylim(63, 95)\n", - " # Use the slowest model to define the x-axis range\n", - " xlim = int(perf_metrics[\"bge-small PyTorch\"][\"time_avg_ms\"] + 12)\n", - " plt.xlim(1, xlim)\n", - " plt.ylabel(\"Accuracy (%)\")\n", - " plt.xlabel(\"Average latency (ms)\")\n", - " plt.show()\n", - "\n", - "\n", - "plot_metrics(perf_metrics, \"bge-small\")" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## 4. Eval DistillBERT as refrence" - ], - "metadata": { - "id": "b-yd2LtbSbXj" - }, - "id": "b-yd2LtbSbXj" - }, - { - "cell_type": "code", - "source": [ - "from transformers import pipeline\n", - "\n", - "#hide_output\n", - "finetuned_model = \"distilbert-base-uncased-finetuned-sst-2-english\"\n", - "pipe = pipeline(\"text-classification\", model=finetuned_model)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 145, - "referenced_widgets": [ - "eb6e6d91566044f18e953d8df93ae3a8", - "29dbf37d879a4f05904b076378d190bf", - "1cfb1e7981264c328e02e49c4080c504", - "e1157a679f354eaba73d4fea3cbda6fe", - "a9eafe51348847b0be895b8502a21e56", - "f338c914be9641a4aee76971c0e9cf72", - "746f408b617a4974ab48665f5b0c6c01", - "561acc35f4e245e8adf81b1efa510fd2", - "85377644ee164abebcfb24c7fdeaab8a", - "b800f7b6b2fe448486f63a24066694aa", - "0661f3e76c9a490db89c29c35d8c0d1b", - "7612716749c14069bc5989f3ee09c00a", - "0e9bfe8c83a44537bb59c444744466ef", - "5e8641e8da904126a3680828c83774e8", - "2f99603c364b4af48ea25b8d43cd1941", - "26ae8872c49642c588330b34ff470609", - "a8ee2fe1c3d04082acdafbf39ba0076b", - "5e9bad63ccc14fe2b358b7c7f0bbae68", - "50597a7caa0242df82fafa89bf831827", - "48ba967e29ad45a1ac8f7ba659fb5a5f", - "788dc688a9e9449e9aeb431b06384d9f", - "c82bc76766fd420a9557c4dcd78cff0c", - "f52944fb4da049afae19d9d1f4a6b37a", - "9cee60b1935441e4a81d26cc5caa319e", - "4621e316677d44caab491fd8a7b1433f", - "39bea84d46004ecd9dae38e81c4bb2d8", - "5859b86234164b55aa97852b4368bd66", - "9fd724ea0d6947be92361604c9f318a7", - "996b172b2f434b5a95de4a839c2ae4ae", - "ff8e804c7b5e472187685d1fdcc5d0ff", - "764f957750064a7eb61d5e6b28e0e49d", - "d46affe5b0ff4914bc2504c2737c6d40", - "350d6653f3d947b8a125b051b36e9d4a", - "e9e9cb6c63c54831acf6922003883421", - "1d9e7ff2e4d74891a3f70798bb5266eb", - "55135c120cc94702be910b033f6df07d", - "a01975d1f8154082bd903bb55b3d49d6", - "26db4649e9924203912f92e0e1c994bb", - "2261900d8f594e7b965fe8d4d6264e35", - "14f43841b87646dd9c8eca2966488900", - "c7a0c077c06e4344bdffe4848de56e15", - "581e9a90552543188ec3c95b4cdab508", - "15b87e982dcf45139029b93eb6a7f857", - "dc42e38e51464a2095b179359da4c816" - ] - }, - "id": "3o6VdvFwvRUM", - "outputId": "7a853fe0-5d8f-4361-f99b-620f984d23dc" - }, - "id": "3o6VdvFwvRUM", - "execution_count": null, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "Downloading (…)lve/main/config.json: 0%| | 0.00/629 [00:00:30: MatplotlibDeprecationWarning: The legendHandles attribute was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use legend_handles instead.\n", - " for handle in legend.legendHandles:\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAukAAAG2CAYAAADC9Bi0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABOKUlEQVR4nO3daXgUVf728buz0tkhIRsmISQQdlllG5GRKCCDbCIg/lmCgBBFVGAGFQU3BsYF0RkQZcJmEDdQHAUBZVNEZFcwQAQCkoCDJCGQPfW8yEOPLVsCSbqgv5/r6kuq6tSpX6fT9t2VU6cshmEYAgAAAGAaLo4uAAAAAIA9QjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMg4N6WfOnNG4ceMUFRUlq9Wq9u3ba+vWrbbtQ4cOlcVisXt07drVgRUDAAAAlc/NkQd/4IEH9MMPP2jRokUKDw/X4sWLFR8fr71796pWrVqSpK5duyopKcm2j6enp6PKBQAAAKqExTAMwxEHzs3Nla+vrz7++GN1797dtr5ly5bq1q2bnn/+eQ0dOlSZmZlavny5I0oEAAAAHMJhZ9KLiopUXFysatWq2a23Wq3atGmTbXndunUKDg5W9erVdfvtt+v5559XYGDgJfvNz89Xfn6+bbmkpES//fabAgMDZbFYKv6JAACACmcYhs6cOaPw8HC5uHAJHZyPw86kS1L79u3l4eGh5ORkhYSEaMmSJRoyZIhiY2OVkpKid999V15eXoqOjlZqaqqeeOIJ+fj4aPPmzXJ1db1on1OmTNHUqVOr+JkAAIDKcPToUd10002OLgOocg4N6ampqUpISNCGDRvk6uqqFi1aqF69etq2bZv27dt3Qfuff/5ZMTExWrNmjTp37nzRPv94Jj0rK0uRkZE6evSo/Pz8Ku25AACAipOdna2IiAhlZmbK39/f0eUAVc6hF47GxMRo/fr1Onv2rLKzsxUWFqb+/furTp06F21fp04dBQUF6eDBg5cM6Z6enhe9uNTPz4+QDgDAdYahqnBWphjk5e3trbCwMJ0+fVqrVq1Sz549L9ru2LFjOnXqlMLCwqq4QgAAAKDqOPRM+qpVq2QYhuLi4nTw4EFNmDBB9evX17Bhw5STk6OpU6eqb9++Cg0NVWpqqiZOnKjY2Fh16dLFkWUDAAAAlcqhZ9KzsrKUmJio+vXra/DgwfrTn/6kVatWyd3dXa6urtq9e7fuvvtu1atXT8OHD1fLli21ceNG5koHAADADc2hF45WhezsbPn7+ysrK4sx6QAAXCfK+vldXFyswsLCKqwMuHoeHh5lnlLUocNdAAAAroZhGMrIyFBmZqajSwHKzMXFRdHR0fLw8LhiW0I6AAC47pwP6MHBwfLy8mIWGJheSUmJjh8/rvT0dEVGRl7xd5aQDgAArivFxcW2gH65u5ADZlOzZk0dP35cRUVFcnd3v2xbU0zBCAAAUFbnx6B7eXk5uBKgfM4PcykuLr5iW0I6AAC4LjHEBdeb8vzOEtIBAAAAkyGkAwAAVIFOnTpp3Lhxji6jysyfP18BAQG25SlTpqhZs2YOq6cy1K5dWzNnzqyUvgnpAAAAcLjDhw/LYrHYHoGBgbrzzju1Y8eOMu1fu3Ztu/3/+Bg6dGjlPoEKxuwuAAAAMI01a9aoUaNGOnbsmMaOHatu3brpp59+sjsrfzFbt261XZD5zTffqG/fvkpJSbHdDMtqtZarjsLCwivOwFKZOJMOAACc1umzBTr037M6fbagSo5XVFSkhx56SP7+/goKCtLkyZP1+5u/p6enq3v37rJarYqOjlZycvIFQyoyMzP1wAMPqGbNmvLz89Ptt9+uXbt2Xfa469at0y233CJvb28FBASoQ4cOOnLkiKT/DUP597//rcjISPn4+GjMmDEqLi7WjBkzFBoaquDgYL3wwgt2fb7yyitq0qSJvL29FRERoTFjxignJ+eaf0aBgYEKDQ1Vq1at9NJLL+nEiRPasmWLnn32WTVu3PiC9s2aNdPkyZNVs2ZNhYaGKjQ0VDVq1JAkBQcH29YlJycrJiZGHh4eiouL06JFi+z6sVgsmj17tu6++255e3vbnu+KFSvUunVrVatWTUFBQerdu7fdfufOnVNCQoJ8fX0VGRmpuXPnXvPPQCKkAwAAJ5RXWKwPth3V9JU/6dXVKZq+8id9sO2o8gqvPDXetViwYIHc3Nz03Xff6bXXXtMrr7yit99+27Z98ODBOn78uNatW6cPP/xQc+fO1cmTJ+366Nevn06ePKnPP/9c27ZtU4sWLdS5c2f99ttvFz1mUVGRevXqpdtuu027d+/W5s2bNXLkSLuZRlJTU/X5559r5cqVWrJkiebNm6fu3bvr2LFjWr9+vaZPn66nnnpKW7Zsse3j4uKiWbNm6ccff9SCBQv05ZdfauLEiRX68zp/9rugoEAJCQnat2+ftm7datu+Y8cO7d69W8OGDbtsP8uWLdMjjzyixx9/XD/88INGjRqlYcOG6auvvrJrN2XKFPXu3Vt79uxRQkKC/vOf/6h379666667tGPHDq1du1a33HKL3T4vv/yyWrVqpR07dmjMmDEaPXq0UlJSrv3JGze4rKwsQ5KRlZXl6FIAAEAZXe7zOzc319i7d6+Rm5t71f2//32aMXLhVmPSh7uNaZ/tNSZ9uNsYuXCr8f73addS9mXddtttRoMGDYySkhLbur/+9a9GgwYNDMMwjH379hmSjK1bt9q2HzhwwJBkvPrqq4ZhGMbGjRsNPz8/Iy8vz67vmJgY480337zocU+dOmVIMtatW3fR7c8884zh5eVlZGdn29Z16dLFqF27tlFcXGxbFxcXZ0ybNu2Sz+/99983AgMDbctJSUmGv7+/3XFuvvnmS+5/6NAhQ5KxY8cOwzAM4/Tp00bv3r0NHx8fIyMjwzAMw+jWrZsxevRo2z4PP/yw0alTpwv6+uqrrwxJxunTpw3DMIz27dsbI0aMsGvTr18/46677rItSzLGjRtn16Zdu3bGoEGDLllzVFSUcf/999uWS0pKjODgYGP27NkXbV+e313OpAMAAKdy+myBvj98WoHenqrp6ylPN1fV9PVUoLenth0+XalDX9q2bWt3Brtdu3Y6cOCAiouLlZKSIjc3N7Vo0cK2PTY2VtWrV7ct79q1Szk5OQoMDJSPj4/tcejQIaWmpiotLc1u/YsvvqgaNWpo6NCh6tKli3r06KHXXntN6enpdnXVrl1bvr6+tuWQkBA1bNhQLi4udut+f1Z/zZo16ty5s2rVqiVfX1/93//9n06dOqVz585d08+offv28vHxUfXq1bVr1y4tXbpUISEhkqQRI0ZoyZIlysvLU0FBgZKTk5WQkHDFPvft26cOHTrYrevQoYP27dtnt65Vq1Z2yzt37lTnzp0v23fTpk1t/7ZYLAoNDb3grx9XgwtHAQCAU8nMLdS5giKFB9hfSOhnddPxzFxl5haqureHg6q7vJycHIWFhWndunUXbAsICFBAQIB27txpW3d+bHZSUpLGjh2rlStXaunSpXrqqae0evVqtW3bVpIuuEDSYrFcdF1JSYmk0plY/vKXv2j06NF64YUXVKNGDW3atEnDhw9XQUHBNd0NdunSpWrYsKECAwMvuFi0R48e8vT01LJly+Th4aHCwkLdc889V32sP/L29rZbLsvFppf7OV0LQjoAAHAqAVZ3eXm4KTu3SDV9XW3rs3OL5O3hpgBr5c3o8fsx3ZL07bffqm7dunJ1dVVcXJyKioq0Y8cOtWzZUpJ08OBBnT592ta+RYsWysjIkJubm2rXrn3RY8TGxl50ffPmzdW8eXNNmjRJ7dq1U3Jysi2kl9e2bdtUUlKil19+2Xa2/b333ruqvv4oIiJCMTExF93m5uamIUOGKCkpSR4eHhowYECZgnSDBg309ddfa8iQIbZ1X3/9tRo2bHjZ/Zo2baq1a9deccx7ZSCkAwAAp1Ld20OtalfX6r0nJJWeQc/OLdKps/m6o2FIpZ5FT0tL02OPPaZRo0Zp+/btev311/Xyyy9LkurXr6/4+HiNHDlSs2fPlru7ux5//HFZrVbbEJn4+Hi1a9dOvXr10owZM1SvXj0dP37cdoHjH4drSNKhQ4c0d+5c3X333QoPD1dKSooOHDigwYMHX/XziI2NVWFhoV5//XX16NFDX3/9tebMmXPV/ZXHAw88oAYNGkgqDdplMWHCBN17771q3ry54uPjtWLFCn300Udas2bNZfd75pln1LlzZ8XExGjAgAEqKirSZ599pr/+9a/X/DyuhDHpAADA6fylabjuaBgiwzB0PDNXhmHojoYh+kvT8Eo97uDBg5Wbm6tbbrlFiYmJeuSRRzRy5Ejb9oULFyokJEQdO3ZU7969NWLECPn6+qpatWqSSodSfPbZZ+rYsaOGDRumevXqacCAATpy5Iht3PYfeXl56aefflLfvn1Vr149jRw5UomJiRo1atRVP4+bb75Zr7zyiqZPn67GjRvrnXfe0bRp0666v/KoW7eu2rdvr/r166tNmzZl2qdXr1567bXX9NJLL6lRo0Z68803lZSUpE6dOl12v06dOun999/XJ598ombNmun222/Xd999VwHP4sos//9q1htWdna2/P39lZWVZZvMHgAAmNvlPr/z8vJ06NAhRUdH28Lr1Tp9tkCZuYUKsLqbchz6sWPHFBERYbtIE5JhGKpbt67GjBmjxx57zNHllEt5fncZ7gIAAJxWdW8PU4XzL7/8Ujk5OWrSpInS09M1ceJE1a5dWx07dnR0aabw66+/6t1331VGRoZDxolXJUI6AACASRQWFuqJJ57Qzz//LF9fX7Vv317vvPOOQ29PbybBwcEKCgrS3Llz7aamvBER0gEAAEyiS5cu6tKli6PLMK0bfJS2HS4cBQAAAEyGkA4AAACYDCEdAAAAMBlCOgAAAGAyhHQAAADAZAjpAAAAgMkQ0gEAABykU6dOGjdunG25du3amjlz5lX3N3/+fAUEBNiWp0yZombNmtmWhw4dql69el3y+LiyP/6MKwshHQAAwCS2bt2qkSNHlqntxQJ9//79tX///kqorPz++IXgcu0sFossFos8PDwUGxurZ599VkVFRVfcd/78+bZ9L/U4fPjwtT8ZB+BmRgAAOMLRrdJ/D0hnT0jFBZKnn+R/kxTZTvIOcnR1cJCaNWte0/5Wq1VWq7WCqrk6xcXFslgs5dqna9euSkpKUn5+vj777DMlJibK3d1dkyZNuux+/fv3V9euXW3Lffr0UePGjfXss8/a1pXnZ1pQUCAPD49y1V5ZOJMOAEBVOftfKS+r9N/7V0p73pMOf10a2A+ulXa/J51KlQxDOn1EKi50bL3OwDCk/Jwq+VmfPXtWgwcPlo+Pj8LCwvTyyy9f0Ob3Z8cNw9CUKVMUGRkpT09PhYeHa+zYsZJKh6kcOXJEjz76qO2MsXR1QzGKior00EMPyd/fX0FBQZo8ebLdnT3z8/M1fvx41apVS97e3mrTpo3WrVtn237+mJ988okaNmwoT09PJSQkaMGCBfr4449t9f1+nz/y9PRUaGiooqKiNHr0aMXHx+uTTz7R2bNn5efnpw8++MCu/fLly+Xt7a2ioiKFhobaHh4eHvLy8rItFxQUqE+fPvLx8ZGfn5/uvfdenThxwtbP+eFAb7/9tqKjo1WtWjVJUmZmpkaNGqWQkBBVq1ZNjRs31qeffmpXw6pVq9SgQQP5+Pioa9euSk9PL9fP/Uo4kw4AQGUrKZEOrZf2fSIF1ZPaPCjVaiFlpklu1SSLS+nZdA8fKaiu9NvP0uZ/Sv4R0s39Jb9wRz+DG9PJfdKPy6ScE5KnvxTZVoqNl9wq50zqhAkTtH79en388ccKDg7WE088oe3bt9uNGf+9Dz/8UK+++qreffddNWrUSBkZGdq1a5ck6aOPPtLNN9+skSNHasSIEddU14IFCzR8+HB99913+v777zVy5EhFRkba+n3ooYe0d+9evfvuuwoPD9eyZcvUtWtX7dmzR3Xr1pUknTt3TtOnT9fbb7+twMBAhYWFKTc3V9nZ2UpKSpIk1ahRo8w1Wa1WnTp1St7e3howYICSkpJ0zz332LafX/b19b1kHyUlJerZs6d8fHy0fv16FRUVKTExUf3797f7wnDw4EF9+OGH+uijj+Tq6qqSkhJ169ZNZ86c0eLFixUTE6O9e/fK1dXVts+5c+f00ksvadGiRXJxcdH999+v8ePH65133inzc7wSQjoAAJWpuFDa84F0YJV07jepqEDK/kWq82epZn3JK1BydZfyz0hF+ZJXjdKz6r+lSlnHpLMnpZbDpJr1HP1MbixZv0hb50lunpJ3cOm6nz6VivKkxn0q/HA5OTmaN2+eFi9erM6dO0sqDcc33XTTJfdJS0tTaGio4uPj5e7ursjISN1yyy2SSgOvq6urfH19FRoaek21RURE6NVXX5XFYlFcXJz27NmjV199VSNGjFBaWpqSkpKUlpam8PDSL4vjx4/XypUrlZSUpBdffFGSVFhYqH/961+6+eabbf1arVbl5+eXqz7DMLR27VqtWrVKDz/8sCTpgQceUPv27ZWenq6wsDCdPHlSn332mdasWXPZvtauXas9e/bo0KFDioiIkCQtXLhQjRo10tatW9W6dWtJpUNcFi5caBsW88UXX+i7777Tvn37VK9e6fuuTp06dn0XFhZqzpw5iomJkVT6Reb3Q2wqAsNdAACoLIYh7V0upXwmFeZKPiFS9G2Sd03Jw0sKjJGsAZKHt+QbKlWPKt0vvLl00y2Si2vp8Jetb5eedUfFOfKNVC1AumWkdPuTUscJUvVo6cSP/xuSVIFSU1NVUFCgNm3a2NbVqFFDcXFxl9ynX79+ys3NVZ06dTRixAgtW7asTBdTllfbtm3txpC3a9dOBw4cUHFxsfbs2aPi4mLVq1dPPj4+tsf69euVmppq28fDw0NNmza96ho+/fRT+fj4qFq1aurWrZv69++vKVOmSJJuueUWNWrUSAsWLJAkLV68WFFRUerYseNl+9y3b58iIiJsAV2SGjZsqICAAO3bt8+2Lioqym7c+s6dO3XTTTfZAvrFeHl52QK6JNuXh4pESAcAoLIUnpOO7yz9r1eg1Hq41LRf6dnbywmKlTqMlaI6lA6FyTkhZfxQJSU7jfRdUn72/74YuXmU/rUi52TptQMmEBERoZSUFP3rX/+S1WrVmDFj1LFjRxUWVt21Cjk5OXJ1ddW2bdu0c+dO22Pfvn167bXXbO2sVmu5Lxb9vT//+c/auXOnDhw4oNzcXC1YsEDe3t627Q888IDmz58vqXSoy7Bhw67peL/3++NIKtOFt+7u7nbLFovFbhx/RSCkA3BaJSUV+z9U4AIe3lK7h6T6f5Ga3Vc65rmswcLTV2o5RIrtXDrcpe4dlVurswm7uXRGndNHSpeLCqRf90s+wZUyu05MTIzc3d21ZcsW27rTp09fcbpEq9WqHj16aNasWVq3bp02b96sPXv2SCo9e11cXHzNtf2+Jkn69ttvVbduXbm6uqp58+YqLi7WyZMnFRsba/e40jCW8tTn7e2t2NhYRUZGys3twtHY999/v44cOaJZs2Zp7969GjJkyBX7bNCggY4ePaqjR4/a1u3du1eZmZlq2LDhJfdr2rSpjh075vCpLBmTDsBpFBaXKCXjjLYdOa20386qoKhEHm4uiqzhrZZR1RUX6it3V85doIL5hUmthl3dvp6+0i3XdlEgLiGqvXR0i/TdXMndq3Rd9i+lF45W86/ww/n4+Gj48OGaMGGCAgMDFRwcrCeffFIuLpf+f878+fNVXFysNm3ayMvLS4sXL5bValVUVOnZ/9q1a2vDhg0aMGCAPD09FRR0dV8u0tLS9Nhjj2nUqFHavn27Xn/9ddvMM/Xq1dOgQYM0ePBgvfzyy2revLl+/fVXrV27Vk2bNlX37t0v2W/t2rW1atUqpaSkKDAwUP7+/hecgS6r6tWrq0+fPpowYYLuvPPOy47lPy8+Pl5NmjTRoEGDNHPmTBUVFWnMmDG67bbb1KpVq0vud9ttt6ljx47q27evXnnlFcXGxuqnn36SxWKxm+6xsvFpBOCGZxiGdqSd1iurUzRnfaq2/HxKOXnFKi6RcvKKteXnU5qzPlWvrE7RjrTTji4XN4q0b6Ufl5eOKb9Whbml/e1MlnL5Ha0Q/rVKhx95eJdenFuUX/oXj/p/qbRD/uMf/9Ctt96qHj16KD4+Xn/605/UsmXLS7YPCAjQW2+9pQ4dOqhp06Zas2aNVqxYocDAQEnSs88+q8OHDysmJuaa5lcfPHiwcnNzdcsttygxMVGPPPKI3Q2VkpKSNHjwYD3++OOKi4tTr169tHXrVkVGRl623xEjRiguLk6tWrVSzZo19fXXX191jZI0fPhwFRQUKCEhoUztLRaLPv74Y1WvXl0dO3ZUfHy86tSpo6VLl15x3w8//FCtW7fWwIED1bBhQ02cOLFC/mpRHhajogfQmEx2drb8/f2VlZUlPz8/R5cDoIoZhqFNB/+rZdt/UWFJicL9rarm7npBu7zCYh3PypW7i4t6t6ilW+te2w1FAG18RTq8UQqsK3V+WvL0ufq+Ur+Uti+SSoqkWx8rvbD0Bne5z++8vDwdOnTIbl7rq2YYUsHZ0usEXK/uLC+qxqJFi/Too4/q+PHjprnhUHmV53eXM+kAbmg7j2Zq2fZf5OZqUZ0gn4sGdEmq5u6qOkE+cnO1aNmOX7TzaGbVFoobz9lfJcv//307P5ziann4lAZ0ScrLvra+YM9iKf0CRUA3rXPnzik1NVV///vfNWrUqOs2oJcXIR3ADauwuERf7M1QYUmJwvzLdpvsMH9r6X4/ZqiwuKSSK8QNyzD+dwdL92rSZcYdl4nr70JJCXchhXOZMWOG6tevr9DQUE2aNMnR5VQZQjqAG1ZKxhkd/S1X4WUM6OeF+1uV9ts5pWScqaTKcMOzWP53ZrYwr/SOo9eiuKA0+EuSC2d84VymTJmiwsJCrV27Vj4+1zBs7DpDSAdww9p25LRKSoxLDnG5lGruriopMbTtCBfo4Rp415SM/3+hWeG5a+urIKc09FssUjWurwKcAVMwArhhpf12Vr7Vru6so081N6X9do3BCs4tqr1Uo44U2uTaLhqVpMh2kru39NvPpXfFhCRV+M1jgMpWnt9ZQjqAG1JJiaGCohK5ulzdHelcXSwqKCpWSYkhl6vsA04usm3F9eVulSLblD5gm2v73LlzZbo7JGAWBQUFkiRX1yv/hZeQDuCG5OJikYebi3Lyrm5e2+ISQ14ebgR0XLvsdGn/SimorlT7T+XbNy9b2r20dBrH2h2YgeT/c3V1VUBAgE6ePClJ8vLyqrBbxAOVpaSkRL/++qu8vLwuelfVPyKkA7hhRdbw1pafTynUv/zzKOfkFalReMXfdRBOpuCstPkN6defJJ8QycVNimhTOrb8SvLPSNsXls61fnhj6bj2+ndVfs3XifO3pD8f1IHrgYuLiyIjI8v0pdKhIf3MmTOaPHmyli1bppMnT6p58+Z67bXX1Lp1a0ml43aeeeYZvfXWW8rMzFSHDh00e/Zs1a1b15FlA7hOtIyqrq2Hf1NeYXG5Lh7NKyyWi4tFLaOqV2J1cAruXlJ4MykzTTp3Sto6TzqdJjXqWXrznEv578HSM+gnfpCMEsk3TAprWmVlXw8sFovCwsIUHByswkKmpcT1wcPDQy5lnJLVoSH9gQce0A8//KBFixYpPDxcixcvVnx8vPbu3atatWppxowZmjVrlhYsWKDo6GhNnjxZXbp00d69e6/9DmMAbnhxob6KqGHVsdO5qhNU9gv3jmflKrKGl+JCfSuxOjgFi0Vq2EsqLpIOrJJyTkiH1ku120vWGlL2cck7sHQe9Lzs0qkWq0dJx3dIx76TXD2lwFipVYLkf5Ojn40pubq6lml8L3C9sRgOujQ6NzdXvr6++vjjj9W9e3fb+pYtW6pbt2567rnnFB4erscff1zjx4+XJGVlZSkkJETz58/XgAEDynScy91WGMCNb0faaS3afERurpYy3dAoPStXRSWGBrerrWYRAZVfIJxDSUlpON/3iRQUJ7UZJaV8Jv24XHKrJllcSgO6h4/050lS7mlp8z+lgAipaX/JL9zRz6DK8fkNZ+ewM+lFRUUqLi6+4Iy41WrVpk2bdOjQIWVkZCg+Pt62zd/fX23atNHmzZsvGdLz8/OVn59vW87O5vbJgDNrFhGgnPwiLdv+i37+b47C/a0XHfqSV1is41m5cnd1Ue/mtQjoqFguLlLMn0unYzw/3/kv26Wzv5aOUz+vIEf67wEp4hbpT+NKh7lwsSjglBwW0n19fdWuXTs999xzatCggUJCQrRkyRJt3rxZsbGxysjIkCSFhITY7RcSEmLbdjHTpk3T1KlTK7V2ANcPi8WiW+vWlG81d33xY4bSfjunkhJDPtXc5OpiUXGJoZy8Irm4WBRZw0t3NgoloKPyeAf9799x3aQaMdLZk1JxvuTpJ/lHSIExpSE+INJxdQJwOIeOSV+0aJESEhJUq1Ytubq6qkWLFho4cKC2bdt21X1OmjRJjz32mG05OztbERERFVEugOtYs4gANQr3U0rGGW07clppv51TQVGxvDzc1CjcXy2jqisu1FfurtyIGVXkplalDwC4CIeG9JiYGK1fv15nz55Vdna2wsLC1L9/f9WpU8c2tdKJEycUFhZm2+fEiRNq1qzZJfv09PSUp+dlrpgH4LTcXV3UuJa/GtcqnVqRGxUBAMzKFKeMvL29FRYWptOnT2vVqlXq2bOnoqOjFRoaqrVr19raZWdna8uWLWrXrp0DqwVwoyCgAwDMyqFn0letWiXDMBQXF6eDBw9qwoQJql+/voYNGyaLxaJx48bp+eefV926dW1TMIaHh6tXr16OLBsAAACoVA4N6VlZWZo0aZKOHTumGjVqqG/fvnrhhRfk7l56JfvEiRN19uxZjRw5UpmZmfrTn/6klStXMkc6AAAAbmgOmye9qjDPKgAA1x8+v+HsTDEmHQAAAMD/ENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZh4b04uJiTZ48WdHR0bJarYqJidFzzz0nwzBsbYYOHSqLxWL36Nq1qwOrBgAAACqXmyMPPn36dM2ePVsLFixQo0aN9P3332vYsGHy9/fX2LFjbe26du2qpKQk27Knp6cjygUAAACqhEND+jfffKOePXuqe/fukqTatWtryZIl+u677+zaeXp6KjQ01BElAgAAAFXOocNd2rdvr7Vr12r//v2SpF27dmnTpk3q1q2bXbt169YpODhYcXFxGj16tE6dOuWIcgEAAIAq4dAz6X/729+UnZ2t+vXry9XVVcXFxXrhhRc0aNAgW5uuXbuqT58+io6OVmpqqp544gl169ZNmzdvlqur6wV95ufnKz8/37acnZ1dJc8FAAAAqCgODenvvfee3nnnHSUnJ6tRo0bauXOnxo0bp/DwcA0ZMkSSNGDAAFv7Jk2aqGnTpoqJidG6devUuXPnC/qcNm2apk6dWmXPAQAAAKhoFuP3U6lUsYiICP3tb39TYmKibd3zzz+vxYsX66effrrkfjVr1tTzzz+vUaNGXbDtYmfSIyIilJWVJT8/v4p9AgAAoFJkZ2fL39+fz284LYeeST937pxcXOyHxbu6uqqkpOSS+xw7dkynTp1SWFjYRbd7enoy+wsAAACuaw4N6T169NALL7ygyMhINWrUSDt27NArr7yihIQESVJOTo6mTp2qvn37KjQ0VKmpqZo4caJiY2PVpUsXR5YOAAAAVBqHDnc5c+aMJk+erGXLlunkyZMKDw/XwIED9fTTT8vDw0O5ubnq1auXduzYoczMTIWHh+vOO+/Uc889p5CQkDIdgz+XAQBw/eHzG87OoSG9KvAmBwDg+sPnN5ydQ+dJBwAAAHAhQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGTcytO4pKRE69ev18aNG3XkyBGdO3dONWvWVPPmzRUfH6+IiIjKqhMAAABwGmU6k56bm6vnn39eERERuuuuu/T5558rMzNTrq6uOnjwoJ555hlFR0frrrvu0rffflvZNQMAAAA3tDKdSa9Xr57atWunt956S3fccYfc3d0vaHPkyBElJydrwIABevLJJzVixIgKLxYAAABwBhbDMIwrNdq3b58aNGhQpg4LCwuVlpammJiYay6uImRnZ8vf319ZWVny8/NzdDkAAKAM+PyGsyvTcJeyBnRJcnd3N01ABwAAAK5H5bpw9PeKior05ptvat26dSouLlaHDh2UmJioatWqVWR9AAAAgNO56pA+duxY7d+/X3369FFhYaEWLlyo77//XkuWLKnI+gAAAACnU+aQvmzZMvXu3du2/MUXXyglJUWurq6SpC5duqht27YVXyEAAADgZMp8M6N///vf6tWrl44fPy5JatGihR588EGtXLlSK1as0MSJE9W6detKKxQAAABwFmUO6StWrNDAgQPVqVMnvf7665o7d678/Pz05JNPavLkyYqIiFBycnJl1goAAAA4hTJNwfh7mZmZmjhxonbt2qU5c+aoefPmlVVbhWAKJwAArj98fsPZlflM+nkBAQGaO3eu/vGPf2jw4MGaMGGC8vLyKqM2AAAAwCmVOaSnpaXp3nvvVZMmTTRo0CDVrVtX27Ztk5eXl26++WZ9/vnnlVknAAAA4DTKPNylU6dOCg0N1dChQ7Vq1Sqlpqbqk08+kVR6R9JRo0YpNDRU7733XqUWXF78uQwAgOsPn99wdmWegvH777/Xrl27FBMToy5duig6Otq2rUGDBtqwYYPmzp1bKUUCAAAAzqTMIb1ly5Z6+umnNWTIEK1Zs0ZNmjS5oM3IkSMrtDgAAADAGZV5TPrChQuVn5+vRx99VL/88ovefPPNyqwLAAAAcFplPpMeFRWlDz74oDJrAQAAAKAynkk/e/ZsuTotb3sAAAAA/1OmkB4bG6u///3vSk9Pv2QbwzC0evVqdevWTbNmzaqwAgEAAABnU6bhLuvWrdMTTzyhKVOm6Oabb1arVq0UHh6uatWq6fTp09q7d682b94sNzc3TZo0SaNGjarsugEAAIAbVpnnSZdKb2j0/vvva+PGjTpy5Ihyc3MVFBSk5s2bq0uXLurWrZtcXV0rs95yY55VAACuP3x+w9mVK6Rfj3iTAwBw/eHzG86uzFMwAgAAAKgahHQAAADAZAjpAAAAgMkQ0gEAAACTIaQDAAAAJlPukF67dm09++yzSktLu+aDFxcXa/LkyYqOjpbValVMTIyee+45/X7CGcMw9PTTTyssLExWq1Xx8fE6cODANR8bAAAAMKtyh/Rx48bpo48+Up06dXTHHXfo3XffVX5+/lUdfPr06Zo9e7beeOMN7du3T9OnT9eMGTP0+uuv29rMmDFDs2bN0pw5c7RlyxZ5e3urS5cuysvLu6pjAgAAAGZ31fOkb9++XfPnz9eSJUtUXFys++67TwkJCWrRokWZ+/jLX/6ikJAQzZs3z7aub9++slqtWrx4sQzDUHh4uB5//HGNHz9ekpSVlaWQkBDNnz9fAwYMuOIxmGcVAIDrD5/fcHZXPSa9RYsWmjVrlo4fP65nnnlGb7/9tlq3bq1mzZrp3//+t8qS/du3b6+1a9dq//79kqRdu3Zp06ZN6tatmyTp0KFDysjIUHx8vG0ff39/tWnTRps3b75on/n5+crOzrZ7AAAAANcTt6vdsbCwUMuWLVNSUpJWr16ttm3bavjw4Tp27JieeOIJrVmzRsnJyZft429/+5uys7NVv359ubq6qri4WC+88IIGDRokScrIyJAkhYSE2O0XEhJi2/ZH06ZN09SpU6/2aQEAAAAOV+6Qvn37diUlJWnJkiVycXHR4MGD9eqrr6p+/fq2Nr1791br1q2v2Nd7772nd955R8nJyWrUqJF27typcePGKTw8XEOGDClvaZKkSZMm6bHHHrMtZ2dnKyIi4qr6AgAAAByh3CG9devWuuOOOzR79mz16tVL7u7uF7SJjo4u03jxCRMm6G9/+5utbZMmTXTkyBFNmzZNQ4YMUWhoqCTpxIkTCgsLs+134sQJNWvW7KJ9enp6ytPTs7xPCwAAADCNcof0n3/+WVFRUZdt4+3traSkpCv2de7cObm42A+Ld3V1VUlJiaTSsB8aGqq1a9faQnl2dra2bNmi0aNHl7d0AAAA4LpQ7pB+8uRJZWRkqE2bNnbrt2zZIldXV7Vq1arMffXo0UMvvPCCIiMj1ahRI+3YsUOvvPKKEhISJEkWi0Xjxo3T888/r7p16yo6OlqTJ09WeHi4evXqVd7SAQAAgOtCuWd3SUxM1NGjRy9Y/8svvygxMbFcfb3++uu65557NGbMGDVo0EDjx4/XqFGj9Nxzz9naTJw4UQ8//LBGjhyp1q1bKycnRytXrlS1atXKWzoAAABwXSj3POk+Pj7avXu36tSpY7f+0KFDatq0qc6cOVOhBV4r5lkFAOD6w+c3nF25z6R7enrqxIkTF6xPT0+Xm9tVz+gIAAAA4P8rd0i/8847NWnSJGVlZdnWZWZm6oknntAdd9xRocUBAAAAzqjcp75feukldezYUVFRUWrevLkkaefOnQoJCdGiRYsqvEAAAADA2ZQ7pNeqVUu7d+/WO++8o127dslqtWrYsGEaOHDgRedMBwAAAFA+VzWI3NvbWyNHjqzoWgAAAADoKkO6JO3du1dpaWkqKCiwW3/33Xdfc1EAAACAM7uqO4727t1be/bskcVi0fkZHC0WiySpuLi4YisEAAAAnEy5Z3d55JFHFB0drZMnT8rLy0s//vijNmzYoFatWmndunWVUCIAAADgXMp9Jn3z5s368ssvFRQUJBcXF7m4uOhPf/qTpk2bprFjx2rHjh2VUScAAADgNMp9Jr24uFi+vr6SpKCgIB0/flySFBUVpZSUlIqtDgAAAHBC5T6T3rhxY+3atUvR0dFq06aNZsyYIQ8PD82dO1d16tSpjBoBAAAAp1LukP7UU0/p7NmzkqRnn31Wf/nLX3TrrbcqMDBQS5curfACAQAAAGdjMc5Pz3INfvvtN1WvXt02w4uZZGdny9/fX1lZWfLz83N0OQAAoAz4/IazK9eY9MLCQrm5uemHH36wW1+jRg1TBnQAAADgelSukO7u7q7IyEjmQgcAAAAqUblnd3nyySf1xBNP6LfffquMegAAAACnV+4LR9944w0dPHhQ4eHhioqKkre3t9327du3V1hxAAAAgDMqd0jv1atXJZQBAAAA4LwKmd3FzLg6HACA6w+f33B25R6TDgAAAKBylXu4i4uLy2WnW2TmFwAAAODalDukL1u2zG65sLBQO3bs0IIFCzR16tQKKwwAAABwVhU2Jj05OVlLly7Vxx9/XBHdVRjGtAEAcP3h8xvOrsLGpLdt21Zr166tqO4AAAAAp1UhIT03N1ezZs1SrVq1KqI7AAAAwKmVe0x69erV7S4cNQxDZ86ckZeXlxYvXlyhxQEAAADOqNwh/dVXX7UL6S4uLqpZs6batGmj6tWrV2hxAAAAgDMqd0gfOnRoJZQBAAAA4Lxyj0lPSkrS+++/f8H6999/XwsWLKiQogAAAABnVu6QPm3aNAUFBV2wPjg4WC+++GKFFAUAAAA4s3KH9LS0NEVHR1+wPioqSmlpaRVSFAAAAODMyh3Sg4ODtXv37gvW79q1S4GBgRVSFAAAAODMyh3SBw4cqLFjx+qrr75ScXGxiouL9eWXX+qRRx7RgAEDKqNGAAAAwKmUe3aX5557TocPH1bnzp3l5la6e0lJiQYPHsyYdAAAAKACWAzDMK5mxwMHDmjnzp2yWq1q0qSJoqKiKrq2CpGdnS1/f39lZWXJz8/P0eUAAIAy4PMbzq7cZ9LPq1u3rurWrVuRtQAAAADQVYxJ79u3r6ZPn37B+hkzZqhfv34VUhQAAADgzMod0jds2KC77rrrgvXdunXThg0bKqQoAAAAwJmVO6Tn5OTIw8PjgvXu7u7Kzs6ukKIAAAAAZ1bukN6kSRMtXbr0gvXvvvuuGjZsWCFFAQAAAM6s3BeOTp48WX369FFqaqpuv/12SdLatWu1ZMkSvf/++xVeIAAAAOBsyh3Se/TooeXLl+vFF1/UBx98IKvVqqZNm2rNmjW67bbbKqNGAAAAwKlc9TzpF/PDDz+ocePGFdVdhWCeVQAArj98fsPZlXtM+h+dOXNGc+fO1S233KKbb765ImoCAAAAnNpVh/QNGzZo8ODBCgsL00svvaTbb79d3377bUXWBgAAADilco1Jz8jI0Pz58zVv3jxlZ2fr3nvvVX5+vpYvX87MLgAAAEAFKfOZ9B49eiguLk67d+/WzJkzdfz4cb3++uuVWRsAAADglMoc0j///HMNHz5cU6dOVffu3eXq6nrNB69du7YsFssFj8TERElSp06dLtj24IMPXvNxAQAAADMrc0jftGmTzpw5o5YtW6pNmzZ644039N///veaDr5161alp6fbHqtXr5Yk9evXz9ZmxIgRdm1mzJhxTccEAAAAzK7MIb1t27Z66623lJ6erlGjRundd99VeHi4SkpKtHr1ap05c6bcB69Zs6ZCQ0Ntj08//VQxMTF28617eXnZtWEaJgAAANzoyj27i7e3txISErRp0ybt2bNHjz/+uP7+978rODhYd99991UXUlBQoMWLFyshIUEWi8W2/p133lFQUJAaN26sSZMm6dy5c5ftJz8/X9nZ2XYPAAAA4HpyTfOkx8XFacaMGTp27JiWLFlyTYUsX75cmZmZGjp0qG3dfffdp8WLF+urr77SpEmTtGjRIt1///2X7WfatGny9/e3PSIiIq6pLgAAAKCqVegdR69Fly5d5OHhoRUrVlyyzZdffqnOnTvr4MGDiomJuWib/Px85efn25azs7MVERHBHcsAALiOcMdROLtyzZNeWY4cOaI1a9boo48+umy7Nm3aSNJlQ7qnp6c8PT0rvEYAAACgqlzTcJeKkpSUpODgYHXv3v2y7Xbu3ClJCgsLq4KqAAAAAMdw+Jn0kpISJSUlaciQIXJz+185qampSk5O1l133aXAwEDt3r1bjz76qDp27KimTZs6sGIAAACgcjk8pK9Zs0ZpaWlKSEiwW+/h4aE1a9Zo5syZOnv2rCIiItS3b1899dRTDqoUAAAAqBqmuXC0snDhCQAA1x8+v+HsTDEmHQAAAMD/ENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMoR0AAAAwGQI6QAAAIDJENIBAAAAkyGkAwAAACZDSAcAAABMhpAOAAAAmAwhHQAAADAZQjoAAABgMg4N6bVr15bFYrngkZiYKEnKy8tTYmKiAgMD5ePjo759++rEiROOLBkAAACodA4N6Vu3blV6errtsXr1aklSv379JEmPPvqoVqxYoffff1/r16/X8ePH1adPH0eWDAAAAFQ6i2EYhqOLOG/cuHH69NNPdeDAAWVnZ6tmzZpKTk7WPffcI0n66aef1KBBA23evFlt27YtU5/Z2dny9/dXVlaW/Pz8KrN8AABQQfj8hrMzzZj0goICLV68WAkJCbJYLNq2bZsKCwsVHx9va1O/fn1FRkZq8+bNl+wnPz9f2dnZdg8AAADgemKakL58+XJlZmZq6NChkqSMjAx5eHgoICDArl1ISIgyMjIu2c+0adPk7+9ve0RERFRi1QAAAEDFM01Inzdvnrp166bw8PBr6mfSpEnKysqyPY4ePVpBFQIAAABVw83RBUjSkSNHtGbNGn300Ue2daGhoSooKFBmZqbd2fQTJ04oNDT0kn15enrK09OzMssFAAAAKpUpzqQnJSUpODhY3bt3t61r2bKl3N3dtXbtWtu6lJQUpaWlqV27do4oEwAAAKgSDj+TXlJSoqSkJA0ZMkRubv8rx9/fX8OHD9djjz2mGjVqyM/PTw8//LDatWtX5pldAAAAgOuRw0P6mjVrlJaWpoSEhAu2vfrqq3JxcVHfvn2Vn5+vLl266F//+pcDqgQAAACqjqnmSa8MzLMKAMD1h89vODtTjEkHAAAA8D+EdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJiMw0P6L7/8ovvvv1+BgYGyWq1q0qSJvv/+e9v2oUOHymKx2D26du3qwIoBAACAyuXmyIOfPn1aHTp00J///Gd9/vnnqlmzpg4cOKDq1avbtevatauSkpJsy56enlVdKgAAAFBlHBrSp0+froiICLsAHh0dfUE7T09PhYaGVmVpAAAAgMM4dLjLJ598olatWqlfv34KDg5W8+bN9dZbb13Qbt26dQoODlZcXJxGjx6tU6dOXbLP/Px8ZWdn2z0AAACA64lDQ/rPP/+s2bNnq27dulq1apVGjx6tsWPHasGCBbY2Xbt21cKFC7V27VpNnz5d69evV7du3VRcXHzRPqdNmyZ/f3/bIyIioqqeDgAAAFAhLIZhGI46uIeHh1q1aqVvvvnGtm7s2LHaunWrNm/efNF9fv75Z8XExGjNmjXq3LnzBdvz8/OVn59vW87OzlZERISysrLk5+dX8U8CAABUuOzsbPn7+/P5Dafl0DPpYWFhatiwod26Bg0aKC0t7ZL71KlTR0FBQTp48OBFt3t6esrPz8/uAQAAAFxPHBrSO3TooJSUFLt1+/fvV1RU1CX3OXbsmE6dOqWwsLDKLg8AAABwCIeG9EcffVTffvutXnzxRR08eFDJycmaO3euEhMTJUk5OTmaMGGCvv32Wx0+fFhr165Vz549FRsbqy5dujiydAAAAKDSODSkt27dWsuWLdOSJUvUuHFjPffcc5o5c6YGDRokSXJ1ddXu3bt19913q169eho+fLhatmypjRs3Mlc6AAAAblgOvXC0KnDhCQAA1x8+v+HsHHomHQAAAMCFCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMAAAAmQ0gHAAAATIaQDgAAAJgMIR0AAAAwGTdHF1DZDMOQJGVnZzu4EgAAUFbnP7fPf44DzuaGD+lnzpyRJEVERDi4EgAAUF6nTp2Sv7+/o8sAqpzFuMG/opaUlOj48ePy9fWVxWJxdDnXtezsbEVEROjo0aPy8/NzdDn4HV4b8+K1MTdeH/PKyspSZGSkTp8+rYCAAEeXA1S5G/5MuouLi2666SZHl3FD8fPz48PMpHhtzIvXxtx4fczLxYXL5+Cc+M0HAAAATIaQDgAAAJgMIR1l5unpqWeeeUaenp6OLgV/wGtjXrw25sbrY168NnB2N/yFowAAAMD1hjPpAAAAgMkQ0gEAAACTIaQDAAAAJkNIBwAAAEyGkI7LmjJliiwWi92jfv36ji7LaW3YsEE9evRQeHi4LBaLli9fbrfdMAw9/fTTCgsLk9VqVXx8vA4cOOCYYp3MlV6boUOHXvBe6tq1q2OKdTLTpk1T69at5evrq+DgYPXq1UspKSl2bfLy8pSYmKjAwED5+Piob9++OnHihIMqdh5leW06dep0wXvnwQcfdFDFQNUhpOOKGjVqpPT0dNtj06ZNji7JaZ09e1Y333yz/vnPf150+4wZMzRr1izNmTNHW7Zskbe3t7p06aK8vLwqrtT5XOm1kaSuXbvavZeWLFlShRU6r/Xr1ysxMVHffvutVq9ercLCQt155506e/asrc2jjz6qFStW6P3339f69et1/Phx9enTx4FVO4eyvDaSNGLECLv3zowZMxxUMVB13BxdAMzPzc1NoaGhji4Dkrp166Zu3bpddJthGJo5c6aeeuop9ezZU5K0cOFChYSEaPny5RowYEBVlup0LvfanOfp6cl7yQFWrlxptzx//nwFBwdr27Zt6tixo7KysjRv3jwlJyfr9ttvlyQlJSWpQYMG+vbbb9W2bVtHlO0UrvTanOfl5cV7B06HM+m4ogMHDig8PFx16tTRoEGDlJaW5uiScBGHDh1SRkaG4uPjbev8/f3Vpk0bbd682YGV4bx169YpODhYcXFxGj16tE6dOuXokpxSVlaWJKlGjRqSpG3btqmwsNDuvVO/fn1FRkby3qlif3xtznvnnXcUFBSkxo0ba9KkSTp37pwjygOqFGfScVlt2rTR/PnzFRcXp/T0dE2dOlW33nqrfvjhB/n6+jq6PPxORkaGJCkkJMRufUhIiG0bHKdr167q06ePoqOjlZqaqieeeELdunXT5s2b5erq6ujynEZJSYnGjRunDh06qHHjxpJK3zseHh4KCAiwa8t7p2pd7LWRpPvuu09RUVEKDw/X7t279de//lUpKSn66KOPHFgtUPkI6bis3//5vmnTpmrTpo2ioqL03nvvafjw4Q6sDLi+/H64UZMmTdS0aVPFxMRo3bp16ty5swMrcy6JiYn64YcfuLbGhC712owcOdL27yZNmigsLEydO3dWamqqYmJiqrpMoMow3AXlEhAQoHr16ungwYOOLgV/cH685h9npDhx4gRjOU2oTp06CgoK4r1UhR566CF9+umn+uqrr3TTTTfZ1oeGhqqgoECZmZl27XnvVJ1LvTYX06ZNG0nivYMbHiEd5ZKTk6PU1FSFhYU5uhT8QXR0tEJDQ7V27VrbuuzsbG3ZskXt2rVzYGW4mGPHjunUqVO8l6qAYRh66KGHtGzZMn355ZeKjo62296yZUu5u7vbvXdSUlKUlpbGe6eSXem1uZidO3dKEu8d3PAY7oLLGj9+vHr06KGoqCgdP35czzzzjFxdXTVw4EBHl+aUcnJy7M4eHTp0SDt37lSNGjUUGRmpcePG6fnnn1fdunUVHR2tyZMnKzw8XL169XJc0U7icq9NjRo1NHXqVPXt21ehoaFKTU3VxIkTFRsbqy5dujiwaueQmJio5ORkffzxx/L19bWNM/f395fVapW/v7+GDx+uxx57TDVq1JCfn58efvhhtWvXjpldKtmVXpvU1FQlJyfrrrvuUmBgoHbv3q1HH31UHTt2VNOmTR1cPVDJDOAy+vfvb4SFhRkeHh5GrVq1jP79+xsHDx50dFlO66uvvjIkXfAYMmSIYRiGUVJSYkyePNkICQkxPD09jc6dOxspKSmOLdpJXO61OXfunHHnnXcaNWvWNNzd3Y2oqChjxIgRRkZGhqPLdgoXe10kGUlJSbY2ubm5xpgxY4zq1asbXl5eRu/evY309HTHFe0krvTapKWlGR07djRq1KhheHp6GrGxscaECROMrKwsxxYOVAGLYRhGVX4pAAAAAHB5jEkHAAAATIaQDgAAAJgMIR0AAAAwGUI6AAAAYDKEdAAAAMBkCOkAAACAyRDSAQAAAJMhpAMwlcOHD8tisdhu/X0jKigoUGxsrL755ptKO8acOXPUo0ePSusfAFC5COnAdWDz5s1ydXVV9+7dHV2KKQ0dOlS9evVydBllNmfOHEVHR6t9+/aVdoyEhARt375dGzdurLRjAAAqDyEduA7MmzdPDz/8sDZs2KDjx49X6rEMw1BRUVGlHsOZGYahN954Q8OHD6/U43h4eOi+++7TrFmzKvU4AIDKQUgHTC4nJ0dLly7V6NGj1b17d82fP9+27b777lP//v3t2hcWFiooKEgLFy6UJJWUlGjatGmKjo6W1WrVzTffrA8++MDWft26dbJYLPr888/VsmVLeXp6atOmTUpNTVXPnj0VEhIiHx8ftW7dWmvWrLE7Vnp6urp37y6r1aro6GglJyerdu3amjlzpq1NZmamHnjgAdWsWVN+fn66/fbbtWvXrjI//+LiYg0fPtxWf1xcnF577TXb9ilTpmjBggX6+OOPZbFYZLFYtG7dOknS0aNHde+99yogIEA1atRQz549dfjwYdu+58/Av/TSSwoLC1NgYKASExNVWFhoa5Ofn6+//vWvioiIkKenp2JjYzVv3jwZhqHY2Fi99NJLdvXu3LlTFotFBw8evOjz2bZtm1JTU+3+KnJ+iM97772nW2+9VVarVa1bt9b+/fu1detWtWrVSj4+PurWrZt+/fVXu9fulltukbe3twICAtShQwcdOXLEtr1Hjx765JNPlJubW+afNwDAJAwApjZv3jyjVatWhmEYxooVK4yYmBijpKTEMAzD+PTTTw2r1WqcOXPG1n7FihWG1Wo1srOzDcMwjOeff96oX7++sXLlSiM1NdVISkoyPD09jXXr1hmGYRhfffWVIclo2rSp8cUXXxgHDx40Tp06ZezcudOYM2eOsWfPHmP//v3GU089ZVSrVs04cuSI7Vjx8fFGs2bNjG+//dbYtm2bcdtttxlWq9V49dVX7dr06NHD2Lp1q7F//37j8ccfNwIDA41Tp05d9PkeOnTIkGTs2LHDMAzDKCgoMJ5++mlj69atxs8//2wsXrzY8PLyMpYuXWoYhmGcOXPGuPfee42uXbsa6enpRnp6upGfn28UFBQYDRo0MBISEozdu3cbe/fuNe677z4jLi7OyM/PNwzDMIYMGWL4+fkZDz74oLFv3z5jxYoVhpeXlzF37lxbPffee68RERFhfPTRR0ZqaqqxZs0a49133zUMwzBeeOEFo2HDhnb1jx071ujYseMlX89XXnnFqF+//kWf8/nXae/evUbbtm2Nli1bGp06dTI2bdpkbN++3YiNjTUefPBBwzAMo7Cw0PD39zfGjx9vHDx40Ni7d68xf/58u9fn7NmzhouLi/HVV19dsh4AgDkR0gGTa9++vTFz5kzDMEqDWVBQkC10nV9euHChrf3AgQON/v37G4ZhGHl5eYaXl5fxzTff2PU5fPhwY+DAgYZh/C+kL1++/Iq1NGrUyHj99dcNwzCMffv2GZKMrVu32rYfOHDAkGQL6Rs3bjT8/PyMvLw8u35iYmKMN99886LH+GNIv5jExESjb9++tuUhQ4YYPXv2tGuzaNEiIy4uzvaFxjAMIz8/37BarcaqVats+0VFRRlFRUW2Nv369bP9/FJSUgxJxurVqy9axy+//GK4uroaW7ZsMQyj9AtFUFCQMX/+/EvW/sgjjxi33377RZ/z22+/bVu3ZMkSQ5Kxdu1a27pp06YZcXFxhmEYxqlTpwxJti9bl1K9evXL1gMAMCeGuwAmlpKSou+++04DBw6UJLm5ual///6aN2+ebfnee+/VO++8I0k6e/asPv74Yw0aNEiSdPDgQZ07d0533HGHfHx8bI+FCxcqNTXV7litWrWyW87JydH48ePVoEEDBQQEyMfHR/v27VNaWpqtNjc3N7Vo0cK2T2xsrKpXr25b3rVrl3JychQYGGh3/EOHDl1w/Mv55z//qZYtW6pmzZry8fHR3LlzbXVcyq5du3Tw4EH5+vrajlujRg3l5eXZHbtRo0ZydXW1LYeFhenkyZOSSoeuuLq66rbbbrvoMcLDw9W9e3f9+9//liStWLFC+fn56tev3yXrys3NVbVq1S66rWnTprZ/h4SESJKaNGlit+58bTVq1NDQoUPVpUsX9ejRQ6+99prS09Mv6NNqtercuXOXrAcAYE5uji4AwKXNmzdPRUVFCg8Pt60zDEOenp5644035O/vr0GDBum2227TyZMntXr1almtVnXt2lVSadCWpP/85z+qVauWXd+enp52y97e3nbL48eP1+rVq/XSSy8pNjZWVqtV99xzjwoKCspcf05OjsLCwmxjxH8vICCgTH28++67Gj9+vF5++WW1a9dOvr6++sc//qEtW7Zc8dgtW7a0fYH5vZo1a9r+7e7ubrfNYrGopKREUmnAvZIHHnhA//d//6dXX31VSUlJ6t+/v7y8vC7ZPigoSHv27Lnott/XYrFYLrrufG2SlJSUpLFjx2rlypVaunSpnnrqKa1evVpt27a1tfntt9/sni8A4PpASAdMqqioSAsXLtTLL7+sO++8025br169tGTJEj344INq3769IiIitHTpUn3++efq16+fLdg1bNhQnp6eSktLu+TZ4Ev5+uuvNXToUPXu3VtSaej9/UWXcXFxKioq0o4dO9SyZUtJpWfuT58+bWvTokULZWRkyM3NTbVr176Kn0JpHe3bt9eYMWNs6/54Ft7Dw0PFxcV261q0aKGlS5cqODhYfn5+V3XsJk2aqKSkROvXr1d8fPxF29x1113y9vbW7NmztXLlSm3YsOGyfTZv3lyzZ8+WYRi2IH4tmjdvrubNm2vSpElq166dkpOTbSE9NTVVeXl5at68+TUfBwBQtRjuApjUp59+qtOnT2v48OFq3Lix3aNv3762IS9S6Swvc+bM0erVq21DXSTJ19dX48eP16OPPqoFCxYoNTVV27dv1+uvv64FCxZc9vh169bVRx99pJ07d2rXrl2677777M7i1q9fX/Hx8Ro5cqS+++477dixQyNHjpTVarWFz/j4eLVr1069evXSF198ocOHD+ubb77Rk08+qe+//75MP4e6devq+++/16pVq7R//35NnjxZW7dutWtTu3Zt7d69WykpKfrvf/+rwsJCDRo0SEFBQerZs6c2btyoQ4cOad26dRo7dqyOHTtWpmPXrl1bQ4YMUUJCgpYvX27r47333rO1cXV11dChQzVp0iTVrVtX7dq1u2yff/7zn5WTk6Mff/yxTDVcyqFDhzRp0iRt3rxZR44c0RdffKEDBw6oQYMGtjYbN25UnTp1FBMTc03HAgBUPUI6YFLz5s1TfHy8/P39L9jWt29fff/999q9e7ckadCgQdq7d69q1aqlDh062LV97rnnNHnyZE2bNk0NGjRQ165d9Z///EfR0dGXPf4rr7yi6tWrq3379urRo4e6dOliN/5ckhYuXKiQkBB17NhRvXv31ogRI+Tr62sbc22xWPTZZ5+pY8eOGjZsmOrVq6cBAwboyJEjtjHXVzJq1Cj16dNH/fv3V5s2bXTq1Cm7s+qSNGLECMXFxalVq1aqWbOmvv76a3l5eWnDhg2KjIxUnz591KBBAw0fPlx5eXnlOrM+e/Zs3XPPPRozZozq16+vESNG6OzZs3Zthg8froKCAg0bNuyK/QUGBqp3794XHYZTHl5eXvrpp5/Ut29f1atXTyNHjlRiYqJGjRpla7NkyRKNGDHimo4DAHAMi2EYhqOLAHBjOHbsmCIiIrRmzRp17tzZ0eVUmY0bN6pz5846evRomb587N69W3fccYdSU1Pl4+NTKTX9+OOPuv3227V///6LftEDAJgbIR3AVfvyyy+Vk5OjJk2aKD09XRMnTtQvv/yi/fv3X3BB5o0oPz9fv/76q4YMGaLQ0NBynR2fP3++WrZsaTd7S0Vas2aNiouL1aVLl0rpHwBQuQjpAK7aqlWr9Pjjj+vnn3+Wr6+v2rdvr5kzZyoqKsrRpVWJ+fPna/jw4WrWrJk++eSTC2bQAQDgahHSAQAAAJPhwlEAAADAZAjpAAAAgMkQ0gEAAACTIaQDAAAAJkNIBwAAAEyGkA4AAACYDCEdAAAAMBlCOgAAAGAyhHQAAADAZP4fAbK/kONKb2kAAAAASUVORK5CYII=\n" - }, - "metadata": {} - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## 5. Compressing further with Optimum ONNX and TensorrtExecutionProvider" - ], - "metadata": { - "id": "AiPUhOCNWRny" - }, - "id": "AiPUhOCNWRny" - }, - { - "cell_type": "code", - "source": [ - "#@title We'll be using Optimum's ONNX Runtime support with `CUDAExecutionProvider` [because it's fast while also supporting dynamic shapes](https://github.com/huggingface/optimum-benchmark/tree/main/examples/fast-mteb#notes)\n", - "\n", - "!pip install optimum[onnxruntime-gpu]" - ], - "metadata": { - "id": "NEnwnsEQWRn8", - "cellView": "form" - }, - "execution_count": null, - "outputs": [], - "id": "NEnwnsEQWRn8" - }, - { - "cell_type": "code", - "source": [ - "#@title [`optimum-cli`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/optimization#optimizing-a-model-during-the-onnx-export) makes it extremely easy to export a model to ONNX and apply SOTA graph optimizations / kernel fusions\n", - "\n", - "!optimum-cli export onnx \\\n", - " --model moshew/bge-small-en-v1.5_setfit-sst2-english \\\n", - " --task feature-extraction \\\n", - " --optimize O4 \\\n", - " --device cuda \\\n", - " bge_auto_opt_O4 # output folder" - ], - "metadata": { - "id": "hPqEcDi8WRn8" - }, - "execution_count": null, - "outputs": [], - "id": "hPqEcDi8WRn8" - }, - { - "cell_type": "code", - "source": [ - "onnx_path = Path(\"onnx\")" - ], - "metadata": { - "id": "XVt5Gm-nWRn8" - }, - "execution_count": null, - "outputs": [], - "id": "XVt5Gm-nWRn8" - }, - { - "cell_type": "code", - "source": [ - "class OnnxPerformanceBenchmark(PerformanceBenchmark):\n", - " def __init__(self, *args, model_path, **kwargs):\n", - " super().__init__(*args, **kwargs)\n", - " self.model_path = model_path\n", - "\n", - " def compute_size(self):\n", - " size_mb = Path(self.model_path).stat().st_size / (1024 * 1024)\n", - " print(f\"Model size (MB) - {size_mb:.2f}\")\n", - " return {\"size_mb\": size_mb}\n", - "\n", - " def compute_accuracy(self):\n", - " preds = []\n", - " chunk_size = 100\n", - " for i in tqdm(range(0, len(self.dataset[\"text\"]), chunk_size)):\n", - " preds.extend(self.model.predict(self.dataset[\"text\"][i : i + chunk_size]))\n", - " labels = self.dataset[\"label\"]\n", - " accuracy = metric.compute(predictions=preds, references=labels)\n", - " print(f\"Accuracy on test set - {accuracy['accuracy']:.3f}\")\n", - " return accuracy" - ], - "metadata": { - "id": "8hvfl3xvlnEs" - }, - "id": "8hvfl3xvlnEs", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "#@title Based on the example given in [BAAI/bge-base-en-v1.5](https://huggingface.co/BAAI/bge-base-en-v1.5#using-huggingface-transformers)\n", - "import time\n", - "import torch\n", - "from transformers import AutoTokenizer\n", - "from optimum.onnxruntime import ORTModelForFeatureExtraction\n", - "\n", - "# Load model from HuggingFace Hub\n", - "tokenizer = AutoTokenizer.from_pretrained('/content/bge_auto_opt_O4')\n", - "#ort_model = ORTModelForFeatureExtraction.from_pretrained('/content/bge_auto_opt_O4', provider=\"CUDAExecutionProvider\")\n", - "ort_model = ORTModelForFeatureExtraction.from_pretrained('/content/bge_auto_opt_O4', provider=\"TensorrtExecutionProvider\")\n", - "\n", - "ort_model.save_pretrained(onnx_path)\n", - "tokenizer.save_pretrained(onnx_path)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "IpoDwkPiWRn8", - "outputId": "b3d90e98-0d6c-4e55-cf6c-9dbf1b873d61" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "EP Error /onnxruntime_src/onnxruntime/core/session/provider_bridge_ort.cc:1193 onnxruntime::Provider& onnxruntime::ProviderLibrary::Get() [ONNXRuntimeError] : 1 : FAIL : Failed to load library libonnxruntime_providers_tensorrt.so with error: libnvinfer.so.8: cannot open shared object file: No such file or directory\n", - " when using ['TensorrtExecutionProvider', 'CUDAExecutionProvider']\n", - "Falling back to ['CUDAExecutionProvider', 'CPUExecutionProvider'] and retrying.\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "('onnx/tokenizer_config.json',\n", - " 'onnx/special_tokens_map.json',\n", - " 'onnx/vocab.txt',\n", - " 'onnx/added_tokens.json',\n", - " 'onnx/tokenizer.json')" - ] - }, - "metadata": {}, - "execution_count": 19 - } - ], - "id": "IpoDwkPiWRn8" - }, - { - "cell_type": "code", - "source": [ - "from setfit.exporters.utils import mean_pooling\n", - "from sklearn.linear_model import LogisticRegression\n", - "\n", - "\n", - "class OnnxSetFitModel_1:\n", - " def __init__(self, ort_model, tokenizer, model_head):\n", - " self.ort_model = ort_model\n", - " self.tokenizer = tokenizer\n", - " self.model_head = model_head\n", - "\n", - " def predict(self, inputs):\n", - " encoded_inputs = self.tokenizer(\n", - " inputs, padding=True, truncation=True, return_tensors=\"pt\"\n", - " ).to(\"cuda\")\n", - "\n", - " outputs = self.ort_model(**encoded_inputs)\n", - " embeddings = mean_pooling(\n", - " outputs[\"last_hidden_state\"], encoded_inputs[\"attention_mask\"]\n", - " )\n", - " return self.model_head.predict(torch.Tensor.cpu(embeddings))\n", - "\n", - " def __call__(self, inputs):\n", - " return self.predict(inputs)" - ], - "metadata": { - "id": "enaQpBF9WRn9" - }, - "execution_count": null, - "outputs": [], - "id": "enaQpBF9WRn9" - }, - { - "cell_type": "code", - "source": [ - "model = SetFitModel.from_pretrained(\"moshew/bge-small-en-v1.5_setfit-sst2-english\")\n", - "onnx_setfit_model = OnnxSetFitModel_1(ort_model, tokenizer, model.model_head)\n", - "onnx_setfit_model(test_dataset[\"text\"][:2])" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "19a9c233-d94a-4134-eb38-51a0c1e3b749", - "id": "qRviEk2WWRn9" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n" - ] - }, - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "array([0, 0])" - ] - }, - "metadata": {}, - "execution_count": 21 - } - ], - "id": "qRviEk2WWRn9" - }, - { - "cell_type": "code", - "source": [ - "pb = OnnxPerformanceBenchmark(\n", - " onnx_setfit_model,\n", - " test_dataset,\n", - " \"bge-small_TensorRT (optimum onnx)\",\n", - " model_path=\"onnx/model.onnx\",\n", - ")" - ], - "metadata": { - "id": "O8jpZ3gdWRn9" - }, - "execution_count": null, - "outputs": [], - "id": "O8jpZ3gdWRn9" - }, - { - "cell_type": "code", - "source": [ - "perf_metrics.update(pb.run_benchmark())\n", - "plot_metrics(perf_metrics, \"bge-small (optimum onnx)\")" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 574, - "referenced_widgets": [ - "12a4e79070444acbbb228aba4e34a25e", - "f8bfdfce5edc4c6397e22b90eea50817", - "edd2f3f9a691462dbfb703ca072fd644", - "70fd4d6c76d348fa82ac29e416c94f2c", - "8cefab222736437289c985ffbd6a1ed5", - "b3eaa19e1a97474995de388a44bc18f1", - "71a105f4761549fc898c0a4b03fbb043", - "d3c2759bf4324ad78eda1aa61d9c3fb0", - "37d8b6b1ea044b7680b715ec31b48d8e", - "dfbde307f9f84be1b413b47248410438", - "bc3d582d8e394397897f44efe51dd333" - ] - }, - "id": "tpjtxQQlZQPa", - "outputId": "515c2532-1f34-4abc-a016-dc20c10cf1db" - }, - "id": "tpjtxQQlZQPa", - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Model size (MB) - 65.93\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - " 0%| | 0/9 [00:00:30: MatplotlibDeprecationWarning: The legendHandles attribute was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use legend_handles instead.\n", - " for handle in legend.legendHandles:\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2kAAAG2CAYAAADlf851AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABhBklEQVR4nO3deVyU5f7/8fewOuwCshkCguK+757UoxQux+OWaVm4JR6jzMrsWFpamSdPpWnfY3ky3M06pa1mamFqpuZaaiRkoglZKiCyyHL//uDn1OQGCswor+fjMY+a677u6/7MjDC+ve77uk2GYRgCAAAAANgFB1sXAAAAAAD4HSENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsiE1D2tmzZzVhwgSFhYXJbDarU6dO2rlzp2X7iBEjZDKZrB49e/a0YcUAAAAAULmcbHnw++67T999952WLl2qkJAQLVu2TDExMTp48KBq164tSerZs6cSExMt+7i6utqqXAAAAACodCbDMAxbHDgvL0+enp56//331adPH0t769at1atXLz333HMaMWKEMjMztWbNGluUCAAAAABVzmYzaUVFRSouLlaNGjWs2s1ms7Zs2WJ5npSUpICAANWsWVPdu3fXc889Jz8/v8uOW1BQoIKCAsvzkpISnT59Wn5+fjKZTBX/QgAAQIUzDENnz55VSEiIHBy4hB5A9WKzmTRJ6tSpk1xcXLRixQoFBgZq5cqVGj58uKKiopScnKy33npLbm5uioiIUGpqqp544gl5eHho27ZtcnR0vOSY06ZN0/Tp06v4lQAAgMpw7Ngx3XLLLbYuAwCqlE1DWmpqqkaNGqUvv/xSjo6OatWqlerXr69du3bp0KFDF/X/8ccfFRkZqQ0bNqhHjx6XHPPPM2lZWVmqU6eOjh07Ji8vr0p7LQAAoOJkZ2crNDRUmZmZ8vb2tnU5AFClbLpwSGRkpDZt2qRz584pOztbwcHBGjJkiOrWrXvJ/nXr1pW/v79SUlIuG9JcXV0vubiIl5cXIQ0AgBsMlyoAqI7s4iRvd3d3BQcH68yZM1q3bp369et3yX7Hjx/XqVOnFBwcXMUVAgAAAEDVsOlM2rp162QYhqKjo5WSkqLHHntMDRo00MiRI5WTk6Pp06dr0KBBCgoKUmpqqiZNmqSoqCjFxsbasmwAAAAAqDQ2nUnLyspSQkKCGjRooLi4OP3lL3/RunXr5OzsLEdHR+3fv19///vfVb9+fY0ePVqtW7fW5s2buVcaAAAAgJuWTRcOqQrZ2dny9vZWVlYW16QBAHCDKOv3d3FxsQoLC6uwMgAovwuTUGVl09MdAQAAroVhGMrIyFBmZqatSwGAMvHx8VFQUFCZFkQipAEAgBvOhYAWEBAgNzc3VoEEYLcMw1Bubq5OnjwpSWVaBJGQBgAAbijFxcWWgObn52frcgDgqsxmsyTp5MmTCggIuOqpj3axBD8AAEBZXbgGzc3NzcaVAEDZXfidVZbraAlpAADghsQpjgBuJOX5nUVIAwAAAAA7QkgDAACoAt26ddOECRNsXUaVWbRokXx8fCzPp02bphYtWtisnsoQHh6uOXPm2LoM3IQIaQAAALC5n376SSaTyfLw8/PT7bffrj179pRp//DwcKv9//wYMWJE5b4AoAKxuiMAAADsxoYNG9S4cWMdP35c48ePV69evfT9999bzcpdys6dO1VcXCxJ+uqrrzRo0CAlJydbboZ+YXW9siosLJSzs/M1vQbgejGTBgAAqq0z587ryG/ndObc+So5XlFRkR544AF5e3vL399fU6dOlWEYlu3p6enq06ePzGazIiIitGLFiotOqcvMzNR9992nWrVqycvLS927d9e+ffuueNykpCS1a9dO7u7u8vHxUefOnXX06FFJv5+G+Oabb6pOnTry8PDQ/fffr+LiYs2aNUtBQUEKCAjQjBkzrMZ8+eWX1bRpU7m7uys0NFT333+/cnJyrvs98vPzU1BQkNq0aaMXX3xRv/zyi7Zv365nnnlGTZo0uah/ixYtNHXqVNWqVUtBQUEKCgqSr6+vJCkgIMDStmLFCkVGRsrFxUXR0dFaunSp1Tgmk0nz58/X3//+d7m7u1te74cffqi2bduqRo0a8vf314ABA6z2y83N1ahRo+Tp6ak6depowYIF1/0eAIQ0AABQ7eQXFut/u47phU+/1+z1yXrh0+/1v13HlF9YXKnHXbx4sZycnLRjxw698sorevnll/XGG29YtsfFxenEiRNKSkrSu+++qwULFlhugHvB4MGDdfLkSa1du1a7du1Sq1at1KNHD50+ffqSxywqKlL//v3VtWtX7d+/X9u2bVN8fLzVSnOpqalau3atPv30U61cuVILFy5Unz59dPz4cW3atEkvvPCCpkyZou3bt1v2cXBw0Ny5c3XgwAEtXrxYn3/+uSZNmlSh79eF2a/z589r1KhROnTokHbu3GnZvmfPHu3fv18jR4684jirV6/WQw89pEcffVTfffedxo4dq5EjR+qLL76w6jdt2jQNGDBA3377rUaNGqWPP/5YAwYMUO/evbVnzx5t3LhR7dq1s9rnpZdeUps2bbRnzx7df//9GjdunJKTkyvoHUC1ZdzksrKyDElGVlaWrUsBAABldKXv77y8POPgwYNGXl7eNY//zjdpRvySncbkd/cbMz85aEx+d78Rv2Sn8c43addT9hV17drVaNiwoVFSUmJpe/zxx42GDRsahmEYhw4dMiQZO3futGw/fPiwIcmYPXu2YRiGsXnzZsPLy8vIz8+3GjsyMtJ4/fXXL3ncU6dOGZKMpKSkS25/+umnDTc3NyM7O9vSFhsba4SHhxvFxcWWtujoaGPmzJmXfX3vvPOO4efnZ3memJhoeHt7Wx2nefPml93/yJEjhiRjz549hmEYxpkzZ4wBAwYYHh4eRkZGhmEYhtGrVy9j3Lhxln0efPBBo1u3bheN9cUXXxiSjDNnzhiGYRidOnUyxowZY9Vn8ODBRu/evS3PJRkTJkyw6tOxY0dj2LBhl605LCzMuOeeeyzPS0pKjICAAGP+/PmX3QfVV3l+dzGTBgAAqpUz587rm5/OyM/dVbU8XeXq5Khanq7yc3fVrp/OVOqpjx06dLCawerYsaMOHz6s4uJiJScny8nJSa1atbJsj4qKUs2aNS3P9+3bp5ycHPn5+cnDw8PyOHLkiFJTU5WWlmbV/vzzz8vX11cjRoxQbGys+vbtq1deeUXp6elWdYWHh8vT09PyPDAwUI0aNZKDg4NV2x9n9TZs2KAePXqodu3a8vT01L333qtTp04pNzf3ut6jTp06ycPDQzVr1tS+ffu0atUqBQYGSpLGjBmjlStXKj8/X+fPn9eKFSs0atSoq4556NAhde7c2aqtc+fOOnTokFVbmzZtrJ7v3btXPXr0uOLYzZo1s/y/yWRSUFDQRbOfQHmxcAgAAKhWMvMKlXu+SCE+1gtJeJmddCIzT5l5harp7mKj6q4sJydHwcHBSkpKumibj4+PfHx8tHfvXkvbhWuzEhMTNX78eH366adatWqVpkyZovXr16tDhw6SdNECGSaT6ZJtJSUlkkpXYvzb3/6mcePGacaMGfL19dWWLVs0evRonT9/Xm5ubtf8GletWqVGjRrJz8/vosVC+vbtK1dXV61evVouLi4qLCzUHXfccc3H+jN3d3er52VZbORK7xNwrQhpAACgWvExO8vNxUnZeUWq5eloac/OK5K7i5N8zJW3ot8fr+mSpK+//lr16tWTo6OjoqOjVVRUpD179qh169aSpJSUFJ05c8bSv1WrVsrIyJCTk5PCw8MveYyoqKhLtrds2VItW7bU5MmT1bFjR61YscIS0spr165dKikp0UsvvWSZbXv77bevaaw/Cw0NVWRk5CW3OTk5afjw4UpMTJSLi4uGDh1apiDVsGFDbd26VcOHD7e0bd26VY0aNbrifs2aNdPGjRuves0bUNEIaQAAoFqp6e6iNuE1tf7gL5JKZ9Cy84p06lyBbmsUWKmzaGlpaXrkkUc0duxY7d69W/PmzdNLL70kSWrQoIFiYmIUHx+v+fPny9nZWY8++qjMZrPlFMmYmBh17NhR/fv316xZs1S/fn2dOHHCssDFn0/Xk6QjR45owYIF+vvf/66QkBAlJyfr8OHDiouLu+bXERUVpcLCQs2bN099+/bV1q1b9dprr13zeOVx3333qWHDhpJKg1ZZPPbYY7rzzjvVsmVLxcTE6MMPP9R7772nDRs2XHG/p59+Wj169FBkZKSGDh2qoqIiffLJJ3r88cev+3UAV8I1aQAAoNr5W7MQ3dYoUIZh6ERmngzD0G2NAvW3ZiGVety4uDjl5eWpXbt2SkhI0EMPPaT4+HjL9iVLligwMFBdunTRgAEDNGbMGHl6eqpGjRqSSk+l++STT9SlSxeNHDlS9evX19ChQ3X06FHLdVt/5ubmpu+//16DBg1S/fr1FR8fr4SEBI0dO/aaX0fz5s318ssv64UXXlCTJk20fPlyzZw585rHK4969eqpU6dOatCggdq3b1+mffr3769XXnlFL774oho3bqzXX39diYmJ6tat2xX369atm9555x198MEHatGihbp3764dO3ZUwKsArsxkGH+4OcdNKDs7W97e3srKyrLczBAAANi3K31/5+fn68iRI4qIiLCEl2t15tx5ZeYVysfsbJfXoR0/flyhoaGWRTogGYahevXq6f7779cjjzxi63KAMivP7y5OdwQAANVWTXcXuwpnn3/+uXJyctS0aVOlp6dr0qRJCg8PV5cuXWxdml349ddf9dZbbykjI4PrxHBTI6QBAADYicLCQj3xxBP68ccf5enpqU6dOmn58uUXrSBYXQUEBMjf318LFiywujUBcLMhpAEAANiJ2NhYxcbG2roMu3WTX6UDWLBwCAAAAADYEUIaAAAAANgRQhoAAAAA2BFCGgAAAADYEUIaAAAAANgRQhoAAAAA2BFCGgAAgI1069ZNEyZMsDwPDw/XnDlzrnm8RYsWycfHx/J82rRpatGiheX5iBEj1L9//8seH1f35/cYqAyENAAAqpJhSDm/ShnfSj8mSSkbSv+b8W1pO/eBqtZ27typ+Pj4MvW9VKAbMmSIfvjhh0qorPz+HAiv1M9kMslkMsnFxUVRUVF65plnVFRUdNV9Fy1aZNn3co+ffvrp+l8MUMW4mTUAAFWhMF9K3ycd/Uo6lSKdz5GMEkkmSYZkcpBcPCS/KCmskxTcXHKuYeuqUcVq1ap1XfubzWaZzeYKqubaFBcXy2QylWufnj17KjExUQUFBfrkk0+UkJAgZ2dnTZ48+Yr7DRkyRD179rQ8HzhwoJo0aaJnnnnG0lae9/T8+fNycXEpV+1AZWAmDQCAyvZbirT1Fenr/0gZ+yUX99IwFtBICmhY+l+/qNL2jP2l/ba+Ip1KtXXlN7/c06Xvc+7pSj/UuXPnFBcXJw8PDwUHB+ull166qM8fZ8cMw9C0adNUp04dubq6KiQkROPHj5dUepri0aNH9fDDD1tmjKRrOxWvqKhIDzzwgLy9veXv76+pU6fK+MOMbkFBgSZOnKjatWvL3d1d7du3V1JSkmX7hWN+8MEHatSokVxdXTVq1CgtXrxY77//vqW+P+7zZ66urgoKClJYWJjGjRunmJgYffDBBzp37py8vLz0v//9z6r/mjVr5O7urqKiIgUFBVkeLi4ucnNzszw/f/68Bg4cKA8PD3l5eenOO+/UL7/8Yhnnwumgb7zxhiIiIlSjRuk/jGRmZmrs2LEKDAxUjRo11KRJE3300UdWNaxbt04NGzaUh4eHevbsqfT09HK978CVMJMGAEBlMYzSUxm/e1fKz5Z8IySny8yOOThJ5pqlj6J86ZcDUtYxqckgqW43qZwzE7iKwjzpwGop7Wvp/LnSgFyng9R4gORcOTNRjz32mDZt2qT3339fAQEBeuKJJ7R7926ra8b+6N1339Xs2bP11ltvqXHjxsrIyNC+ffskSe+9956aN2+u+Ph4jRkz5rrqWrx4sUaPHq0dO3bom2++UXx8vOrUqWMZ94EHHtDBgwf11ltvKSQkRKtXr1bPnj317bffql69epKk3NxcvfDCC3rjjTfk5+en4OBg5eXlKTs7W4mJiZIkX1/fMtdkNpt16tQpubu7a+jQoUpMTNQdd9xh2X7huaen52XHKCkpUb9+/eTh4aFNmzapqKhICQkJGjJkiFVgTElJ0bvvvqv33ntPjo6OKikpUa9evXT27FktW7ZMkZGROnjwoBwdHS375Obm6sUXX9TSpUvl4OCge+65RxMnTtTy5cvL/BqBKyGkAQBQWX5MkvaukBxdpFoNyh60nGqU9s/+uXR/SYr8a6WVWS0dWC19/7HkHiB531Iaor//uHRbi7sr/HA5OTlauHChli1bph49ekgqDUe33HLLZfdJS0tTUFCQYmJi5OzsrDp16qhdu3aSSgOPo6OjPD09FRQUdF21hYaGavbs2TKZTIqOjta3336r2bNna8yYMUpLS1NiYqLS0tIUEhIiSZo4caI+/fRTJSYm6vnnn5ckFRYW6j//+Y+aN29uGddsNqugoKBc9RmGoY0bN2rdunV68MEHJUn33XefOnXqpPT0dAUHB+vkyZP65JNPtGHDhiuOtXHjRn377bc6cuSIQkNDJUlLlixR48aNtXPnTrVt21ZS6SmOS5YssZwW+dlnn2nHjh06dOiQ6tevL0mqW7eu1diFhYV67bXXFBkZKak0yP7xFEvgenG6IwAAleG3lNIZNEeX0hBQ3pkwk6l0P0eX0nF+S6mcOquj3NOlM2juAZJHQGko9ggofZ72daWc+piamqrz58+rffv2ljZfX19FR0dfdp/BgwcrLy9PdevW1ZgxY7R69eoyLaZRXh06dLC6hqxjx446fPiwiouL9e2336q4uFj169eXh4eH5bFp0yalpv5+Oq6Li4uaNWt2zTV89NFH8vDwUI0aNdSrVy8NGTJE06ZNkyS1a9dOjRs31uLFiyVJy5YtU1hYmLp06XLFMQ8dOqTQ0FBLQJOkRo0aycfHR4cOHbK0hYWFWV23tnfvXt1yyy2WgHYpbm5uloAmyRIegYpCSAMAoKIV5v9+iqNX7esby6t26TjfvVs6Lq5f3pnSUxxreFm31/Aqbc87Y5u6/iQ0NFTJycn6z3/+I7PZrPvvv19dunRRYWFhldWQk5MjR0dH7dq1S3v37rU8Dh06pFdeecXSz2w2l3uxkD/661//qr179+rw4cPKy8vT4sWL5e7ubtl+3333adGiRZJKT3UcOXLkdR3vj/54HEllWnjF2dnZ6rnJZLK6jg+4XoQ0ANVWSQlfqKgk6fukkwdLr0G73r9Imkyl45w8WLqoCK6fuWbpNWj52dbt+dml7eaaFX7IyMhIOTs7a/v27Za2M2fOXHW5fLPZrL59+2ru3LlKSkrStm3b9O2330oqnb0qLi6+7tr+WJMkff3116pXr54cHR3VsmVLFRcX6+TJk4qKirJ6XO00xvLU5+7urqioKNWpU0dOThdfjXPPPffo6NGjmjt3rg4ePKjhw4dfdcyGDRvq2LFjOnbsmKXt4MGDyszMVKNGjS67X7NmzXT8+HG7uZUBqieuSQNQbRQWlyg546x2HT2jtNPndL6oRC5ODqrj667WYTUVHeQpZ0f+7QrXyTBKl9k3OVx+kZDycqpRGtZ+2ird0pZFRK6Xm2/pIiEXrkGr4VUa0M6dlBr0Kd1ewTw8PDR69Gg99thj8vPzU0BAgJ588kk5OFz+d86iRYtUXFys9u3by83NTcuWLZPZbFZYWJik0pUgv/zySw0dOlSurq7y9/e/ptrS0tL0yCOPaOzYsdq9e7fmzZtnWXmyfv36GjZsmOLi4vTSSy+pZcuW+vXXX7Vx40Y1a9ZMffr0uey44eHhWrdunZKTk+Xn5ydvb++LZqDKqmbNmho4cKAee+wx3X777Ve8lu+CmJgYNW3aVMOGDdOcOXNUVFSk+++/X127dlWbNm0uu1/Xrl3VpUsXDRo0SC+//LKioqL0/fffy2QyWS33D1Qm/jYC4KZnGIb2pJ3Ry+uT9dqmVG3/8ZRy8otVXCLl5Bdr+4+n9NqmVL28Pll70uzjNCfcwM79VnofNPfru9/VRdwDSsfNPVWx41ZXjQeUBjKjWMo6XvrfBn1K2yvJv//9b916663q27evYmJi9Je//EWtW7e+bH8fHx/997//VefOndWsWTNt2LBBH374ofz8/CRJzzzzjH766SdFRkZe1/3V4uLilJeXp3bt2ikhIUEPPfSQ1Q21ExMTFRcXp0cffVTR0dHq37+/du7cqTp16lxx3DFjxig6Olpt2rRRrVq1tHXr1muuUZJGjx6t8+fPa9SoUWXqbzKZ9P7776tmzZrq0qWLYmJiVLduXa1ateqq+7777rtq27at7rrrLjVq1EiTJk2qkFlLoKxMxk1+Am12dra8vb2VlZUlLy+vq+9QQfKK8pR8OlmHzxxWblGu3JzcVK9mPUX7RsvsZNubTALViWEY2pLym1bv/lmFJSUK8TarhrPjRf3yC4t1IitPzg4OGtCqtm6tV8F/wUb1kfGt9OWLpfc9c6jAE1ZKikpDWpeJUlDTihvXTl3p+zs/P19Hjhyxuq/VNcs9XXoNmrlmpcygoeIsXbpUDz/8sE6cOMENp3FDKs/vLk53rAQHfjugT458ovRz6ZIhOTs6q7C4UNvStynYPVi9I3qrsX9jW5cJVAt7j2Vq9e6f5eRoUqivx2X71XB2VF1/D6Vn5Wn1np/lWcNZLUJ9qq5Q3DxyT0lGScUGNKl0PKOEmbSK5uZLOLNzubm5Sk9P17/+9S+NHTuWgIZqgdMdK9iB3w5oVfIq/Zr3q8I8wxRVM0phXv//v55h+i3vN61KXqUDvx2wdanATa+wuESfHcxQYUmJgr3LNoMd7G0u3e9AhgqLSyq5QtyUSookVdY1Y6b/Pz5QfcyaNUsNGjRQUFCQJk+ebOtygCpBSKtAeUV5+uTIJ8ovzleYZ5icHa0vjnV2dFYdzzoqKC7QJ0c+UV5Rno0qBaqH5IyzOnY6TyFlDGgXhHiblXY6V8kZZyupMtzUHJwkVdaVBEbFz9ABdm7atGkqLCzUxo0b5eFx+TMigJsJIa0CJZ9OVvq5dNV2r33Ze3eYTCaFuIco/Vy6kk8nV3GFQPWy6+gZlZQYl7wG7UpqODuqpMTQrqMsIoJr4OZXurJjRc94lRSVjuvmV7HjAgDsDiGtAh0+c9hyDdqVODs6S4aUkplSRZUB1VPa6XPyrHFtyz171HBS2uncCq4I1YJHkOTiIRVU8ExswdnScT2DK3ZcAIDdIaRVoNyi3KsGtAucHZ11rvBcJVcEVF8lJYbOF5XI0eHarg1ydDDpfFExN7xG+bn7l67seO7Xih333EnJP4qZNACoBghpFcjNyU2FxYVl6ltYXCh3Z/dKrgiovhwcTHJxclDxNYas4hJDLk6OcrjGkIdqzGSSwjqVrsRYlF8xYxbll94kO6wzN7IGgGqAkFaB6tWsJ5l01aBWWFwomaQon6gqqgyonur4uutsftn+4eTPcvKLVMfXrYIrQrUR3FwKaCSdPlIarq6HYZSOE9BICmpWMfUBAOyaTUPa2bNnNWHCBIWFhclsNqtTp07auXOnZbthGHrqqacUHBwss9msmJgYHT582IYVX1m0b7SC3YN14twJXe4e4YZh6MS5Ewp2D1a0b3QVVwhUL63DasrBwaT8wuJy7ZdfWCwHB5Nah9WspMpw03OuITUZJNXwkrJ/vr6xsn8uHafpHaXjAgBuejYNaffdd5/Wr1+vpUuX6ttvv9Xtt9+umJgY/fxz6RfarFmzNHfuXL322mvavn273N3dFRsbq/z8Cjp9pIKZnczqHdFbro6uSjubdtGMWmFxodLOpsnV0VW9I3rL7FS+ZcEBlE90kKdCfc06kVW+212cyMpTHV83RQd5VlJlqBb8o0qDWvF5Ket4+WfUDKN0v+LzpeP4RVZOnagy3bp104QJE2xdRpVZtGiRfHx8LM+nTZumFi1a2KyeG9m9996r559/vkqOZTKZtGbNmio51s1k6NCheumllypsPJuFtLy8PL377ruaNWuWunTpoqioKE2bNk1RUVGaP3++DMPQnDlzNGXKFPXr10/NmjXTkiVLdOLECbv+g9PYv7GGRA+Rv9lfR88eVcqZFB3NPqqUzBQdPXtU/mZ/DYkeosb+jW1dKnDTc3Z00O2NguTs4KD0Mga19Ky80v0aB8nZkTPCcZ3qdpNa3F16Hdmv35f9GrWi/NL+JlPp/nW7VWaVgF0JDw+XyWS67GPEiBG2LvGSunXrZqmxRo0aql+/vmbOnCnDMDRt2rQrvqbL3bpJkvbt26dPPvlE48ePr9B6Lxea09PT1atXrwo9VnUwZcoUzZgxQ1lZWRUyns3uiFlUVKTi4mLVqGF96obZbNaWLVt05MgRZWRkKCYmxrLN29tb7du317Zt2zR06NBLjltQUKCCggLL8+zs7Mp5AVfQ2L+x6vrUVfLpZKVkpuhc4Tm5O7sryidKDXwbqIYTp6sAVaVFqI9yCoq0evfP+vG3HIV4my9537T8wmKd+P8BbUDL2moR6lP1xeLmYzJJkX+VvEOl796VTh4sbXMPkFw9rW9MXVJUusz+uZOls2iBjUtPcWQGDdXMzp07VVxcepr6V199pUGDBik5OVleXl6SSv+uaE/Onz8vFxcXSdKYMWP0zDPPqKCgQJ9//rni4+Pl4+OjiRMn6h//+Idln7Zt2yo+Pl5jxoy56vjz5s3T4MGDq+xG3kFBQVVynJtNkyZNFBkZqWXLlikhIeG6x7PZPxN7enqqY8eOevbZZ3XixAkVFxdr2bJl2rZtm9LT05WRkSFJCgwMtNovMDDQsu1SZs6cKW9vb8sjNDS0Ul/H5ZidzGoR0EJ31L9DwxsP1x3171CLgBYENKCKmUwm3VqvluI6hSu0ppt+zszT4V/OKj0rTyfP5is9q/T5z5l5Cq3ppriO4bq1Xi1bl42bjX+U1PkhqWOCFNRcOn9OOpVSGtpOHir976mU0vag5qX9Oj9EQKsCmfmZOpp9VJn5mVVyvKKiIj3wwAPy9vaWv7+/pk6danUde3p6uvr06SOz2ayIiAitWLFC4eHhmjNnzu81Z2bqvvvuU61ateTl5aXu3btr3759VzxuUlKS2rVrJ3d3d/n4+Khz5846evSopN9nVN58803VqVNHHh4euv/++1VcXKxZs2YpKChIAQEBmjFjhtWYL7/8spo2bSp3d3eFhobq/vvvV05OznW/R7Vq1VJQUJCCgoLk6+srSQoICLC0JSUlqVWrVqpRo4bq1q2r6dOnq6jo95vHm0wmvfHGGxowYIDc3NxUr149ffDBB5btZ86c0bBhw1SrVi2ZzWbVq1dPiYmJlu3ffvutunfvLrPZLD8/P8XHx1u9rhEjRqh///6aMWOGQkJCFB39+xoDbm5uCgoKUlhYmEaOHKlmzZpp/fr18vDwsNQfFBQkR0dHeXp6WrVdSnFxsf73v/+pb9++Vu1nzpxRXFycatasKTc3N/Xq1ctq3YYLp5quWbNG9erVU40aNRQbG6tjx45Ztk+fPl379u2zzOQtWrTI8v5dOGvtp59+kslk0ttvv61bb71VZrNZbdu21Q8//KCdO3eqTZs28vDwUK9evfTrr7/fduRSp/b279/fahY0PDxczz33nOLi4uTh4aGwsDB98MEH+vXXX9WvXz95eHioWbNm+uabby753lyQlpZm6e/l5aU777xTv/zyi2X7hT/fS5cuVXh4uLy9vTV06FCdPfv7vSy7deum8ePHa9KkSfL19VVQUJCmTZtm2Z6UlCQXFxdt3rzZ0jZr1iwFBARYHatv37566623rlhvWdn0XJ6lS5fKMAzVrl1brq6umjt3ru666y45OFx7WZMnT1ZWVpblceEPI4DqrUWojx6+rb7+0TVS7ev6ybOGs5wcTPKs4az2df30j66Revi2+sygofI415BC20l/mSDdNl3qMlFqO1pqHVf63y4TS9v/MqG0H4uEVKr8ony9n/K+5uyeo//b83+as3uO3k95X/kVdduEy1i8eLGcnJy0Y8cOvfLKK3r55Zf1xhtvWLbHxcXpxIkTSkpK0rvvvqsFCxbo5MmTVmMMHjxYJ0+e1Nq1a7Vr1y61atVKPXr00OnTpy95zKKiIvXv319du3bV/v37tW3bNsXHx1udYpeamqq1a9fq008/1cqVK7Vw4UL16dNHx48f16ZNm/TCCy9oypQp2r59u2UfBwcHzZ07VwcOHNDixYv1+eefa9KkSRX8jlnbvHmz4uLi9NBDD+ngwYN6/fXXtWjRoosC5PTp03XnnXdq//796t27t4YNG2Z5f6ZOnaqDBw9q7dq1OnTokObPny9/f39J0rlz5xQbG6uaNWtq586deuedd7RhwwY98MADVuNv3LhRycnJWr9+vT766KOL6jQMQ5s3b9b3339vmWW7Fvv371dWVpbatGlj1T5ixAh98803+uCDD7Rt2zYZhqHevXursPD39RByc3M1Y8YMLVmyRFu3blVmZqblTLQhQ4bo0UcfVePGjZWenq709HQNGTLksnU8/fTTmjJlinbv3i0nJyfdfffdmjRpkl555RVt3rxZKSkpeuqpp8r9+mbPnq3OnTtrz5496tOnj+69917FxcXpnnvu0e7duxUZGam4uLjLLshXUlKifv366fTp09q0aZPWr1+vH3/88aLXkpqaqjVr1uijjz7SRx99pE2bNulf//qXVZ/FixfL3d1d27dv16xZs/TMM89o/fr1kn4Pnffee6+ysrK0Z88eTZ06VW+88YbVhFK7du20Y8cOq7P6rplhB3JycowTJ04YhmEYd955p9G7d28jNTXVkGTs2bPHqm+XLl2M8ePHl3nsrKwsQ5KRlZVVkSUDuAkUF5fYugQAl3Gl7++8vDzj4MGDRl5e3jWPv+bwGmP8xvHGM189Y7z8zcvGM189Y4zfON5Yc3jN9ZR9RV27djUaNmxolJT8/rvn8ccfNxo2bGgYhmEcOnTIkGTs3LnTsv3w4cOGJGP27NmGYRjG5s2bDS8vLyM/P99q7MjISOP111+/5HFPnTplSDKSkpIuuf3pp5823NzcjOzsbEtbbGysER4ebhQXF1vaoqOjjZkzZ1729b3zzjuGn5+f5XliYqLh7e1tdZzmzZtfdv9L+eKLLwxJxpkzZwzDMIwePXoYzz//vFWfpUuXGsHBwZbnkowpU6ZYnufk5BiSjLVr1xqGYRh9+/Y1Ro4cecnjLViwwKhZs6aRk5Njafv4448NBwcHIyMjwzAMwxg+fLgRGBhoFBQUWO3btWtXw9nZ2XB3dzecnZ0NSUaNGjWMrVu3XnScsLAwy2d6JatXrzYcHR2t/sz88MMPhiSrcX/77TfDbDYbb7/9tmEYpe+9JOPrr7+29Lnw52v79u2GYVz+85BkrF692jAMwzhy5IghyXjjjTcs21euXGlIMjZu3GhpmzlzphEdHW31Xjz00ENW4/br188YPny41Xtwzz33WJ6np6cbkoypU6da2rZt22ZIMtLT0y/5/nz22WeGo6OjkZaWZmk7cOCAIcnYsWOH5XX++c/3Y489ZrRv396q3r/85S9WY7dt29Z4/PHHLc8LCgqMFi1aGHfeeafRqFEjY8yYMRfVs2/fPkOS8dNPP12y3vL87rKLq+Ld3d0VHBysM2fOaN26derXr58iIiIUFBSkjRs3WvplZ2dr+/bt6tixow2rBXCz4EbVQPWUmZ+pPSf3yN/sLz+zn1wdXeVn9pO/2V97Tu6p1FMfO3ToYDWD1bFjRx0+fFjFxcVKTk6Wk5OTWrVqZdkeFRWlmjV/vx3Ivn37lJOTIz8/P3l4eFgeR44cUWpqqtLS0qzan3/+efn6+mrEiBGKjY1V37599corryg9Pd2qrvDwcHl6/r6ibWBgoBo1amR1dlNgYKDVrN6GDRvUo0cP1a5dW56enrr33nt16tQp5ebmVuh79kf79u3TM888Y/Uax4wZo/T0dKvjNmv2+z0F3d3d5eXlZal93Lhxeuutt9SiRQtNmjRJX331laXvoUOH1Lx5c7m7u1vaOnfurJKSEiUnJ1vamjZteskZsmHDhmnv3r3aunWrevXqpSeffFKdOnW65tebl5cnV1dXqz8zhw4dkpOTk9q3b29p8/PzU3R0tA4dOmRpc3JyUtu2bS3PGzRoIB8fH6s+ZfXH9/PCzFHTpk2t2v4841tR40q67NiHDh1SaGio1eVNjRo1uuh1/vnPd3Bw8EVj/rGWS/VxcXHR8uXL9e677yo/P1+zZ8++qJ4L10tWxM+AzRYOkaR169bJMAxFR0crJSVFjz32mBo0aKCRI0fKZDJpwoQJeu6551SvXj1FRERo6tSpCgkJUf/+/W1ZNgAAuIFlnc9SbmGugj2Crdo9XDyUcS5DWeez5FPDxzbFXUVOTo6Cg4OVlJR00TYfHx/5+Pho7969lrYL13QlJiZq/Pjx+vTTT7Vq1SpNmTJF69evV4cOHSRJzs7OVmOZTKZLtpWUlEgqvVbpb3/7m8aNG6cZM2bI19dXW7Zs0ejRo3X+/Hm5ublV4Kv+XU5OjqZPn66BAwdetO2Pi9FdqfZevXrp6NGj+uSTT7R+/Xr16NFDCQkJevHFF8tcxx9D3B95e3srKipKkvT2228rKipKHTp0sFoIrzz8/f2Vm5trtTiJLfzx/bwQGP/cduH9lUpPhTX+dIriH0/FLM+4kqzGvhZX+vNQnj4XAv3p06d1+vTpi/4cXDiltlat67+23aYzaVlZWUpISFCDBg0UFxenv/zlL1q3bp3lTZo0aZIefPBBxcfHq23btsrJydGnn3560YqQAAAAZeXt4i03ZzflnLde5CLnfI7MTmZ5u3hX2rH/eE2XJH399deqV6+eHB0dFR0draKiIu3Zs8eyPSUlRWfOnLE8b9WqlTIyMuTk5KSoqCirh7+//0XtF0KaJLVs2VKTJ0/WV199pSZNmmjFihXX/Dp27dqlkpISvfTSS+rQoYPq16+vEydOXPN4ZdWqVSslJydf9NqjoqLKtaZBrVq1NHz4cC1btkxz5szRggULJEkNGzbUvn37dO7cOUvfrVu3ysHBwWqBkLLw8PDQQw89pIkTJ172mqqrubBE/sGDBy1tDRs2VFFRkdWfpVOnTik5OVmNGjWytBUVFVktupGcnKzMzEw1bNhQUunM0IVVNCtarVq1rGZri4uL9d1331X4cRo2bKhjx45ZrUFx8OBBZWZmWr0XFSE1NVUPP/yw/vvf/6p9+/YaPnz4RSHuu+++0y233GK5xvF62DSk3XnnnUpNTVVBQYHS09P16quvytv791+MJpNJzzzzjDIyMpSfn68NGzaofv36NqwYAADc6Hxq+KhlQEv9lvebTuWdUkFxgU7lndJveb+pZUDLSp1FS0tL0yOPPKLk5GStXLlS8+bN00MPPSSp9HS0mJgYxcfHa8eOHdqzZ4/i4+NlNpstMwoxMTHq2LGj+vfvr88++0w//fSTvvrqKz355JOXXQXvyJEjmjx5srZt26ajR4/qs88+0+HDhy1/Wb8WUVFRKiws1Lx58/Tjjz9q6dKleu211655vLJ66qmntGTJEk2fPl0HDhzQoUOH9NZbb2nKlCnlGuP9999XSkqKDhw4oI8++sjyXgwbNkw1atTQ8OHD9d133+mLL77Qgw8+qHvvvfeiFcfLYuzYsfrhhx/07rvvlntfqTTstGrVSlu2bLG01atXT/369dOYMWO0ZcsW7du3T/fcc49q166tfv36Wfo5OzvrwQcf1Pbt27Vr1y6NGDFCHTp0ULt27SSVngJ45MgR7d27V7/99lvFLHbx/3Xv3l0ff/yxPv74Y33//fcaN26cMjMzK2z8C2JiYtS0aVMNGzZMu3fv1o4dOxQXF6euXbtetNjK9SguLtY999yj2NhYjRw5UomJidq/f/9FN6/evHmzbr/99go5pl1ckwYAAFCVYsNj1b1OdxUbxco4l6Fio1jd63RXbHhspR43Li5OeXl5ateunRISEvTQQw8pPj7esn3JkiUKDAxUly5dNGDAAI0ZM0aenp6Ws4hMJpM++eQTdenSRSNHjlT9+vU1dOhQHT169LIhws3NTd9//70GDRqk+vXrKz4+XgkJCRo7duw1v47mzZvr5Zdf1gsvvKAmTZpo+fLlmjlz5jWPV1axsbH66KOP9Nlnn6lt27bq0KGDZs+erbCwsDKP4eLiosmTJ6tZs2bq0qWLHB0dLcumu7m5ad26dTp9+rTatm2rO+64Qz169NCrr756TfX6+voqLi5O06ZNu+ZT9u677z4tX77cqi0xMVGtW7fW3/72N3Xs2FGGYeiTTz6xOmXPzc1Njz/+uO6++2517txZHh4eWrVqlWX7oEGD1LNnT/31r39VrVq1tHLlymuq71JGjRql4cOHWwJT3bp19de//rXCxr/AZDLp/fffV82aNdWlSxfFxMSobt26Vq+zIsyYMUNHjx7V66+/Lqn0erUFCxZoypQplttf5Ofna82aNWW6911ZmIxrnX+9QWRnZ8vb21tZWVmWmyACAAD7dqXv7/z8fB05ckQRERHXfQlEZn6mss5nydvF2y6vQzt+/LhCQ0Mti3Sg+snLy1N0dLRWrVpV5sXzFi1apAkTJlTK7BUubf78+Vq9erU+++yzy/Ypz+8umy4cAgAAYEs+NXzsKpx9/vnnysnJUdOmTZWenq5JkyYpPDxcXbp0sXVpsBGz2awlS5bot99+s3UpuAJnZ2fNmzevwsYjpAEAANiJwsJCPfHEE/rxxx/l6empTp06afny5RetPHej8/DwuOy2tWvX6tZbb63Cauxft27dbF0CruK+++6r0PE43REAANidqjrdEbaRkpJy2W21a9e23G8KuJlwuiMAAADs1oV7iQG4NFZ3BAAAN6Sb/GQgADeZ8vzOIqQBAIAbyoXrs3Jzc21cCQCU3YXfWWW5xpTTHQEAwA3F0dFRPj4+OnnypKTS+0FduNkzANgbwzCUm5urkydPysfHR46Ojlfdh5AGAABuOEFBQZJkCWoAYO98fHwsv7uuhpAGAABuOCaTScHBwQoICFBhYaGtywGAK3J2di7TDNoFhDQAAHDDcnR0LNdffADgRsDCIQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgR2wa0oqLizV16lRFRETIbDYrMjJSzz77rAzDsPQZMWKETCaT1aNnz542rBoAAAAAKo+TLQ/+wgsvaP78+Vq8eLEaN26sb775RiNHjpS3t7fGjx9v6dezZ08lJiZanru6utqiXAAAAACodDYNaV999ZX69eunPn36SJLCw8O1cuVK7dixw6qfq6urgoKCbFEiAAAAAFQpm57u2KlTJ23cuFE//PCDJGnfvn3asmWLevXqZdUvKSlJAQEBio6O1rhx43Tq1ClblAsAAAAAlc6mM2n//Oc/lZ2drQYNGsjR0VHFxcWaMWOGhg0bZunTs2dPDRw4UBEREUpNTdUTTzyhXr16adu2bXJ0dLxozIKCAhUUFFieZ2dnV8lrAQAAAICKYNOQ9vbbb2v58uVasWKFGjdurL1792rChAkKCQnR8OHDJUlDhw619G/atKmaNWumyMhIJSUlqUePHheNOXPmTE2fPr3KXgMAAAAAVCST8celFKtYaGio/vnPfyohIcHS9txzz2nZsmX6/vvvL7tfrVq19Nxzz2ns2LEXbbvUTFpoaKiysrLk5eVVsS8AAABUiuzsbHl7e/P9DaBasulMWm5urhwcrC+Lc3R0VElJyWX3OX78uE6dOqXg4OBLbnd1dWX1RwAAAAA3LJuGtL59+2rGjBmqU6eOGjdurD179ujll1/WqFGjJEk5OTmaPn26Bg0apKCgIKWmpmrSpEmKiopSbGysLUsHAAAAgEph09Mdz549q6lTp2r16tU6efKkQkJCdNddd+mpp56Si4uL8vLy1L9/f+3Zs0eZmZkKCQnR7bffrmeffVaBgYFlOganSwAAcOPh+xtAdWbTkFYV+CUPAMCNh+9vANWZTe+TBgAAAACwRkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA7QkgDAAAAADtCSAMAAAAAO0JIAwAAAAA74lSeziUlJdq0aZM2b96so0ePKjc3V7Vq1VLLli0VExOj0NDQyqoTAAAAAKqFMs2k5eXl6bnnnlNoaKh69+6ttWvXKjMzU46OjkpJSdHTTz+tiIgI9e7dW19//XVl1wwAAAAAN60yzaTVr19fHTt21H//+1/ddtttcnZ2vqjP0aNHtWLFCg0dOlRPPvmkxowZU+HFAgAAAMDNzmQYhnG1TocOHVLDhg3LNGBhYaHS0tIUGRl53cVVhOzsbHl7eysrK0teXl62LgcAAJQB398AqrMyne5Y1oAmSc7OznYT0AAAAADgRlOuhUP+qKioSK+//rqSkpJUXFyszp07KyEhQTVq1KjI+gAAAACgWrnmkDZ+/Hj98MMPGjhwoAoLC7VkyRJ98803WrlyZUXWBwAAAADVSplD2urVqzVgwADL888++0zJyclydHSUJMXGxqpDhw4VXyEAAAAAVCNlvpn1m2++qf79++vEiROSpFatWukf//iHPv30U3344YeaNGmS2rZtW2mFAgAAAEB1UOaQ9uGHH+quu+5St27dNG/ePC1YsEBeXl568sknNXXqVIWGhmrFihWVWSsAAAAA3PTKtAT/H2VmZmrSpEnat2+fXnvtNbVs2bKyaqsQLOELAMCNh+9vANVZmWfSLvDx8dGCBQv073//W3FxcXrssceUn59fGbUBAAAAQLVT5pCWlpamO++8U02bNtWwYcNUr1497dq1S25ubmrevLnWrl1bmXUCAAAAQLVQ5tMdu3XrpqCgII0YMULr1q1TamqqPvjgA0nSoUOHNHbsWAUFBentt9+u1ILLi9MlAAC48fD9DaA6K/MS/N9884327dunyMhIxcbGKiIiwrKtYcOG+vLLL7VgwYJKKRIAAAAAqosyh7TWrVvrqaee0vDhw7VhwwY1bdr0oj7x8fEVWhwAAAAAVDdlviZtyZIlKigo0MMPP6yff/5Zr7/+emXWBQAAAADVUpln0sLCwvS///2vMmsBAAAAgGqvTDNp586dK9eg5e0PAAAAAChVppAWFRWlf/3rX0pPT79sH8MwtH79evXq1Utz586tsAIBAAAAoDop0+mOSUlJeuKJJzRt2jQ1b95cbdq0UUhIiGrUqKEzZ87o4MGD2rZtm5ycnDR58mSNHTu2susGAAAAgJtSme+TJpXe0Pqdd97R5s2bdfToUeXl5cnf318tW7ZUbGysevXqJUdHx8qst9y4zwoAADcevr8BVGflCmk3In7JAwBw4+H7G0B1VuYl+AEAAAAAlY+QBgAAAAB2hJAGAAAAAHaEkAYAAAAAdoSQBgAAAAB2pNwhLTw8XM8884zS0tKu++DFxcWaOnWqIiIiZDabFRkZqWeffVZ/XHDSMAw99dRTCg4OltlsVkxMjA4fPnzdxwYAAAAAe1TukDZhwgS99957qlu3rm677Ta99dZbKigouKaDv/DCC5o/f75effVVHTp0SC+88IJmzZqlefPmWfrMmjVLc+fO1Wuvvabt27fL3d1dsbGxys/Pv6ZjAgAAAIA9u+b7pO3evVuLFi3SypUrVVxcrLvvvlujRo1Sq1atyjzG3/72NwUGBmrhwoWWtkGDBslsNmvZsmUyDEMhISF69NFHNXHiRElSVlaWAgMDtWjRIg0dOvSqx+A+KwAA3Hj4/gZQnV3zNWmtWrXS3LlzdeLECT399NN644031LZtW7Vo0UJvvvmmypL9OnXqpI0bN+qHH36QJO3bt09btmxRr169JElHjhxRRkaGYmJiLPt4e3urffv22rZt2yXHLCgoUHZ2ttUDAAAAAG4UTte6Y2FhoVavXq3ExEStX79eHTp00OjRo3X8+HE98cQT2rBhg1asWHHFMf75z38qOztbDRo0kKOjo4qLizVjxgwNGzZMkpSRkSFJCgwMtNovMDDQsu3PZs6cqenTp1/rywIAAAAAmyp3SNu9e7cSExO1cuVKOTg4KC4uTrNnz1aDBg0sfQYMGKC2bdteday3335by5cv14oVK9S4cWPt3btXEyZMUEhIiIYPH17e0iRJkydP1iOPPGJ5np2drdDQ0GsaCwAAAACqWrlDWtu2bXXbbbdp/vz56t+/v5ydnS/qExERUabrxR577DH985//tPRt2rSpjh49qpkzZ2r48OEKCgqSJP3yyy8KDg627PfLL7+oRYsWlxzT1dVVrq6u5X1ZAAAAAGAXyh3SfvzxR4WFhV2xj7u7uxITE686Vm5urhwcrC+Lc3R0VElJiaTSsBcUFKSNGzdaQll2dra2b9+ucePGlbd0AAAAALB75Q5pJ0+eVEZGhtq3b2/Vvn37djk6OqpNmzZlHqtv376aMWOG6tSpo8aNG2vPnj16+eWXNWrUKEmSyWTShAkT9Nxzz6levXqKiIjQ1KlTFRISov79+5e3dAAAAACwe+Ve3TEhIUHHjh27qP3nn39WQkJCucaaN2+e7rjjDt1///1q2LChJk6cqLFjx+rZZ5+19Jk0aZIefPBBxcfHq23btsrJydGnn36qGjVqlLd0AAAAALB75b5PmoeHh/bv36+6detatR85ckTNmjXT2bNnK7TA68V9VgAAuPHw/Q2gOiv3TJqrq6t++eWXi9rT09Pl5HTNK/oDAAAAAHQNIe3222/X5MmTlZWVZWnLzMzUE088odtuu61CiwMAAACA6qbcU18vvviiunTporCwMLVs2VKStHfvXgUGBmrp0qUVXiAAAAAAVCflDmm1a9fW/v37tXz5cu3bt09ms1kjR47UXXfddcl7pgEAAAAAyu6aLiJzd3dXfHx8RdcCAAAAANXeNa/0cfDgQaWlpen8+fNW7X//+9+vuygAAAAAqK7KHdJ+/PFHDRgwQN9++61MJpMurOBvMpkkScXFxRVbIQAAAABUI+Ve3fGhhx5SRESETp48KTc3Nx04cEBffvml2rRpo6SkpEooEQAAAACqj3LPpG3btk2ff/65/P395eDgIAcHB/3lL3/RzJkzNX78eO3Zs6cy6gQAAACAaqHcM2nFxcXy9PSUJPn7++vEiROSpLCwMCUnJ1dsdQAAAABQzZR7Jq1Jkybat2+fIiIi1L59e82aNUsuLi5asGCB6tatWxk1AgAAAEC1Ue6QNmXKFJ07d06S9Mwzz+hvf/ubbr31Vvn5+WnVqlUVXiAAAAAAVCcm48LyjNfh9OnTqlmzpmWFR3uSnZ0tb29vZWVlycvLy9blAACAMuD7G0B1Vq5r0goLC+Xk5KTvvvvOqt3X19cuAxoAAAAA3GjKFdKcnZ1Vp04d7oUGAAAAAJWk3Ks7Pvnkk3riiSd0+vTpyqgHAAAAAKq1ci8c8uqrryolJUUhISEKCwuTu7u71fbdu3dXWHEAAAAAUN2UO6T179+/EsoAAAAAAEgVtLqjPWN1KAAAbjx8fwOozsp9TRoAAAAAoPKU+3RHBweHKy63z8qPAAAAAHDtyh3SVq9ebfW8sLBQe/bs0eLFizV9+vQKKwwAAAAAqqMKuyZtxYoVWrVqld5///2KGK7CcE47AAA3Hr6/AVRnFXZNWocOHbRx48aKGg4AAAAAqqUKCWl5eXmaO3euateuXRHDAQAAAEC1Ve5r0mrWrGm1cIhhGDp79qzc3Ny0bNmyCi0OAAAAAKqbcoe02bNnW4U0BwcH1apVS+3bt1fNmjUrtDgAAAAAqG7KHdJGjBhRCWUAAAAAAKRruCYtMTFR77zzzkXt77zzjhYvXlwhRQEAAABAdVXukDZz5kz5+/tf1B4QEKDnn3++QooCAAAAgOqq3CEtLS1NERERF7WHhYUpLS2tQooCAAAAgOqq3CEtICBA+/fvv6h937598vPzq5CiAAAAAKC6KndIu+uuuzR+/Hh98cUXKi4uVnFxsT7//HM99NBDGjp0aGXUCAAAAADVRrlXd3z22Wf1008/qUePHnJyKt29pKREcXFxXJMGAAAAANfJZBiGcS07Hj58WHv37pXZbFbTpk0VFhZW0bVViOzsbHl7eysrK0teXl62LgcAAJQB398AqrNyz6RdUK9ePdWrV68iawEAAACAaq/c16QNGjRIL7zwwkXts2bN0uDBgyukKAAAAACorsod0r788kv17t37ovZevXrpyy+/rJCiAAAAAKC6KndIy8nJkYuLy0Xtzs7Oys7OrpCiAAAAAKC6KndIa9q0qVatWnVR+1tvvaVGjRpVSFEAAAAAUF2Ve+GQqVOnauDAgUpNTVX37t0lSRs3btTKlSv1zjvvVHiBAAAAAFCdlDuk9e3bV2vWrNHzzz+v//3vfzKbzWrWrJk2bNigrl27VkaNAAAAAFBtXPN90i7lu+++U5MmTSpquArBfVYAALjx8P0NoDor9zVpf3b27FktWLBA7dq1U/PmzSuiJgAAAACotq45pH355ZeKi4tTcHCwXnzxRXXv3l1ff/11RdYGAAAAANVOua5Jy8jI0KJFi7Rw4UJlZ2frzjvvVEFBgdasWcPKjgAAAABQAco8k9a3b19FR0dr//79mjNnjk6cOKF58+ZVZm0AAAAAUO2UOaStXbtWo0eP1vTp09WnTx85Ojpe98HDw8NlMpkueiQkJEiSunXrdtG2f/zjH9d9XAAAAACwV2UOaVu2bNHZs2fVunVrtW/fXq+++qp+++236zr4zp07lZ6ebnmsX79ekjR48GBLnzFjxlj1mTVr1nUdEwAAAADsWZlDWocOHfTf//5X6enpGjt2rN566y2FhISopKRE69ev19mzZ8t98Fq1aikoKMjy+OijjxQZGWl1vzU3NzerPizDCwAAAOBmVu7VHd3d3TVq1Cht2bJF3377rR599FH961//UkBAgP7+979fcyHnz5/XsmXLNGrUKJlMJkv78uXL5e/vryZNmmjy5MnKzc294jgFBQXKzs62egAAAADAjeK67pMWHR2tWbNm6fjx41q5cuV1FbJmzRplZmZqxIgRlra7775by5Yt0xdffKHJkydr6dKluueee644zsyZM+Xt7W15hIaGXlddAAAAAFCVTIZhGLYuQpJiY2Pl4uKiDz/88LJ9Pv/8c/Xo0UMpKSmKjIy8ZJ+CggIVFBRYnmdnZys0NFRZWVmcKgkAwA0iOztb3t7efH8DqJbKdZ+0ynL06FFt2LBB77333hX7tW/fXpKuGNJcXV3l6upa4TUCAAAAQFW4rtMdK0piYqICAgLUp0+fK/bbu3evJCk4OLgKqgIAAACAqmfzmbSSkhIlJiZq+PDhcnL6vZzU1FStWLFCvXv3lp+fn/bv36+HH35YXbp0UbNmzWxYMQAAAABUHpuHtA0bNigtLU2jRo2yandxcdGGDRs0Z84cnTt3TqGhoRo0aJCmTJlio0oBAAAAoPLZzcIhlYULjwEAuPHw/Q2gOrOLa9IAAAAAAKUIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgR2wa0sLDw2UymS56JCQkSJLy8/OVkJAgPz8/eXh4aNCgQfrll19sWTIAAAAAVCqbhrSdO3cqPT3d8li/fr0kafDgwZKkhx9+WB9++KHeeecdbdq0SSdOnNDAgQNtWTIAAAAAVCqTYRiGrYu4YMKECfroo490+PBhZWdnq1atWlqxYoXuuOMOSdL333+vhg0batu2berQoUOZxszOzpa3t7eysrLk5eVVmeUDAIAKwvc3gOrMbq5JO3/+vJYtW6ZRo0bJZDJp165dKiwsVExMjKVPgwYNVKdOHW3btu2y4xQUFCg7O9vqAQAAAAA3CrsJaWvWrFFmZqZGjBghScrIyJCLi4t8fHys+gUGBiojI+Oy48ycOVPe3t6WR2hoaCVWDQAAAAAVy25C2sKFC9WrVy+FhIRc1ziTJ09WVlaW5XHs2LEKqhAAAAAAKp+TrQuQpKNHj2rDhg167733LG1BQUE6f/68MjMzrWbTfvnlFwUFBV12LFdXV7m6ulZmuQAAAABQaexiJi0xMVEBAQHq06ePpa1169ZydnbWxo0bLW3JyclKS0tTx44dbVEmAAAAAFQ6m8+klZSUKDExUcOHD5eT0+/leHt7a/To0XrkkUfk6+srLy8vPfjgg+rYsWOZV3YEAAAAgBuNzUPahg0blJaWplGjRl20bfbs2XJwcNCgQYNUUFCg2NhY/ec//7FBlQAAAABQNezqPmmVgfusAABw4+H7G0B1ZhfXpAEAAAAAShHSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCOENIAAAAAwI4Q0gAAAADAjhDSAAAAAMCO2Dyk/fzzz7rnnnvk5+cns9mspk2b6ptvvrFsHzFihEwmk9WjZ8+eNqwYAAAAACqPky0PfubMGXXu3Fl//etftXbtWtWqVUuHDx9WzZo1rfr17NlTiYmJlueurq5VXSoAAAAAVAmbhrQXXnhBoaGhVgEsIiLion6urq4KCgqqytIAAAAAwCZserrjBx98oDZt2mjw4MEKCAhQy5Yt9d///veifklJSQoICFB0dLTGjRunU6dOXXbMgoICZWdnWz0AAAAA4EZh05D2448/av78+apXr57WrVuncePGafz48Vq8eLGlT8+ePbVkyRJt3LhRL7zwgjZt2qRevXqpuLj4kmPOnDlT3t7elkdoaGhVvRwAAAAAuG4mwzAMWx3cxcVFbdq00VdffWVpGz9+vHbu3Klt27Zdcp8ff/xRkZGR2rBhg3r06HHR9oKCAhUUFFieZ2dnKzQ0VFlZWfLy8qr4FwEAACpcdna2vL29+f4GUC3ZdCYtODhYjRo1smpr2LCh0tLSLrtP3bp15e/vr5SUlEtud3V1lZeXl9UDAAAAAG4UNg1pnTt3VnJyslXbDz/8oLCwsMvuc/z4cZ06dUrBwcGVXR4AAAAAVDmbhrSHH35YX3/9tZ5//nmlpKRoxYoVWrBggRISEiRJOTk5euyxx/T111/rp59+0saNG9WvXz9FRUUpNjbWlqUDAAAAQKWwaUhr27atVq9erZUrV6pJkyZ69tlnNWfOHA0bNkyS5OjoqP379+vvf/+76tevr9GjR6t169bavHkz90oDAAAAcFOy6cIhVYELjwEAuPHw/Q2gOrPpTBoAAAAAwBohDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOwIIQ0AAAAA7AghDQAAAADsCCENAAAAAOyIk60LqGyGYUiSsrOzbVwJAAAoqwvf2xe+xwGgOrnpQ9rZs2clSaGhoTauBAAAlNepU6fk7e1t6zIAoEqZjJv8n6hKSkp04sQJeXp6ymQy2bqcG1p2drZCQ0N17NgxeXl52boc/AGfjf3is7FvfD72KysrS3Xq1NGZM2fk4+Nj63IAoErd9DNpDg4OuuWWW2xdxk3Fy8uLv8zYKT4b+8VnY9/4fOyXgwOXzwOofvjNBwAAAAB2hJAGAAAAAHaEkIYyc3V11dNPPy1XV1dbl4I/4bOxX3w29o3Px37x2QCozm76hUMAAAAA4EbCTBoAAAAA2BFCGgAAAADYEUIaAAAAANgRQhoAAAAA2BFCGq5o2rRpMplMVo8GDRrYuqxq68svv1Tfvn0VEhIik8mkNWvWWG03DENPPfWUgoODZTabFRMTo8OHD9um2Grmap/NiBEjLvpZ6tmzp22KrWZmzpyptm3bytPTUwEBAerfv7+Sk5Ot+uTn5yshIUF+fn7y8PDQoEGD9Msvv9io4uqjLJ9Nt27dLvrZ+cc//mGjigGgahDScFWNGzdWenq65bFlyxZbl1RtnTt3Ts2bN9f//d//XXL7rFmzNHfuXL322mvavn273N3dFRsbq/z8/CqutPq52mcjST179rT6WVq5cmUVVlh9bdq0SQkJCfr666+1fv16FRYW6vbbb9e5c+csfR5++GF9+OGHeuedd7Rp0yadOHFCAwcOtGHV1UNZPhtJGjNmjNXPzqxZs2xUMQBUDSdbFwD75+TkpKCgIFuXAUm9evVSr169LrnNMAzNmTNHU6ZMUb9+/SRJS5YsUWBgoNasWaOhQ4dWZanVzpU+mwtcXV35WbKBTz/91Or5okWLFBAQoF27dqlLly7KysrSwoULtWLFCnXv3l2SlJiYqIYNG+rrr79Whw4dbFF2tXC1z+YCNzc3fnYAVCvMpOGqDh8+rJCQENWtW1fDhg1TWlqarUvCJRw5ckQZGRmKiYmxtHl7e6t9+/batm2bDSvDBUlJSQoICFB0dLTGjRunU6dO2bqkaikrK0uS5OvrK0natWuXCgsLrX52GjRooDp16vCzU8X+/NlcsHz5cvn7+6tJkyaaPHmycnNzbVEeAFQZZtJwRe3bt9eiRYsUHR2t9PR0TZ8+Xbfeequ+++47eXp62ro8/EFGRoYkKTAw0Ko9MDDQsg2207NnTw0cOFARERFKTU3VE088oV69emnbtm1ydHS0dXnVRklJiSZMmKDOnTurSZMmkkp/dlxcXOTj42PVl5+dqnWpz0aS7r77boWFhSkkJET79+/X448/ruTkZL333ns2rBYAKhchDVf0x9O3mjVrpvbt2yssLExvv/22Ro8ebcPKgBvLH083bdq0qZo1a6bIyEglJSWpR48eNqyseklISNB3333HtbV26HKfTXx8vOX/mzZtquDgYPXo0UOpqamKjIys6jIBoEpwuiPKxcfHR/Xr11dKSoqtS8GfXLhe488r0v3yyy9cy2GH6tatK39/f36WqtADDzygjz76SF988YVuueUWS3tQUJDOnz+vzMxMq/787FSdy302l9K+fXtJ4mcHwE2NkIZyycnJUWpqqoKDg21dCv4kIiJCQUFB2rhxo6UtOztb27dvV8eOHW1YGS7l+PHjOnXqFD9LVcAwDD3wwANavXq1Pv/8c0VERFhtb926tZydna1+dpKTk5WWlsbPTiW72mdzKXv37pUkfnYA3NQ43RFXNHHiRPXt21dhYWE6ceKEnn76aTk6Ouquu+6ydWnVUk5OjtW/Hh85ckR79+6Vr6+v6tSpowkTJui5555TvXr1FBERoalTpyokJET9+/e3XdHVxJU+G19fX02fPl2DBg1SUFCQUlNTNWnSJEVFRSk2NtaGVVcPCQkJWrFihd5//315enparjPz9vaW2WyWt7e3Ro8erUceeUS+vr7y8vLSgw8+qI4dO7KyYyW72meTmpqqFStWqHfv3vLz89P+/fv18MMPq0uXLmrWrJmNqweASmQAVzBkyBAjODjYcHFxMWrXrm0MGTLESElJsXVZ1dYXX3xhSLroMXz4cMMwDKOkpMSYOnWqERgYaLi6uho9evQwkpOTbVt0NXGlzyY3N9e4/fbbjVq1ahnOzs5GWFiYMWbMGCMjI8PWZVcLl/pcJBmJiYmWPnl5ecb9999v1KxZ03BzczMGDBhgpKen267oauJqn01aWprRpUsXw9fX13B1dTWioqKMxx57zMjKyrJt4QBQyUyGYRhVGQoBAAAAAJfHNWkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQAAAABgRwhpAAAAAGBHCGkAAAAAYEcIaQDsyk8//SSTyaS9e/faupRKc/78eUVFRemrr76qtGO89tpr6tu3b6WNDwAAKg8hDbgBbNu2TY6OjurTp4+tS7FLI0aMUP/+/W1dRpm99tprioiIUKdOnSrtGKNGjdLu3bu1efPmSjsGAACoHIQ04AawcOFCPfjgg/ryyy914sSJSj2WYRgqKiqq1GNUZ4Zh6NVXX9Xo0aMr9TguLi66++67NXfu3Eo9DgAAqHiENMDO5eTkaNWqVRo3bpz69OmjRYsWWbbdfffdGjJkiFX/wsJC+fv7a8mSJZKkkpISzZw5UxERETKbzWrevLn+97//WfonJSXJZDJp7dq1at26tVxdXbVlyxalpqaqX79+CgwMlIeHh9q2basNGzZYHSs9PV19+vSR2WxWRESEVqxYofDwcM2ZM8fSJzMzU/fdd59q1aolLy8vde/eXfv27Svz6y8uLtbo0aMt9UdHR+uVV16xbJ82bZoWL16s999/XyaTSSaTSUlJSZKkY8eO6c4775SPj498fX3Vr18//fTTT5Z9L8zAvfjiiwoODpafn58SEhJUWFho6VNQUKDHH39coaGhcnV1VVRUlBYuXCjDMBQVFaUXX3zRqt69e/fKZDIpJSXlkq9n165dSk1NtZoVvXCK59tvv61bb71VZrNZbdu21Q8//KCdO3eqTZs28vDwUK9evfTrr79afXbt2rWTu7u7fHx81LlzZx09etSyvW/fvvrggw+Ul5dX5vcbAADYHiENsHNvv/22GjRooOjoaN1zzz168803ZRiGJGnYsGH68MMPlZOTY+m/bt065ebmasCAAZKkmTNnasmSJXrttdd04MABPfzww7rnnnu0adMmq+P885//1L/+9S8dOnRIzZo1U05Ojnr37q2NGzdqz5496tmzp/r27au0tDTLPnFxcTpx4oSSkpL07rvvasGCBTp58qTVuIMHD9bJkye1du1a7dq1S61atVKPHj10+vTpMr3+kpIS3XLLLXrnnXd08OBBPfXUU3riiSf09ttvS5ImTpyoO++8Uz179lR6errS09PVqVMnFRYWKjY2Vp6entq8ebO2bt0qDw8P9ezZU+fPn7eM/8UXXyg1NVVffPGFFi9erEWLFlkF4bi4OK1cuVJz587VoUOH9Prrr8vDw0Mmk0mjRo1SYmKiVb2JiYnq0qWLoqKiLvl6Nm/erPr168vT0/OibU8//bSmTJmi3bt3y8nJSXfffbcmTZqkV155RZs3b1ZKSoqeeuopSVJRUZH69++vrl27av/+/dq2bZvi4+NlMpks47Vp00ZFRUXavn17md5rAABgJwwAdq1Tp07GnDlzDMMwjMLCQsPf39/44osvrJ4vWbLE0v+uu+4yhgwZYhiGYeTn5xtubm7GV199ZTXm6NGjjbvuusswDMP44osvDEnGmjVrrlpL48aNjXnz5hmGYRiHDh0yJBk7d+60bD98+LAhyZg9e7ZhGIaxefNmw8vLy8jPz7caJzIy0nj99dcveYwjR44Ykow9e/Zcto6EhARj0KBBlufDhw83+vXrZ9Vn6dKlRnR0tFFSUmJpKygoMMxms7Fu3TrLfmFhYUZRUZGlz+DBgy3vX3JysiHJWL9+/SXr+Pnnnw1HR0dj+/bthmEYxvnz5w1/f39j0aJFl639oYceMrp3737J1/zGG29Y2lauXGlIMjZu3GhpmzlzphEdHW0YhmGcOnXKkGQkJSVd9liGYRg1a9a8Yj0AAMD+MJMG2LHk5GTt2LFDd911lyTJyclJQ4YM0cKFCy3P77zzTi1fvlySdO7cOb3//vsaNmyYJCklJUW5ubm67bbb5OHhYXksWbJEqampVsdq06aN1fOcnBxNnDhRDRs2lI+Pjzw8PHTo0CHLTFpycrKcnJzUqlUryz5RUVGqWbOm5fm+ffuUk5MjPz8/q+MfOXLkouNfyf/93/+pdevWqlWrljw8PLRgwQKrGb1L2bdvn1JSUuTp6Wk5rq+vr/Lz862O3bhxYzk6OlqeBwcHW2YD9+7dK0dHR3Xt2vWSxwgJCVGfPn305ptvSpI+/PBDFRQUaPDgwZetKy8vTzVq1LjktmbNmln+PzAwUJLUtGlTq7YLtfn6+mrEiBGKjY1V37599corryg9Pf2iMc1ms3Jzcy9bDwAAsD9Oti4AwOUtXLhQRUVFCgkJsbQZhiFXV1e9+uqr8vb21rBhw9S1a1edPHlS69evl9lsVs+ePSXJchrkxx9/rNq1a1uN7erqavXc3d3d6vnEiRO1fv16vfjii4qKipLZbNYdd9xhdarg1eTk5Cg4ONhyjdgf+fj4lGmMt956SxMnTtRLL72kjh07ytPTU//+97+vegpfTk6OWrdubQmwf1SrVi3L/zs7O1ttM5lMKikpkVQacK7mvvvu07333qvZs2crMTFRQ4YMkZub22X7+/v769tvv73ktj/WcuG0xT+3XahNKj21cvz48fr000+1atUqTZkyRevXr1eHDh0sfU6fPm31egEAgP0jpAF2qqioSEuWLNFLL72k22+/3Wpb//79tXLlSv3jH/9Qp06dFBoaqlWrVmnt2rUaPHiw5S/2jRo1kqurq9LS0i47G3Q5W7du1YgRIyzXtuXk5FgtuhEdHa2ioiLt2bNHrVu3llQ6c3fmzBlLn1atWikjI0NOTk4KDw+/hnehtI5OnTrp/vvvt7T9eRbOxcVFxcXFVm2tWrXSqlWrFBAQIC8vr2s6dtOmTVVSUqJNmzYpJibmkn169+4td3d3zZ8/X59++qm+/PLLK47ZsmVLzZ8/X4ZhWF0/dq1atmypli1bavLkyerYsaNWrFhhCWmpqanKz89Xy5Ytr/s4AACg6nC6I2CnPvroI505c0ajR49WkyZNrB6DBg2ynPIola7y+Nprr2n9+vWWUx0lydPTUxMnTtTDDz+sxYsXKzU1Vbt379a8efO0ePHiKx6/Xr16eu+997R3717t27dPd999t9UsToMGDRQTE6P4+Hjt2LFDe/bsUXx8vMxmsyV8xMTEqGPHjurfv78+++wz/fTTT/rqq6/05JNP6ptvvinT+1CvXj198803WrdunX744QdNnTpVO3futOoTHh6u/fv3Kzk5Wb/99psKCws1bNgw+fv7q1+/ftq8ebOOHDmipKQkjR8/XsePHy/TscPDwzV8+HCNGjVKa9assYxxYdESSXJ0dNSIESM0efJk1atXTx07drzimH/961+Vk5OjAwcOlKmGyzly5IgmT56sbdu26ejRo/rss890+PBhNWzY0NJn8+bNqlu3riIjI6/rWAAAoGoR0gA7tXDhQsXExMjb2/uibYMGDdI333yj/fv3Sypd5fHgwYOqXbu2OnfubNX32Wef1dSpUzVz5kw1bNhQPXv21Mcff6yIiIgrHv/ll19WzZo11alTJ/Xt21exsbFW159J0pIlSxQYGKguXbpowIABGjNmjDw9PS3XXJlMJn3yySfq0qWLRo4cqfr162vo0KE6evSo5Zqrqxk7dqwGDhyoIUOGqH379jp16pTVrJokjRkzRtHR0WrTpo1q1aqlrVu3ys3NTV9++aXq1KmjgQMHqmHDhho9erTy8/PLNbM2f/583XHHHbr//vvVoEEDjRkzRufOnbPqM3r0aJ0/f14jR4686nh+fn4aMGDAJU/DLA83Nzd9//33GjRokOrXr6/4+HglJCRo7Nixlj4rV67UmDFjrus4AACg6pkM4/+v5Q0A1+n48eMKDQ3Vhg0b1KNHD1uXU2U2b96sHj166NixY2UKn/v379dtt92m1NRUeXh4VEpNBw4cUPfu3fXDDz9cMugDAAD7RUgDcM0+//xz5eTkqGnTpkpPT9ekSZP0888/64cffrhoQY6bUUFBgX799VcNHz5cQUFB5ZodW7RokVq3bm21emNF2rBhg4qLixUbG1sp4wMAgMpDSANwzdatW6dHH31UP/74ozw9PdWpUyfNmTNHYWFhti6tSixatEijR49WixYt9MEHH1y0giYAAMC1IKQBAAAAgB1h4RAAAAAAsCOENAAAAACwI4Q0AAAAALAjhDQAAAAAsCOENAAAAACwI4Q0AAAAALAjhDQAAAAAsCOENAAAAACwI4Q0AAAAALAj/w9Ovd8N32EWsgAAAABJRU5ErkJggg==\n" - }, - "metadata": {} - } - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "vscode": { - "interpreter": { - "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" - } - }, - "colab": { - "provenance": [], - "collapsed_sections": [ - "ffbea843-2e86-4f14-961c-b8895f9de77d", - "1402c1ba-aa7f-4b0b-9db5-1e6f0d301e70", - "b-yd2LtbSbXj", - "AiPUhOCNWRny" - ], - "machine_shape": "hm", - "gpuType": "T4" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "c22db575e86b45229ab97eb5f5193fe1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "VBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "VBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "VBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_1da311f97a57441988a622d5b019fa5b", - "IPY_MODEL_458a3ee5ff8d44fca9ca6f4888d79174", - "IPY_MODEL_09d3ebe126e2484d8cc50653f7d8f8fb", - "IPY_MODEL_f867a0c844b547e8bd1d294245c4fa13", - "IPY_MODEL_1993399b93224ec6865eaa01b6a73d8b" - ], - "layout": "IPY_MODEL_b35a241e895c41ca8be0f334615df0a3" - } - }, - "1da311f97a57441988a622d5b019fa5b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_99ef6333ea314434a1dd3ef05fa0e42b", - "placeholder": "​", - "style": "IPY_MODEL_ac1617fabea24f8a912e7cb6bb9abca4", - "value": "

Copy a token from your Hugging Face\ntokens page and paste it below.
Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file.
" - } - }, - "458a3ee5ff8d44fca9ca6f4888d79174": { - "model_module": "@jupyter-widgets/controls", - "model_name": "PasswordModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "PasswordModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "PasswordView", - "continuous_update": true, - "description": "Token:", - "description_tooltip": null, - "disabled": false, - "layout": "IPY_MODEL_108c04b373e243218924ab3b101e8908", - "placeholder": "​", - "style": "IPY_MODEL_16019f898e814155ae1c27ab2475565a", - "value": "" - } - }, - "09d3ebe126e2484d8cc50653f7d8f8fb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "CheckboxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "CheckboxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "CheckboxView", - "description": "Add token as git credential?", - "description_tooltip": null, - "disabled": false, - "indent": true, - "layout": "IPY_MODEL_d1df847b4fbe4ab992c64d5d2262cbcf", - "style": "IPY_MODEL_eeb36e1cf46b46328725481ce4b9a23b", - "value": true - } - }, - "f867a0c844b547e8bd1d294245c4fa13": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ButtonModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ButtonModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ButtonView", - "button_style": "", - "description": "Login", - "disabled": false, - "icon": "", - "layout": "IPY_MODEL_58c9a6969edf4b5ab636c0fdcb9b5b5b", - "style": "IPY_MODEL_abe2bd79c6574e5aafa865bd18f14221", - "tooltip": "" - } - }, - "1993399b93224ec6865eaa01b6a73d8b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_317f736683394e5cb974117fd8f5e7fe", - "placeholder": "​", - "style": "IPY_MODEL_a34b80886d56489cb82c36ba1415841c", - "value": "\nPro Tip: If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks.
" - } - }, - "b35a241e895c41ca8be0f334615df0a3": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": "center", - "align_self": null, - "border": null, - "bottom": null, - "display": "flex", - "flex": null, - "flex_flow": "column", - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": "50%" - } - }, - "99ef6333ea314434a1dd3ef05fa0e42b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ac1617fabea24f8a912e7cb6bb9abca4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "108c04b373e243218924ab3b101e8908": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "16019f898e814155ae1c27ab2475565a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "d1df847b4fbe4ab992c64d5d2262cbcf": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "eeb36e1cf46b46328725481ce4b9a23b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "58c9a6969edf4b5ab636c0fdcb9b5b5b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "abe2bd79c6574e5aafa865bd18f14221": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ButtonStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ButtonStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "button_color": null, - "font_weight": "" - } - }, - "317f736683394e5cb974117fd8f5e7fe": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a34b80886d56489cb82c36ba1415841c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "25405341ff8949ad8da33761ed6f66fc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_494db21467a242abae3367fe33ae22e2", - "IPY_MODEL_5d7cbb32534a4237bc92aac8ea7b5f6a", - "IPY_MODEL_d9e0294296c744d19d89a99199de3498" - ], - "layout": "IPY_MODEL_c79628e8178041688dd3bf655160e36c" - } - }, - "494db21467a242abae3367fe33ae22e2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4254f2566bb84309bfc1f44485e07fd2", - "placeholder": "​", - "style": "IPY_MODEL_d1b2a7d252ae4a8992dc66578aad690b", - "value": "Downloading builder script: 100%" - } - }, - "5d7cbb32534a4237bc92aac8ea7b5f6a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_147f2cbac9414145a9d25c4c7279098c", - "max": 4203, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_f9a6c4b4ff744f7f9f88e9d9db1476ce", - "value": 4203 - } - }, - "d9e0294296c744d19d89a99199de3498": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9378346306f241de9a7820cb4d3083a6", - "placeholder": "​", - "style": "IPY_MODEL_f8f367707cfb4c51a06a96dfc73a5f8c", - "value": " 4.20k/4.20k [00:00<00:00, 367kB/s]" - } - }, - "c79628e8178041688dd3bf655160e36c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4254f2566bb84309bfc1f44485e07fd2": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d1b2a7d252ae4a8992dc66578aad690b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "147f2cbac9414145a9d25c4c7279098c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f9a6c4b4ff744f7f9f88e9d9db1476ce": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "9378346306f241de9a7820cb4d3083a6": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f8f367707cfb4c51a06a96dfc73a5f8c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "af6c3c3e2bc240cab72aaa1e3f924925": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_5e23e345354845b6aa8bb09240ed6121", - "IPY_MODEL_447b76e1644548e18c5a2c7252acd6fa", - "IPY_MODEL_e84886bc35784d45b07f12278d85a051" - ], - "layout": "IPY_MODEL_b1a98e3cdb1c4065824791888dae60dc" - } - }, - "5e23e345354845b6aa8bb09240ed6121": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8aac54d700ab440a95c1f1027316c7ab", - "placeholder": "​", - "style": "IPY_MODEL_aa7b1c90b7504bad8dcb81ad14aeabba", - "value": "Downloading readme: 100%" - } - }, - "447b76e1644548e18c5a2c7252acd6fa": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4903060e8ef24a039941fed7bfd249c1", - "max": 378, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_d9c1fcbb414345988cc5dbb18bf0090f", - "value": 378 - } - }, - "e84886bc35784d45b07f12278d85a051": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_69dace5d98fb4916a3d8afbe1b95c291", - "placeholder": "​", - "style": "IPY_MODEL_4c37b8182e1d42c784564c8f592434a7", - "value": " 378/378 [00:00<00:00, 34.2kB/s]" - } - }, - "b1a98e3cdb1c4065824791888dae60dc": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8aac54d700ab440a95c1f1027316c7ab": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "aa7b1c90b7504bad8dcb81ad14aeabba": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "4903060e8ef24a039941fed7bfd249c1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d9c1fcbb414345988cc5dbb18bf0090f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "69dace5d98fb4916a3d8afbe1b95c291": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4c37b8182e1d42c784564c8f592434a7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "82b80be2ae3444318c0b371696a0e7e4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c89f41dfdc11499790eb7b2898421788", - "IPY_MODEL_4e6a444b8787437bad081a0c601db38f", - "IPY_MODEL_15896bcd72ed4f2b83164986e4cf6a89" - ], - "layout": "IPY_MODEL_5a0fb282d639488ca6542405582552a4" - } - }, - "c89f41dfdc11499790eb7b2898421788": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_883a493d3294437ca0d5f575dd567b6a", - "placeholder": "​", - "style": "IPY_MODEL_59b070f51c7247699cb1d08ff826983e", - "value": "Downloading data files: 100%" - } - }, - "4e6a444b8787437bad081a0c601db38f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6140854720f242169351615b62d69ea2", - "max": 3, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_8763052dfc0f49a8a43b01e483c72efa", - "value": 3 - } - }, - "15896bcd72ed4f2b83164986e4cf6a89": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4a7727656e3c4a6c99620c462af236f2", - "placeholder": "​", - "style": "IPY_MODEL_7c99a33d66444ad6801e1ef45282ca77", - "value": " 3/3 [00:01<00:00, 1.98it/s]" - } - }, - "5a0fb282d639488ca6542405582552a4": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "883a493d3294437ca0d5f575dd567b6a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "59b070f51c7247699cb1d08ff826983e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "6140854720f242169351615b62d69ea2": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8763052dfc0f49a8a43b01e483c72efa": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "4a7727656e3c4a6c99620c462af236f2": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7c99a33d66444ad6801e1ef45282ca77": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "8ca983f1de9a42f6ac889638260ee546": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c61f32ac7513430fb4baf02ea477e0ca", - "IPY_MODEL_a1538109caf4497d82bc7a5161dbc132", - "IPY_MODEL_bf62c8aab16d4bcb956ef7aff31a712a" - ], - "layout": "IPY_MODEL_5dfa9e86e0e84c939bfee8a3a81e9f03" - } - }, - "c61f32ac7513430fb4baf02ea477e0ca": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b9ab181750904882abebcd2ea17ade22", - "placeholder": "​", - "style": "IPY_MODEL_b1617dc6515a437b81e21ddcc8e212cf", - "value": "Downloading data: 100%" - } - }, - "a1538109caf4497d82bc7a5161dbc132": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2b2b57b77c514370919c0ed60101b491", - "max": 1071431, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_db866bc03c2440c2acc06b90295d8dc4", - "value": 1071431 - } - }, - "bf62c8aab16d4bcb956ef7aff31a712a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fd08faeb036f49179c9c409575b6829e", - "placeholder": "​", - "style": "IPY_MODEL_b8d05609e0c2462c9c29909b72e4ce4d", - "value": " 1.07M/1.07M [00:00<00:00, 3.39MB/s]" - } - }, - "5dfa9e86e0e84c939bfee8a3a81e9f03": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b9ab181750904882abebcd2ea17ade22": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b1617dc6515a437b81e21ddcc8e212cf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2b2b57b77c514370919c0ed60101b491": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "db866bc03c2440c2acc06b90295d8dc4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "fd08faeb036f49179c9c409575b6829e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b8d05609e0c2462c9c29909b72e4ce4d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "a168593bb357422e9e5707a5180e26f5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_a2414abd5de84802b6c30255d3aca777", - "IPY_MODEL_212c74bfd17b4b41b8c03b42d33672d5", - "IPY_MODEL_a62f55c8225641e69d21e5bdb072cb2f" - ], - "layout": "IPY_MODEL_869e78300e044d889fb3eb70ec6c3498" - } - }, - "a2414abd5de84802b6c30255d3aca777": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_dbaba5bf8be84be2b41a3b66461716c9", - "placeholder": "​", - "style": "IPY_MODEL_728fef012809452fa4edde3ecd114fd2", - "value": "Downloading data: 100%" - } - }, - "212c74bfd17b4b41b8c03b42d33672d5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_32344c7f62a246a08b804b3b2697b858", - "max": 136292, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e9c9fbb0955548a1a95a06e1a50a4bba", - "value": 136292 - } - }, - "a62f55c8225641e69d21e5bdb072cb2f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0e00b9fb00b645ebbbd4444edfacaad7", - "placeholder": "​", - "style": "IPY_MODEL_fe79098c2b044816b8bc12cf5181d0b6", - "value": " 136k/136k [00:00<00:00, 255kB/s]" - } - }, - "869e78300e044d889fb3eb70ec6c3498": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "dbaba5bf8be84be2b41a3b66461716c9": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "728fef012809452fa4edde3ecd114fd2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "32344c7f62a246a08b804b3b2697b858": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e9c9fbb0955548a1a95a06e1a50a4bba": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "0e00b9fb00b645ebbbd4444edfacaad7": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fe79098c2b044816b8bc12cf5181d0b6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "fcb883d5a42f44d28f5c9024b2c7cb3c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_88a2de0b046443858bcf64207436ef61", - "IPY_MODEL_d5ce1c09377d4d65abe05a43b7248ef2", - "IPY_MODEL_4c1558b7aa3a4df0888bbbf53f7ace2e" - ], - "layout": "IPY_MODEL_4498e6513e374d1db7d4fdf3e19db8b2" - } - }, - "88a2de0b046443858bcf64207436ef61": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e90fd1f7a97f471d8d0e1002ea662fb5", - "placeholder": "​", - "style": "IPY_MODEL_a9e4fa39283f46ed97f005c170860e1e", - "value": "Downloading data: 100%" - } - }, - "d5ce1c09377d4d65abe05a43b7248ef2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ccaf3c7cbc404bcdb2df8766cafc6ca3", - "max": 281037, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_5ba057f74d6b4803aa10a274214516e1", - "value": 281037 - } - }, - "4c1558b7aa3a4df0888bbbf53f7ace2e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_03ee849ef4ce489cb38d7d1ef197c7f8", - "placeholder": "​", - "style": "IPY_MODEL_83a4acbc55fd4d829118a8623e886c34", - "value": " 281k/281k [00:00<00:00, 506kB/s]" - } - }, - "4498e6513e374d1db7d4fdf3e19db8b2": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e90fd1f7a97f471d8d0e1002ea662fb5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a9e4fa39283f46ed97f005c170860e1e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ccaf3c7cbc404bcdb2df8766cafc6ca3": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5ba057f74d6b4803aa10a274214516e1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "03ee849ef4ce489cb38d7d1ef197c7f8": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "83a4acbc55fd4d829118a8623e886c34": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "be18b797b8f94720b8bbbef4ed6f8294": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c5525330a50a492fa3ab4f5953add4a5", - "IPY_MODEL_b16f4c82e60644dcb3377816d368e350", - "IPY_MODEL_2e98dbfb059f45a29253745b04ce7571" - ], - "layout": "IPY_MODEL_cad57d29986d4326ab150f5b4d2c5410" - } - }, - "c5525330a50a492fa3ab4f5953add4a5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_92fd04a760f4480ebdc1716f26af1fea", - "placeholder": "​", - "style": "IPY_MODEL_198efd15631d4c7d8ef37368e261ccc6", - "value": "Extracting data files: 100%" - } - }, - "b16f4c82e60644dcb3377816d368e350": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_04a5bc70b79c487a9f766d6b799710f0", - "max": 3, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_3ee22e5d88404568a798c33422eac2a1", - "value": 3 - } - }, - "2e98dbfb059f45a29253745b04ce7571": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_98e214e341144e518975429816963527", - "placeholder": "​", - "style": "IPY_MODEL_7e6cb607d3a94c72b57b985c19167a73", - "value": " 3/3 [00:00<00:00, 142.19it/s]" - } - }, - "cad57d29986d4326ab150f5b4d2c5410": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "92fd04a760f4480ebdc1716f26af1fea": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "198efd15631d4c7d8ef37368e261ccc6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "04a5bc70b79c487a9f766d6b799710f0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3ee22e5d88404568a798c33422eac2a1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "98e214e341144e518975429816963527": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7e6cb607d3a94c72b57b985c19167a73": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "7b3fa3ef1631414caa526a500f10548b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_412d2001df7b478ca4ab15da3e1e5387", - "IPY_MODEL_a8ce332fc16140c88cf982276f3f8f2b", - "IPY_MODEL_f0d2bda0937f4e4bac51f7bf98de5188" - ], - "layout": "IPY_MODEL_673b1fcb790044c99d8b965b20ba5530" - } - }, - "412d2001df7b478ca4ab15da3e1e5387": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e24a51f9c8af474c91dbc82675757eee", - "placeholder": "​", - "style": "IPY_MODEL_8d03a24919f4489a8f026a95a0d98a29", - "value": "Generating train split: " - } - }, - "a8ce332fc16140c88cf982276f3f8f2b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_93ffb9e29cd54ce1b397a5eed8b1ceef", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_04228d1cb9094b6ba5c90d072ded66e1", - "value": 1 - } - }, - "f0d2bda0937f4e4bac51f7bf98de5188": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_09ffa024197c4654a080ea720281c685", - "placeholder": "​", - "style": "IPY_MODEL_c141d1452e08451baa5ab8a69f38bc80", - "value": " 6920/0 [00:00<00:00, 163091.53 examples/s]" - } - }, - "673b1fcb790044c99d8b965b20ba5530": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e24a51f9c8af474c91dbc82675757eee": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8d03a24919f4489a8f026a95a0d98a29": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "93ffb9e29cd54ce1b397a5eed8b1ceef": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": "20px" - } - }, - "04228d1cb9094b6ba5c90d072ded66e1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "09ffa024197c4654a080ea720281c685": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c141d1452e08451baa5ab8a69f38bc80": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "5f9b9c8ed9764beba4ca6f99326deacd": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_b228adde847e4601b78b16554cb1060b", - "IPY_MODEL_f07333bb1174419b95d09a306e0969c4", - "IPY_MODEL_ae97fac586b94d3fbb448c37cd350538" - ], - "layout": "IPY_MODEL_fbd20e710796474fa92fd91c4f80da3e" - } - }, - "b228adde847e4601b78b16554cb1060b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_3e6f0aea7fd044fab818aa0fb7517e1f", - "placeholder": "​", - "style": "IPY_MODEL_d5a4f8aecc5d43c29697b0a9052e339a", - "value": "Generating validation split: " - } - }, - "f07333bb1174419b95d09a306e0969c4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_eeaf377dcfea4954940b0da7f87cf008", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_31fef98c822544e3923aa51018d5d3cc", - "value": 1 - } - }, - "ae97fac586b94d3fbb448c37cd350538": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_45e6bfbf3c67469a974d4bd4d9b91595", - "placeholder": "​", - "style": "IPY_MODEL_ff710d6f22d049d6b20d96a3d4ae082b", - "value": " 872/0 [00:00<00:00, 42874.28 examples/s]" - } - }, - "fbd20e710796474fa92fd91c4f80da3e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3e6f0aea7fd044fab818aa0fb7517e1f": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d5a4f8aecc5d43c29697b0a9052e339a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "eeaf377dcfea4954940b0da7f87cf008": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": "20px" - } - }, - "31fef98c822544e3923aa51018d5d3cc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "45e6bfbf3c67469a974d4bd4d9b91595": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ff710d6f22d049d6b20d96a3d4ae082b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c16bb3d240f64a6dbfbdbd91b7fa27ca": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_50806132e381403d95214aa1ea8dcdda", - "IPY_MODEL_3c7feaf56edc404782bf6e7c1e02a31d", - "IPY_MODEL_de812255fa0044e7bb1f6d30f264203e" - ], - "layout": "IPY_MODEL_8e7325aae85441bfb5613a4a3bc9bd13" - } - }, - "50806132e381403d95214aa1ea8dcdda": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9930de3e02d946689e717f6821bf5bd0", - "placeholder": "​", - "style": "IPY_MODEL_3f102e34df7b4be2a85a6db22bbc2d88", - "value": "Generating test split: " - } - }, - "3c7feaf56edc404782bf6e7c1e02a31d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_c9fe4932ebb6450195d0c5a936cd34b8", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_2b3d9ab563104e8db5259079d4610760", - "value": 1 - } - }, - "de812255fa0044e7bb1f6d30f264203e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b1c9975483d04fd49a32a941846b3c19", - "placeholder": "​", - "style": "IPY_MODEL_88455de695bb4339abcafc29adb1e427", - "value": " 1821/0 [00:00<00:00, 82528.28 examples/s]" - } - }, - "8e7325aae85441bfb5613a4a3bc9bd13": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9930de3e02d946689e717f6821bf5bd0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3f102e34df7b4be2a85a6db22bbc2d88": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c9fe4932ebb6450195d0c5a936cd34b8": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": "20px" - } - }, - "2b3d9ab563104e8db5259079d4610760": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "b1c9975483d04fd49a32a941846b3c19": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "88455de695bb4339abcafc29adb1e427": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "dc146e9b07ba4c7b90fe5118cb5aa0a9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_99a79a8f99724020b4d66e895cc9a772", - "IPY_MODEL_9f9d6787f0ea4acc8b9428513c92bcc5", - "IPY_MODEL_4500ae3ea21b4499b62262ef01cb0a24" - ], - "layout": "IPY_MODEL_a0b1fb701a5243c4bfaa02b19c7f347e" - } - }, - "99a79a8f99724020b4d66e895cc9a772": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d6d0461b752e4d95a7ee368f783c7908", - "placeholder": "​", - "style": "IPY_MODEL_6d06a18ec5e54eafb47df9ddd595a6c1", - "value": "Downloading (…)lve/main/config.json: 100%" - } - }, - "9f9d6787f0ea4acc8b9428513c92bcc5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_bcac97d19e044d25acab914b12ada9f7", - "max": 748, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a0e50ee4759b421598d3e78d523642fa", - "value": 748 - } - }, - "4500ae3ea21b4499b62262ef01cb0a24": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_eae6863c13ed4179a19862dd9f8dd78e", - "placeholder": "​", - "style": "IPY_MODEL_8040a35914dc47519ea680c14a621903", - "value": " 748/748 [00:00<00:00, 66.5kB/s]" - } - }, - "a0b1fb701a5243c4bfaa02b19c7f347e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d6d0461b752e4d95a7ee368f783c7908": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6d06a18ec5e54eafb47df9ddd595a6c1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "bcac97d19e044d25acab914b12ada9f7": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a0e50ee4759b421598d3e78d523642fa": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "eae6863c13ed4179a19862dd9f8dd78e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8040a35914dc47519ea680c14a621903": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "399418e7502445b98d7192e24ee94351": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_3a13cdf78533435b963cb43ad9fa5965", - "IPY_MODEL_380eca1553d144eb849dff5857015b02", - "IPY_MODEL_77da5e6e677a4d5d864184907cf3ec38" - ], - "layout": "IPY_MODEL_00ae52637a424295ac0d3330fb6d1744" - } - }, - "3a13cdf78533435b963cb43ad9fa5965": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d4d33ec061bc4901aaa088f2b9be67b3", - "placeholder": "​", - "style": "IPY_MODEL_d460b19e61ee446388a6e2abca5da9d0", - "value": "Downloading (…)831ea/.gitattributes: 100%" - } - }, - "380eca1553d144eb849dff5857015b02": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_adf02a94243f44ffa376f72c76bcc6ee", - "max": 1519, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_2e80bdbdbb6f4895ab7823dd9d0c7264", - "value": 1519 - } - }, - "77da5e6e677a4d5d864184907cf3ec38": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d6a0b910a6064ec296805253ff7d4353", - "placeholder": "​", - "style": "IPY_MODEL_32bd5b2693a14cbf8b2934d0809d75a7", - "value": " 1.52k/1.52k [00:00<00:00, 141kB/s]" - } - }, - "00ae52637a424295ac0d3330fb6d1744": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d4d33ec061bc4901aaa088f2b9be67b3": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d460b19e61ee446388a6e2abca5da9d0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "adf02a94243f44ffa376f72c76bcc6ee": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2e80bdbdbb6f4895ab7823dd9d0c7264": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "d6a0b910a6064ec296805253ff7d4353": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "32bd5b2693a14cbf8b2934d0809d75a7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "4995e67d7019480f9b638c86e1890434": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f8b3819f461548138f5f944a62f3975a", - "IPY_MODEL_98ab5169667a4655b5690d3b7ef20163", - "IPY_MODEL_4f35804a8a634ff58049168aecccb5c5" - ], - "layout": "IPY_MODEL_d9cf86bf5cb141f9b9b9788630aeb628" - } - }, - "f8b3819f461548138f5f944a62f3975a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4c32f663032c40c3995507e02bbe2a8b", - "placeholder": "​", - "style": "IPY_MODEL_3def1ccbd0f2496190c05cbf1f6a6d56", - "value": "Downloading (…)_Pooling/config.json: 100%" - } - }, - "98ab5169667a4655b5690d3b7ef20163": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8abb3e4520474d5f84eb2985d4945935", - "max": 190, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_97c6990e29774cbca80699adde648e47", - "value": 190 - } - }, - "4f35804a8a634ff58049168aecccb5c5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_64001a83afb24177b6c3648a453bb06a", - "placeholder": "​", - "style": "IPY_MODEL_712b0a9c7db04af18680fe57b12a6399", - "value": " 190/190 [00:00<00:00, 18.2kB/s]" - } - }, - "d9cf86bf5cb141f9b9b9788630aeb628": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4c32f663032c40c3995507e02bbe2a8b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3def1ccbd0f2496190c05cbf1f6a6d56": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "8abb3e4520474d5f84eb2985d4945935": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "97c6990e29774cbca80699adde648e47": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "64001a83afb24177b6c3648a453bb06a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "712b0a9c7db04af18680fe57b12a6399": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c6c88bd7a68044cc8b6c79666bd3f604": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_3719de4ae5ac4ed58a359c713c6351bf", - "IPY_MODEL_1ce663c3d8ac43f38e9a1cdb660fc328", - "IPY_MODEL_0f752900bf484d8bb36f0c3f6a81c65e" - ], - "layout": "IPY_MODEL_f8a0d2ea13294358988c6ef6849e8745" - } - }, - "3719de4ae5ac4ed58a359c713c6351bf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_c620f73097504444aba0d692f0b1698f", - "placeholder": "​", - "style": "IPY_MODEL_9ad9981dd31b41fc9792a95dc3ca096f", - "value": "Downloading (…)1c222831ea/README.md: 100%" - } - }, - "1ce663c3d8ac43f38e9a1cdb660fc328": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a7aa45bdc2a24b18adbacf20921eb344", - "max": 2222, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_abab565070934031b0112ef27ac01d7b", - "value": 2222 - } - }, - "0f752900bf484d8bb36f0c3f6a81c65e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ce2b600a91da4b4e829f4f4e96d863cd", - "placeholder": "​", - "style": "IPY_MODEL_745525d6b9c24149bfc54e7f10dd3540", - "value": " 2.22k/2.22k [00:00<00:00, 212kB/s]" - } - }, - "f8a0d2ea13294358988c6ef6849e8745": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c620f73097504444aba0d692f0b1698f": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9ad9981dd31b41fc9792a95dc3ca096f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "a7aa45bdc2a24b18adbacf20921eb344": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "abab565070934031b0112ef27ac01d7b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "ce2b600a91da4b4e829f4f4e96d863cd": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "745525d6b9c24149bfc54e7f10dd3540": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "9c7ef01f9d6e4561a7822a063edfc26a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_3d37acbb3b68444dbd68849e46d55a61", - "IPY_MODEL_e25e63d49be94535a41a6425796e8093", - "IPY_MODEL_bb1219f567c24b0e84231b30b0d4670e" - ], - "layout": "IPY_MODEL_81907ab8a98b4763b26d23caf1f7252d" - } - }, - "3d37acbb3b68444dbd68849e46d55a61": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1a4a4cae5ef948ca950d3ac9b4ff9947", - "placeholder": "​", - "style": "IPY_MODEL_30a0606700fc424b8bd30a194002ee03", - "value": "Downloading (…)222831ea/config.json: 100%" - } - }, - "e25e63d49be94535a41a6425796e8093": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ca995a79154848c6b0d64214832cf29c", - "max": 748, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_327f0119d2934623bb31eec9afb92717", - "value": 748 - } - }, - "bb1219f567c24b0e84231b30b0d4670e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a246c6dc9d6c44579bcabed1fd63e052", - "placeholder": "​", - "style": "IPY_MODEL_da4e4803b1ee43899aa7c6cf5bce3c0e", - "value": " 748/748 [00:00<00:00, 67.7kB/s]" - } - }, - "81907ab8a98b4763b26d23caf1f7252d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1a4a4cae5ef948ca950d3ac9b4ff9947": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "30a0606700fc424b8bd30a194002ee03": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ca995a79154848c6b0d64214832cf29c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "327f0119d2934623bb31eec9afb92717": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "a246c6dc9d6c44579bcabed1fd63e052": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "da4e4803b1ee43899aa7c6cf5bce3c0e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "0f2fc48655bf4039ae0f82c1aed4ecd6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_62250f331dc541ebbb347c9bf5cb89e7", - "IPY_MODEL_40c6fb85a6454b7080ba69307c4c05d9", - "IPY_MODEL_4d48891cfe0749dba0e2b4ed3aec7b81" - ], - "layout": "IPY_MODEL_15a8d50ddfa74c87aa64cd59d6708275" - } - }, - "62250f331dc541ebbb347c9bf5cb89e7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5cc27f55bf5544cd9441e2f222183a9b", - "placeholder": "​", - "style": "IPY_MODEL_9eae3d0f8d6545bdaafca20189423455", - "value": "Downloading (…)ce_transformers.json: 100%" - } - }, - "40c6fb85a6454b7080ba69307c4c05d9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5eddb4176e6f4f9ba8a210af56faaeaa", - "max": 124, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_d244786bc24147bf85682feaa17aab16", - "value": 124 - } - }, - "4d48891cfe0749dba0e2b4ed3aec7b81": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2c0644b1f1714c2bbdcf143684ec8f9b", - "placeholder": "​", - "style": "IPY_MODEL_5a01c037c4074fbbb86597a3c77e9ba8", - "value": " 124/124 [00:00<00:00, 9.36kB/s]" - } - }, - "15a8d50ddfa74c87aa64cd59d6708275": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5cc27f55bf5544cd9441e2f222183a9b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9eae3d0f8d6545bdaafca20189423455": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "5eddb4176e6f4f9ba8a210af56faaeaa": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d244786bc24147bf85682feaa17aab16": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "2c0644b1f1714c2bbdcf143684ec8f9b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5a01c037c4074fbbb86597a3c77e9ba8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "cd1768dee5c248948510056b2509901c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_77693468af0e4e049a034586c91fbd8e", - "IPY_MODEL_76750d33664b40069f5f326c4e2ea1bb", - "IPY_MODEL_dcc1295b86c44aa3aef9ca95bf1de1f0" - ], - "layout": "IPY_MODEL_05c33c32b81d4b4999b9528ee26b61f5" - } - }, - "77693468af0e4e049a034586c91fbd8e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ee07c09131ca4d918262a8356f48be1d", - "placeholder": "​", - "style": "IPY_MODEL_9d1bdad0bca1488ea52b1b25023b1264", - "value": "Downloading model_head.pkl: 100%" - } - }, - "76750d33664b40069f5f326c4e2ea1bb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8b8c2dbc3bf5407db755cdf32a238833", - "max": 3919, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_9271a7f8736f4c6d932197eff416edeb", - "value": 3919 - } - }, - "dcc1295b86c44aa3aef9ca95bf1de1f0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_43614ce55026485cb6cccb9164f2f9f3", - "placeholder": "​", - "style": "IPY_MODEL_668f3676156d4da4ab628634dd6788a1", - "value": " 3.92k/3.92k [00:00<00:00, 340kB/s]" - } - }, - "05c33c32b81d4b4999b9528ee26b61f5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ee07c09131ca4d918262a8356f48be1d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9d1bdad0bca1488ea52b1b25023b1264": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "8b8c2dbc3bf5407db755cdf32a238833": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9271a7f8736f4c6d932197eff416edeb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "43614ce55026485cb6cccb9164f2f9f3": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "668f3676156d4da4ab628634dd6788a1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "aa8820f46e7d46138e5d772dc52b9944": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_5a2e9682ea2c42dd980d465f91236b82", - "IPY_MODEL_110d200e8f454daf98575056aa9e27f1", - "IPY_MODEL_edfec7c81c6e4ed19da71dde6f0689a6" - ], - "layout": "IPY_MODEL_0d86d77967fb4c009ad6988aefed07d4" - } - }, - "5a2e9682ea2c42dd980d465f91236b82": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_54720c63b00d4515a49a00d469e0e81b", - "placeholder": "​", - "style": "IPY_MODEL_f505a56350ea42618c3ea7a60dc017ec", - "value": "Downloading pytorch_model.bin: 100%" - } - }, - "110d200e8f454daf98575056aa9e27f1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1a77f99e83d047f482eb3506aab7c61c", - "max": 133507174, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_4f0cff377f414596b7d6d61256896be1", - "value": 133507174 - } - }, - "edfec7c81c6e4ed19da71dde6f0689a6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d95b5633c5e74bdab7791f427537ac12", - "placeholder": "​", - "style": "IPY_MODEL_a1d201343b324c4386c044aa32d0d076", - "value": " 134M/134M [00:03<00:00, 53.3MB/s]" - } - }, - "0d86d77967fb4c009ad6988aefed07d4": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "54720c63b00d4515a49a00d469e0e81b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f505a56350ea42618c3ea7a60dc017ec": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "1a77f99e83d047f482eb3506aab7c61c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4f0cff377f414596b7d6d61256896be1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "d95b5633c5e74bdab7791f427537ac12": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a1d201343b324c4386c044aa32d0d076": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "4ce6dfd9090b433982962b6c23332bc9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_d5b25ac6c2b540d29583dbf3cf118ee5", - "IPY_MODEL_9cdfee8f567a4c459cd229e9641b6d37", - "IPY_MODEL_39cf3db0efda4676bf5ec7aad7ca37be" - ], - "layout": "IPY_MODEL_40390da1ecb24416b031a72af0ff82ab" - } - }, - "d5b25ac6c2b540d29583dbf3cf118ee5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1e06fe633c2c4ed79e219c3541e6740f", - "placeholder": "​", - "style": "IPY_MODEL_f468411955b648fb8b472f4aafa26765", - "value": "Downloading (…)nce_bert_config.json: 100%" - } - }, - "9cdfee8f567a4c459cd229e9641b6d37": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a64b96e1935e46339b02ef664f693c6d", - "max": 52, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_55c23dad05c14e0fa5255ea3cc3d774f", - "value": 52 - } - }, - "39cf3db0efda4676bf5ec7aad7ca37be": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_401fb1beebf042c18160a7b5eac59841", - "placeholder": "​", - "style": "IPY_MODEL_b48520d0aa7d40339cf7f52105d8c396", - "value": " 52.0/52.0 [00:00<00:00, 4.85kB/s]" - } - }, - "40390da1ecb24416b031a72af0ff82ab": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1e06fe633c2c4ed79e219c3541e6740f": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f468411955b648fb8b472f4aafa26765": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "a64b96e1935e46339b02ef664f693c6d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "55c23dad05c14e0fa5255ea3cc3d774f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "401fb1beebf042c18160a7b5eac59841": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b48520d0aa7d40339cf7f52105d8c396": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "33c005a5f12448ceb283f358d2cf4adc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_6770b60a09804294a8b4f966888db00f", - "IPY_MODEL_e54de119f490455e8c4fc164975b4e3e", - "IPY_MODEL_f92f2aa2c16d49a993ccfc2bc2a6c405" - ], - "layout": "IPY_MODEL_3b4dc1919db346b8bcd384cc1261438d" - } - }, - "6770b60a09804294a8b4f966888db00f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_55039a4737164130ab1ee6cb29ffa786", - "placeholder": "​", - "style": "IPY_MODEL_df30813516c644dab488bd09a4aa1a86", - "value": "Downloading (…)cial_tokens_map.json: 100%" - } - }, - "e54de119f490455e8c4fc164975b4e3e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_88d26f1a452c4733bc1427e56e62f8dc", - "max": 125, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_865fe252fd594d7d927365e14573cd1e", - "value": 125 - } - }, - "f92f2aa2c16d49a993ccfc2bc2a6c405": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_c6d7fbc357e2468f94811a2a0249cb01", - "placeholder": "​", - "style": "IPY_MODEL_2bdd98ffe4e942f08f6d81fd61dad7ee", - "value": " 125/125 [00:00<00:00, 10.5kB/s]" - } - }, - "3b4dc1919db346b8bcd384cc1261438d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "55039a4737164130ab1ee6cb29ffa786": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "df30813516c644dab488bd09a4aa1a86": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "88d26f1a452c4733bc1427e56e62f8dc": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "865fe252fd594d7d927365e14573cd1e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "c6d7fbc357e2468f94811a2a0249cb01": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2bdd98ffe4e942f08f6d81fd61dad7ee": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2fe48b1d327544b78aee28e1f871d7fc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_44880c7b99764d14add9f31903bea9ce", - "IPY_MODEL_e16088132c1c4ba7886449e35f338372", - "IPY_MODEL_52f05d88a6254d398eca1196f5e031b2" - ], - "layout": "IPY_MODEL_9a66a3c87df14440b547ac9772899b38" - } - }, - "44880c7b99764d14add9f31903bea9ce": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_3514155994664dca8a9e1cfd136d1f32", - "placeholder": "​", - "style": "IPY_MODEL_1bed6c4ad69a48b19180c1e6e7af6123", - "value": "Downloading (…)831ea/tokenizer.json: 100%" - } - }, - "e16088132c1c4ba7886449e35f338372": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e079d8a609e745a4a701e6b998119610", - "max": 711649, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_6b81b872347b484d8624422f729c357f", - "value": 711649 - } - }, - "52f05d88a6254d398eca1196f5e031b2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7316bc0130fa40359ad26efba5b22b3e", - "placeholder": "​", - "style": "IPY_MODEL_391b543200884ccd85dbc4ea89b8917d", - "value": " 712k/712k [00:00<00:00, 4.35MB/s]" - } - }, - "9a66a3c87df14440b547ac9772899b38": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3514155994664dca8a9e1cfd136d1f32": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1bed6c4ad69a48b19180c1e6e7af6123": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "e079d8a609e745a4a701e6b998119610": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6b81b872347b484d8624422f729c357f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "7316bc0130fa40359ad26efba5b22b3e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "391b543200884ccd85dbc4ea89b8917d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "047f10baf53c4023af28934a847021e2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_1fae7f1a5d9c492296f505877445ef0b", - "IPY_MODEL_65d3ca6bc3e44628b4ad7527d4c02294", - "IPY_MODEL_67782dd666ee4eaab8960e46826677fa" - ], - "layout": "IPY_MODEL_32c40d3b63c74f0da5d64bed6d216f2c" - } - }, - "1fae7f1a5d9c492296f505877445ef0b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0520ac5668a445c585d9476f83ec7088", - "placeholder": "​", - "style": "IPY_MODEL_4a84b8b89dc443999b362a06c762569b", - "value": "Downloading (…)okenizer_config.json: 100%" - } - }, - "65d3ca6bc3e44628b4ad7527d4c02294": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e7f9835056ce44cf974a987ff571aaf5", - "max": 1270, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_1a2f951678b54010a96e393738c4a2f5", - "value": 1270 - } - }, - "67782dd666ee4eaab8960e46826677fa": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_54eac4acad6a4f44b9839442866b1641", - "placeholder": "​", - "style": "IPY_MODEL_7fc2664adffe40e295fb14bffc3c53e3", - "value": " 1.27k/1.27k [00:00<00:00, 120kB/s]" - } - }, - "32c40d3b63c74f0da5d64bed6d216f2c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0520ac5668a445c585d9476f83ec7088": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4a84b8b89dc443999b362a06c762569b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "e7f9835056ce44cf974a987ff571aaf5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1a2f951678b54010a96e393738c4a2f5": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "54eac4acad6a4f44b9839442866b1641": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7fc2664adffe40e295fb14bffc3c53e3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "e2f4617e834644d0b3211e9a1d964bbe": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_009b5b89fa4947319825034d458786ff", - "IPY_MODEL_7062248300d049fcb6aab73529535864", - "IPY_MODEL_7cecaeee3d2a44ed844bce978daef136" - ], - "layout": "IPY_MODEL_45efd6ebc60645dd83797c225184a92c" - } - }, - "009b5b89fa4947319825034d458786ff": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_819fd904bf4d44a09eebdc05bbf8d39b", - "placeholder": "​", - "style": "IPY_MODEL_e00957ae947f4ca298aa9563ceb50446", - "value": "Downloading (…)1c222831ea/vocab.txt: 100%" - } - }, - "7062248300d049fcb6aab73529535864": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b69d0bc437b449ce803e56d97a2d57f0", - "max": 231508, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b86c025377c64e218717075bcdbcac33", - "value": 231508 - } - }, - "7cecaeee3d2a44ed844bce978daef136": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b9c0e0201bca498eb65eddae56412b1b", - "placeholder": "​", - "style": "IPY_MODEL_e5e361b8bdf7435cb92149993df8ddf9", - "value": " 232k/232k [00:00<00:00, 2.83MB/s]" - } - }, - "45efd6ebc60645dd83797c225184a92c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "819fd904bf4d44a09eebdc05bbf8d39b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e00957ae947f4ca298aa9563ceb50446": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b69d0bc437b449ce803e56d97a2d57f0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b86c025377c64e218717075bcdbcac33": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "b9c0e0201bca498eb65eddae56412b1b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e5e361b8bdf7435cb92149993df8ddf9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "5572121bcbd94107929ed5dff76a6cbd": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_5fbb7145a5f74e88bfeadc30f6ea96ef", - "IPY_MODEL_5a10ccd98b724b7a862b343e1e0010c9", - "IPY_MODEL_a548ed5b6a3e42fbb4d9e288a97e9b27" - ], - "layout": "IPY_MODEL_11bef2742d6a4f53a37640ab7ca84c49" - } - }, - "5fbb7145a5f74e88bfeadc30f6ea96ef": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_097d394b44cb4e318a11f955d0c63ef2", - "placeholder": "​", - "style": "IPY_MODEL_9de476dd14c94060b50ed96c70120747", - "value": "Downloading (…)22831ea/modules.json: 100%" - } - }, - "5a10ccd98b724b7a862b343e1e0010c9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fdfdce50a81f451aa2dbeb02fe226a5f", - "max": 349, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_f2ca6bddd40a468396c6b0fa24d92785", - "value": 349 - } - }, - "a548ed5b6a3e42fbb4d9e288a97e9b27": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7602b79489c6406eb087f045ecef9e31", - "placeholder": "​", - "style": "IPY_MODEL_55ef22f22dbc44d59ce1abd5bed5d174", - "value": " 349/349 [00:00<00:00, 29.3kB/s]" - } - }, - "11bef2742d6a4f53a37640ab7ca84c49": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "097d394b44cb4e318a11f955d0c63ef2": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9de476dd14c94060b50ed96c70120747": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "fdfdce50a81f451aa2dbeb02fe226a5f": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f2ca6bddd40a468396c6b0fa24d92785": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "7602b79489c6406eb087f045ecef9e31": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "55ef22f22dbc44d59ce1abd5bed5d174": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c24ded7ea4674b9ab404a541f82f5e23": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_987d0f782cc84d24a723f0640c2c6b7b", - "IPY_MODEL_5bbe003313d541b196b62765ffc8cdeb", - "IPY_MODEL_c69a869a500b4a31bd82166a5f5320c6" - ], - "layout": "IPY_MODEL_88bf04ac3b9448c9b4bd1f1f381fa4f3" - } - }, - "987d0f782cc84d24a723f0640c2c6b7b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5f0725d5ecd2452ab2dabbb9a6809329", - "placeholder": "​", - "style": "IPY_MODEL_2322b2da29634875b67851042da03a9a", - "value": "Downloading model_head.pkl: 100%" - } - }, - "5bbe003313d541b196b62765ffc8cdeb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cfa79548ffee42949bf10d718321adfd", - "max": 3919, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_4a776c2df1d249f8a61f9663092de047", - "value": 3919 - } - }, - "c69a869a500b4a31bd82166a5f5320c6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1223136c2471432e84b1d700d9dcee72", - "placeholder": "​", - "style": "IPY_MODEL_9e7eaa3b76a54091b189c3465248460b", - "value": " 3.92k/3.92k [00:00<00:00, 333kB/s]" - } - }, - "88bf04ac3b9448c9b4bd1f1f381fa4f3": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5f0725d5ecd2452ab2dabbb9a6809329": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2322b2da29634875b67851042da03a9a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "cfa79548ffee42949bf10d718321adfd": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4a776c2df1d249f8a61f9663092de047": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "1223136c2471432e84b1d700d9dcee72": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9e7eaa3b76a54091b189c3465248460b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "eb6e6d91566044f18e953d8df93ae3a8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_29dbf37d879a4f05904b076378d190bf", - "IPY_MODEL_1cfb1e7981264c328e02e49c4080c504", - "IPY_MODEL_e1157a679f354eaba73d4fea3cbda6fe" - ], - "layout": "IPY_MODEL_a9eafe51348847b0be895b8502a21e56" - } - }, - "29dbf37d879a4f05904b076378d190bf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_f338c914be9641a4aee76971c0e9cf72", - "placeholder": "​", - "style": "IPY_MODEL_746f408b617a4974ab48665f5b0c6c01", - "value": "Downloading (…)lve/main/config.json: 100%" - } - }, - "1cfb1e7981264c328e02e49c4080c504": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_561acc35f4e245e8adf81b1efa510fd2", - "max": 629, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_85377644ee164abebcfb24c7fdeaab8a", - "value": 629 - } - }, - "e1157a679f354eaba73d4fea3cbda6fe": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b800f7b6b2fe448486f63a24066694aa", - "placeholder": "​", - "style": "IPY_MODEL_0661f3e76c9a490db89c29c35d8c0d1b", - "value": " 629/629 [00:00<00:00, 56.8kB/s]" - } - }, - "a9eafe51348847b0be895b8502a21e56": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f338c914be9641a4aee76971c0e9cf72": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "746f408b617a4974ab48665f5b0c6c01": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "561acc35f4e245e8adf81b1efa510fd2": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "85377644ee164abebcfb24c7fdeaab8a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "b800f7b6b2fe448486f63a24066694aa": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0661f3e76c9a490db89c29c35d8c0d1b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "7612716749c14069bc5989f3ee09c00a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_0e9bfe8c83a44537bb59c444744466ef", - "IPY_MODEL_5e8641e8da904126a3680828c83774e8", - "IPY_MODEL_2f99603c364b4af48ea25b8d43cd1941" - ], - "layout": "IPY_MODEL_26ae8872c49642c588330b34ff470609" - } - }, - "0e9bfe8c83a44537bb59c444744466ef": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a8ee2fe1c3d04082acdafbf39ba0076b", - "placeholder": "​", - "style": "IPY_MODEL_5e9bad63ccc14fe2b358b7c7f0bbae68", - "value": "Downloading model.safetensors: 100%" - } - }, - "5e8641e8da904126a3680828c83774e8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_50597a7caa0242df82fafa89bf831827", - "max": 267832558, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_48ba967e29ad45a1ac8f7ba659fb5a5f", - "value": 267832558 - } - }, - "2f99603c364b4af48ea25b8d43cd1941": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_788dc688a9e9449e9aeb431b06384d9f", - "placeholder": "​", - "style": "IPY_MODEL_c82bc76766fd420a9557c4dcd78cff0c", - "value": " 268M/268M [00:00<00:00, 476MB/s]" - } - }, - "26ae8872c49642c588330b34ff470609": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a8ee2fe1c3d04082acdafbf39ba0076b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5e9bad63ccc14fe2b358b7c7f0bbae68": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "50597a7caa0242df82fafa89bf831827": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "48ba967e29ad45a1ac8f7ba659fb5a5f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "788dc688a9e9449e9aeb431b06384d9f": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c82bc76766fd420a9557c4dcd78cff0c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f52944fb4da049afae19d9d1f4a6b37a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_9cee60b1935441e4a81d26cc5caa319e", - "IPY_MODEL_4621e316677d44caab491fd8a7b1433f", - "IPY_MODEL_39bea84d46004ecd9dae38e81c4bb2d8" - ], - "layout": "IPY_MODEL_5859b86234164b55aa97852b4368bd66" - } - }, - "9cee60b1935441e4a81d26cc5caa319e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9fd724ea0d6947be92361604c9f318a7", - "placeholder": "​", - "style": "IPY_MODEL_996b172b2f434b5a95de4a839c2ae4ae", - "value": "Downloading (…)okenizer_config.json: 100%" - } - }, - "4621e316677d44caab491fd8a7b1433f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ff8e804c7b5e472187685d1fdcc5d0ff", - "max": 48, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_764f957750064a7eb61d5e6b28e0e49d", - "value": 48 - } - }, - "39bea84d46004ecd9dae38e81c4bb2d8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d46affe5b0ff4914bc2504c2737c6d40", - "placeholder": "​", - "style": "IPY_MODEL_350d6653f3d947b8a125b051b36e9d4a", - "value": " 48.0/48.0 [00:00<00:00, 4.45kB/s]" - } - }, - "5859b86234164b55aa97852b4368bd66": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9fd724ea0d6947be92361604c9f318a7": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "996b172b2f434b5a95de4a839c2ae4ae": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ff8e804c7b5e472187685d1fdcc5d0ff": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "764f957750064a7eb61d5e6b28e0e49d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "d46affe5b0ff4914bc2504c2737c6d40": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "350d6653f3d947b8a125b051b36e9d4a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "e9e9cb6c63c54831acf6922003883421": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_1d9e7ff2e4d74891a3f70798bb5266eb", - "IPY_MODEL_55135c120cc94702be910b033f6df07d", - "IPY_MODEL_a01975d1f8154082bd903bb55b3d49d6" - ], - "layout": "IPY_MODEL_26db4649e9924203912f92e0e1c994bb" - } - }, - "1d9e7ff2e4d74891a3f70798bb5266eb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2261900d8f594e7b965fe8d4d6264e35", - "placeholder": "​", - "style": "IPY_MODEL_14f43841b87646dd9c8eca2966488900", - "value": "Downloading (…)solve/main/vocab.txt: 100%" - } - }, - "55135c120cc94702be910b033f6df07d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_c7a0c077c06e4344bdffe4848de56e15", - "max": 231508, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_581e9a90552543188ec3c95b4cdab508", - "value": 231508 - } - }, - "a01975d1f8154082bd903bb55b3d49d6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_15b87e982dcf45139029b93eb6a7f857", - "placeholder": "​", - "style": "IPY_MODEL_dc42e38e51464a2095b179359da4c816", - "value": " 232k/232k [00:00<00:00, 2.79MB/s]" - } - }, - "26db4649e9924203912f92e0e1c994bb": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2261900d8f594e7b965fe8d4d6264e35": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "14f43841b87646dd9c8eca2966488900": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c7a0c077c06e4344bdffe4848de56e15": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "581e9a90552543188ec3c95b4cdab508": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "15b87e982dcf45139029b93eb6a7f857": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "dc42e38e51464a2095b179359da4c816": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "12a4e79070444acbbb228aba4e34a25e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f8bfdfce5edc4c6397e22b90eea50817", - "IPY_MODEL_edd2f3f9a691462dbfb703ca072fd644", - "IPY_MODEL_70fd4d6c76d348fa82ac29e416c94f2c" - ], - "layout": "IPY_MODEL_8cefab222736437289c985ffbd6a1ed5" - } - }, - "f8bfdfce5edc4c6397e22b90eea50817": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b3eaa19e1a97474995de388a44bc18f1", - "placeholder": "​", - "style": "IPY_MODEL_71a105f4761549fc898c0a4b03fbb043", - "value": "100%" - } - }, - "edd2f3f9a691462dbfb703ca072fd644": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d3c2759bf4324ad78eda1aa61d9c3fb0", - "max": 9, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_37d8b6b1ea044b7680b715ec31b48d8e", - "value": 9 - } - }, - "70fd4d6c76d348fa82ac29e416c94f2c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_dfbde307f9f84be1b413b47248410438", - "placeholder": "​", - "style": "IPY_MODEL_bc3d582d8e394397897f44efe51dd333", - "value": " 9/9 [00:00<00:00, 19.41it/s]" - } - }, - "8cefab222736437289c985ffbd6a1ed5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b3eaa19e1a97474995de388a44bc18f1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "71a105f4761549fc898c0a4b03fbb043": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "d3c2759bf4324ad78eda1aa61d9c3fb0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "37d8b6b1ea044b7680b715ec31b48d8e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "dfbde307f9f84be1b413b47248410438": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "bc3d582d8e394397897f44efe51dd333": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - } - } - }, - "accelerator": "GPU" - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file diff --git a/notebooks/setfit-onnx-optimum.ipynb b/notebooks/setfit-onnx-optimum.ipynb new file mode 100644 index 00000000..a7634df1 --- /dev/null +++ b/notebooks/setfit-onnx-optimum.ipynb @@ -0,0 +1,774 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "76571396-8f54-40ed-9e81-6c7531e6eaee", + "metadata": { + "id": "76571396-8f54-40ed-9e81-6c7531e6eaee" + }, + "source": [ + "# Efficiently run SetFit Models with Optimum" + ] + }, + { + "cell_type": "markdown", + "id": "24fd5853-812f-45a4-8a7b-0a0c9a60d0a2", + "metadata": { + "id": "24fd5853-812f-45a4-8a7b-0a0c9a60d0a2" + }, + "source": [ + "[SetFit](https://github.com/huggingface/setfit) is a technique for few-shot text classification that uses contrastive learning to fine-tune Sentence Transformers in domains where little to no labeled data is available. It achieves comparable performance to existing state-of-the-art methods based on large language models, yet requires no prompts and is efficient to train (typically a few seconds on a GPU to minutes on a CPU).\n", + "\n", + "In this notebook you'll learn how to further compress SetFit models for faster inference & deployment on GPU using Optimum Onnx." + ] + }, + { + "cell_type": "markdown", + "id": "a3b30b35-7875-498f-a771-068132f4084f", + "metadata": { + "id": "a3b30b35-7875-498f-a771-068132f4084f", + "tags": [] + }, + "source": [ + "## 1. Setup development environment" + ] + }, + { + "cell_type": "markdown", + "id": "dc40c7af-1f4f-4324-847c-dc9b7797b60c", + "metadata": { + "id": "dc40c7af-1f4f-4324-847c-dc9b7797b60c" + }, + "source": [ + "Our first step is to install SetFit. Running the following cell will install all the required packages for us." + ] + }, + { + "cell_type": "code", + "execution_count": 156, + "id": "Cu9et-iSaU0i", + "metadata": { + "id": "Cu9et-iSaU0i" + }, + "outputs": [], + "source": [ + "!pip install setfit accelerate -qqq" + ] + }, + { + "cell_type": "markdown", + "id": "ffbea843-2e86-4f14-961c-b8895f9de77d", + "metadata": { + "id": "ffbea843-2e86-4f14-961c-b8895f9de77d" + }, + "source": [ + "## 2. Create a performance benchmark\n", + "\n", + "Before we train and optimize any models, let's define a performance benchmark that we can use to compare our models. In general, deploying ML models in production environments involves a tradeoff among several constraints:\n", + "\n", + "* Model performance: how well does the model perform on a well crafted test set?\n", + "* Latency: how fast can our model deliver predictions?\n", + "* Memory: on what cloud instance or device can we store and load our model?\n", + "\n", + "The class below defines a simple benchmark that measure each quantity for a given SetFit model and test dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": 157, + "id": "55756fec-fc22-4590-84d7-2f3df37b9256", + "metadata": { + "id": "55756fec-fc22-4590-84d7-2f3df37b9256" + }, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from time import perf_counter\n", + "\n", + "import evaluate\n", + "import numpy as np\n", + "import torch\n", + "from tqdm.auto import tqdm\n", + "\n", + "metric = evaluate.load(\"accuracy\")\n", + "\n", + "\n", + "class PerformanceBenchmark:\n", + " def __init__(self, model, dataset, optim_type):\n", + " self.model = model\n", + " self.dataset = dataset\n", + " self.optim_type = optim_type\n", + "\n", + " def compute_accuracy(self):\n", + " preds = self.model.predict(self.dataset[\"text\"])\n", + " labels = self.dataset[\"label\"]\n", + " accuracy = metric.compute(predictions=preds, references=labels)\n", + " print(f\"Accuracy on test set - {accuracy['accuracy']:.3f}\")\n", + " return accuracy\n", + "\n", + " def compute_size(self):\n", + " state_dict = self.model.model_body.state_dict()\n", + " tmp_path = Path(\"model.pt\")\n", + " torch.save(state_dict, tmp_path)\n", + " # Calculate size in megabytes\n", + " size_mb = Path(tmp_path).stat().st_size / (1024 * 1024)\n", + " # Delete temporary file\n", + " tmp_path.unlink()\n", + " print(f\"Model size (MB) - {size_mb:.2f}\")\n", + " return {\"size_mb\": size_mb}\n", + "\n", + " def time_model(self, query=\"that loves its characters and communicates something rather beautiful about human nature\"):\n", + " latencies = []\n", + " # Warmup\n", + " for _ in range(10):\n", + " _ = self.model([query])\n", + " # Timed run\n", + " for _ in range(100):\n", + " start_time = perf_counter()\n", + " _ = self.model([query])\n", + " latency = perf_counter() - start_time\n", + " latencies.append(latency)\n", + " # Compute run statistics\n", + " time_avg_ms = 1000 * np.mean(latencies)\n", + " time_std_ms = 1000 * np.std(latencies)\n", + " print(rf\"Average latency (ms) - {time_avg_ms:.2f} +\\- {time_std_ms:.2f}\")\n", + " return {\"time_avg_ms\": time_avg_ms, \"time_std_ms\": time_std_ms}\n", + "\n", + " def run_benchmark(self):\n", + " metrics = {}\n", + " metrics[self.optim_type] = self.compute_size()\n", + " metrics[self.optim_type].update(self.compute_accuracy())\n", + " metrics[self.optim_type].update(self.time_model())\n", + " return metrics" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4QkMcqR0qcBZ" + }, + "source": [ + "Beyond that, we'll create a simple function to plot the performances reported by this benchmark." + ], + "id": "4QkMcqR0qcBZ" + }, + { + "cell_type": "code", + "execution_count": 194, + "metadata": { + "id": "PurksLh3qcBa" + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", + "\n", + "\n", + "def plot_metrics(perf_metrics):\n", + " df = pd.DataFrame.from_dict(perf_metrics, orient=\"index\")\n", + "\n", + " for idx in df.index:\n", + " df_opt = df.loc[idx]\n", + " plt.errorbar(\n", + " df_opt[\"time_avg_ms\"],\n", + " df_opt[\"accuracy\"] * 100,\n", + " xerr=df_opt[\"time_std_ms\"],\n", + " fmt=\"o\",\n", + " alpha=0.5,\n", + " ms=df_opt[\"size_mb\"] / 15,\n", + " label=idx,\n", + " capsize=5,\n", + " capthick=1,\n", + " )\n", + "\n", + " legend = plt.legend(loc=\"lower right\")\n", + "\n", + " plt.ylim(63, 95)\n", + " # Use the slowest model to define the x-axis range\n", + " xlim = max([metrics[\"time_avg_ms\"] for metrics in perf_metrics.values()]) * 1.2\n", + " plt.xlim(0, xlim)\n", + " plt.ylabel(\"Accuracy (%)\")\n", + " plt.xlabel(\"Average latency with batch_size=1 (ms)\")\n", + " plt.show()\n" + ], + "id": "PurksLh3qcBa" + }, + { + "cell_type": "markdown", + "id": "1402c1ba-aa7f-4b0b-9db5-1e6f0d301e70", + "metadata": { + "id": "1402c1ba-aa7f-4b0b-9db5-1e6f0d301e70" + }, + "source": [ + "## 3. Train/evaluate bge-small SetFit models" + ] + }, + { + "cell_type": "markdown", + "id": "5a850dfb-fffb-4e03-b468-b1f78d434705", + "metadata": { + "id": "5a850dfb-fffb-4e03-b468-b1f78d434705" + }, + "source": [ + "Before we optimize any models, let's train a few baselines as a point of reference. We'll use the [sst-2](https://huggingface.co/datasets/SetFit/sst2) dataset, which is a collection of sentiment text catagorized into 2 classes: positive, negative\n", + "\n", + "Let's start by loading the dataset from the Hub:" + ] + }, + { + "cell_type": "code", + "execution_count": 159, + "id": "7850d846-07c8-48eb-9aa6-2ce1af276ff4", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7850d846-07c8-48eb-9aa6-2ce1af276ff4", + "outputId": "056eff7c-293f-4fd8-e76e-29d261abff1b" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/repocard.py:105: UserWarning: Repo card metadata block was not found. Setting CardData to empty.\n", + " warnings.warn(\"Repo card metadata block was not found. Setting CardData to empty.\")\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['text', 'label', 'label_text'],\n", + " num_rows: 6920\n", + " })\n", + " validation: Dataset({\n", + " features: ['text', 'label', 'label_text'],\n", + " num_rows: 872\n", + " })\n", + " test: Dataset({\n", + " features: ['text', 'label', 'label_text'],\n", + " num_rows: 1821\n", + " })\n", + "})" + ] + }, + "metadata": {}, + "execution_count": 159 + } + ], + "source": [ + "from datasets import load_dataset\n", + "\n", + "dataset = load_dataset(\"SetFit/sst2\")\n", + "dataset" + ] + }, + { + "cell_type": "markdown", + "id": "a714cf95-a831-41b7-8f04-ffe0350d4659", + "metadata": { + "id": "a714cf95-a831-41b7-8f04-ffe0350d4659" + }, + "source": [ + "We train a SetFit model with the full dataset. Recall that SetFit excels with few-shot scenario, but this time we are interested to achieve maximum accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": 160, + "id": "a628cbfa-cfcd-4e4f-ab48-b454e5695ac8", + "metadata": { + "id": "a628cbfa-cfcd-4e4f-ab48-b454e5695ac8" + }, + "outputs": [], + "source": [ + "train_dataset = dataset[\"train\"]\n", + "test_dataset = dataset[\"validation\"]" + ] + }, + { + "cell_type": "markdown", + "id": "P8yY_SBbWlv9", + "metadata": { + "id": "P8yY_SBbWlv9" + }, + "source": [ + "Use the following line code to download the [already finetuned model](https://huggingface.co/moshew/bge-small-en-v1.5_setfit-sst2-english) and evaluate. Alternatively, uncomment the code below it to fine-tune the base model from scratch.\n", + "\n", + "Note that we perform the evaluations on Google Colab using the free T4 GPU." + ] + }, + { + "cell_type": "code", + "execution_count": 197, + "id": "u-w99Y2qW4lU", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "u-w99Y2qW4lU", + "outputId": "57f0b8f7-6dad-4e90-c779-658a7de6e960" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Model size (MB) - 127.33\n", + "Accuracy on test set - 0.906\n", + "Average latency (ms) - 13.43 +\\- 1.62\n" + ] + } + ], + "source": [ + "# Evaluate the uploaded model!\n", + "from setfit import SetFitModel\n", + "\n", + "small_model = SetFitModel.from_pretrained(\"moshew/bge-small-en-v1.5_setfit-sst2-english\")\n", + "pb = PerformanceBenchmark(model=small_model, dataset=test_dataset, optim_type=\"bge-small (PyTorch)\")\n", + "perf_metrics = pb.run_benchmark()" + ] + }, + { + "cell_type": "code", + "execution_count": 162, + "id": "52b3bc70-dc3e-4c23-a152-7a149b8b46fe", + "metadata": { + "id": "52b3bc70-dc3e-4c23-a152-7a149b8b46fe" + }, + "outputs": [], + "source": [ + "# # Fine-tune the base model and Evaluate!\n", + "# from setfit import SetFitModel, Trainer, TrainingArguments\n", + "\n", + "# # Load pretrained model from the Hub\n", + "# small_model = SetFitModel.from_pretrained(\n", + "# \"BAAI/bge-small-en-v1.5\"\n", + "# )\n", + "# args = TrainingArguments(num_iterations=20)\n", + "\n", + "# # Create trainer\n", + "# small_trainer = Trainer(\n", + "# model=small_model, args=args, train_dataset=train_dataset\n", + "# )\n", + "# # Train!\n", + "# small_trainer.train()\n", + "\n", + "# # Evaluate!\n", + "# pb = PerformanceBenchmark(\n", + "# model=small_trainer.model, dataset=test_dataset, optim_type=\"bge-small (base)\"\n", + "# )\n", + "# perf_metrics = pb.run_benchmark()" + ] + }, + { + "cell_type": "markdown", + "id": "82bf3e15-2804-4669-97d6-87e1bbef7223", + "metadata": { + "id": "82bf3e15-2804-4669-97d6-87e1bbef7223" + }, + "source": [ + "Let's plot the results to visualise the performance:" + ] + }, + { + "cell_type": "code", + "execution_count": 198, + "id": "89d0a144-d463-4a61-b78a-861d0d8cd061", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 455 + }, + "id": "89d0a144-d463-4a61-b78a-861d0d8cd061", + "outputId": "9ba81223-b2d2-4b10-f78d-2691846782a2" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAG2CAYAAAByJ/zDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABMHUlEQVR4nO3deVhUZf8/8PcwwDCsIshWbAKCC+4b2qOWGJq5pLmXIppplGKlqWVZLqSl+Wh9NctwR1vU0jIVCpdyV1xCUZDABSQXGNYBZu7fH/6cxwlEloGBw/t1Xee6nLN+7jPjzJtz7nOOTAghQERERCQhJsYugIiIiMjQGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyjBpwcnJyEBERAU9PTyiVSnTr1g0nTpzQTQ8NDYVMJtMb+vbta8SKiYiIqD4wNebGJ06ciAsXLmDjxo1wc3PDpk2bEBwcjISEBDzxxBMAgL59+yIqKkq3jEKhMFa5REREVE/IjPWwzYKCAtjY2ODHH39E//79deM7dOiAfv36YcGCBQgNDUVWVhZ27txpjBKJiIionjLaEZySkhJoNBpYWFjojVcqlTh8+LDudVxcHJycnGBvb49nnnkGCxYsgIODwyPXq1aroVarda+1Wi3u3r0LBwcHyGQywzeEiIiIDE4IgZycHLi5ucHEpAo9aoQRBQUFiZ49e4obN26IkpISsXHjRmFiYiKaNWsmhBAiOjpa/Pjjj+LcuXNix44donnz5qJTp06ipKTkkev84IMPBAAOHDhw4MCBgwSGa9euVSljGO0UFQAkJycjLCwMBw8ehFwuR/v27dGsWTOcOnUKFy9eLDX/1atX4ePjg5iYGPTu3bvMdf77CE52djY8PDxw7do12Nra1lhbiIiIyHBUKhXc3d2RlZUFOzu7Si9v1E7GPj4+OHDgAPLy8qBSqeDq6ooRI0agadOmZc7ftGlTODo6Iikp6ZEBR6FQlNkR2dbWlgGHiIionqlq95I6cR8cKysruLq64t69e9i7dy8GDRpU5nzXr1/HnTt34OrqWssVEhERUX1i1CM4e/fuhRAC/v7+SEpKwowZMxAQEIDx48cjNzcXH374IYYOHQoXFxckJydj5syZ8PX1RUhIiDHLJiIiojrOqEdwsrOzER4ejoCAAIwdOxZPPfUU9u7dCzMzM8jlcpw7dw4DBw5Es2bNMGHCBHTo0AGHDh3ivXCIiIioXEbtZFwbVCoV7OzskJ2dzT44RERE9UR1f7/rRB8cIiIiIkNiwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIqPLVZfgSPId5KpLjF1KtZRotCgo0qBEozV2KdUihffD1NgFEBER5atLcPTqHfg0sYK1on79NGm1Aldv5+FM2j2cu56NEq0WpiYmaP2kHdp52KOpoxVMTGTGLrNS6vP78UD9rJqIiKgOUJdo8FP8TfyRdBuFxVrYWZrBXG6CIo0WsRcz8UfSbXT3dcTAtm5QmMqNXW6DwoBDRERUBVqtwE/xNxF7MRPOtgp4OpjrTXextUBWfhFiL2YCAIa2f7LeHcmpz9gHh4iIqAqu3s7DH0m34WyrQCNL8zLnaWRpDmdbBf5Iuo2UO3m1XGHDxiM4RERUJ2i0AkUaLYpK6kcH3RN/30WeWoMn7S2h0YpHzmdjYYb07EKcSLkLd3vLWqyw6orqeSdpgAGHiIjqiBN/30VhsQZW9aBTq1YrcODyP9AIAVVh8WPnz1WXIPp4Gm7cK6gXp6ny1CX14n0oD09RERERVZJGCGiFgFxWsbAil8mg0QpoxKOP9JBh1e94RkREktHJqzFGdnaHk42FsUt5rBKNFtkFxSgq0cLZ9vH13lIVQmFqgtef9oWpvO4fW8jMKcR3J68bu4xqYcAhIqI6QW4ig7ncBOamdT8AmJuaoK17I8RcvAW3RsrHzp9TWIIuLZxgWU9O+5jXgxD2OEZtQU5ODiIiIuDp6QmlUolu3brhxIkTuulCCLz//vtwdXWFUqlEcHAwrly5YsSKiYiI7mvnYQ+lmRxZ+UXlzpeVXwQLMxO087CvpcoIMHLAmThxIvbv34+NGzfi/PnzePbZZxEcHIwbN24AAJYsWYIVK1Zg9erVOHbsGKysrBASEoLCwkJjlk1ERISmjlbo7uuIWyr1I0NOVn4RbqnU6O7rCG8Hq1qusGEzWsApKCjADz/8gCVLlqBHjx7w9fXFvHnz4Ovri1WrVkEIgeXLl+O9997DoEGD0Lp1a2zYsAE3b97Ezp07jVU2ERERAMDERIaBbd3Qu7kTsguKkZiRgwxVIe7mFSFDVYjEjBxkFxSjd3MnDGzrVi+unpISo50MLCkpgUajgYWFfucspVKJw4cPIyUlBRkZGQgODtZNs7OzQ5cuXXDkyBGMHDmyzPWq1Wqo1Wrda5VKVTMNICKiBk9hKsfQ9k+ivac9zqTdw/nr2SjWaGFhaoJOLZzQzsMe3g7171lUUmC0gGNjY4OgoCDMnz8fzZs3h7OzM6Kjo3HkyBH4+voiIyMDAODs7Ky3nLOzs25aWSIjI/Hhhx/WaO1ERGRYlgpTdG3qUG864T7MxEQGnybW8GlijcFtn0CxRsBMLqsXV0s9Sn1+Px4w6t7fuHEjhBB44oknoFAosGLFCowaNQomJlUva/bs2cjOztYN165dM2DFRERUE6wVpgjycai3T65+wFRuAqW5vF6HG0Aa74dR3wEfHx8cOHAAubm5uHbtGo4fP47i4mI0bdoULi4uAIBbt27pLXPr1i3dtLIoFArY2trqDURERNSw1ImIaWVlBVdXV9y7dw979+7FoEGD4O3tDRcXF8TGxurmU6lUOHbsGIKCgoxYLREREdV1Rj32tHfvXggh4O/vj6SkJMyYMQMBAQEYP348ZDIZIiIisGDBAvj5+cHb2xtz586Fm5sbBg8ebMyyiYiIqI4zasDJzs7G7Nmzcf36dTRu3BhDhw7FwoULYWZmBgCYOXMm8vLyMGnSJGRlZeGpp57Cr7/+WurKKyIiIqKHyYSQ9pO/VCoV7OzskJ2dzf44RERE9UR1f7/rRB8cIiIiIkNiwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIskxasDRaDSYO3cuvL29oVQq4ePjg/nz50MIoZsnNDQUMplMb+jbt68RqyYiIqK6ztSYG1+8eDFWrVqF9evXo2XLljh58iTGjx8POzs7TJ06VTdf3759ERUVpXutUCiMUS4RERHVE0YNOH/++ScGDRqE/v37AwC8vLwQHR2N48eP682nUCjg4uJijBKJiIioHjLqKapu3bohNjYWly9fBgCcPXsWhw8fRr9+/fTmi4uLg5OTE/z9/TFlyhTcuXPnketUq9VQqVR6AxERETUsRj2CM2vWLKhUKgQEBEAul0Oj0WDhwoUYM2aMbp6+fftiyJAh8Pb2RnJyMubMmYN+/frhyJEjkMvlpdYZGRmJDz/8sDabQURERHWMTDzco7eWbd26FTNmzMAnn3yCli1bIj4+HhEREVi2bBnGjRtX5jJXr16Fj48PYmJi0Lt371LT1Wo11Gq17rVKpYK7uzuys7Nha2tbY20hIiIiw1GpVLCzs6vy77dRj+DMmDEDs2bNwsiRIwEAgYGBSE1NRWRk5CMDTtOmTeHo6IikpKQyA45CoWAnZCIiogbOqH1w8vPzYWKiX4JcLodWq33kMtevX8edO3fg6upa0+URERFRPWXUIzgDBgzAwoUL4eHhgZYtW+LMmTNYtmwZwsLCAAC5ubn48MMPMXToULi4uCA5ORkzZ86Er68vQkJCjFk6ERER1WFG7YOTk5ODuXPnYseOHcjMzISbmxtGjRqF999/H+bm5igoKMDgwYNx5swZZGVlwc3NDc8++yzmz58PZ2fnCm2juufwiIiIqPZV9/fbqAGnNjDgEBER1T/V/f3ms6iIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIckwrM7NWq8WBAwdw6NAhpKamIj8/H02aNEG7du0QHBwMd3f3mqqTiIiIqMIqdASnoKAACxYsgLu7O5577jns2bMHWVlZkMvlSEpKwgcffABvb28899xzOHr0aE3XTERERFSuCh3BadasGYKCgvDVV1+hT58+MDMzKzVPamoqtmzZgpEjR+Ldd9/FK6+8YvBiiYiIiCpCJoQQj5vp4sWLaN68eYVWWFxcjLS0NPj4+FS7OENQqVSws7NDdnY2bG1tjV0OERERVUB1f78rdIqqouEGAMzMzOpMuCEiIqKGqVKdjB9WUlKCL7/8EnFxcdBoNOjevTvCw8NhYWFhyPqIiIiIKq3KAWfq1Km4fPkyhgwZguLiYmzYsAEnT55EdHS0IesjIiIiqrQKB5wdO3bghRde0L3et28fEhMTIZfLAQAhISHo2rWr4SskIiIiqqQK3+jvm2++weDBg3Hz5k0AQPv27TF58mT8+uuv2LVrF2bOnIlOnTrVWKFEREREFVXhgLNr1y6MGjUKvXr1wsqVK7FmzRrY2tri3Xffxdy5c+Hu7o4tW7bUZK1EREREFVKhy8QflpWVhZkzZ+Ls2bNYvXo12rVrV1O1GQQvEyciIqp/auUy8Yc1atQIa9aswSeffIKxY8dixowZKCwsrPSGiYiIiGpKhQNOWloahg8fjsDAQIwZMwZ+fn44deoULC0t0aZNG+zZs6cm6yQiIiKqsAqfourVqxdcXFwQGhqKvXv3Ijk5GT/99BOA+3c6fvXVV+Hi4oJvv/22RguuLJ6iIiIiqn+q+/td4cvET548ibNnz8LHxwchISHw9vbWTWvevDkOHjyINWvWVLoAIiIiIkOrcMDp0KED3n//fYwbNw4xMTEIDAwsNc+kSZMMWhwRERFRVVS4D86GDRugVqsxffp03LhxA19++WVN1kVERERUZRU+guPp6Ynvv/++JmshIiIiMogKHcHJy8ur1EorOz8RERGRIVUo4Pj6+uLjjz9Genr6I+cRQmD//v3o168fVqxYYbACiYiIiCqrQqeo4uLiMGfOHMybNw9t2rRBx44d4ebmBgsLC9y7dw8JCQk4cuQITE1NMXv2bLz66qs1XTcRERHRI1XqUQ1paWn47rvvcOjQIaSmpqKgoACOjo5o164dQkJC0K9fP93TxesK3geHiIio/qnu73eln0VV3zDgEBER1T+1/iwqIiIiorqOAYeIiIgkhwGHiIiIJIcBh4iIiCSHAYeIiIgkp9IBx8vLCx999BHS0tKqvXGNRoO5c+fC29sbSqUSPj4+mD9/Ph6+sEsIgffffx+urq5QKpUIDg7GlStXqr1tIiIikq5KB5yIiAhs374dTZs2RZ8+fbB161ao1eoqbXzx4sVYtWoVPv/8c1y8eBGLFy/GkiVLsHLlSt08S5YswYoVK7B69WocO3YMVlZWCAkJQWFhYZW2SURERNJX5fvgnD59GuvWrUN0dDQ0Gg1Gjx6NsLAwtG/fvsLreP755+Hs7Iy1a9fqxg0dOhRKpRKbNm2CEAJubm5466238PbbbwMAsrOz4ezsjHXr1mHkyJGP3Qbvg0NERFT/GO0+OO3bt8eKFStw8+ZNfPDBB/j666/RqVMntG3bFt988w0qkpu6deuG2NhYXL58GQBw9uxZHD58GP369QMApKSkICMjA8HBwbpl7Ozs0KVLFxw5cqTMdarVaqhUKr2BiIiIGpYKPYuqLMXFxdixYweioqKwf/9+dO3aFRMmTMD169cxZ84cxMTEYMuWLeWuY9asWVCpVAgICIBcLodGo8HChQsxZswYAEBGRgYAwNnZWW85Z2dn3bR/i4yMxIcffljVZhEREZEEVDrgnD59GlFRUYiOjoaJiQnGjh2Lzz77DAEBAbp5XnjhBXTq1Omx6/r222+xefNmbNmyBS1btkR8fDwiIiLg5uaGcePGVbY0AMDs2bPx5ptv6l6rVCq4u7tXaV1ERERUP1U64HTq1Al9+vTBqlWrMHjwYJiZmZWax9vbu0L9Y2bMmIFZs2bp5g0MDERqaioiIyMxbtw4uLi4AABu3boFV1dX3XK3bt1C27Zty1ynQqGAQqGobLOIiIhIQiodcK5evQpPT89y57GyskJUVNRj15Wfnw8TE/1uQHK5HFqtFsD9oOTi4oLY2FhdoFGpVDh27BimTJlS2dKJiIiogah0wMnMzERGRga6dOmiN/7YsWOQy+Xo2LFjhdc1YMAALFy4EB4eHmjZsiXOnDmDZcuWISwsDAAgk8kQERGBBQsWwM/PD97e3pg7dy7c3NwwePDgypZOREREDUSlr6IKDw/HtWvXSo2/ceMGwsPDK7WulStX4sUXX8Rrr72G5s2b4+2338arr76K+fPn6+aZOXMm3njjDUyaNAmdOnVCbm4ufv31V1hYWFS2dCIiImogKn0fHGtra5w7dw5NmzbVG5+SkoLWrVsjJyfHoAVWF++DQ0REVP/U+n1wFAoFbt26VWp8eno6TE2rfNU5ERERkcFUOuA8++yzmD17NrKzs3XjsrKyMGfOHPTp08egxRERERFVRaUPuXz66afo0aMHPD090a5dOwBAfHw8nJ2dsXHjRoMXSERERFRZlQ44TzzxBM6dO4fNmzfj7NmzUCqVGD9+PEaNGlXmPXGIiIiIaluVOs1YWVlh0qRJhq6FiIiIyCCq3Cs4ISEBaWlpKCoq0hs/cODAahdFREREVB1VupPxCy+8gPPnz0Mmk+meGi6TyQAAGo3GsBUSERERVVKlr6KaNm0avL29kZmZCUtLS/z11184ePAgOnbsiLi4uBookYiIiKhyKn0E58iRI/jtt9/g6OgIExMTmJiY4KmnnkJkZCSmTp2KM2fO1ESdRERERBVW6SM4Go0GNjY2AABHR0fcvHkTAODp6YnExETDVkdERERUBZU+gtOqVSucPXsW3t7e6NKlC5YsWQJzc3OsWbOm1OMbiIiIiIyh0gHnvffeQ15eHgDgo48+wvPPP4///Oc/cHBwwLZt2wxeIBEREVFlVfphm2W5e/cu7O3tdVdS1SV82CYREVH9U6sP2ywuLoapqSkuXLigN75x48Z1MtwQERFRw1SpgGNmZgYPDw/e64aIiIjqtEpfRfXuu+9izpw5uHv3bk3UQ0RERFRtle5k/PnnnyMpKQlubm7w9PSElZWV3vTTp08brDgiIiKiqqh0wBk8eHANlEFERERkOAa5iqou41VURERE9U+tXkVFREREVB9U+hSViYlJuZeE8worIiIiMrZKB5wdO3bovS4uLsaZM2ewfv16fPjhhwYrjIiIiKiqDNYHZ8uWLdi2bRt+/PFHQ6zOYNgHh4iIqP6pM31wunbtitjYWEOtjoiIiKjKDBJwCgoKsGLFCjzxxBOGWB0RERFRtVS6D86/H6ophEBOTg4sLS2xadMmgxZHREREVBWVDjifffaZXsAxMTFBkyZN0KVLF9jb2xu0OCIiIqKqqHTACQ0NrYEyiIiIiAyn0n1woqKi8N1335Ua/91332H9+vUGKYqIiIioOiodcCIjI+Ho6FhqvJOTExYtWmSQooiIiIiqo9IBJy0tDd7e3qXGe3p6Ii0tzSBFEREREVVHpQOOk5MTzp07V2r82bNn4eDgYJCiiIiIiKqj0gFn1KhRmDp1Kn7//XdoNBpoNBr89ttvmDZtGkaOHFkTNRIRERFVSqWvopo/fz7+/vtv9O7dG6am9xfXarUYO3Ys++AQERFRnVDlZ1FduXIF8fHxUCqVCAwMhKenp6FrMwg+i4qIiKj+qe7vd6WP4Dzg5+cHPz+/qi5OREREVGMq3Qdn6NChWLx4canxS5YswbBhwwxSFBEREVF1VDrgHDx4EM8991yp8f369cPBgwcNUhQRERFRdVQ64OTm5sLc3LzUeDMzM6hUKoMURURERFQdlQ44gYGB2LZtW6nxW7duRYsWLQxSFBEREVF1VLqT8dy5czFkyBAkJyfjmWeeAQDExsYiOjq6zGdUEREREdW2SgecAQMGYOfOnVi0aBG+//57KJVKtG7dGjExMejZs2dN1EhERERUKVW+D05ZLly4gFatWhlqdQbB++AQERHVP9X9/a50H5x/y8nJwZo1a9C5c2e0adOmuqsjIiIiqrYqB5yDBw9i7NixcHV1xaeffopnnnkGR48eNWRtRERERFVSqT44GRkZWLduHdauXQuVSoXhw4dDrVZj586dvIKKiIiI6owKH8EZMGAA/P39ce7cOSxfvhw3b97EypUra7I2IiIioiqpcMDZs2cPJkyYgA8//BD9+/eHXC6v9sa9vLwgk8lKDeHh4QCAXr16lZo2efLkam+XiIiIpK3CAefw4cPIyclBhw4d0KVLF3z++ee4fft2tTZ+4sQJpKen64b9+/cDgN4zrV555RW9eZYsWVKtbRIREZH0VTjgdO3aFV999RXS09Px6quvYuvWrXBzc4NWq8X+/fuRk5NT6Y03adIELi4uumH37t3w8fHRu5+OpaWl3jy81JuIiIgep9JXUVlZWSEsLAyHDx/G+fPn8dZbb+Hjjz+Gk5MTBg4cWOVCioqKsGnTJoSFhUEmk+nGb968GY6OjmjVqhVmz56N/Pz8ctejVquhUqn0BiIiImpYqnUfHH9/fyxZsgTXr19HdHR0tQrZuXMnsrKyEBoaqhs3evRobNq0Cb///jtmz56NjRs34qWXXip3PZGRkbCzs9MN7u7u1aqLiIiI6h+D3sm4OkJCQmBubo5du3Y9cp7ffvsNvXv3RlJSEnx8fMqcR61WQ61W616rVCq4u7vzTsZERET1SHXvZFzpZ1HVhNTUVMTExGD79u3lztelSxcAKDfgKBQKKBQKg9dIRERE9Ue1H9VgCFFRUXByckL//v3LnS8+Ph4A4OrqWgtVERERUX1l9CM4Wq0WUVFRGDduHExN/1dOcnIytmzZgueeew4ODg44d+4cpk+fjh49eqB169ZGrJiIiIjqOqMHnJiYGKSlpSEsLExvvLm5OWJiYrB8+XLk5eXB3d0dQ4cOxXvvvWekSomIiKi+qDOdjGtKdTspERERUe2r7u93neiDQ0RERGRIDDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOUYNOF5eXpDJZKWG8PBwAEBhYSHCw8Ph4OAAa2trDB06FLdu3TJmyURERFQPGDXgnDhxAunp6bph//79AIBhw4YBAKZPn45du3bhu+++w4EDB3Dz5k0MGTLEmCUTERFRPSATQghjF/FAREQEdu/ejStXrkClUqFJkybYsmULXnzxRQDApUuX0Lx5cxw5cgRdu3at0DpVKhXs7OyQnZ0NW1vbmiyfiIiIDKS6v991pg9OUVERNm3ahLCwMMhkMpw6dQrFxcUIDg7WzRMQEAAPDw8cOXLkketRq9VQqVR6AxERETUsdSbg7Ny5E1lZWQgNDQUAZGRkwNzcHI0aNdKbz9nZGRkZGY9cT2RkJOzs7HSDu7t7DVZNREREdVGdCThr165Fv3794ObmVq31zJ49G9nZ2brh2rVrBqqQiIiI6gtTYxcAAKmpqYiJicH27dt141xcXFBUVISsrCy9ozi3bt2Ci4vLI9elUCigUChqslwiIiKq4+rEEZyoqCg4OTmhf//+unEdOnSAmZkZYmNjdeMSExORlpaGoKAgY5RJRERE9YTRj+BotVpERUVh3LhxMDX9Xzl2dnaYMGEC3nzzTTRu3Bi2trZ44403EBQUVOErqIiIiKhhMnrAiYmJQVpaGsLCwkpN++yzz2BiYoKhQ4dCrVYjJCQE//d//2eEKomIiKg+qVP3wakJvA8OERFR/SOZ++AQERERGQoDDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUmOqbELICKisgkhUFJSAo1GY+xSiAxOLpfD1NQUMpmsRtbPgENEVAcVFRUhPT0d+fn5xi6FqMZYWlrC1dUV5ubmBl83Aw4RUR2j1WqRkpICuVwONzc3mJub19hfuUTGIIRAUVER/vnnH6SkpMDPzw8mJobtNcOAQ0RUxxQVFUGr1cLd3R2WlpbGLoeoRiiVSpiZmSE1NRVFRUWwsLAw6PrZyZiIqI4y9F+0RHVNTX7G+b+HiEgCctUlOJJ8B7nqklLTSjRaFBRpUKLRVmo5ovrM6AHnxo0beOmll+Dg4AClUonAwECcPHlSNz00NBQymUxv6Nu3rxErJiKqe/LVJTh69Q7y/39Q0WoFkjJz8d3Ja/hwVwI+2v0XPtyVgO9OXkNSZi60WlHmckRSYdSAc+/ePXTv3h1mZmbYs2cPEhISsHTpUtjb2+vN17dvX6Snp+uG6OhoI1VMRFT3qUs0+OH0dXz+2xXEXsxEkUYLUxMTFGm0iL2Yic9/u4IfTl+HusTwl5/36tULERERBl9vXbVu3To0atRI93revHlo27btY5ebO3cuJk2aVHOF1bB/t7sss2bNwhtvvFE7BZXBqJ2MFy9eDHd3d0RFRenGeXt7l5pPoVDAxcWlQutUq9VQq9W61yqVqvqFEhHVE1oh8FP8TcRezISzrQKeDvqX37rYWiArvwixFzMBAP/xczRGmQ1aRkYG/vvf/+L8+fO6caGhoVi/fj0AwMzMDB4eHhg7dizmzJkDU9Pyf6rXrVuH8ePHlztPSkoKvLy8ql17Zbz99tto2rQppk+fjqZNm9bqtgEjH8H56aef0LFjRwwbNgxOTk5o164dvvrqq1LzxcXFwcnJCf7+/pgyZQru3LnzyHVGRkbCzs5ON7i7u9dkE4iI6pS0O/n4I+k2nG0VaGRZ9r1FGlmaw9lWgT+SbuPaXd5np7Z9/fXX6NatGzw9PfXGPzhbceXKFbz11luYN28ePvnkk8eub8SIEXpnOYKCgvDKK6/ojavMb2FRUVGl21QWR0dHhISEYNWqVQZZX2UZNeBcvXoVq1atgp+fH/bu3YspU6Zg6tSpuhQL3H/DN2zYgNjYWCxevBgHDhxAv379Hnlnz9mzZyM7O1s3XLt2rbaaQ0RkVBqtQPy1LOSpNbCxMINGKx452FiYIb9IgzPXsgxeR0lJCV5//XXY2dnB0dERc+fOhRBCNz09PR39+/eHUqmEt7c3tmzZAi8vLyxfvlw3T1ZWFiZOnIgmTZrA1tYWzzzzDM6ePVvuduPi4tC5c2dYWVmhUaNG6N69O1JTUwH879TRN998Aw8PD1hbW+O1116DRqPBkiVL4OLiAicnJyxcuFBvncuWLUNgYCCsrKzg7u6O1157Dbm5udXaP1u3bsWAAQNKjX9wtsLT0xNTpkxBcHAwfvrpJ+Tl5cHW1hbff/+93vw7d+6ElZUVSkpK4OLiohvMzc1haWmpe11UVIQhQ4bA2toatra2GD58OG7duqVbz4N98/XXX8Pb21t3uXZWVhZeffVVODs7w8LCAq1atcLu3bv1ati7dy+aN28Oa2trXUB72IABA7B169Zq7a+qMuopKq1Wi44dO2LRokUAgHbt2uHChQtYvXo1xo0bBwAYOXKkbv7AwEC0bt0aPj4+iIuLQ+/evUutU6FQQKFQ1E4DiIjqkOMpd5BTWAITExlUhcWPnT9XXYLrZ/LRO8DZoHWsX78eEyZMwPHjx3Hy5ElMmjQJHh4eeOWVVwAAY8eOxe3btxEXFwczMzO8+eabyMzM1FvHsGHDoFQqsWfPHtjZ2eHLL79E7969cfnyZTRu3LjUNktKSjB48GC88soriI6ORlFREY4fP653g8Tk5GTs2bMHv/76K5KTk/Hiiy/i6tWraNasGQ4cOIA///wTYWFhCA4ORpcuXQDcv4x5xYoV8Pb2xtWrV/Haa69h5syZ+L//+78q7Zu7d+8iISEBHTt2fOy8SqUSd+7cgZWVFUaOHImoqCi8+OKLuukPXtvY2DxyHVqtFoMGDYK1tTUOHDiAkpIShIeHY8SIEYiLi9PNl5SUhB9++AHbt2+HXC6HVqtFv379kJOTg02bNsHHxwcJCQmQy+W6ZfLz8/Hpp59i48aNMDExwUsvvYS3334bmzdv1s3TuXNnXL9+HX///XetnyIzasBxdXVFixYt9MY1b94cP/zwwyOXadq0KRwdHZGUlFRmwCEiaqi04n4fHDNZxQ7Oy2UyFGm00Dx0dMUQ3N3d8dlnn0Emk8Hf3x/nz5/HZ599hldeeQWXLl1CTEwMTpw4ofuR//rrr+Hn56db/vDhwzh+/DgyMzN1f7B++umn2LlzJ77//vsyO+eqVCpkZ2fj+eefh4+PD4D7vycP02q1+Oabb2BjY4MWLVrg6aefRmJiIn755ReYmJjA398fixcvxu+//64LOA93mPby8sKCBQswefLkKgectLQ0CCHg5ub2yHmEEIiNjcXevXt1nXQnTpyIbt26IT09Ha6ursjMzMQvv/yCmJiYcrcXGxuL8+fPIyUlRXeaasOGDWjZsiVOnDiBTp06Abh/WmrDhg1o0qQJAGDfvn04fvw4Ll68iGbNmgFAqX40xcXFWL16tW5/v/766/joo4/05nnQztTU1IYVcLp3747ExES9cZcvXy51XvJh169fx507d+Dq6lrT5RER1SudvRojQ1UAuYkJnG0ff1fYW6pCaIWA3MCPgejatavekZOgoCAsXboUGo0GiYmJMDU1Rfv27XXTfX199a6ePXv2LHJzc+Hg4KC33oKCAiQnJyMtLU3vj+M5c+Zgzpw5CA0NRUhICPr06YPg4GAMHz5c77fCy8tL72iHs7Mz5HK53s3mnJ2d9Y4mxcTEIDIyEpcuXYJKpUJJSQkKCwuRn59fpbtMFxQUAECZd+3dvXs3rK2tUVxcDK1Wi9GjR2PevHkA7h8JadmyJdavX49Zs2Zh06ZN8PT0RI8ePcrd3sWLF+Hu7q7XB6dFixZo1KgRLl68qAs4np6eunADAPHx8XjyySd14aYslpaWunADQBe8HqZUKgHAKM9UM2ofnOnTp+Po0aNYtGgRkpKSsGXLFqxZswbh4eEAgNzcXMyYMQNHjx7F33//jdjYWAwaNAi+vr4ICQkxZulERHWOmakJWrrZQVVYDLmJ7LFDTmEJWrrZwsSkbj3nKjc3F66uroiPj9cbEhMTMWPGDLi5uemNnzx5MoD7p2yOHDmCbt26Ydu2bWjWrBmOHj2qW6+ZmZnedmQyWZnjtNr7N0T8+++/8fzzz6N169b44YcfcOrUKXzxxRcAqt4R19Hx/lVr9+7dKzXt6aefRnx8PK5cuYKCggKsX78eVlZWuukTJ07EunXrdG0dP368wZ5R9vB2gP8Fk/KUte/Ev44G3r17FwD0wlNtMWrA6dSpE3bs2IHo6Gi0atUK8+fPx/LlyzFmzBgA9x+lfu7cOQwcOBDNmjXDhAkT0KFDBxw6dIj9bIiIyhD4hB2UZnJk5Zf/A5yVXwQLMxMEPmFn8BqOHTum9/ro0aPw8/ODXC6Hv78/SkpKcObMGd30pKQkvR/89u3bIyMjA6ampvD19dUbHB0dS41/uE9Ou3btMHv2bPz5559o1aoVtmzZUuV2nDp1ClqtFkuXLkXXrl3RrFkz3Lx5s8rrAwAfHx/Y2toiISGh1DQrKyv4+vrCw8OjzEvDX3rpJaSmpmLFihVISEjQ9VUtT/PmzXHt2jW9C24SEhKQlZVVqovIw1q3bo3r16/j8uXLFWxZ2S5cuAAzMzO0bNmyWuupCqM/bPP555/H888/X+Y0pVKJvXv31nJFRET1l4eDJbr7Ouruc1PWpeJZ+UW4pVKjd3MnuDe2BPDoW29URVpaGt588028+uqrOH36NFauXImlS5cCAAICAhAcHIxJkyZh1apVMDMzw1tvvQWlUqk7GhEcHIygoCAMHjwYS5Ys0QWLn3/+GS+88EKZHXRTUlKwZs0aDBw4EG5ubkhMTMSVK1cwduzYKrfD19cXxcXFWLlyJQYMGIA//vgDq1evrvL6gPudloODg3H48GEMHjy4Usva29tjyJAhmDFjBp599lk8+eSTj10mODgYgYGBGDNmDJYvX46SkhK89tpr6NmzZ7kdnXv27IkePXpg6NChWLZsGXx9fXHp0qVKP03g0KFD+M9//lOhI0KGZvRHNRARkeGYyGQY2NYNvZs7IbugGIkZOchQFeJuXhEyVIVIzMhBdkExejd3wsC2bjAxcP8b4P5VUgUFBejcuTPCw8Mxbdo0vY7BGzZsgLOzM3r06IEXXngBr7zyCmxsbHT9UmQyGX755Rf06NED48ePR7NmzTBy5EikpqbC2bnsK74sLS1x6dIlDB06FM2aNcOkSZMQHh6OV199tcrtaNOmDZYtW4bFixejVatW2Lx5MyIjI6u8vgcmTpyIrVu36k6FVcaECRNQVFSEsLCwCs0vk8nw448/wt7eHj169EBwcDCaNm2Kbdu2PXbZH374AZ06dcKoUaPQokULzJw585G3aHmUrVu36q6eq20y8e8TZhKjUqlgZ2eH7Oxs2NraGrscIqLHKiwsREpKit49SR4nU1WIzcfSMKaLB5xsLaDVCqTcycOZtHs4fz0bxRotzOQmCHzSDu087OHtYAUTE1mp5Yzh+vXrcHd3R0xMTIO4OlYIgS5dumD69OkYNWpUpZbduHEjpk+fjps3b8LcvOwbOdYVe/bswVtvvYVz58498m7M5X3Wq/v7bfRTVEREVH2WClN0beoAS8X9r3UTExl8mljDp4k1Brd9AsUaATO5DKZyk3KXqw2//fYbcnNzERgYiPT0dMycORNeXl6PvSJIKmQyGdasWaP3qIbHyc/PR3p6Oj7++GO8+uqrdT7cAEBeXh6ioqIe+6iJmsKAQ0QkAdYKUwT5OJQ5zVRuAlN5mZPKXa6mFBcXY86cObh69SpsbGzQrVs3bN68udRVOVLWtm3bCj2U84ElS5Zg4cKF6NGjB2bPnl1zhRnQwzclNAaeoiIiqmOqcoqKqD6qyVNU7GRMREREksOAQ0RUR0n8ADtRjX7GGXCIiOqYB31RjHF7e6La9OAzXhP9r9jJmIiojpHL5WjUqJHuuT6WlpYGuyU/UV0ghEB+fj4yMzPRqFEjvaeUGwoDDhFRHeTi4gIApR5eSCQljRo10n3WDY0Bh4ioDpLJZHB1dYWTkxOKi4uNXQ6RwZmZmdXIkZsHGHCIiOowuVxeoz8CRFLFTsZEREQkOQw4REREJDkMOERERCQ5ku+D8+AmQiqVysiVEBERUUU9+N2u6s0AJR9w7ty5AwBwd3c3ciVERERUWTk5ObCzs6v0cpIPOI0bNwYApKWlVWkH1UcqlQru7u64du1ag3rAKNvdcNrdENsMNMx2N8Q2Aw2z3f9usxACOTk5cHNzq9L6JB9wTEzudzOys7NrMB+SB2xtbRtcmwG2uyFpiG0GGma7G2KbgYbZ7ofbXJ0DE+xkTERERJLDgENERESSI/mAo1Ao8MEHH0ChUBi7lFrTENsMsN0Nqd0Nsc1Aw2x3Q2wz0DDbbeg2y0RVr78iIiIiqqMkfwSHiIiIGh4GHCIiIpIcBhwiIiKSHAYcIiIikhxJB5wvvvgCXl5esLCwQJcuXXD8+HFjl1SjIiMj0alTJ9jY2MDJyQmDBw9GYmKiscuqVR9//DFkMhkiIiKMXUqNu3HjBl566SU4ODhAqVQiMDAQJ0+eNHZZNUqj0WDu3Lnw9vaGUqmEj48P5s+fX+Vn1dRFBw8exIABA+Dm5gaZTIadO3fqTRdC4P3334erqyuUSiWCg4Nx5coV4xRrQOW1u7i4GO+88w4CAwNhZWUFNzc3jB07Fjdv3jRewQbwuPf6YZMnT4ZMJsPy5ctrrb6aUpF2X7x4EQMHDoSdnR2srKzQqVMnpKWlVWo7kg0427Ztw5tvvokPPvgAp0+fRps2bRASEoLMzExjl1ZjDhw4gPDwcBw9ehT79+9HcXExnn32WeTl5Rm7tFpx4sQJfPnll2jdurWxS6lx9+7dQ/fu3WFmZoY9e/YgISEBS5cuhb29vbFLq1GLFy/GqlWr8Pnnn+PixYtYvHgxlixZgpUrVxq7NIPJy8tDmzZt8MUXX5Q5fcmSJVixYgVWr16NY8eOwcrKCiEhISgsLKzlSg2rvHbn5+fj9OnTmDt3Lk6fPo3t27cjMTERAwcONEKlhvO49/qBHTt24OjRo1V+ZEFd87h2Jycn46mnnkJAQADi4uJw7tw5zJ07FxYWFpXbkJCozp07i/DwcN1rjUYj3NzcRGRkpBGrql2ZmZkCgDhw4ICxS6lxOTk5ws/PT+zfv1/07NlTTJs2zdgl1ah33nlHPPXUU8Yuo9b1799fhIWF6Y0bMmSIGDNmjJEqqlkAxI4dO3SvtVqtcHFxEZ988oluXFZWllAoFCI6OtoIFdaMf7e7LMePHxcARGpqau0UVcMe1ebr16+LJ554Qly4cEF4enqKzz77rNZrq0lltXvEiBHipZdeqva6JXkEp6ioCKdOnUJwcLBunImJCYKDg3HkyBEjVla7srOzAfzvgaNSFh4ejv79++u951L2008/oWPHjhg2bBicnJzQrl07fPXVV8Yuq8Z169YNsbGxuHz5MgDg7NmzOHz4MPr162fkympHSkoKMjIy9D7ndnZ26NKlS4P6bgPuf7/JZDI0atTI2KXUGK1Wi5dffhkzZsxAy5YtjV1OrdBqtfj555/RrFkzhISEwMnJCV26dCn39N2jSDLg3L59GxqNBs7OznrjnZ2dkZGRYaSqapdWq0VERAS6d++OVq1aGbucGrV161acPn0akZGRxi6l1ly9ehWrVq2Cn58f9u7diylTpmDq1KlYv369sUurUbNmzcLIkSMREBAAMzMztGvXDhERERgzZoyxS6sVD76/GvJ3GwAUFhbinXfewahRoyT9IMrFixfD1NQUU6dONXYptSYzMxO5ubn4+OOP0bdvX+zbtw8vvPAChgwZggMHDlRqXZJ/mnhDFR4ejgsXLuDw4cPGLqVGXbt2DdOmTcP+/fsrf362HtNqtejYsSMWLVoEAGjXrh0uXLiA1atXY9y4cUauruZ8++232Lx5M7Zs2YKWLVsiPj4eERERcHNzk3S76X+Ki4sxfPhwCCGwatUqY5dTY06dOoX//ve/OH36NGQymbHLqTVarRYAMGjQIEyfPh0A0LZtW/z5559YvXo1evbsWeF1SfIIjqOjI+RyOW7duqU3/tatW3BxcTFSVbXn9ddfx+7du/H777/jySefNHY5NerUqVPIzMxE+/btYWpqClNTUxw4cAArVqyAqakpNBqNsUusEa6urmjRooXeuObNm1f6KoP6ZsaMGbqjOIGBgXj55Zcxffr0BnP07sH3V0P9bnsQblJTU7F//35JH705dOgQMjMz4eHhoftuS01NxVtvvQUvLy9jl1djHB0dYWpqapDvN0kGHHNzc3To0AGxsbG6cVqtFrGxsQgKCjJiZTVLCIHXX38dO3bswG+//QZvb29jl1TjevfujfPnzyM+Pl43dOzYEWPGjEF8fDzkcrmxS6wR3bt3L3ULgMuXL8PT09NIFdWO/Px8mJjof23J5XLdX31S5+3tDRcXF73vNpVKhWPHjkn6uw34X7i5cuUKYmJi4ODgYOySatTLL7+Mc+fO6X23ubm5YcaMGdi7d6+xy6sx5ubm6NSpk0G+3yR7iurNN9/EuHHj0LFjR3Tu3BnLly9HXl4exo8fb+zSakx4eDi2bNmCH3/8ETY2Nrpz8nZ2dlAqlUaurmbY2NiU6mNkZWUFBwcHSfc9mj59Orp164ZFixZh+PDhOH78ONasWYM1a9YYu7QaNWDAACxcuBAeHh5o2bIlzpw5g2XLliEsLMzYpRlMbm4ukpKSdK9TUlIQHx+Pxo0bw8PDAxEREViwYAH8/Pzg7e2NuXPnws3NDYMHDzZe0QZQXrtdXV3x4osv4vTp09i9ezc0Go3u+61x48YwNzc3VtnV8rj3+t8hzszMDC4uLvD396/tUg3qce2eMWMGRowYgR49euDpp5/Gr7/+il27diEuLq5yG6r2dVh12MqVK4WHh4cwNzcXnTt3FkePHjV2STUKQJlDVFSUsUurVQ3hMnEhhNi1a5do1aqVUCgUIiAgQKxZs8bYJdU4lUolpk2bJjw8PISFhYVo2rSpePfdd4VarTZ2aQbz+++/l/n/eNy4cUKI+5eKz507Vzg7OwuFQiF69+4tEhMTjVu0AZTX7pSUlEd+v/3+++/GLr3KHvde/5tULhOvSLvXrl0rfH19hYWFhWjTpo3YuXNnpbcjE0JCtwAlIiIigkT74BAREVHDxoBDREREksOAQ0RERJLDgENERESSw4BDREREksOAQ0RERJLDgENERESSw4BDREREksOAQ1RH/P3335DJZIiPjzd2KUYXFxcHmUyGrKyscufz8vLC8uXLK7XuXr16ISIiosq1VVVNvr8V3V91TVFREXx9ffHnn3/W2DZWr16NAQMG1Nj6qe5iwKE67ciRI5DL5ejfv7+xS6mTQkND6/0ziMrSrVs3pKenw87ODgCwbt06NGrUyLhFPaSuBYp/76/atn37djz77LNwcHCoVIhbvXo1vL290a1btxqrLSwsDKdPn8ahQ4dqbBtUNzHgUJ22du1avPHGGzh48CBu3rxZo9sSQqCkpKRGt0EVY25uDhcXF8hkMmOXUi8Ye3/l5eXhqaeewuLFiyu8jBACn3/+OSZMmFCDld3fN6NHj8aKFStqdDtU9zDgUJ2Vm5uLbdu2YcqUKejfvz/WrVunmzZ69GiMGDFCb/7i4mI4Ojpiw4YNAACtVovIyEh4e3tDqVSiTZs2+P7773XzP/grfM+ePejQoQMUCgUOHz6M5ORkDBo0CM7OzrC2tkanTp0QExOjt6309HT0798fSqUS3t7e2LJlS6nTJVlZWZg4cSKaNGkCW1tbPPPMMzh79myF26/RaDBhwgRd/f7+/vjvf/+rmz5v3jysX78eP/74I2QyGWQyme5pu9euXcPw4cPRqFEjNG7cGIMGDcLff/+tW/bBkZ9PP/0Urq6ucHBwQHh4OIqLi3XzqNVqvPPOO3B3d4dCoYCvry/Wrl0LIQR8fX3x6aef6tUbHx8PmUym95TgBy5cuAATExP8888/AIC7d+/CxMQEI0eO1M2zYMECPPXUU3rvTVZWFuLi4jB+/HhkZ2fr2jlv3jzdcvn5+QgLC4ONjQ08PDwq9ET1kpISvP7667Czs4OjoyPmzp2Lhx/Lt3HjRnTs2BE2NjZwcXHB6NGjkZmZCeD+qaann34aAGBvbw+ZTIbQ0FAA9z9zS5Ysga+vLxQKBTw8PLBw4UK9bV+9ehVPP/00LC0t0aZNGxw5cuSx9QJAamoqBgwYAHt7e1hZWaFly5b45ZdfSu0v4P5puAf76uHhwWegup/Nf3v55Zfx/vvvIzg4uMLLnDp1CsnJyXpHZx+cxvv222/xn//8B0qlEp06dcLly5dx4sQJdOzYEdbW1ujXr5/us/Sg/Z07d4aVlRUaNWqE7t27IzU1VTd9wIAB+Omnn1BQUFDlNlI9ZJBHgxLVgLVr14qOHTsKIe4/OdvHx0dotVohhBC7d+8WSqVS5OTk6ObftWuXUCqVQqVSCSGEWLBggQgICBC//vqrSE5OFlFRUUKhUIi4uDghxP+eaNu6dWuxb98+kZSUJO7cuSPi4+PF6tWrxfnz58Xly5fFe++9JywsLERqaqpuW8HBwaJt27bi6NGj4tSpU6Jnz55CqVTqPek3ODhYDBgwQJw4cUJcvnxZvPXWW8LBwUHcuXOnzPY+eGLymTNnhBBCFBUViffff1+cOHFCXL16VWzatElYWlqKbdu2CSGEyMnJEcOHDxd9+/YV6enpIj09XajValFUVCSaN28uwsLCxLlz50RCQoIYPXq08Pf31z11e9y4ccLW1lZMnjxZXLx4UezatUtYWlrqPZF8+PDhwt3dXWzfvl0kJyeLmJgYsXXrViGEEAsXLhQtWrTQq3/q1KmiR48eZbZNq9UKR0dH8d133wkhhNi5c6dwdHQULi4uevvr3Xff1Xtv7t27J9RqtVi+fLmwtbXVtfPB++7p6SkaN24svvjiC3HlyhURGRkpTExMxKVLl8qsQ4j7T5u3trYW06ZNE5cuXdLt14fbvnbtWvHLL7+I5ORkceTIEREUFCT69esnhBCipKRE/PDDDwKASExMFOnp6SIrK0sIIcTMmTOFvb29WLdunUhKShKHDh0SX331ld77GxAQIHbv3i0SExPFiy++KDw9PUVxcfEj632gf//+ok+fPuLcuXMiOTlZ7Nq1Sxw4cKDU/hJCiDt37uj2VXp6uhgyZIjw9/cX+fn5un1d3mfz4MGDwsrKqtxh06ZNpWr892e4PMuWLRMBAQFlLv/g/21CQoLo2rWr6NChg+jVq5c4fPiwOH36tPD19RWTJ08WQghRXFws7OzsxNtvvy2SkpJEQkKCWLdund7/17y8PGFiYlKvnzxOlceAQ3VWt27dxPLly4UQ97/EHB0ddV9QD15v2LBBN/+oUaPEiBEjhBBCFBYWCktLS/Hnn3/qrXPChAli1KhRQoj//Sjs3LnzsbW0bNlSrFy5UgghxMWLFwUAceLECd30K1euCAC6gHPo0CFha2srCgsL9dbj4+MjvvzyyzK3UZEfh/DwcDF06FDd63HjxolBgwbpzbNx40bh7++vC4NCCKFWq4VSqRR79+7VLefp6SlKSkp08wwbNky3/xITEwUAsX///jLruHHjhpDL5eLYsWNCiPthzNHRUaxbt+6RtQ8ZMkSEh4cLIYSIiIgQM2bMEPb29uLixYuiqKhIWFpain379gkhSv9gR0VFCTs7u1Lr9PT0FC+99JLutVarFU5OTmLVqlWPrKNnz56iefPmevvnnXfeEc2bN3/kMidOnBAAdMHq3/UJIYRKpRIKhUIXaP7twfv79ddf68b99ddfAoC4ePHiI7f9QGBgoJg3b16Z08qq54Fly5aJRo0aicTERCFExT6b+fn54sqVK+UOD/6QKKuNFQk406ZNE88880yZyz+8j6KjowUAERsbqxsXGRkp/P39hRD3wxwA3R8uj/IgeFLDYVqLB4uIKiwxMRHHjx/Hjh07AACmpqYYMWIE1q5di169esHU1BTDhw/H5s2b8fLLLyMvLw8//vgjtm7dCgBISkpCfn4++vTpo7feoqIitGvXTm9cx44d9V7n5uZi3rx5+Pnnn5Geno6SkhIUFBQgLS1NV5upqSnat2+vW8bX1xf29va612fPnkVubi4cHBz01l1QUIDk5OQK74cvvvgC33zzDdLS0lBQUICioiK0bdu23GXOnj2LpKQk2NjY6I0vLCzU23bLli0hl8t1r11dXXH+/HkA9083yeVy9OzZs8xtuLm5oX///vjmm2/QuXNn7Nq1C2q1GsOGDXtkXT179tSdPjpw4AAWLVqEy5cvIy4uDnfv3kVxcTG6d+9ebtvK0rp1a92/ZTIZXFxcdKeTHqVr1656/VWCgoKwdOlSaDQayOVynDp1CvPmzcPZs2dx7949aLVaAEBaWhpatGhR5jovXrwItVqN3r17V7heV1dXAEBmZiYCAgLKXW7q1KmYMmUK9u3bh+DgYAwdOlRvXWXZs2cPZs2ahV27dqFZs2YAKvbZVCqV8PX1LXfd1VVQUAALC4sypz3cLmdnZwBAYGCg3rgH73Hjxo0RGhqKkJAQ9OnTB8HBwRg+fLhu3z6gVCqRn59v6GZQHcaAQ3XS2rVrUVJSAjc3N904IQQUCgU+//xz2NnZYcyYMejZsycyMzOxf/9+KJVK9O3bF8D9kAIAP//8M5544gm9dSsUCr3XVlZWeq/ffvtt7N+/H59++il8fX2hVCrx4osvoqioqML15+bmwtXVVdcn5mEVvRpo69atePvtt7F06VIEBQXBxsYGn3zyCY4dO/bYbXfo0AGbN28uNa1Jkya6f5uZmelNk8lkuh9ypVL52PomTpyIl19+GZ999hmioqIwYsQIWFpaPnL+B5dnX7lyBQkJCXjqqadw6dIlxMXF4d69e+jYsWO5yz9Kee2oiry8PISEhCAkJASbN29GkyZNkJaWhpCQkHI/AxXZZ/+u90HIqki9EydOREhICH7++Wfs27cPkZGRWLp0Kd54440y509ISMDIkSPx8ccf49lnn9WNr8hn89ChQ+jXr1+59Xz55ZcYM2bMY+t+FEdHR12g/rey9tG/xz28z6KiojB16lT8+uuv2LZtG9577z3s378fXbt21c1z9+5dvc8/SR8DDtU5JSUl2LBhA5YuXar3xQwAgwcPRnR0NCZPnoxu3brB3d0d27Ztw549ezBs2DDdl2CLFi2gUCiQlpb2yKMQj/LHH38gNDQUL7zwAoD7PwgPd9D19/dHSUkJzpw5gw4dOgC4f8To3r17unnat2+PjIwMmJqawsvLqwp74X4d3bp1w2uvvaYb9++jP+bm5tBoNHrj2rdvj23btsHJyQm2trZV2nZgYCC0Wi0OHDjwyI6jzz33HKysrLBq1Sr8+uuvOHjw4GPXaW9vjwULFqBt27awtrZGr169sHjxYty7dw+9evV65LJltbM6/h0Sjx49Cj8/P8jlcly6dAl37tzBxx9/DHd3dwDAyZMnS9UDQK8mPz8/KJVKxMbGYuLEiQar9WHu7u6YPHkyJk+ejNmzZ+Orr74qM+Dcvn0bAwYMwNChQzF9+nS9aRX5bHbs2PGxl3o/OLJSVe3atcOqVasghDDI1V/t2rVDu3btMHv2bAQFBWHLli26gJOcnIzCwsJSR29J2ngVFdU5u3fvxr179zBhwgS0atVKbxg6dCjWrl2rm3f06NFYvXo19u/fr/fXpI2NDd5++21Mnz4d69evR3JyMk6fPo2VK1di/fr15W7fz88P27dvR3x8PM6ePYvRo0fr/bUYEBCA4OBgTJo0CcePH8eZM2cwadIkKJVK3Rd1cHAwgoKCMHjwYOzbtw9///03/vzzT7z77rulfizLq+PkyZPYu3cvLl++jLlz5+LEiRN683h5eeHcuXNITEzE7du3UVxcjDFjxsDR0RGDBg3CoUOHkJKSgri4OEydOhXXr1+v0La9vLwwbtw4hIWFYefOnbp1fPvtt7p55HI5QkNDMXv2bPj5+SEoKKjcdcpkMvTo0QObN2/WhZnWrVtDrVYjNja23CDq5eWF3NxcxMbG4vbt29U+1ZCWloY333wTiYmJiI6OxsqVKzFt2jQAgIeHB8zNzbFy5UpcvXoVP/30E+bPn6+3vKenJ2QyGXbv3o1//vkHubm5sLCwwDvvvIOZM2diw4YNSE5OxtGjR/U+r9URERGBvXv3IiUlBadPn8bvv/+O5s2blznv0KFDYWlpiXnz5iEjI0M3aDSaCn02H5yiKm94+BTo3bt3ER8fj4SEBAD3T+PGx8cjIyPjke15+umnkZubi7/++qta+yUlJQWzZ8/GkSNHkJqain379uHKlSt6++bQoUNo2rQpfHx8qrUtqmeM3QmI6N+ef/558dxzz5U57dixYwKAOHv2rBBCiISEBAFAeHp66nUaFeJ+h9Ply5cLf39/YWZmJpo0aSJCQkIeeeXJAykpKeLpp58WSqVSuLu7i88//1z07NlTTJs2TTfPzZs3Rb9+/YRCoRCenp5iy5YtwsnJSaxevVo3j0qlEm+88YZwc3MTZmZmwt3dXYwZM0akpaWV2bZ/d9AsLCwUoaGhws7OTjRq1EhMmTJFzJo1S7Rp00a3TGZmpujTp4+wtrYWAHSdsNPT08XYsWOFo6OjUCgUomnTpuKVV14R2dnZQoiyOydPmzZN9OzZU/e6oKBATJ8+Xbi6ugpzc3Ph6+srvvnmG71lkpOTBQCxZMmSMtv0b5999pkAIPbs2aMbN2jQIGFqaqp3RVxZ783kyZOFg4ODACA++OADIcT9TsYPX7kmhBBt2rTRTS9Lz549xWuvvSYmT54sbG1thb29vZgzZ47e52fLli3Cy8tLKBQKERQUJH766adSnWc/+ugj4eLiImQymRg3bpwQQgiNRiMWLFggPD09hZmZmfDw8BCLFi0SQpTdAffevXt671t5Xn/9deHj4yMUCoVo0qSJePnll8Xt27fL3F8AyhxSUlKEEJX/bD5OVFRUmdsr730Q4v6VerNmzdK9LmsflfVZeLjTeUZGhhg8eLDuc+rp6Snef/99odFodPM/++yzIjIyskpto/pLJsRDN38goiq5fv063N3dERMT89hOplJy6NAh9O7dG9euXav2KQtqeM6dO4c+ffogOTkZ1tbWNbKNv/76C8888wwuX75stDs9k3Ew4BBVwW+//Ybc3FwEBgYiPT0dM2fOxI0bN3D58uVSnV6lSK1W459//sG4cePg4uJSZodmoopYt24dOnTooHeVlCHFxMRAo9EgJCSkRtZPdRf74BBVQXFxMebMmYOWLVvihRdeQJMmTRAXF9cgwg0AREdHw9PTE1lZWViyZImxy5GEfv36wdrausxh0aJFxi6vxoSGhtZYuAHu94djuGmYeASHiKgOuHHjxiMfJdC4cWM0bty4lisiqt8YcIiIiEhyeIqKiIiIJIcBh4iIiCSHAYeIiIgkhwGHiIiIJIcBh4iIiCSHAYeIiIgkhwGHiIiIJOf/AUuPmGmgH3wzAAAAAElFTkSuQmCC\n" + }, + "metadata": {} + } + ], + "source": [ + "plot_metrics(perf_metrics)" + ] + }, + { + "cell_type": "markdown", + "id": "AiPUhOCNWRny", + "metadata": { + "id": "AiPUhOCNWRny" + }, + "source": [ + "## 4. Compressing with Optimum ONNX and CUDAExecutionProvider\n", + "\n", + "We'll be using Optimum's ONNX Runtime support with `CUDAExecutionProvider` [because it's fast while also supporting dynamic shapes](https://github.com/huggingface/optimum-benchmark/tree/main/examples/fast-mteb#notes)." + ] + }, + { + "cell_type": "code", + "execution_count": 164, + "id": "NEnwnsEQWRn8", + "metadata": { + "cellView": "form", + "id": "NEnwnsEQWRn8" + }, + "outputs": [], + "source": [ + "!pip install optimum[onnxruntime-gpu] -qqq" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HjeZkCtSqcBe" + }, + "source": [ + "[`optimum-cli`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/optimization#optimizing-a-model-during-the-onnx-export) makes it extremely easy to export a model to ONNX and apply SOTA graph optimizations / kernel fusions." + ], + "id": "HjeZkCtSqcBe" + }, + { + "cell_type": "code", + "execution_count": 165, + "id": "hPqEcDi8WRn8", + "metadata": { + "id": "hPqEcDi8WRn8", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "0e1202d8-aa84-422c-f10f-6bb0b43d1ef8" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "2023-11-27 12:23:25.781950: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-27 12:23:25.782000: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-27 12:23:25.782035: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-27 12:23:26.931536: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "Framework not specified. Using pt to export to ONNX.\n", + "Using the export variant default. Available variants are:\n", + " - default: The default ONNX variant.\n", + "Using framework PyTorch: 2.1.0+cu118\n", + "Overriding 1 configuration item(s)\n", + "\t- use_cache -> False\n", + "2023-11-27 12:23:34.728172634 [W:onnxruntime:, session_state.cc:1162 VerifyEachNodeIsAssignedToAnEp] Some nodes were not assigned to the preferred execution providers which may or may not have an negative impact on performance. e.g. ORT explicitly assigns shape related ops to CPU to improve perf.\n", + "2023-11-27 12:23:34.728200557 [W:onnxruntime:, session_state.cc:1164 VerifyEachNodeIsAssignedToAnEp] Rerunning with verbose output on a non-minimal build will show node assignments.\n", + "Overridding for_gpu=False to for_gpu=True as half precision is available only on GPU.\n", + "/usr/local/lib/python3.10/dist-packages/optimum/onnxruntime/configuration.py:770: FutureWarning: disable_embed_layer_norm will be deprecated soon, use disable_embed_layer_norm_fusion instead, disable_embed_layer_norm_fusion is set to True.\n", + " warnings.warn(\n", + "Optimizing model...\n", + "2023-11-27 12:23:36.378780811 [W:onnxruntime:, session_state.cc:1162 VerifyEachNodeIsAssignedToAnEp] Some nodes were not assigned to the preferred execution providers which may or may not have an negative impact on performance. e.g. ORT explicitly assigns shape related ops to CPU to improve perf.\n", + "2023-11-27 12:23:36.378811421 [W:onnxruntime:, session_state.cc:1164 VerifyEachNodeIsAssignedToAnEp] Rerunning with verbose output on a non-minimal build will show node assignments.\n", + "symbolic shape inference disabled or failed.\n", + "symbolic shape inference disabled or failed.\n", + "Configuration saved in bge_auto_opt_O4/ort_config.json\n", + "Optimized model saved at: bge_auto_opt_O4 (external data format: False; saved all tensor to one file: True)\n", + "Post-processing the exported models...\n", + "Deduplicating shared (tied) weights...\n", + "Validating models in subprocesses...\n", + "2023-11-27 12:23:48.005601: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-27 12:23:48.005666: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-27 12:23:48.005707: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-27 12:23:50.980859: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "Validating ONNX model bge_auto_opt_O4/model.onnx...\n", + "2023-11-27 12:23:54.208836299 [W:onnxruntime:, session_state.cc:1162 VerifyEachNodeIsAssignedToAnEp] Some nodes were not assigned to the preferred execution providers which may or may not have an negative impact on performance. e.g. ORT explicitly assigns shape related ops to CPU to improve perf.\n", + "2023-11-27 12:23:54.208860681 [W:onnxruntime:, session_state.cc:1164 VerifyEachNodeIsAssignedToAnEp] Rerunning with verbose output on a non-minimal build will show node assignments.\n", + "\t-[✓] ONNX model output names match reference model (last_hidden_state)\n", + "\t- Validating ONNX Model output \"last_hidden_state\":\n", + "\t\t-[✓] (2, 16, 384) matches (2, 16, 384)\n", + "\t\t-[x] values not close enough, max diff: 2.1155929565429688 (atol: 0.0001)\n", + "The ONNX export succeeded with the warning: The maximum absolute difference between the output of the reference model and the ONNX exported model is not within the set tolerance 0.0001:\n", + "- last_hidden_state: max diff = 2.1155929565429688.\n", + " The exported model was saved at: bge_auto_opt_O4\n" + ] + } + ], + "source": [ + "!optimum-cli export onnx \\\n", + " --model moshew/bge-small-en-v1.5_setfit-sst2-english \\\n", + " --task feature-extraction \\\n", + " --optimize O4 \\\n", + " --device cuda \\\n", + " bge_auto_opt_O4" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IYkxQOTRqcBe" + }, + "source": [ + "We may see some warnings, but these are not ones to be concerned about. We'll see later that it does not affect the model performance.\n", + "\n", + "First of all, we'll create a subclass of our performance benchmark to also allow benchmarking ONNX models." + ], + "id": "IYkxQOTRqcBe" + }, + { + "cell_type": "code", + "execution_count": 166, + "id": "8hvfl3xvlnEs", + "metadata": { + "id": "8hvfl3xvlnEs" + }, + "outputs": [], + "source": [ + "class OnnxPerformanceBenchmark(PerformanceBenchmark):\n", + " def __init__(self, *args, model_path, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " self.model_path = model_path\n", + "\n", + " def compute_size(self):\n", + " size_mb = Path(self.model_path).stat().st_size / (1024 * 1024)\n", + " print(f\"Model size (MB) - {size_mb:.2f}\")\n", + " return {\"size_mb\": size_mb}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4ht5U1qUqcBe" + }, + "source": [ + "Then, we can load the converted SentenceTransformer model with the `\"CUDAExecutionProvider\"` provider. Feel free to also experiment with other providers, such as `\"TensorrtExecutionProvider\"` and `\"CPUExecutionProvider\"`. The former may be even faster than `\"CUDAExecutionProvider\"`, but requires more installation." + ], + "id": "4ht5U1qUqcBe" + }, + { + "cell_type": "code", + "execution_count": 169, + "id": "IpoDwkPiWRn8", + "metadata": { + "id": "IpoDwkPiWRn8" + }, + "outputs": [], + "source": [ + "import torch\n", + "from transformers import AutoTokenizer\n", + "from optimum.onnxruntime import ORTModelForFeatureExtraction\n", + "\n", + "# Load model from HuggingFace Hub\n", + "tokenizer = AutoTokenizer.from_pretrained('bge_auto_opt_O4', model_max_length=512)\n", + "ort_model = ORTModelForFeatureExtraction.from_pretrained('bge_auto_opt_O4', provider=\"CUDAExecutionProvider\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Odn2lSPJqcBf" + }, + "source": [ + "And let's make a class that uses the tokenizer, ONNX Runtime (ORT) model and a model head." + ], + "id": "Odn2lSPJqcBf" + }, + { + "cell_type": "code", + "execution_count": 170, + "id": "enaQpBF9WRn9", + "metadata": { + "id": "enaQpBF9WRn9" + }, + "outputs": [], + "source": [ + "from setfit.exporters.utils import mean_pooling\n", + "\n", + "\n", + "class OnnxSetFitModel:\n", + " def __init__(self, ort_model, tokenizer, model_head):\n", + " self.ort_model = ort_model\n", + " self.tokenizer = tokenizer\n", + " self.model_head = model_head\n", + "\n", + " def predict(self, inputs):\n", + " encoded_inputs = self.tokenizer(\n", + " inputs, padding=True, truncation=True, return_tensors=\"pt\"\n", + " ).to(self.ort_model.device)\n", + "\n", + " outputs = self.ort_model(**encoded_inputs)\n", + " embeddings = mean_pooling(\n", + " outputs[\"last_hidden_state\"], encoded_inputs[\"attention_mask\"]\n", + " )\n", + " return self.model_head.predict(embeddings.cpu())\n", + "\n", + " def __call__(self, inputs):\n", + " return self.predict(inputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "N1TDdcOkqcBh" + }, + "source": [ + "We can initialize this model like so:" + ], + "id": "N1TDdcOkqcBh" + }, + { + "cell_type": "code", + "execution_count": 171, + "id": "qRviEk2WWRn9", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qRviEk2WWRn9", + "outputId": "33f010a8-376e-4f0c-b21b-97fe25bf1a81" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "array([0, 0])" + ] + }, + "metadata": {}, + "execution_count": 171 + } + ], + "source": [ + "model = SetFitModel.from_pretrained(\"moshew/bge-small-en-v1.5_setfit-sst2-english\")\n", + "onnx_setfit_model = OnnxSetFitModel(ort_model, tokenizer, model.model_head)\n", + "\n", + "# Perform inference\n", + "onnx_setfit_model(test_dataset[\"text\"][:2])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3DPl1ZpYqcBh" + }, + "source": [ + "Time to benchmark this ONNX model." + ], + "id": "3DPl1ZpYqcBh" + }, + { + "cell_type": "code", + "execution_count": 201, + "id": "O8jpZ3gdWRn9", + "metadata": { + "id": "O8jpZ3gdWRn9", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "8d31c81a-67e4-4074-cf35-9f56d6dcdd20" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Model size (MB) - 63.39\n", + "Accuracy on test set - 0.906\n", + "Average latency (ms) - 2.19 +\\- 0.50\n" + ] + } + ], + "source": [ + "pb = OnnxPerformanceBenchmark(\n", + " onnx_setfit_model,\n", + " test_dataset,\n", + " \"bge-small (optimum ONNX)\",\n", + " model_path=\"bge_auto_opt_O4/model.onnx\",\n", + ")\n", + "perf_metrics.update(pb.run_benchmark())" + ] + }, + { + "cell_type": "code", + "execution_count": 202, + "id": "tpjtxQQlZQPa", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 455 + }, + "id": "tpjtxQQlZQPa", + "outputId": "01efad97-4780-4c47-f10f-3afa7e819d15" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAG2CAYAAAByJ/zDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbEklEQVR4nO3deVwV9f4/8NfhAIcDAio7xSYgouK+gV5XFM1My9yy3DWNUjQ1MTVLDbVc0vpqluK+tGlpuUHhcsVdXEJREEETJBc4IPs5n98f/DzXI4jsB4bX8/GYB56Zz8y8Z86R82LmMzMyIYQAERERkYQY6LsAIiIioorGgENERESSw4BDREREksOAQ0RERJLDgENERESSw4BDREREksOAQ0RERJLDgENERESSw4BDREREksOAQ0RERJKj14CTnp6OoKAguLi4QKlUws/PD2fOnNFOHzVqFGQymc7Qu3dvPVZMRERENYGhPlc+btw4XLlyBVu2bIGjoyO2bt0Kf39/REdH46WXXgIA9O7dG6Ghodp5FAqFvsolIiKiGkKmr4dtZmVlwdzcHL/++iv69u2rHd+6dWv06dMHCxcuxKhRo5Camoo9e/boo0QiIiKqofR2BCc/Px9qtRomJiY645VKJY4fP659HRERAVtbW9SrVw/du3fHwoULYWVl9dzl5uTkICcnR/tao9Hg4cOHsLKygkwmq/gNISIiogonhEB6ejocHR1hYFCGHjVCj3x9fUWXLl3EP//8I/Lz88WWLVuEgYGBaNiwoRBCiB07dohff/1VXLp0SezevVt4e3uLtm3bivz8/Ocu85NPPhEAOHDgwIEDBw4SGG7fvl2mjKG3U1QAEBcXhzFjxuDo0aOQy+Vo1aoVGjZsiHPnzuHq1auF2t+8eRPu7u4ICwtDjx49ilzms0dw0tLS4OzsjNu3b8PCwqLStoWIiIgqjkqlgpOTE1JTU2FpaVnq+fXaydjd3R1HjhzB48ePoVKp4ODggCFDhqBBgwZFtm/QoAGsra0RGxv73ICjUCiK7IhsYWHBgENERFTDlLV7SbW4D46ZmRkcHBzw6NEjHDx4EP379y+y3Z07d/DgwQM4ODhUcYVERERUk+j1CM7BgwchhICXlxdiY2MxY8YMNGrUCKNHj0ZGRgY+/fRTDBw4EPb29oiLi8PMmTPh4eGBgIAAfZZNRERE1Zxej+CkpaUhMDAQjRo1wogRI9CpUyccPHgQRkZGkMvluHTpEl577TU0bNgQY8eORevWrXHs2DHeC4eIiIiKpddOxlVBpVLB0tISaWlp7INDRERUQ5T3+7ta9MEhIiIiqkgMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMOERERCQ5DDhEREQkOQw4REREJDkMODVNTjoQf6zgZ21aNxFJWkZOPiLjHiAjJ1/fpZRLvlqDrFw18tUafZdSLlJ4Pwz1XQCVUu5j4NZxwNoTUJjXnnUTkaRl5uTj5M0HcLcxQx1Fzfpq0mgEbt5/jAuJj3DpThryNRoYGhig2cuWaOlcDw2szWBgINN3maVSk9+PJ2pm1URERNVATr4av0XdxX9j7yM7TwNLUyMYyw2Qq9Yg/GoK/ht7Hx09rPFaC0coDOX6LrdWYcCpTdT5QN5jwMgMkPOtJyIqD41G4Leouwi/mgI7CwVcrIx1pttbmCA1MxfhV1MAAANbvVzjjuTUZPyWqw00GiDhOBD7J5D5ADC1Ajy6Ay6dAAN2wyIiKoub9x/jv7H3YWehQF1T4yLbPBn/39j7aOVSD+42daqyxFqNAacm0qiB/NyCoSRuHQcubAZkcsCkLpB2Gzi7AVCrAdeOJV9vSddHRFQGao1ArlqD3Pya0UH3zK2HeJyjxsv1TKHWiOe2MzcxQlJaNs7EP4RTPdMqrLDscmt4J2mAAadmSjwB5GUBihL8JSA0QPxRIEcFKOsB6UkF47MeAUcWA4mdAVkJj+LkZJRsnUREZXDm1kNk56lhVgM6tWo0Akeu/wu1EFBl572wfUZOPnacTsQ/j7JqxGmqxzn5NeJ9KA7PT0idOhfIywQMTXTHG5oUhCQ1j8oQEZWWWghohIBcVrKwIpfJoNYIqMXzj/RQxarZ8ay2cvYDWo0AzO1f3FaTX3AUR3UXqOf2v/GP4gELR6DLR4BBCT8G6clA1Lay1UxE9AJtXetjaDsn2JqbvLixnuWrNUjLykNuvgZ2Fi+u954qGwpDA7zfzQOG8up/bCElPRs/nr2j7zLKhQGnJjKQA4bGBcMLGQOePYFzG4HURMDEEshOAyAAz16AcSnOB5dofUREZSM3kMFYbgBjw+ofAIwNDdDCqS7Crt6DY13lC9unZ+ejfWNbmNaQ0z7GNSCEvYhetyA9PR1BQUFwcXGBUqmEn58fzpw5o50uhMC8efPg4OAApVIJf39/3LhxQ48V11AunYDWowuO2KizC362Hg24lKKDMRER6WjpXA9KIzlSM4s/1Z+amQsTIwO0dK5XRZURoOcjOOPGjcOVK1ewZcsWODo6YuvWrfD390d0dDReeuklLF26FKtWrcKmTZvg5uaGuXPnIiAgANHR0TAxqf6HMKsNAwPA7T+Asy/vg0NEVEEaWJuho4e19j43RV0qnpqZi3uqHPTwtoWblVlVl1ir6e0ITlZWFn7++WcsXboUnTt3hoeHB+bPnw8PDw+sWbMGQgisXLkSc+bMQf/+/dGsWTNs3rwZd+/exZ49e/RVds0mNyw4RcVwQ0RUbgYGMrzWwhE9vG2RlpWHmOR0JKuy8fBxLpJV2YhJTkdaVh56eNvitRaONeLqKSnR2zddfn4+1Gp1oSMxSqUSx48fR3x8PJKTk+Hv76+dZmlpifbt2yMyMhJDhw4tcrk5OTnIycnRvlapVJWzAUREVOspDOUY2OpltHKphwuJj3D5Thry1BqYGBqgbWNbtHSuBzermvcsKinQW8AxNzeHr68vFixYAG9vb9jZ2WHHjh2IjIyEh4cHkpOTAQB2dnY689nZ2WmnFSUkJASffvpppdauV8ZmgGungp+1ad1EJGmmCkN0aGBVYzrhPs3AQAZ3mzpwt6mDAS1eQp5awEguqxFXSz1PTX4/ntDr3t+yZQuEEHjppZegUCiwatUqDBs2DAbleHxAcHAw0tLStMPt27crsOJqQGFe0J9GH0/z1ue6iUjS6igM4etuVWOfXP2EodwASmN5jQ43gDTeD72+A+7u7jhy5AgyMjJw+/ZtnD59Gnl5eWjQoAHs7Qvu8XLv3j2dee7du6edVhSFQgELCwudgYiIiGqXahExzczM4ODggEePHuHgwYPo378/3NzcYG9vj/DwcG07lUqFU6dOwdfXV4/VEhERUXWn12NPBw8ehBACXl5eiI2NxYwZM9CoUSOMHj0aMpkMQUFBWLhwITw9PbWXiTs6OmLAgAH6LJuIiIiqOb0GnLS0NAQHB+POnTuoX78+Bg4ciEWLFsHIyAgAMHPmTDx+/BgTJkxAamoqOnXqhAMHDvAeOERERFQsmRDSfvKXSqWCpaUl0tLS2B+HiIiohijv93e16INDREREVJEYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHIYcIiIiEhyGHCIiIhIchhwiIiISHL0GnDUajXmzp0LNzc3KJVKuLu7Y8GCBRBCaNuMGjUKMplMZ+jdu7ceqyYiIqLqzlCfK1+yZAnWrFmDTZs2oUmTJjh79ixGjx4NS0tLTJ48Wduud+/eCA0N1b5WKBT6KJeIiIhqCL0GnBMnTqB///7o27cvAMDV1RU7duzA6dOnddopFArY29vro0QiIiKqgfR6isrPzw/h4eG4fv06AODixYs4fvw4+vTpo9MuIiICtra28PLywqRJk/DgwYPnLjMnJwcqlUpnICIiotpFr0dwZs2aBZVKhUaNGkEul0OtVmPRokUYPny4tk3v3r3xxhtvwM3NDXFxcZg9ezb69OmDyMhIyOXyQssMCQnBp59+WpWbQURERNWMTDzdo7eK7dy5EzNmzMAXX3yBJk2aICoqCkFBQVi+fDlGjhxZ5Dw3b96Eu7s7wsLC0KNHj0LTc3JykJOTo32tUqng5OSEtLQ0WFhYVNq2EBERUcVRqVSwtLQs8/e3Xo/gzJgxA7NmzcLQoUMBAD4+PkhISEBISMhzA06DBg1gbW2N2NjYIgOOQqFgJ2QiIqJaTq99cDIzM2FgoFuCXC6HRqN57jx37tzBgwcP4ODgUNnlERERUQ2l1yM4/fr1w6JFi+Ds7IwmTZrgwoULWL58OcaMGQMAyMjIwKeffoqBAwfC3t4ecXFxmDlzJjw8PBAQEKDP0omIiKga02sfnPT0dMydOxe7d+9GSkoKHB0dMWzYMMybNw/GxsbIysrCgAEDcOHCBaSmpsLR0RG9evXCggULYGdnV6J1lPccHhEREVW98n5/6zXgVAUGHCIiopqnvN/ffBYVERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSQ4DDhEREUkOAw4RERFJDgMOERERSY5haRprNBocOXIEx44dQ0JCAjIzM2FjY4OWLVvC398fTk5OlVUnERERUYmV6AhOVlYWFi5cCCcnJ7zyyivYv38/UlNTIZfLERsbi08++QRubm545ZVXcPLkycqumYiIiKhYJTqC07BhQ/j6+uK7775Dz549YWRkVKhNQkICtm/fjqFDh+Ljjz/G+PHjK7xYIiIiopKQCSHEixpdvXoV3t7eJVpgXl4eEhMT4e7uXu7iKoJKpYKlpSXS0tJgYWGh73KIiIioBMr7/V2iU1QlDTcAYGRkVG3CDREREdVOpepk/LT8/Hx8++23iIiIgFqtRseOHREYGAgTE5OKrI+IiIio1MoccCZPnozr16/jjTfeQF5eHjZv3oyzZ89ix44dFVkfERERUamVOODs3r0br7/+uvb1oUOHEBMTA7lcDgAICAhAhw4dKr5CIiIiolIq8Y3+NmzYgAEDBuDu3bsAgFatWmHixIk4cOAA9u7di5kzZ6Jt27aVVigRERFRSZU44OzduxfDhg1D165dsXr1aqxbtw4WFhb4+OOPMXfuXDg5OWH79u2VWSsRERFRiZToMvGnpaamYubMmbh48SLWrl2Lli1bVlZtFYKXiRMREdU8VXKZ+NPq1q2LdevW4YsvvsCIESMwY8YMZGdnl3rFRERERJWlxAEnMTERgwcPho+PD4YPHw5PT0+cO3cOpqamaN68Ofbv31+ZdRIRERGVWIlPUXXt2hX29vYYNWoUDh48iLi4OPz2228ACu50/O6778Le3h4//PBDpRZcWjxFRUREVPOU9/u7xJeJnz17FhcvXoS7uzsCAgLg5uamnebt7Y2jR49i3bp1pS6AiIiIqKKVOOC0bt0a8+bNw8iRIxEWFgYfH59CbSZMmFChxRERERGVRYn74GzevBk5OTmYOnUq/vnnH3z77beVWRcRERFRmZX4CI6Liwt++umnyqyFiIiIqEKU6AjO48ePS7XQ0rYnIiIiqkglCjgeHh5YvHgxkpKSnttGCIHDhw+jT58+WLVqVYUVSERERFRaJTpFFRERgdmzZ2P+/Plo3rw52rRpA0dHR5iYmODRo0eIjo5GZGQkDA0NERwcjHfffbey6yYiIiJ6rlI9qiExMRE//vgjjh07hoSEBGRlZcHa2hotW7ZEQEAA+vTpo326eHXB++AQERHVPOX9/i71s6hqGgYcIiKimqfKn0VFREREVN0x4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHklDrguLq64rPPPkNiYmK5V65WqzF37ly4ublBqVTC3d0dCxYswNMXdgkhMG/ePDg4OECpVMLf3x83btwo97qJiIhIukodcIKCgvDLL7+gQYMG6NmzJ3bu3ImcnJwyrXzJkiVYs2YNvv76a1y9ehVLlizB0qVLsXr1am2bpUuXYtWqVVi7di1OnToFMzMzBAQEIDs7u0zrJCIiIukr831wzp8/j40bN2LHjh1Qq9V46623MGbMGLRq1arEy3j11VdhZ2eH9evXa8cNHDgQSqUSW7duhRACjo6O+PDDDzF9+nQAQFpaGuzs7LBx40YMHTr0hevgfXCIiIhqHr3dB6dVq1ZYtWoV7t69i08++QTff/892rZtixYtWmDDhg0oSW7y8/NDeHg4rl+/DgC4ePEijh8/jj59+gAA4uPjkZycDH9/f+08lpaWaN++PSIjI4tcZk5ODlQqlc5AREREtUuJnkVVlLy8POzevRuhoaE4fPgwOnTogLFjx+LOnTuYPXs2wsLCsH379mKXMWvWLKhUKjRq1AhyuRxqtRqLFi3C8OHDAQDJyckAADs7O5357OzstNOeFRISgk8//bSsm0VEREQSUOqAc/78eYSGhmLHjh0wMDDAiBEjsGLFCjRq1Ejb5vXXX0fbtm1fuKwffvgB27Ztw/bt29GkSRNERUUhKCgIjo6OGDlyZGlLAwAEBwdj2rRp2tcqlQpOTk5lWhYRERHVTKUOOG3btkXPnj2xZs0aDBgwAEZGRoXauLm5lah/zIwZMzBr1ixtWx8fHyQkJCAkJAQjR46Evb09AODevXtwcHDQznfv3j20aNGiyGUqFAooFIrSbhYRERFJSKkDzs2bN+Hi4lJsGzMzM4SGhr5wWZmZmTAw0O0GJJfLodFoABQEJXt7e4SHh2sDjUqlwqlTpzBp0qTSlk5ERES1RKkDTkpKCpKTk9G+fXud8adOnYJcLkebNm1KvKx+/fph0aJFcHZ2RpMmTXDhwgUsX74cY8aMAQDIZDIEBQVh4cKF8PT0hJubG+bOnQtHR0cMGDCgtKUTERFRLVHqq6gCAwNx+/btQuP/+ecfBAYGlmpZq1evxptvvon33nsP3t7emD59Ot59910sWLBA22bmzJn44IMPMGHCBLRt2xYZGRk4cOAATExMSls6ERER1RKlvg9OnTp1cOnSJTRo0EBnfHx8PJo1a4b09PQKLbC8eB8cIiKimqfK74OjUChw7969QuOTkpJgaFjmq86JiIiIKkypA06vXr0QHByMtLQ07bjU1FTMnj0bPXv2rNDiiIiIiMqi1IdcvvzyS3Tu3BkuLi5o2bIlACAqKgp2dnbYsmVLhRdIREREVFqlDjgvvfQSLl26hG3btuHixYtQKpUYPXo0hg0bVuQ9cYiIiIiqWpk6zZiZmWHChAkVXQsRERFRhShzr+Do6GgkJiYiNzdXZ/xrr71W7qKIiIiIyqNMdzJ+/fXXcfnyZchkMu1Tw2UyGQBArVZXbIVEREREpVTqq6imTJkCNzc3pKSkwNTUFH///TeOHj2KNm3aICIiohJKJCIiIiqdUh/BiYyMxJ9//glra2sYGBjAwMAAnTp1QkhICCZPnowLFy5URp1EREREJVbqIzhqtRrm5uYAAGtra9y9excA4OLigpiYmIqtjoiIiKgMSn0Ep2nTprh48SLc3NzQvn17LF26FMbGxli3bl2hxzcQERER6UOpA86cOXPw+PFjAMBnn32GV199Ff/5z39gZWWFXbt2VXiBRERERKVV6odtFuXhw4eoV6+e9kqq6oQP2yQiIqp5qvRhm3l5eTA0NMSVK1d0xtevX79ahhsiIiKqnUoVcIyMjODs7Mx73RAREVG1VuqrqD7++GPMnj0bDx8+rIx6iIiIiMqt1J2Mv/76a8TGxsLR0REuLi4wMzPTmX7+/PkKK46IiIioLEodcAYMGFAJZRARERFVnAq5iqo641VURERENU+VXkVFREREVBOU+hSVgYFBsZeE8worIiIi0rdSB5zdu3frvM7Ly8OFCxewadMmfPrppxVWGBEREVFZVVgfnO3bt2PXrl349ddfK2JxFYZ9cIiIiGqeatMHp0OHDggPD6+oxRERERGVWYUEnKysLKxatQovvfRSRSyOiIiIqFxK3Qfn2YdqCiGQnp4OU1NTbN26tUKLIyIiIiqLUgecFStW6AQcAwMD2NjYoH379qhXr16FFkdERERUFqUOOKNGjaqEMoiIiIgqTqn74ISGhuLHH38sNP7HH3/Epk2bKqQoIiIiovIodcAJCQmBtbV1ofG2trb4/PPPK6QoIiIiovIodcBJTEyEm5tbofEuLi5ITEyskKKIiIiIyqPUAcfW1haXLl0qNP7ixYuwsrKqkKKIiIiIyqPUAWfYsGGYPHky/vrrL6jVaqjVavz555+YMmUKhg4dWhk1EhEREZVKqa+iWrBgAW7duoUePXrA0LBgdo1GgxEjRrAPDhEREVULZX4W1Y0bNxAVFQWlUgkfHx+4uLhUdG0Vgs+iIiIiqnnK+/1d6iM4T3h6esLT07OssxMRERFVmlL3wRk4cCCWLFlSaPzSpUsxaNCgCimKiIiIqDxKHXCOHj2KV155pdD4Pn364OjRoxVSFBEREVF5lDrgZGRkwNjYuNB4IyMjqFSqCimKiIiIqDxKHXB8fHywa9euQuN37tyJxo0bV0hRREREROVR6k7Gc+fOxRtvvIG4uDh0794dABAeHo4dO3YU+YwqIiIioqpW6oDTr18/7NmzB59//jl++uknKJVKNGvWDGFhYejSpUtl1EhERERUKmW+D05Rrly5gqZNm1bU4ioE74NDRERU85T3+7vUfXCelZ6ejnXr1qFdu3Zo3rx5eRdHREREVG5lDjhHjx7FiBEj4ODggC+//BLdu3fHyZMnK7I2IiIiojIpVR+c5ORkbNy4EevXr4dKpcLgwYORk5ODPXv28AoqIiIiqjZKfASnX79+8PLywqVLl7By5UrcvXsXq1evrszaiIiIiMqkxAFn//79GDt2LD799FP07dsXcrm83Ct3dXWFTCYrNAQGBgIAunbtWmjaxIkTy71eIiIikrYSB5zjx48jPT0drVu3Rvv27fH111/j/v375Vr5mTNnkJSUpB0OHz4MADrPtBo/frxOm6VLl5ZrnURERCR9JQ44HTp0wHfffYekpCS8++672LlzJxwdHaHRaHD48GGkp6eXeuU2Njawt7fXDvv27YO7u7vO/XRMTU112vBSbyIiInqRUl9FZWZmhjFjxuD48eO4fPkyPvzwQyxevBi2trZ47bXXylxIbm4utm7dijFjxkAmk2nHb9u2DdbW1mjatCmCg4ORmZlZ7HJycnKgUql0BiIiIqpdynUfHC8vLyxduhR37tzBjh07ylXInj17kJqailGjRmnHvfXWW9i6dSv++usvBAcHY8uWLXj77beLXU5ISAgsLS21g5OTU7nqIiIiopqnQu9kXB4BAQEwNjbG3r17n9vmzz//RI8ePRAbGwt3d/ci2+Tk5CAnJ0f7WqVSwcnJiXcyJiIiqkHKeyfjUj+LqjIkJCQgLCwMv/zyS7Ht2rdvDwDFBhyFQgGFQlHhNRIREVHNUe5HNVSE0NBQ2Nraom/fvsW2i4qKAgA4ODhUQVVERERUU+n9CI5Go0FoaChGjhwJQ8P/lRMXF4ft27fjlVdegZWVFS5duoSpU6eic+fOaNasmR4rJiIioupO7wEnLCwMiYmJGDNmjM54Y2NjhIWFYeXKlXj8+DGcnJwwcOBAzJkzR0+VEhERUU1RbToZV5bydlIiIiKiqlfe7+9q0QeHiIiIqCIx4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHk6DXguLq6QiaTFRoCAwMBANnZ2QgMDISVlRXq1KmDgQMH4t69e/osmYiIiGoAvQacM2fOICkpSTscPnwYADBo0CAAwNSpU7F37178+OOPOHLkCO7evYs33nhDnyUTERFRDSATQgh9F/FEUFAQ9u3bhxs3bkClUsHGxgbbt2/Hm2++CQC4du0avL29ERkZiQ4dOpRomSqVCpaWlkhLS4OFhUVllk9EREQVpLzf39WmD05ubi62bt2KMWPGQCaT4dy5c8jLy4O/v7+2TaNGjeDs7IzIyMjnLicnJwcqlUpnICIiotql2gScPXv2IDU1FaNGjQIAJCcnw9jYGHXr1tVpZ2dnh+Tk5OcuJyQkBJaWltrBycmpEqsmIiKi6qjaBJz169ejT58+cHR0LNdygoODkZaWph1u375dQRUSERFRTWGo7wIAICEhAWFhYfjll1+04+zt7ZGbm4vU1FSdozj37t2Dvb39c5elUCigUCgqs1wiIiKq5qrFEZzQ0FDY2tqib9++2nGtW7eGkZERwsPDteNiYmKQmJgIX19ffZRJRERENYTej+BoNBqEhoZi5MiRMDT8XzmWlpYYO3Yspk2bhvr168PCwgIffPABfH19S3wFFREREdVOeg84YWFhSExMxJgxYwpNW7FiBQwMDDBw4EDk5OQgICAA//d//6eHKomIiKgmqVb3wakMvA8OERFRzSOZ++AQERERVRQGHCIiIpIcBhwiIiKSHAYcIiIikhwGHCIiIpIcBhwiIiKSHAYcIiIikhwGHCIiIpIcBhwiIiKSHAYcIiIikhwGHCIiIpIcBhwiIiKSHAYcIiIikhwGHCIiIpIcBhwiIiKSHAYcIiIikhwGHCIiIpIcBhwiIiKSHAYcIiIikhwGHCIiIpIcQ30XQERUXmq1Gnl5efoug4hKQS6Xw9DQEDKZrFKWz4BDRDVaRkYG7ty5AyGEvksholIyNTWFg4MDjI2NK3zZDDhEVGOp1WrcuXMHpqamsLGxqbS/BImoYgkhkJubi3///Rfx8fHw9PSEgUHF9pphwCGiGisvLw9CCNjY2ECpVOq7HCIqBaVSCSMjIyQkJCA3NxcmJiYVunx2MiaiGo9Hbohqpoo+aqOz7EpbMhFRNZORk4/IuAfIyMkvNC1frUFWrhr5ak2p5iOi6omnqIio1sjMycfJmw/gbmOGOgpDaDQCN+8/xoXER7h0Jw35Gg0MDQzQ7GVLtHSuhwbWZjAwkBWaj4iqPx7BIaJaKSdfjZ/P38HXf95A+NUU5KoLwk2uWoPwqyn4+s8b+Pn8HeTkqyt83V27dkVQUFCFL7e62rhxI+rWrat9PX/+fLRo0eKF882dOxcTJkyovMIq2bPbXZRZs2bhgw8+qJqCahkGHCKqdTRC4Leouwi/mgJLpRG87M1hb2GC+mbGsLcwgZe9OSyVRgi/moLfou5Cw0vQq1xycjK++uorfPzxx9pxo0aNgkwmg0wmg7GxMTw8PPDZZ58hP//Fpw43btyonfd5w61btypxi4o2ffp0bNq0CTdv3qzydUsdAw4R1TqJDzLx39j7sLNQoK5p0fffqGtqDDsLBf4bex+3H2ZWcYX0/fffw8/PDy4uLjrje/fujaSkJNy4cQMffvgh5s+fjy+++OKFyxsyZAiSkpK0g6+vL8aPH68zzsnJqcT15ebmlnqbimJtbY2AgACsWbOmQpZH/8OAQ0S1ilojEHU7FY9z1DA3MYJaI547mJsYITNXjQu3Uyu8jvz8fLz//vuwtLSEtbU15s6dq3OzwqSkJPTt2xdKpRJubm7Yvn07XF1dsXLlSm2b1NRUjBs3DjY2NrCwsED37t1x8eLFYtcbERGBdu3awczMDHXr1kXHjh2RkJAA4H+njjZs2ABnZ2fUqVMH7733HtRqNZYuXQp7e3vY2tpi0aJFOstcvnw5fHx8YGZmBicnJ7z33nvIyMgo1/7ZuXMn+vXrV2i8QqGAvb09XFxcMGnSJPj7++O3337D48ePYWFhgZ9++kmn/Z49e2BmZob8/HzY29trB2NjY5iammpf5+bm4o033kCdOnVgYWGBwYMH4969e9rlPNk333//Pdzc3LSXNKempuLdd9+FnZ0dTExM0LRpU+zbt0+nhoMHD8Lb2xt16tTRBrSn9evXDzt37izX/qLC2FuOiGqV0/EPkJ6dDwMDGVTZL368Q0ZOPu5cyESPRnYVWsemTZswduxYnD59GmfPnsWECRPg7OyM8ePHAwBGjBiB+/fvIyIiAkZGRpg2bRpSUlJ0ljFo0CAolUrs378flpaW+Pbbb9GjRw9cv34d9evXL7TO/Px8DBgwAOPHj8eOHTuQm5uL06dP61xmHxcXh/379+PAgQOIi4vDm2++iZs3b6Jhw4Y4cuQITpw4gTFjxsDf3x/t27cHUHCp76pVq+Dm5oabN2/ivffew8yZM/F///d/Zdo3Dx8+RHR0NNq0afPCtkqlEg8ePICZmRmGDh2K0NBQvPnmm9rpT16bm5s/dxkajQb9+/dHnTp1cOTIEeTn5yMwMBBDhgxBRESEtl1sbCx+/vln/PLLL5DL5dBoNOjTpw/S09OxdetWuLu7Izo6GnK5XDtPZmYmvvzyS2zZsgUGBgZ4++23MX36dGzbtk3bpl27drhz5w5u3boFV1fX0u0sei4GHCKqVTSioA+OkaxkB7DlMhly1RqoK7gfjpOTE1asWAGZTAYvLy9cvnwZK1aswPjx43Ht2jWEhYXhzJkz2i/577//Hp6entr5jx8/jtOnTyMlJQUKhQIA8OWXX2LPnj346aefiuycq1KpkJaWhldffRXu7u4AAG9vb502Go0GGzZsgLm5ORo3boxu3bohJiYGf/zxBwwMDODl5YUlS5bgr7/+0gacpztMu7q6YuHChZg4cWKZA05iYiKEEHB0dHxuGyEEwsPDcfDgQW0n3XHjxsHPzw9JSUlwcHBASkoK/vjjD4SFhRW7vvDwcFy+fBnx8fHa01SbN29GkyZNcObMGbRt2xZAwWmpzZs3w8bGBgBw6NAhnD59GlevXkXDhg0BAA0aNNBZdl5eHtauXavd3++//z4+++wznTZPtjMhIYEBpwIx4BBRrdLOtT6SVVmQGxjAzuLFd069p8qGRgjIK/hmgh06dNA5cuLr64tly5ZBrVYjJiYGhoaGaNWqlXa6h4cH6tWrp3198eJFZGRkwMrKSme5WVlZiIuLQ2JiIho3bqwdP3v2bMyePRujRo1CQEAAevbsCX9/fwwePBgODg7adq6urjpHO+zs7CCXy3VuyGZnZ6dzNCksLAwhISG4du0aVCoV8vPzkZ2djczMTJiampZ632RlZQFAkXe23bdvH+rUqYO8vDxoNBq89dZbmD9/PoCCIyFNmjTBpk2bMGvWLGzduhUuLi7o3Llzseu7evUqnJycdPrgNG7cGHXr1sXVq1e1AcfFxUUbbgAgKioKL7/8sjbcFMXU1FQbbgBog9fTntyFOzOTfb0qEvvgEFGtYmRogCaOllBl50FuIHvhkJ6djyaOFjAwqF53S87IyICDgwOioqJ0hpiYGMyYMQOOjo464ydOnAig4JRNZGQk/Pz8sGvXLjRs2BAnT57ULtfIyEhnPTKZrMhxGk3BDRFv3bqFV199Fc2aNcPPP/+Mc+fO4ZtvvgFQ9o641tbWAIBHjx4VmtatWzdERUXhxo0byMrKwqZNm2BmZqadPm7cOGzcuFG7raNHj66wO10/vR4AJXo8SFH77tkHwz58+BAAdMITlR8DDhHVOj4vWUJpJEdqZvFfwKmZuTAxMoDPS5YVXsOpU6d0Xp88eRKenp6Qy+Xw8vJCfn4+Lly4oJ0eGxur84XfqlUrJCcnw9DQEB4eHjqDtbV1ofFP98lp2bIlgoODceLECTRt2hTbt28v83acO3cOGo0Gy5YtQ4cOHdCwYUPcvXu3zMsDAHd3d1hYWCA6OrrQNDMzM3h4eMDZ2RmGhoVPQrz99ttISEjAqlWrEB0djZEjR75wfd7e3rh9+zZu376tHRcdHY3U1FSdo2DPatasGe7cuYPr16+XcMuKduXKFRgZGaFJkyblWg7pYsAholrH2coUHT2scU+V89yQk5qZi3uqHHT0sIZT/dKfZnmRxMRETJs2DTExMdixYwdWr16NKVOmAAAaNWoEf39/TJgwAadPn8aFCxcwYcIEKJVK7dEIf39/+Pr6YsCAATh06BBu3bqFEydO4OOPP8bZs2eLXGd8fDyCg4MRGRmJhIQEHDp0CDdu3CjUD6c0PDw8kJeXh9WrV+PmzZvYsmUL1q5dW+blAQWdlv39/XH8+PFSz1uvXj288cYbmDFjBnr16oWXX375hfP4+/vDx8cHw4cPx/nz53H69GmMGDECXbp0Kbajc5cuXdC5c2cMHDgQhw8fRnx8vLaDdmkcO3YM//nPf/jA2ArGgENEtY6BTIbXWjiih7ct0rLyEJOcjmRVNh4+zkWyKhsxyelIy8pDD29bvNbCEQaV8DDPESNGICsrC+3atUNgYCCmTJmi0zF48+bNsLOzQ+fOnfH6669j/PjxMDc31/ZLkclk+OOPP9C5c2eMHj0aDRs2xNChQ5GQkAA7u6Kv+DI1NcW1a9cwcOBANGzYEBMmTEBgYCDefffdMm9H8+bNsXz5cixZsgRNmzbFtm3bEBISUublPTFu3Djs3LlTeyqsNMaOHYvc3FyMGTOmRO1lMhl+/fVX1KtXD507d4a/vz8aNGiAXbt2vXDen3/+GW3btsWwYcPQuHFjzJw5E2p16e5+vXPnTu3Vc1RxZOLZk4ESo1KpYGlpibS0NFhYWOi7HCKqQNnZ2YiPj9e5L0lxUlTZ2HYqEcPbO8PWwgQajUD8g4JnUV2+k4Y8tQZGcgP4/P9nUblZFTyL6tn59OHOnTtwcnJCWFgYevTooZcaqpIQAu3bt8fUqVMxbNiwUs27ZcsWTJ06FXfv3oWxcdE3cqwu9u/fjw8//BCXLl0q8pSb1BX3f7i839+1b28SUa1lqjBEhwZWMP3/D8w0MJDB3aYO3G3qYECLl5CnFjCSy2AoNyh2vqrw559/IiMjAz4+PkhKSsLMmTPh6ur6wiuCpEImk2HdunW4fPlyiefJzMxEUlISFi9ejHfffbfahxsAePz4MUJDQ2tluKls3KNEVGvUURjC192qyGmGcgMYyoucVOx8lSUvLw+zZ8/GzZs3YW5uDj8/P2zbtq3QVTlS1qJFixI9lPOJpUuXYtGiRejcuTOCg4Mrr7AK9PRNCali8RQVEdVYpT1FRUTVS2WeomInYyIiIpIcBhwiIiKSHAYcIiIikhwGHCIiIpIcBhwiqj1y0oH4YwU/q2I+ItIbvQecf/75B2+//TasrKygVCrh4+Ojc5vxUaNGQSaT6Qy9e/fWY8VEVGPlPgZuHS/4WRXzEZHe6DXgPHr0CB07doSRkRH279+P6OhoLFu2DPXq1dNp17t3byQlJWmHHTt26KliIqLy69q1K4KCgvRdRpXZuHEj6tatq309f/78Et3fZu7cuTqPr6hMrq6uWLlyZZWsq7YYOnQoli1bprf16zXgLFmyBE5OTggNDUW7du3g5uaGXr16wd3dXaedQqGAvb29dng2AD0tJycHKpVKZyAieiF1PpCdVvCT9C45ORlfffUVPv744wpd7rNh64kzZ85UWZiqKH///TcGDx4MGxsbKBQKNGzYEPPmzUNmZqZOO1dXV8hkMpw8eVJnfFBQELp27ap9PX/+fMhkMkycOFGnXVRUFGQyGW7dugUA+OOPP2BsbIzz58/rtFu2bBmsra2RnJwMAJgzZw4WLVqEtLS0Ctri0tFrwPntt9/Qpk0bDBo0CLa2tmjZsiW+++67Qu0iIiJga2sLLy8vTJo0CQ8ePHjuMkNCQmBpaakdnJycKnMTiKim02iA+KPAnwuBg3MKfsYfLRhPevP999/Dz88PLi4uVbI+GxsbmJpW/FPjK8vJkyfRvn175Obm4vfff8f169exaNEibNy4ET179kRubq5OexMTE3z00UcvXK6JiQnWr1+PGzduPLfNK6+8ghEjRmDEiBHIyckBAERHR2POnDn45ptvYG9vDwBo2rQp3N3dsXXr1nJsadnpNeDcvHkTa9asgaenJw4ePIhJkyZh8uTJ2LRpk7ZN7969sXnzZoSHh2PJkiU4cuQI+vTp89yntQYHByMtLU073L59u6o2h4hqAo0ayM/933AzAji7AUi7DciNC36e3QDcPKLbroLl5+fj/fffh6WlJaytrTF37lw8fWP5pKQk9O3bF0qlEm5ubti+fXuh0yipqakYN24cbGxsYGFhge7du+PixYvFrjciIgLt2rWDmZkZ6tati44dOyIhIQHA/04dbdiwAc7OzqhTpw7ee+89qNVqLF26FPb29rC1tcWiRYt0lrl8+XL4+PjAzMwMTk5OeO+995CRkVGu/bNz507069dPZ1xOTg4mT54MW1tbmJiYoFOnTjhz5ozOtslkMvz+++9o1qwZTExM0KFDB1y5ckU7ffTo0UhLS9P26Zw/fz6AwqeoZDIZvv32W7z66qswNTWFt7c3IiMjERsbi65du8LMzAx+fn6Ii4vTzjNq1CgMGDBAp+Znj5J07doVH3zwAYKCglCvXj3Y2dnhu+++w+PHjzF69GiYm5vDw8MD+/fvf+6+EUJg7Nix8Pb2xi+//IJ27drBxcUFgwYNwt69exEZGYkVK1bozDNhwgScPHkSf/zxR7H73cvLC926dXvhkbMVK1YgIyMDn3zyCfLz8zFy5Ej069cPQ4YM0WnXr18/7Ny5s9hlVRa9BhyNRoNWrVrh888/R8uWLTFhwgSMHz8ea9eu1bYZOnQoXnvtNfj4+GDAgAHYt28fzpw5g4iIiCKXqVAoYGFhoTMQEWklngBOfQscWwYc/QI4sgT4NwZITwL+vfr/f8YARxYXTD+2rKB9Bdu0aRMMDQ1x+vRpfPXVV1i+fDm+//577fQRI0bg7t27iIiIwM8//4x169YhJSVFZxmDBg1CSkoK9u/fj3PnzqFVq1bo0aMHHj58WOQ68/PzMWDAAHTp0gWXLl1CZGQkJkyYAJlMpm0TFxeH/fv348CBA9ixYwfWr1+Pvn374s6dOzhy5AiWLFmCOXPm4NSpU9p5DAwMsGrVKvz999/YtGkT/vzzT8ycObPM++bhw4eIjo5GmzZtdMbPnDkTP//8MzZt2oTz58/Dw8MDAQEBhbZ3xowZWLZsGc6cOQMbGxv069cPeXl58PPzw8qVK2FhYaHt0zl9+vTn1rFgwQKMGDECUVFRaNSoEd566y28++67CA4OxtmzZyGEwPvvv1/q7du0aROsra1x+vRpfPDBB5g0aRIGDRoEPz8/nD9/Hr169cI777xT6FTTE1FRUYiOjsa0adNgYKD7Nd68eXP4+/sX6qvq5uaGiRMnIjg4GJoXHJ1cvHgxfv75Z50Lfp5lbm6ODRs2YNmyZRg+fDhu376NNWvWFGrXrl07nD59WnukpyrpNeA4ODigcePGOuO8vb2RmJj43HkaNGgAa2trxMbGVnZ5RCR16lwgLxMwfOY5VoYmQF5WwfRK4uTkhBUrVsDLywvDhw/HBx98oP2r+9q1awgLC8N3332H9u3bo1WrVvj++++RlZWlnf/48eM4ffo0fvzxR7Rp0waenp748ssvUbduXfz0009FrlOlUiEtLQ2vvvoq3N3d4e3tjZEjR8LZ2VnbRqPRYMOGDWjcuDH69euHbt26ISYmBitXroSXlxdGjx4NLy8v/PXXX9p5goKC0K1bN7i6uqJ79+5YuHAhfvjhhzLvm8TERAgh4OjoqB33+PFjrFmzBl988QX69OmDxo0b47vvvoNSqcT69et15v/kk0/Qs2dP+Pj4YNOmTbh37x52794NY2NjWFpaQiaTaft01qlT57l1jB49GoMHD0bDhg3x0Ucf4datWxg+fDgCAgLg7e2NKVOmPPeP7eI0b94cc+bMgaenJ4KDg2FiYgJra2uMHz8enp6emDdvHh48eIBLly4VOf/169cBFHxfFsXb21vb5mlz5sxBfHw8tm3bVmx9rVq1wuDBg194Sqt79+5488038cMPP2DVqlWwsir8QFpHR0fk5uZq++VUJb0+Tbxjx46IiYnRGXf9+vViz7neuXMHDx48gIODQ2WXR0RS5OwHtBoBmNsDmnxAaADVXaCe2//aPIoHLByBLh8BBoZAejIQVfyXQml16NBB58iJr68vli1bBrVajZiYGBgaGqJVq1ba6R4eHjoXWFy8eBEZGRmFvlSysrIQFxeHxMREnT8gZ8+ejdmzZ2PUqFEICAhAz5494e/vj8GDB+v8PnV1dYW5ubn2tZ2dHeRyuc6RAjs7O52jSWFhYQgJCcG1a9egUqmQn5+P7OxsZGZmlqlfy5Mg9/TDF+Pi4pCXl4eOHTtqxxkZGaFdu3a4evWqzvy+vr7af9evXx9eXl6F2pREs2bNtP+2s7MDAPj4+OiMy87OhkqlKtXZgqeXK5fLYWVlVWi5AAodsXtWaZ+VbWNjg+nTp2PevHmFTiU9a+HChfD29sahQ4dga2tbZJt//vkHBw4cgKmpKY4dO4bBgwcXaqNUKgHguUejKpNej+BMnToVJ0+exOeff47Y2Fhs374d69atQ2BgIAAgIyMDM2bMwMmTJ3Hr1i2Eh4ejf//+2sOSRESlZiAHDI0LBmNTwLMnAAGkJhZcRZWaWPDas1fB9Cdtq5mMjAw4ODggKipKZ4iJicGMGTPg6OioM/7JlTGhoaGIjIyEn58fdu3ahYYNG+pcXWNkZKSzHplMVuS4J6c5bt26hVdffRXNmjXDzz//jHPnzuGbb74BgEIdXUvK2toaQMGtRPTp6e1+EkaLGvdkXxgYGBQKHXl5ecUu98lyilvusxo2bAgAzw1tV69e1bZ51rRp05CVlYX/+7//K3L6E+7u7hg/fjxmzZr13CA1fvx4tG7dGvv27cOaNWtw5MiRQm2enD60sbEpdn2VQa8Bp23btti9ezd27NiBpk2bYsGCBVi5ciWGDx8OoCDZXrp0Ca+99hoaNmyIsWPHonXr1jh27BgUCoU+SyciqXDpBLQeXXDERp1d8LP1aMCl44vnLYen+7AABVfFeHp6Qi6Xw8vLC/n5+bhw4YJ2emxsrM4XfqtWrZCcnAxDQ0N4eHjoDNbW1oXG169fXztvy5YtERwcjBMnTqBp06bYvn17mbfj3Llz0Gg0WLZsGTp06ICGDRvi7t27ZV4eUPDlamFhgejoaJ1xxsbG+O9//6sdl5eXhzNnzhTq6vB0YHv06BGuX7+uPZ1jbGz83ItUysvGxgZJSUk646Kioip8PS1atECjRo2wYsWKQiHo4sWLCAsLw7Bhw4qct06dOpg7dy4WLVqE9PTi78w9b948XL9+vchOwt9//z2OHz+O9evXo1u3bpg0aRLGjBmDx491b4Z55coVvPzyy9rQWpX0fifjV199FZcvX0Z2djauXr2K8ePHa6cplUocPHgQKSkpyM3Nxa1bt7Bu3Trt4TsionIzMADc/gN0nwP0Wljw0+0/BeMrUWJiIqZNm4aYmBjs2LEDq1evxpQpUwAAjRo1gr+/PyZMmIDTp0/jwoULmDBhApRKpfave39/f/j6+mLAgAE4dOgQbt26hRMnTuDjjz9+bufQ+Ph4BAcHIzIyEgkJCTh06BBu3Ljx3L4cJeHh4YG8vDysXr0aN2/exJYtW3QuFCkLAwMD+Pv74/jx49pxZmZmmDRpEmbMmIEDBw4gOjoa48ePR2ZmJsaOHasz/2effYbw8HBcuXIFo0aNgrW1tfbqJldXV2RkZCA8PBz379+v0FMn3bt3x9mzZ7F582bcuHEDn3zyifYKrookk8mwfv16REdHY+DAgTh9+jQSExPx448/ol+/fvD19S32RpITJkyApaXlC4OtnZ0dpk2bhlWrVumMT0hIwLRp0/Dll19qu5QsWbIEMpkMs2bN0ml77Ngx9OrVq2wbWk56DzhERNWC3BAwsSz4WQVGjBiBrKwstGvXDoGBgZgyZYrOjeY2b94MOzs7dO7cGa+//jrGjx8Pc3Nzbb8UmUyGP/74A507d8bo0aPRsGFDDB06FAkJCc/9I9DU1BTXrl3DwIED0bBhQ0yYMAGBgYF49913y7wdzZs3x/Lly7FkyRI0bdoU27ZtQ0hISJmX98S4ceOwc+dOnSMUixcvxsCBA/HOO++gVatWiI2NxcGDBwvd/HXx4sWYMmUKWrdujeTkZOzduxfGxgWnGf38/DBx4kQMGTIENjY2WLp0ablrfSIgIABz587FzJkz0bZtW6Snp2PEiBEVtvyn+fn54eTJk5DL5ejTpw88PDwQHByMkSNH4vDhw8We5TAyMsKCBQuQnZ39wvVMnz5dpyP2k0vUfX19dT6vpqam2Lhxo86pquzsbOzZs0fnwEVVkonS9lKqYVQqFSwtLZGWlsZLxokkJjs7G/Hx8XBzc9PpkPpc6cnA2VCgzeiCTsYlVdb5KtCdO3fg5OSEsLAw9OjRQy81VCUhBNq3b4+pU6c+93TLsyIiItCtWzc8evSoyLsVU9Vas2YNdu/ejUOHDj23TXH/h8v7/a3Xq6iIiKqUsRng2qngZ1XMVw5//vknMjIy4OPjg6SkJMycOROurq7o3LlzldWgTzKZDOvWrcPly5f1XQqVkZGREVavXq239TPgEFHtoTAv6F9TVfOVQ15eHmbPno2bN2/C3Nwcfn5+2LZtW6ErcKSsRYsWJXooJ1VP48aN0+v6GXCIiKqhgIAA3g6jlLp27Vrqe8OQdLGTMREREUkOAw4R1Xj8q52oZqrM/7sMOERUY8nlcgBlv2MuEenXk/sQVUbfMvbBIaIay9DQEKampvj3339hZGRU6MnKRFQ9CSGQmZmJlJQU1K1bV/vHSkViwCGiGksmk8HBwQHx8fFISEjQdzlEVEp169aFvX3l3FuKAYeIajRjY2N4enryNBVRDWNkZFQpR26eYMAhohrPwMCgZHcyJqJagyesiYiISHIYcIiIiEhyGHCIiIhIciTfB+fJTYRUKpWeKyEiIqKSevK9XdabAUo+4Dx48AAA4OTkpOdKiIiIqLTS09NhaWlZ6vkkH3Dq168PAEhMTCzTDqqJVCoVnJyccPv2bVhYWOi7nCrD7a49210btxmondtdG7cZqJ3b/ew2CyGQnp4OR0fHMi1P8gHnyZ1NLS0ta82H5AkLC4tat80At7s2qY3bDNTO7a6N2wzUzu1+epvLc2CCnYyJiIhIchhwiIiISHIkH3AUCgU++eQTKBQKfZdSZWrjNgPc7tq03bVxm4Haud21cZuB2rndFb3NMlHW66+IiIiIqinJH8EhIiKi2ocBh4iIiCSHAYeIiIgkhwGHiIiIJEfSAeebb76Bq6srTExM0L59e5w+fVrfJVWqkJAQtG3bFubm5rC1tcWAAQMQExOj77Kq1OLFiyGTyRAUFKTvUirdP//8g7fffhtWVlZQKpXw8fHB2bNn9V1WpVKr1Zg7dy7c3NygVCrh7u6OBQsWlPlZNdXR0aNH0a9fPzg6OkImk2HPnj0604UQmDdvHhwcHKBUKuHv748bN27op9gKVNx25+Xl4aOPPoKPjw/MzMzg6OiIESNG4O7du/oruAK86L1+2sSJEyGTybBy5coqq6+ylGS7r169itdeew2WlpYwMzND27ZtkZiYWKr1SDbg7Nq1C9OmTcMnn3yC8+fPo3nz5ggICEBKSoq+S6s0R44cQWBgIE6ePInDhw8jLy8PvXr1wuPHj/VdWpU4c+YMvv32WzRr1kzfpVS6R48eoWPHjjAyMsL+/fsRHR2NZcuWoV69evourVItWbIEa9aswddff42rV69iyZIlWLp0KVavXq3v0irM48eP0bx5c3zzzTdFTl+6dClWrVqFtWvX4tSpUzAzM0NAQACys7OruNKKVdx2Z2Zm4vz585g7dy7Onz+PX375BTExMXjttdf0UGnFedF7/cTu3btx8uTJMj+yoLp50XbHxcWhU6dOaNSoESIiInDp0iXMnTsXJiYmpVuRkKh27dqJwMBA7Wu1Wi0cHR1FSEiIHquqWikpKQKAOHLkiL5LqXTp6enC09NTHD58WHTp0kVMmTJF3yVVqo8++kh06tRJ32VUub59+4oxY8bojHvjjTfE8OHD9VRR5QIgdu/erX2t0WiEvb29+OKLL7TjUlNThUKhEDt27NBDhZXj2e0uyunTpwUAkZCQUDVFVbLnbfOdO3fESy+9JK5cuSJcXFzEihUrqry2ylTUdg8ZMkS8/fbb5V62JI/g5Obm4ty5c/D399eOMzAwgL+/PyIjI/VYWdVKS0sD8L8HjkpZYGAg+vbtq/OeS9lvv/2GNm3aYNCgQbC1tUXLli3x3Xff6busSufn54fw8HBcv34dAHDx4kUcP34cffr00XNlVSM+Ph7Jyck6n3NLS0u0b9++Vv1uAwp+v8lkMtStW1ffpVQajUaDd955BzNmzECTJk30XU6V0Gg0+P3339GwYUMEBATA1tYW7du3L/b03fNIMuDcv38farUadnZ2OuPt7OyQnJysp6qqlkajQVBQEDp27IimTZvqu5xKtXPnTpw/fx4hISH6LqXK3Lx5E2vWrIGnpycOHjyISZMmYfLkydi0aZO+S6tUs2bNwtChQ9GoUSMYGRmhZcuWCAoKwvDhw/VdWpV48vurNv9uA4Ds7Gx89NFHGDZsmKQfRLlkyRIYGhpi8uTJ+i6lyqSkpCAjIwOLFy9G7969cejQIbz++ut44403cOTIkVItS/JPE6+tAgMDceXKFRw/flzfpVSq27dvY8qUKTh8+HDpz8/WYBqNBm3atMHnn38OAGjZsiWuXLmCtWvXYuTIkXqurvL88MMP2LZtG7Zv344mTZogKioKQUFBcHR0lPR20//k5eVh8ODBEEJgzZo1+i6n0pw7dw5fffUVzp8/D5lMpu9yqoxGowEA9O/fH1OnTgUAtGjRAidOnMDatWvRpUuXEi9LkkdwrK2tIZfLce/ePZ3x9+7dg729vZ6qqjrvv/8+9u3bh7/++gsvv/yyvsupVOfOnUNKSgpatWoFQ0NDGBoa4siRI1i1ahUMDQ2hVqv1XWKlcHBwQOPGjXXGeXt7l/oqg5pmxowZ2qM4Pj4+eOeddzB16tRac/Tuye+v2vq77Um4SUhIwOHDhyV99ObYsWNISUmBs7Oz9ndbQkICPvzwQ7i6uuq7vEpjbW0NQ0PDCvn9JsmAY2xsjNatWyM8PFw7TqPRIDw8HL6+vnqsrHIJIfD+++9j9+7d+PPPP+Hm5qbvkipdjx49cPnyZURFRWmHNm3aYPjw4YiKioJcLtd3iZWiY8eOhW4BcP36dbi4uOipoqqRmZkJAwPdX1tyuVz7V5/Uubm5wd7eXud3m0qlwqlTpyT9uw34X7i5ceMGwsLCYGVlpe+SKtU777yDS5cu6fxuc3R0xIwZM3Dw4EF9l1dpjI2N0bZt2wr5/SbZU1TTpk3DyJEj0aZNG7Rr1w4rV67E48ePMXr0aH2XVmkCAwOxfft2/PrrrzA3N9eek7e0tIRSqdRzdZXD3Ny8UB8jMzMzWFlZSbrv0dSpU+Hn54fPP/8cgwcPxunTp7Fu3TqsW7dO36VVqn79+mHRokVwdnZGkyZNcOHCBSxfvhxjxozRd2kVJiMjA7GxsdrX8fHxiIqKQv369eHs7IygoCAsXLgQnp6ecHNzw9y5c+Ho6IgBAwbor+gKUNx2Ozg44M0338T58+exb98+qNVq7e+3+vXrw9jYWF9ll8uL3utnQ5yRkRHs7e3h5eVV1aVWqBdt94wZMzBkyBB07twZ3bp1w4EDB7B3715ERESUbkXlvg6rGlu9erVwdnYWxsbGol27duLkyZP6LqlSAShyCA0N1XdpVao2XCYuhBB79+4VTZs2FQqFQjRq1EisW7dO3yVVOpVKJaZMmSKcnZ2FiYmJaNCggfj4449FTk6OvkurMH/99VeR/49HjhwphCi4VHzu3LnCzs5OKBQK0aNHDxETE6PfoitAcdsdHx//3N9vf/31l75LL7MXvdfPkspl4iXZ7vXr1wsPDw9hYmIimjdvLvbs2VPq9ciEkNAtQImIiIgg0T44REREVLsx4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQERGR5DDgEBERkeQw4BAREZHkMOAQVRO3bt2CTCZDVFSUvkvRu4iICMhkMqSmphbbztXVFStXrizVsrt27YqgoKAy11ZWlfn+lnR/VTe5ubnw8PDAiRMnKm0da9euRb9+/Spt+VR9MeBQtRYZGQm5XI6+ffvqu5RqadSoUTX+GURF8fPzQ1JSEiwtLQEAGzduRN26dfVb1FOqW6B4dn9VtV9++QW9evWClZVVqULc2rVr4ebmBj8/v0qrbcyYMTh//jyOHTtWaeug6okBh6q19evX44MPPsDRo0dx9+7dSl2XEAL5+fmVug4qGWNjY9jb20Mmk+m7lBpB3/vr8ePH6NSpE5YsWVLieYQQ+PrrrzF27NhKrKxg37z11ltYtWpVpa6Hqh8GHKq2MjIysGvXLkyaNAl9+/bFxo0btdPeeustDBkyRKd9Xl4erK2tsXnzZgCARqNBSEgI3NzcoFQq0bx5c/z000/a9k/+Ct+/fz9at24NhUKB48ePIy4uDv3794ednR3q1KmDtm3bIiwsTGddSUlJ6Nu3L5RKJdzc3LB9+/ZCp0tSU1Mxbtw42NjYwMLCAt27d8fFixdLvP1qtRpjx47V1u/l5YWvvvpKO33+/PnYtGkTfv31V8hkMshkMu3Tdm/fvo3Bgwejbt26qF+/Pvr3749bt25p531y5OfLL7+Eg4MDrKysEBgYiLy8PG2bnJwcfPTRR3BycoJCoYCHhwfWr18PIQQ8PDzw5Zdf6tQbFRUFmUym85TgJ65cuQIDAwP8+++/AICHDx/CwMAAQ4cO1bZZuHAhOnXqpPPepKamIiIiAqNHj0ZaWpp2O+fPn6+dLzMzE2PGjIG5uTmcnZ1L9ET1/Px8vP/++7C0tIS1tTXmzp2Lpx/Lt2XLFrRp0wbm5uawt7fHW2+9hZSUFAAFp5q6desGAKhXrx5kMhlGjRoFoOAzt3TpUnh4eEChUMDZ2RmLFi3SWffNmzfRrVs3mJqaonnz5oiMjHxhvQCQkJCAfv36oV69ejAzM0OTJk3wxx9/FNpfQMFpuCf76unhyWegvJ/NZ73zzjuYN28e/P39SzzPuXPnEBcXp3N09slpvB9++AH/+c9/oFQq0bZtW1y/fh1nzpxBmzZtUKdOHfTp00f7WXqy/e3atYOZmRnq1q2Ljh07IiEhQTu9X79++O2335CVlVXmbaQaqEIeDUpUCdavXy/atGkjhCh4cra7u7vQaDRCCCH27dsnlEqlSE9P17bfu3evUCqVQqVSCSGEWLhwoWjUqJE4cOCAiIuLE6GhoUKhUIiIiAghxP+eaNusWTNx6NAhERsbKx48eCCioqLE2rVrxeXLl8X169fFnDlzhImJiUhISNCuy9/fX7Ro0UKcPHlSnDt3TnTp0kUolUqdJ/36+/uLfv36iTNnzojr16+LDz/8UFhZWYkHDx4Uub1Pnph84cIFIYQQubm5Yt68eeLMmTPi5s2bYuvWrcLU1FTs2rVLCCFEenq6GDx4sOjdu7dISkoSSUlJIicnR+Tm5gpvb28xZswYcenSJREdHS3eeust4eXlpX3q9siRI4WFhYWYOHGiuHr1qti7d68wNTXVeSL54MGDhZOTk/jll19EXFycCAsLEzt37hRCCLFo0SLRuHFjnfonT54sOnfuXOS2aTQaYW1tLX788UchhBB79uwR1tbWwt7eXmd/ffzxxzrvzaNHj0ROTo5YuXKlsLCw0G7nk/fdxcVF1K9fX3zzzTfixo0bIiQkRBgYGIhr164VWYcQBU+br1OnjpgyZYq4du2adr8+ve3r168Xf/zxh4iLixORkZHC19dX9OnTRwghRH5+vvj5558FABETEyOSkpJEamqqEEKImTNninr16omNGzeK2NhYcezYMfHdd9/pvL+NGjUS+/btEzExMeLNN98ULi4uIi8v77n1PtG3b1/Rs2dPcenSJREXFyf27t0rjhw5Umh/CSHEgwcPtPsqKSlJvPHGG8LLy0tkZmZq93Vxn82jR48KMzOzYoetW7cWqvHZz3Bxli9fLho1alTk/E/+30ZHR4sOHTqI1q1bi65du4rjx4+L8+fPCw8PDzFx4kQhhBB5eXnC0tJSTJ8+XcTGxoro6GixceNGnf+vjx8/FgYGBjX6yeNUegw4VG35+fmJlStXCiEKfolZW1trf0E9eb1582Zt+2HDhokhQ4YIIYTIzs4Wpqam4sSJEzrLHDt2rBg2bJgQ4n9fCnv27HlhLU2aNBGrV68WQghx9epVAUCcOXNGO/3GjRsCgDbgHDt2TFhYWIjs7Gyd5bi7u4tvv/22yHWU5MshMDBQDBw4UPt65MiRon///jpttmzZIry8vLRhUAghcnJyhFKpFAcPHtTO5+LiIvLz87VtBg0apN1/MTExAoA4fPhwkXX8888/Qi6Xi1OnTgkhCsKYtbW12Lhx43Nrf+ONN0RgYKAQQoigoCAxY8YMUa9ePXH16lWRm5srTE1NxaFDh4QQhb+wQ0NDhaWlZaFluri4iLffflv7WqPRCFtbW7FmzZrn1tGlSxfh7e2ts38++ugj4e3t/dx5zpw5IwBog9Wz9QkhhEqlEgqFQhtonvXk/f3++++14/7++28BQFy9evW5637Cx8dHzJ8/v8hpRdXzxPLly0XdunVFTEyMEKJkn83MzExx48aNYocnf0gUtY0lCThTpkwR3bt3L3L+p/fRjh07BAARHh6uHRcSEiK8vLyEEAVhDoD2D5fneRI8qfYwrMKDRUQlFhMTg9OnT2P37t0AAENDQwwZMgTr169H165dYWhoiMGDB2Pbtm1455138PjxY/z666/YuXMnACA2NhaZmZno2bOnznJzc3PRsmVLnXFt2rTReZ2RkYH58+fj999/R1JSEvLz85GVlYXExERtbYaGhmjVqpV2Hg8PD9SrV0/7+uLFi8jIyICVlZXOsrOyshAXF1fi/fDNN99gw4YNSExMRFZWFnJzc9GiRYti57l48SJiY2Nhbm6uMz47O1tn3U2aNIFcLte+dnBwwOXLlwEUnG6Sy+Xo0qVLketwdHRE3759sWHDBrRr1w579+5FTk4OBg0a9Ny6unTpoj19dOTIEXz++ee4fv06IiIi8PDhQ+Tl5aFjx47FbltRmjVrpv23TCaDvb299nTS83To0EGnv4qvry+WLVsGtVoNuVyOc+fOYf78+bh48SIePXoEjUYDAEhMTETjxo2LXObVq1eRk5ODHj16lLheBwcHAEBKSgoaNWpU7HyTJ0/GpEmTcOjQIfj7+2PgwIE6yyrK/v37MWvWLOzduxcNGzYEULLPplKphIeHR7HLLq+srCyYmJgUOe3p7bKzswMA+Pj46Ix78h7Xr18fo0aNQkBAAHr27Al/f38MHjxYu2+fUCqVyMzMrOjNoGqMAYeqpfXr1yM/Px+Ojo7acUIIKBQKfP3117C0tMTw4cPRpUsXpKSk4PDhw1AqlejduzeAgpACAL///jteeuklnWUrFAqd12ZmZjqvp0+fjsOHD+PLL7+Eh4cHlEol3nzzTeTm5pa4/oyMDDg4OGj7xDytpFcD7dy5E9OnT8eyZcvg6+sLc3NzfPHFFzh16tQL1926dWts27at0DQbGxvtv42MjHSmyWQy7Re5Uql8YX3jxo3DO++8gxUrViA0NBRDhgyBqanpc9s/uTz7xo0biI6ORqdOnXDt2jVERETg0aNHaNOmTbHzP09x21EWjx8/RkBAAAICArBt2zbY2NggMTERAQEBxX4GSrLPnq33ScgqSb3jxo1DQEAAfv/9dxw6dAghISFYtmwZPvjggyLbR0dHY+jQoVi8eDF69eqlHV+Sz+axY8fQp0+fYuv59ttvMXz48BfW/TzW1tbaQP2sovbRs+Oe3mehoaGYPHkyDhw4gF27dmHOnDk4fPgwOnTooG3z8OFDnc8/SR8DDlU7+fn52Lx5M5YtW6bzixkABgwYgB07dmDixInw8/ODk5MTdu3ahf3792PQoEHaX4KNGzeGQqFAYmLic49CPM9///tfjBo1Cq+//jqAgi+Epzvoenl5IT8/HxcuXEDr1q0BFBwxevTokbZNq1atkJycDENDQ7i6upZhLxTU4efnh/fee0877tmjP8bGxlCr1TrjWrVqhV27dsHW1hYWFhZlWrePjw80Gg2OHDny3I6jr7zyCszMzLBmzRocOHAAR48efeEy69Wrh4ULF6JFixaoU6cOunbtiiVLluDRo0fo2rXrc+ctajvL49mQePLkSXh6ekIul+PatWt48OABFi9eDCcnJwDA2bNnC9UDQKcmT09PKJVKhIeHY9y4cRVW69OcnJwwceJETJw4EcHBwfjuu++KDDj3799Hv379MHDgQEydOlVnWkk+m23atHnhpd5PjqyUVcuWLbFmzRoIISrk6q+WLVuiZcuWCA4Ohq+vL7Zv364NOHFxccjOzi509JakjVdRUbWzb98+PHr0CGPHjkXTpk11hoEDB2L9+vXatm+99RbWrl2Lw4cP6/w1aW5ujunTp2Pq1KnYtGkT4uLicP78eaxevRqbNm0qdv2enp745ZdfEBUVhYsXL+Ktt97S+WuxUaNG8Pf3x4QJE3D69GlcuHABEyZMgFKp1P6i9vf3h6+vLwYMGIBDhw7h1q1bOHHiBD7++ONCX5bF1XH27FkcPHgQ169fx9y5c3HmzBmdNq6urrh06RJiYmJw//595OXlYfjw4bC2tkb//v1x7NgxxMfHIyIiApMnT8adO3dKtG5XV1eMHDkSY8aMwZ49e7TL+OGHH7Rt5HI5Ro0aheDgYHh6esLX17fYZcpkMnTu3Bnbtm3ThplmzZohJycH4eHhxQZRV1dXZGRkIDw8HPfv3y/3qYbExERMmzYNMTEx2LFjB1avXo0pU6YAAJydnWFsbIzVq1fj5s2b+O2337BgwQKd+V1cXCCTybBv3z78+++/yMjIgImJCT766CPMnDkTmzdvRlxcHE6ePKnzeS2PoKAgHDx4EPHx8Th//jz++usveHt7F9l24MCBMDU1xfz585GcnKwd1Gp1iT6bT05RFTc8fQr04cOHiIqKQnR0NICC07hRUVFITk5+7vZ069YNGRkZ+Pvvv8u1X+Lj4xEcHIzIyEgkJCTg0KFDuHHjhs6+OXbsGBo0aAB3d/dyrYtqGH13AiJ61quvvipeeeWVIqedOnVKABAXL14UQggRHR0tAAgXFxedTqNCFHQ4XblypfDy8hJGRkbCxsZGBAQEPPfKkyfi4+NFt27dhFKpFE5OTuLrr78WXbp0EVOmTNG2uXv3rujTp49QKBTCxcVFbN++Xdja2oq1a9dq26hUKvHBBx8IR0dHYWRkJJycnMTw4cNFYmJikdv2bAfN7OxsMWrUKGFpaSnq1q0rJk2aJGbNmiWaN2+unSclJUX07NlT1KlTRwDQdsJOSkoSI0aMENbW1kKhUIgGDRqI8ePHi7S0NCFE0Z2Tp0yZIrp06aJ9nZWVJaZOnSocHByEsbGx8PDwEBs2bNCZJy4uTgAQS5cuLXKbnrVixQoBQOzfv187rn///sLQ0FDnirii3puJEycKKysrAUB88sknQoiCTsZPX7kmhBDNmzfXTi9Kly5dxHvvvScmTpwoLCwsRL169cTs2bN1Pj/bt28Xrq6uQqFQCF9fX/Hbb78V6jz72WefCXt7eyGTycTIkSOFEEKo1WqxcOFC4eLiIoyMjISzs7P4/PPPhRBFd8B99OiRzvtWnPfff1+4u7sLhUIhbGxsxDvvvCPu379f5P4CUOQQHx8vhCj9Z/NFQkNDi1xfce+DEAVX6s2aNUv7uqh9VNRn4elO58nJyWLAgAHaz6mLi4uYN2+eUKvV2va9evUSISEhZdo2qrlkQjx18wciKpM7d+7AyckJYWFhL+xkKiXHjh1Djx49cPv27XKfsqDa59KlS+jZsyfi4uJQp06dSlnH33//je7du+P69et6u9Mz6QcDDlEZ/Pnnn8jIyICPjw+SkpIwc+ZM/PPPP7h+/XqhTq9SlJOTg3///RcjR46Evb19kR2aiUpi48aNaN26tc5VUhUpLCwMarUaAQEBlbJ8qr7YB4eoDPLy8jB79mw0adIEr7/+OmxsbBAREVErwg0A7NixAy4uLkhNTcXSpUv1XY4k9OnTB3Xq1Cly+Pzzz/VdXqUZNWpUpYUboKA/HMNN7cQjOERE1cA///zz3EcJ1K9fH/Xr16/iiohqNgYcIiIikhyeoiIiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJYcAhIiIiyWHAISIiIslhwCEiIiLJ+X9oulqb3DKLoAAAAABJRU5ErkJggg==\n" + }, + "metadata": {} + } + ], + "source": [ + "plot_metrics(perf_metrics)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "By applying ONNX, we were able to improve the latency from 13.43ms per sample to 2.19ms per sample, for a speedup of 6.13x!\n", + "\n", + "For further improvements, we recommend increasing the inference batch size, as this may also heavily improve the throughput. For example, setting the batch size to 128 reduces the latency further down to 0.3ms, and down to 0.2ms at a batch size of 2048." + ], + "metadata": { + "id": "gvdggvIbvowO" + }, + "id": "gvdggvIbvowO" + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "h5ExEou96k3Z" + }, + "id": "h5ExEou96k3Z", + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From c7f49ad1236e07197e5dfdeff9ec48c4470ef671 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 27 Nov 2023 14:43:20 +0100 Subject: [PATCH 116/183] Add ONNX tutorial to docs --- docs/source/en/_toctree.yml | 2 + docs/source/en/tutorials/onnx.mdx | 313 ++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 docs/source/en/tutorials/onnx.mdx diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 11e9df0f..bc53342f 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -12,6 +12,8 @@ title: Overview - local: tutorials/zero_shot title: Zero-shot Text Classification + - local: tutorials/onnx + title: Efficiently run SetFit with ONNX title: Tutorials - sections: diff --git a/docs/source/en/tutorials/onnx.mdx b/docs/source/en/tutorials/onnx.mdx new file mode 100644 index 00000000..ad1f2e16 --- /dev/null +++ b/docs/source/en/tutorials/onnx.mdx @@ -0,0 +1,313 @@ +# Efficiently run SetFit Models with Optimum + +[SetFit](https://github.com/huggingface/setfit) is a technique for few-shot text classification that uses contrastive learning to fine-tune Sentence Transformers in domains where little to no labeled data is available. It achieves comparable performance to existing state-of-the-art methods based on large language models, yet requires no prompts and is efficient to train (typically a few seconds on a GPU to minutes on a CPU). + +In this notebook you'll learn how to further compress SetFit models for faster inference & deployment on GPU using Optimum Onnx. + +## 1. Setup development environment + +Our first step is to install SetFit. Running the following cell will install all the required packages for us. + +``` +!pip install setfit accelerate -qqq +``` + +## 2. Create a performance benchmark + +Before we train and optimize any models, let's define a performance benchmark that we can use to compare our models. In general, deploying ML models in production environments involves a tradeoff among several constraints: + +* Model performance: how well does the model perform on a well crafted test set? +* Latency: how fast can our model deliver predictions? +* Memory: on what cloud instance or device can we store and load our model? + +The class below defines a simple benchmark that measure each quantity for a given SetFit model and test dataset: + +```py +from pathlib import Path +from time import perf_counter + +import evaluate +import numpy as np +import torch +from tqdm.auto import tqdm + +metric = evaluate.load("accuracy") + + +class PerformanceBenchmark: + def __init__(self, model, dataset, optim_type): + self.model = model + self.dataset = dataset + self.optim_type = optim_type + + def compute_accuracy(self): + preds = self.model.predict(self.dataset["text"]) + labels = self.dataset["label"] + accuracy = metric.compute(predictions=preds, references=labels) + print(f"Accuracy on test set - {accuracy['accuracy']:.3f}") + return accuracy + + def compute_size(self): + state_dict = self.model.model_body.state_dict() + tmp_path = Path("model.pt") + torch.save(state_dict, tmp_path) + # Calculate size in megabytes + size_mb = Path(tmp_path).stat().st_size / (1024 * 1024) + # Delete temporary file + tmp_path.unlink() + print(f"Model size (MB) - {size_mb:.2f}") + return {"size_mb": size_mb} + + def time_model(self, query="that loves its characters and communicates something rather beautiful about human nature"): + latencies = [] + # Warmup + for _ in range(10): + _ = self.model([query]) + # Timed run + for _ in range(100): + start_time = perf_counter() + _ = self.model([query]) + latency = perf_counter() - start_time + latencies.append(latency) + # Compute run statistics + time_avg_ms = 1000 * np.mean(latencies) + time_std_ms = 1000 * np.std(latencies) + print(rf"Average latency (ms) - {time_avg_ms:.2f} +\- {time_std_ms:.2f}") + return {"time_avg_ms": time_avg_ms, "time_std_ms": time_std_ms} + + def run_benchmark(self): + metrics = {} + metrics[self.optim_type] = self.compute_size() + metrics[self.optim_type].update(self.compute_accuracy()) + metrics[self.optim_type].update(self.time_model()) + return metrics +``` + +Beyond that, we'll create a simple function to plot the performances reported by this benchmark. + +```py +import matplotlib.pyplot as plt +import pandas as pd + + +def plot_metrics(perf_metrics): + df = pd.DataFrame.from_dict(perf_metrics, orient="index") + + for idx in df.index: + df_opt = df.loc[idx] + plt.errorbar( + df_opt["time_avg_ms"], + df_opt["accuracy"] * 100, + xerr=df_opt["time_std_ms"], + fmt="o", + alpha=0.5, + ms=df_opt["size_mb"] / 15, + label=idx, + capsize=5, + capthick=1, + ) + + legend = plt.legend(loc="lower right") + + plt.ylim(63, 95) + # Use the slowest model to define the x-axis range + xlim = max([metrics["time_avg_ms"] for metrics in perf_metrics.values()]) * 1.2 + plt.xlim(0, xlim) + plt.ylabel("Accuracy (%)") + plt.xlabel("Average latency with batch_size=1 (ms)") + plt.show() +``` + +## 3. Train/evaluate bge-small SetFit models + +Before we optimize any models, let's train a few baselines as a point of reference. We'll use the [sst-2](https://huggingface.co/datasets/SetFit/sst2) dataset, which is a collection of sentiment text catagorized into 2 classes: positive, negative + +Let's start by loading the dataset from the Hub: + +``` +from datasets import load_dataset + +dataset = load_dataset("SetFit/sst2") +dataset +``` +``` +DatasetDict({ + train: Dataset({ + features: ['text', 'label', 'label_text'], + num_rows: 6920 + }) + validation: Dataset({ + features: ['text', 'label', 'label_text'], + num_rows: 872 + }) + test: Dataset({ + features: ['text', 'label', 'label_text'], + num_rows: 1821 + }) +}) +``` + +We train a SetFit model with the full dataset. Recall that SetFit excels with few-shot scenario, but this time we are interested to achieve maximum accuracy. + +```py +train_dataset = dataset["train"] +test_dataset = dataset["validation"] +``` + +Use the following line code to download the [already finetuned model](https://huggingface.co/moshew/bge-small-en-v1.5_setfit-sst2-english) and evaluate. Alternatively, uncomment the code below it to fine-tune the base model from scratch. + +Note that we perform the evaluations on Google Colab using the free T4 GPU. + +```py +# Evaluate the uploaded model! +from setfit import SetFitModel + +small_model = SetFitModel.from_pretrained("moshew/bge-small-en-v1.5_setfit-sst2-english") +pb = PerformanceBenchmark(model=small_model, dataset=test_dataset, optim_type="bge-small (PyTorch)") +perf_metrics = pb.run_benchmark() +``` +``` +Model size (MB) - 127.33 +Accuracy on test set - 0.906 +Average latency (ms) - 17.42 +\- 4.47 +``` + +```py +# # Fine-tune the base model and Evaluate! +# from setfit import SetFitModel, Trainer, TrainingArguments + +# # Load pretrained model from the Hub +# small_model = SetFitModel.from_pretrained( +# "BAAI/bge-small-en-v1.5" +# ) +# args = TrainingArguments(num_iterations=20) + +# # Create trainer +# small_trainer = Trainer( +# model=small_model, args=args, train_dataset=train_dataset +# ) +# # Train! +# small_trainer.train() + +# # Evaluate! +# pb = PerformanceBenchmark( +# model=small_trainer.model, dataset=test_dataset, optim_type="bge-small (base)" +# ) +# perf_metrics = pb.run_benchmark() +``` + +Let's plot the results to visualise the performance: + +``` +plot_metrics(perf_metrics) +``` + +![setfit_torch](https://github.com/huggingface/setfit/assets/37621491/4786eee6-88c8-46ca-95be-801514697a9d) + +## 4. Compressing with Optimum ONNX and CUDAExecutionProvider + +We'll be using Optimum's ONNX Runtime support with `CUDAExecutionProvider` [because it's fast while also supporting dynamic shapes](https://github.com/huggingface/optimum-benchmark/tree/main/examples/fast-mteb#notes). + +``` +!pip install optimum[onnxruntime-gpu] -qqq +``` + +[`optimum-cli`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/optimization#optimizing-a-model-during-the-onnx-export) makes it extremely easy to export a model to ONNX and apply SOTA graph optimizations / kernel fusions. + +```py +!optimum-cli export onnx \ + --model moshew/bge-small-en-v1.5_setfit-sst2-english \ + --task feature-extraction \ + --optimize O4 \ + --device cuda \ + bge_auto_opt_O4 +``` + +We may see some warnings, but these are not ones to be concerned about. We'll see later that it does not affect the model performance. + +First of all, we'll create a subclass of our performance benchmark to also allow benchmarking ONNX models. + +```py +class OnnxPerformanceBenchmark(PerformanceBenchmark): + def __init__(self, *args, model_path, **kwargs): + super().__init__(*args, **kwargs) + self.model_path = model_path + + def compute_size(self): + size_mb = Path(self.model_path).stat().st_size / (1024 * 1024) + print(f"Model size (MB) - {size_mb:.2f}") + return {"size_mb": size_mb} +``` + +Then, we can load the converted SentenceTransformer model with the `"CUDAExecutionProvider"` provider. Feel free to also experiment with other providers, such as `"TensorrtExecutionProvider"` and `"CPUExecutionProvider"`. The former may be even faster than `"CUDAExecutionProvider"`, but requires more installation. + +```py +import torch +from transformers import AutoTokenizer +from optimum.onnxruntime import ORTModelForFeatureExtraction + +# Load model from HuggingFace Hub +tokenizer = AutoTokenizer.from_pretrained('bge_auto_opt_O4', model_max_length=512) +ort_model = ORTModelForFeatureExtraction.from_pretrained('bge_auto_opt_O4', provider="CUDAExecutionProvider") +``` + +And let's make a class that uses the tokenizer, ONNX Runtime (ORT) model and a SetFit model head. + +```py +from setfit.exporters.utils import mean_pooling + + +class OnnxSetFitModel: + def __init__(self, ort_model, tokenizer, model_head): + self.ort_model = ort_model + self.tokenizer = tokenizer + self.model_head = model_head + + def predict(self, inputs): + encoded_inputs = self.tokenizer( + inputs, padding=True, truncation=True, return_tensors="pt" + ).to(self.ort_model.device) + + outputs = self.ort_model(**encoded_inputs) + embeddings = mean_pooling( + outputs["last_hidden_state"], encoded_inputs["attention_mask"] + ) + return self.model_head.predict(embeddings.cpu()) + + def __call__(self, inputs): + return self.predict(inputs) +``` + +We can initialize this model like so: + +```py +model = SetFitModel.from_pretrained("moshew/bge-small-en-v1.5_setfit-sst2-english") +onnx_setfit_model = OnnxSetFitModel(ort_model, tokenizer, model.model_head) + +# Perform inference +onnx_setfit_model(test_dataset["text"][:2]) +``` +``` +array([0, 0]) +``` + +Time to benchmark this ONNX model. + +```py +pb = OnnxPerformanceBenchmark( + onnx_setfit_model, + test_dataset, + "bge-small (optimum ONNX)", + model_path="bge_auto_opt_O4/model.onnx", +) +perf_metrics.update(pb.run_benchmark()) +``` +```py +plot_metrics(perf_metrics) +``` + +![setfit_onnx](https://github.com/huggingface/setfit/assets/37621491/9907ec1d-d4c6-431d-8695-1adc4247a576) + +By applying ONNX, we were able to improve the latency from 13.43ms per sample to 2.19ms per sample, for a speedup of 6.13x! + +For further improvements, we recommend increasing the inference batch size, as this may also heavily improve the throughput. For example, setting the batch size to 128 reduces the latency further down to 0.3ms, and down to 0.2ms at a batch size of 2048. \ No newline at end of file From 8e0c55cf392932b0b1c1c688c036abab71ff6180 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 10:31:48 +0100 Subject: [PATCH 117/183] Update docstring of from_pretrained! --- docs/source/en/reference/main.mdx | 3 +++ src/setfit/modeling.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/docs/source/en/reference/main.mdx b/docs/source/en/reference/main.mdx index 0b62e376..b746ebfe 100644 --- a/docs/source/en/reference/main.mdx +++ b/docs/source/en/reference/main.mdx @@ -5,6 +5,9 @@ [[autodoc]] SetFitModel - all + - from_pretrained + - save_pretrained + - push_to_hub - __call__ ## SetFitHead diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index c2caa7f0..2ba122b1 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -25,6 +25,7 @@ from torch import nn from torch.utils.data import DataLoader from tqdm.auto import tqdm, trange +from transformers.utils import copy_func from . import logging from .data import SetFitDataset @@ -719,3 +720,24 @@ def _from_pretrained( normalize_embeddings=normalize_embeddings, **model_kwargs, ) + + +# To update the docstring, we need to copy the method +SetFitModel.from_pretrained = copy_func(SetFitModel.from_pretrained) +if SetFitModel.from_pretrained.__doc__ is not None: + docstring = SetFitModel.from_pretrained.__doc__ + cut_index = docstring.find("model_kwargs") + if cut_index != -1: + docstring = ( + docstring[:cut_index] + + """multi_target_strategy (`str`, *optional*): + The strategy to use with multi-label classification. One of "one-vs-rest", "multi-output", + or "classifier-chain". + use_differentiable_head (`bool`, *optional*): + Whether to load SetFit using a differentiable (i.e., Torch) head instead of Logistic Regression. + normalize_embeddings (`bool`, *optional*): + Whether to apply normalization on the embeddings produced by the Sentence Transformer body. + device (`Union[torch.device, str]`, *optional*): + The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`.""" + ) + SetFitModel.from_pretrained.__doc__ = docstring From 19d6d9d20a85809f1dd6b021abe17c8ca36cbf26 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 10:38:55 +0100 Subject: [PATCH 118/183] Revert "Update docstring of from_pretrained!" This reverts commit 8e0c55cf392932b0b1c1c688c036abab71ff6180. --- docs/source/en/reference/main.mdx | 3 --- src/setfit/modeling.py | 22 ---------------------- 2 files changed, 25 deletions(-) diff --git a/docs/source/en/reference/main.mdx b/docs/source/en/reference/main.mdx index b746ebfe..0b62e376 100644 --- a/docs/source/en/reference/main.mdx +++ b/docs/source/en/reference/main.mdx @@ -5,9 +5,6 @@ [[autodoc]] SetFitModel - all - - from_pretrained - - save_pretrained - - push_to_hub - __call__ ## SetFitHead diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 2ba122b1..c2caa7f0 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -25,7 +25,6 @@ from torch import nn from torch.utils.data import DataLoader from tqdm.auto import tqdm, trange -from transformers.utils import copy_func from . import logging from .data import SetFitDataset @@ -720,24 +719,3 @@ def _from_pretrained( normalize_embeddings=normalize_embeddings, **model_kwargs, ) - - -# To update the docstring, we need to copy the method -SetFitModel.from_pretrained = copy_func(SetFitModel.from_pretrained) -if SetFitModel.from_pretrained.__doc__ is not None: - docstring = SetFitModel.from_pretrained.__doc__ - cut_index = docstring.find("model_kwargs") - if cut_index != -1: - docstring = ( - docstring[:cut_index] - + """multi_target_strategy (`str`, *optional*): - The strategy to use with multi-label classification. One of "one-vs-rest", "multi-output", - or "classifier-chain". - use_differentiable_head (`bool`, *optional*): - Whether to load SetFit using a differentiable (i.e., Torch) head instead of Logistic Regression. - normalize_embeddings (`bool`, *optional*): - Whether to apply normalization on the embeddings produced by the Sentence Transformer body. - device (`Union[torch.device, str]`, *optional*): - The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`.""" - ) - SetFitModel.from_pretrained.__doc__ = docstring From 5058e31689bd1acc29fecc4defa939b27cfc9a4d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 10:58:48 +0100 Subject: [PATCH 119/183] Update docstring of from_pretrained! But wrapped in MethodType --- docs/source/en/reference/main.mdx | 3 +++ src/setfit/modeling.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/docs/source/en/reference/main.mdx b/docs/source/en/reference/main.mdx index 0b62e376..b746ebfe 100644 --- a/docs/source/en/reference/main.mdx +++ b/docs/source/en/reference/main.mdx @@ -5,6 +5,9 @@ [[autodoc]] SetFitModel - all + - from_pretrained + - save_pretrained + - push_to_hub - __call__ ## SetFitHead diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index c2caa7f0..c7d945a1 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -1,5 +1,6 @@ import os import tempfile +import types import warnings from dataclasses import dataclass from pathlib import Path @@ -25,6 +26,7 @@ from torch import nn from torch.utils.data import DataLoader from tqdm.auto import tqdm, trange +from transformers.utils import copy_func from . import logging from .data import SetFitDataset @@ -719,3 +721,27 @@ def _from_pretrained( normalize_embeddings=normalize_embeddings, **model_kwargs, ) + + +def set_docstring(cls, method, docstring): + copied_function = copy_func(method) + copied_function.__doc__ = docstring + return types.MethodType(copy_func(copied_function), cls) + + +docstring = SetFitModel.from_pretrained.__doc__ +cut_index = docstring.find("model_kwargs") +if cut_index != -1: + docstring = ( + docstring[:cut_index] + + """multi_target_strategy (`str`, *optional*): + The strategy to use with multi-label classification. One of "one-vs-rest", "multi-output", + or "classifier-chain". + use_differentiable_head (`bool`, *optional*): + Whether to load SetFit using a differentiable (i.e., Torch) head instead of Logistic Regression. + normalize_embeddings (`bool`, *optional*): + Whether to apply normalization on the embeddings produced by the Sentence Transformer body. + device (`Union[torch.device, str]`, *optional*): + The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`.""" + ) + SetFitModel.from_pretrained = set_docstring(SetFitModel, SetFitModel.from_pretrained, docstring) From 3e829ba188a96cc5c9a129b90d2527cf936a0488 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 12:05:14 +0100 Subject: [PATCH 120/183] Update docstring edits of from_pretrained types.MethodType was required for every subclass. Not ideal, but fine enough for now :/ --- src/setfit/modeling.py | 11 ++--------- src/setfit/span/modeling.py | 26 ++++++++++++++++++++++++++ src/setfit/utils.py | 8 ++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index c7d945a1..c6edc047 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -1,6 +1,5 @@ import os import tempfile -import types import warnings from dataclasses import dataclass from pathlib import Path @@ -26,10 +25,10 @@ from torch import nn from torch.utils.data import DataLoader from tqdm.auto import tqdm, trange -from transformers.utils import copy_func from . import logging from .data import SetFitDataset +from .utils import set_docstring logging.set_verbosity_info() @@ -723,12 +722,6 @@ def _from_pretrained( ) -def set_docstring(cls, method, docstring): - copied_function = copy_func(method) - copied_function.__doc__ = docstring - return types.MethodType(copy_func(copied_function), cls) - - docstring = SetFitModel.from_pretrained.__doc__ cut_index = docstring.find("model_kwargs") if cut_index != -1: @@ -744,4 +737,4 @@ def set_docstring(cls, method, docstring): device (`Union[torch.device, str]`, *optional*): The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`.""" ) - SetFitModel.from_pretrained = set_docstring(SetFitModel, SetFitModel.from_pretrained, docstring) + SetFitModel.from_pretrained = set_docstring(SetFitModel.from_pretrained, docstring) diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index f25a72c1..61ddf955 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -1,6 +1,7 @@ import json import os import tempfile +import types from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union @@ -11,6 +12,8 @@ from huggingface_hub.utils import SoftTemporaryDirectory, validate_hf_hub_args from jinja2 import Environment, FileSystemLoader +from setfit.utils import set_docstring + from .. import logging from ..modeling import SetFitModel from .aspect_extractor import AspectExtractor @@ -142,6 +145,21 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None f.write(model_card_content) +docstring = SpanSetFitModel.from_pretrained.__doc__ +cut_index = docstring.find("multi_target_strategy") +if cut_index != -1: + docstring = ( + docstring[:cut_index] + + """use_differentiable_head (`bool`, *optional*): + Whether to load SetFit using a differentiable (i.e., Torch) head instead of Logistic Regression. + normalize_embeddings (`bool`, *optional*): + Whether to apply normalization on the embeddings produced by the Sentence Transformer body. + device (`Union[torch.device, str]`, *optional*): + The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`.""" + ) + SpanSetFitModel.from_pretrained = set_docstring(SpanSetFitModel.from_pretrained, docstring, cls=SpanSetFitModel) + + class AspectModel(SpanSetFitModel): # TODO: Assumes binary SetFitModel with 0 == no aspect, 1 == aspect def __call__(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[bool]: @@ -152,11 +170,19 @@ def __call__(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[b ] +# The set_docstring magic has as a consequences that subclasses need to update the cls in the from_pretrained +# classmethod, otherwise the wrong instance will be instantiated. +AspectModel.from_pretrained = types.MethodType(AspectModel.from_pretrained.__func__, AspectModel) + + @dataclass class PolarityModel(SpanSetFitModel): span_context: int = 3 +PolarityModel.from_pretrained = types.MethodType(PolarityModel.from_pretrained.__func__, PolarityModel) + + @dataclass class AbsaModel: aspect_extractor: AspectExtractor diff --git a/src/setfit/utils.py b/src/setfit/utils.py index d75dc7cf..b10bb71d 100644 --- a/src/setfit/utils.py +++ b/src/setfit/utils.py @@ -1,3 +1,4 @@ +import types from contextlib import contextmanager from dataclasses import dataclass, field from time import monotonic_ns @@ -5,6 +6,7 @@ from datasets import Dataset, DatasetDict, load_dataset from sentence_transformers import losses +from transformers.utils import copy_func from .data import create_fewshot_splits, create_fewshot_splits_multilabel from .losses import SupConLoss @@ -152,3 +154,9 @@ class BestRun(NamedTuple): objective: float hyperparameters: Dict[str, Any] backend: Any = None + + +def set_docstring(method, docstring, cls=None): + copied_function = copy_func(method) + copied_function.__doc__ = docstring + return types.MethodType(copied_function, cls or method.__self__) From d476ce0ee3bb66ff5c0f43a5cdc546b4e7c756f3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 13:27:40 +0100 Subject: [PATCH 121/183] Correctly format docstrings for API reference --- src/setfit/modeling.py | 17 +++++++++++++---- src/setfit/span/modeling.py | 3 +++ src/setfit/span/trainer.py | 37 ++++++++++++++++++------------------ src/setfit/trainer.py | 38 ++++++++++++++++++------------------- src/setfit/training_args.py | 8 ++++---- 5 files changed, 58 insertions(+), 45 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index c6edc047..a1d1180e 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -25,6 +25,7 @@ from torch import nn from torch.utils.data import DataLoader from tqdm.auto import tqdm, trange +from transformers.utils import copy_func from . import logging from .data import SetFitDataset @@ -489,7 +490,8 @@ def predict( as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. - Example: + Example:: + >>> model = SetFitModel.from_pretrained(...) >>> model.predict(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) tensor([0, 1, 1], dtype=torch.int32) @@ -514,7 +516,8 @@ def predict_proba( as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. - Example: + Example:: + >>> model = SetFitModel.from_pretrained(...) >>> model.predict_proba(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) tensor([[0.9367, 0.0633], @@ -544,7 +547,7 @@ def to(self, device: Union[str, torch.device]) -> "SetFitModel": Args: device (Union[str, torch.device]): The identifier of the device to move the model to. - Example: + Example:: >>> model = SetFitModel.from_pretrained(...) >>> model.to("cpu") @@ -596,7 +599,8 @@ def __call__( as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. - Example: + Example:: + >>> model = SetFitModel.from_pretrained(...) >>> model(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) tensor([0, 1, 1], dtype=torch.int32) @@ -738,3 +742,8 @@ def _from_pretrained( The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`.""" ) SetFitModel.from_pretrained = set_docstring(SetFitModel.from_pretrained, docstring) + +SetFitModel.save_pretrained = copy_func(SetFitModel.save_pretrained) +SetFitModel.save_pretrained.__doc__ = SetFitModel.save_pretrained.__doc__.replace( + "~ModelHubMixin._from_pretrained", "SetFitModel.push_to_hub" +) diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index 61ddf955..db72d391 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -154,6 +154,9 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None Whether to load SetFit using a differentiable (i.e., Torch) head instead of Logistic Regression. normalize_embeddings (`bool`, *optional*): Whether to apply normalization on the embeddings produced by the Sentence Transformer body. + span_context (`int`, defaults to `0`): + The number of words before and after the span candidate that should be prepended to the full sentence. + By default, 0 for Aspect models and 3 for Polarity models. device (`Union[torch.device, str]`, *optional*): The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`.""" ) diff --git a/src/setfit/span/trainer.py b/src/setfit/span/trainer.py index 1d362616..2490605c 100644 --- a/src/setfit/span/trainer.py +++ b/src/setfit/span/trainer.py @@ -40,10 +40,10 @@ class AbsaTrainer(ColumnMappingMixin): metric_kwargs (`Dict[str, Any]`, *optional*): Keyword arguments passed to the evaluation function if `metric` is an evaluation string like "f1". For example useful for providing an averaging strategy for computing f1 in a multi-label setting. - callbacks: (`List[~transformers.TrainerCallback]`, *optional*): + callbacks (`List[`[`~transformers.TrainerCallback`]`]`, *optional*): A list of callbacks to customize the training loop. Will add those to the list of default callbacks detailed in [here](https://huggingface.co/docs/transformers/main/en/main_classes/callback). - If you want to remove one of the default callbacks used, use the `Trainer.remove_callback()` method. + If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method. column_mapping (`Dict[str, str]`, *optional*): A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: @@ -224,42 +224,43 @@ def train_polarity( """ self.polarity_trainer.train(args=args, trial=trial, **kwargs) - def add_callback(self, callback): + def add_callback(self, callback: Union[type, TrainerCallback]) -> None: """ - Add a callback to the current list of [`~transformer.TrainerCallback`]. + Add a callback to the current list of [`~transformers.TrainerCallback`]. Args: - callback (`type` or [`~transformer.TrainerCallback`]): - A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the - first case, will instantiate a member of that class. + callback (`type` or [`~transformers.TrainerCallback`]): + A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the + first case, will instantiate a member of that class. """ self.aspect_trainer.add_callback(callback) self.polarity_trainer.add_callback(callback) - def pop_callback(self, callback): + def pop_callback(self, callback: Union[type, TrainerCallback]) -> Tuple[TrainerCallback, TrainerCallback]: """ - Remove a callback from the current list of [`~transformer.TrainerCallback`] and returns it. + Remove a callback from the current list of [`~transformers.TrainerCallback`] and returns it. If the callback is not found, returns `None` (and no error is raised). Args: - callback (`type` or [`~transformer.TrainerCallback`]): - A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the - first case, will pop the first member of that class found in the list of callbacks. + callback (`type` or [`~transformers.TrainerCallback`]): + A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the + first case, will pop the first member of that class found in the list of callbacks. Returns: - [`Tuple[~transformer.TrainerCallback]`]: The callbacks removed from the aspect and polarity trainers, if found. + `Tuple[`[`~transformers.TrainerCallback`], [`~transformers.TrainerCallback`]`]`: The callbacks removed from the + aspect and polarity trainers, if found. """ return self.aspect_trainer.pop_callback(callback), self.polarity_trainer.pop_callback(callback) - def remove_callback(self, callback): + def remove_callback(self, callback: Union[type, TrainerCallback]) -> None: """ - Remove a callback from the current list of [`~transformer.TrainerCallback`]. + Remove a callback from the current list of [`~transformers.TrainerCallback`]. Args: - callback (`type` or [`~transformer.TrainerCallback`]): - A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the - first case, will remove the first member of that class found in the list of callbacks. + callback (`type` or [`~transformers.TrainerCallback`]): + A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the + first case, will remove the first member of that class found in the list of callbacks. """ self.aspect_trainer.remove_callback(callback) self.polarity_trainer.remove_callback(callback) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 723aeaa9..8dad92a5 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -147,7 +147,7 @@ class Trainer(ColumnMappingMixin): The evaluation dataset. model_init (`Callable[[], SetFitModel]`, *optional*): A function that instantiates the model to be used. If provided, each call to - [`~Trainer.train`] will start from a new instance of the model as given by this + [`Trainer.train`] will start from a new instance of the model as given by this function when a `trial` is passed. metric (`str` or `Callable`, *optional*, defaults to `"accuracy"`): The metric to use for evaluation. If a string is provided, we treat it as the metric @@ -156,10 +156,10 @@ class Trainer(ColumnMappingMixin): metric_kwargs (`Dict[str, Any]`, *optional*): Keyword arguments passed to the evaluation function if `metric` is an evaluation string like "f1". For example useful for providing an averaging strategy for computing f1 in a multi-label setting. - callbacks: (`List[~transformers.TrainerCallback]`, *optional*): + callbacks (`List[`[`~transformers.TrainerCallback`]`]`, *optional*): A list of callbacks to customize the training loop. Will add those to the list of default callbacks detailed in [here](https://huggingface.co/docs/transformers/main/en/main_classes/callback). - If you want to remove one of the default callbacks used, use the `Trainer.remove_callback()` method. + If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method. column_mapping (`Dict[str, str]`, *optional*): A mapping from the column names in the dataset to the column names expected by the model. The expected format is a dictionary with the following format: @@ -214,40 +214,40 @@ def __init__( self.add_callback(DEFAULT_PROGRESS_CALLBACK if self.args.show_progress_bar else PrinterCallback) self.control = self.callback_handler.on_init_end(args, self.state, self.control) - def add_callback(self, callback): + def add_callback(self, callback: Union[type, TrainerCallback]) -> None: """ - Add a callback to the current list of [`~transformer.TrainerCallback`]. + Add a callback to the current list of [`~transformers.TrainerCallback`]. Args: - callback (`type` or [`~transformer.TrainerCallback`]): - A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + callback (`type` or [`~transformers.TrainerCallback`]): + A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will instantiate a member of that class. """ self.callback_handler.add_callback(callback) - def pop_callback(self, callback): + def pop_callback(self, callback: Union[type, TrainerCallback]) -> TrainerCallback: """ - Remove a callback from the current list of [`~transformer.TrainerCallback`] and returns it. + Remove a callback from the current list of [`~transformers.TrainerCallback`] and returns it. If the callback is not found, returns `None` (and no error is raised). Args: - callback (`type` or [`~transformer.TrainerCallback`]): - A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + callback (`type` or [`~transformers.TrainerCallback`]): + A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will pop the first member of that class found in the list of callbacks. Returns: - [`~transformer.TrainerCallback`]: The callback removed, if found. + [`~transformers.TrainerCallback`]: The callback removed, if found. """ return self.callback_handler.pop_callback(callback) - def remove_callback(self, callback): + def remove_callback(self, callback: Union[type, TrainerCallback]) -> None: """ - Remove a callback from the current list of [`~transformer.TrainerCallback`]. + Remove a callback from the current list of [`~transformers.TrainerCallback`]. Args: - callback (`type` or [`~transformer.TrainerCallback`]): - A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the + callback (`type` or [`~transformers.TrainerCallback`]): + A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the first case, will remove the first member of that class found in the list of callbacks. """ self.callback_handler.remove_callback(callback) @@ -822,16 +822,16 @@ def hyperparameter_search( Args: hp_space (`Callable[["optuna.Trial"], Dict[str, float]]`, *optional*): A function that defines the hyperparameter search space. Will default to - [`~trainer_utils.default_hp_space_optuna`]. + [`~transformers.trainer_utils.default_hp_space_optuna`]. compute_objective (`Callable[[Dict[str, float]], float]`, *optional*): A function computing the objective to minimize or maximize from the metrics returned by the `evaluate` - method. Will default to [`~trainer_utils.default_compute_objective`] which uses the sum of metrics. + method. Will default to [`~transformers.trainer_utils.default_compute_objective`] which uses the sum of metrics. n_trials (`int`, *optional*, defaults to 100): The number of trial runs to test. direction (`str`, *optional*, defaults to `"maximize"`): Whether to optimize greater or lower objects. Can be `"minimize"` or `"maximize"`, you should pick `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or several metrics. - backend (`str` or [`~training_utils.HPSearchBackend`], *optional*): + backend (`str` or [`~transformers.training_utils.HPSearchBackend`], *optional*): The backend to use for hyperparameter search. Only optuna is supported for now. TODO: add support for ray and sigopt. hp_name (`Callable[["optuna.Trial"], str]]`, *optional*): diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 2b8d2b25..389dfec9 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -101,7 +101,7 @@ class TrainingArguments: Whether to display a progress bar for the training epochs and iterations. seed (`int`, defaults to `42`): Random seed that will be set at the beginning of training. To ensure reproducibility across - runs, use the [`~SetTrainer.model_init`] function to instantiate the model if it has some + runs, use the `model_init` argument to [`Trainer`] to instantiate the model if it has some randomly initialized parameters. report_to (`str` or `List[str]`, *optional*, defaults to `"all"`): The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`, @@ -113,7 +113,7 @@ class TrainingArguments: logging_dir (`str`, *optional*): [TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to *runs/**CURRENT_DATETIME_HOSTNAME***. - logging_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): + logging_strategy (`str` or [`~transformers.trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): The logging strategy to adopt during training. Possible values are: - `"no"`: No logging is done during training. @@ -124,7 +124,7 @@ class TrainingArguments: Whether to log and evaluate the first `global_step` or not. logging_steps (`int`, defaults to 50): Number of update steps between two logs if `logging_strategy="steps"`. - evaluation_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`): + evaluation_strategy (`str` or [`~transformers.trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`): The evaluation strategy to adopt during training. Possible values are: - `"no"`: No evaluation is done during training. @@ -138,7 +138,7 @@ class TrainingArguments: Number of epochs or steps to wait for before the first evaluation can be performed, depending on the evaluation_strategy. - save_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): + save_strategy (`str` or [`~transformers.trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): The checkpoint save strategy to adopt during training. Possible values are: - `"no"`: No save is done during training. From dac52211ddd0db69a6bff9bea50887497aad9c6d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 15:43:48 +0100 Subject: [PATCH 122/183] Also maybe log, evaluate & save at epoch end --- src/setfit/trainer.py | 65 ++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 8dad92a5..bbe33f16 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -624,40 +624,15 @@ def _train_sentence_transformer( self.state.epoch = epoch + (step + 1) / steps_per_epoch self.control = self.callback_handler.on_step_end(args, self.state, self.control) - if self.control.should_log: - learning_rate = scheduler_obj.get_last_lr()[0] - metrics = {"embedding_loss": round(loss_value.item(), 4), "learning_rate": learning_rate} - self.control = self.log(args, metrics) - - eval_loss = None - if self.control.should_evaluate and eval_dataloader is not None: - eval_loss = self._evaluate_with_loss(model_body, eval_dataloader, args, loss_func) - learning_rate = scheduler_obj.get_last_lr()[0] - metrics = {"eval_embedding_loss": round(eval_loss, 4), "learning_rate": learning_rate} - self.control = self.log(args, metrics) - - self.control = self.callback_handler.on_evaluate(args, self.state, self.control, metrics) - - loss_func.zero_grad() - loss_func.train() - - if self.control.should_save: - checkpoint_dir = self._checkpoint( - self.args.output_dir, args.save_total_limit, self.state.global_step - ) - self.control = self.callback_handler.on_save(self.args, self.state, self.control) - - if eval_loss is not None and ( - self.state.best_metric is None or eval_loss < self.state.best_metric - ): - self.state.best_metric = eval_loss - self.state.best_model_checkpoint = checkpoint_dir + self.maybe_log_eval_save(model_body, eval_dataloader, args, scheduler_obj, loss_func, loss_value) if self.control.should_epoch_stop or self.control.should_training_stop: break self.control = self.callback_handler.on_epoch_end(args, self.state, self.control) + self.maybe_log_eval_save(model_body, eval_dataloader, args, scheduler_obj, loss_func, loss_value) + if self.control.should_training_stop: break @@ -675,6 +650,40 @@ def _train_sentence_transformer( self.control = self.callback_handler.on_train_end(args, self.state, self.control) + def maybe_log_eval_save( + self, + model_body: SentenceTransformer, + eval_dataloader: Optional[DataLoader], + args: TrainingArguments, + scheduler_obj, + loss_func, + loss_value: torch.Tensor, + ) -> None: + if self.control.should_log: + learning_rate = scheduler_obj.get_last_lr()[0] + metrics = {"embedding_loss": round(loss_value.item(), 4), "learning_rate": learning_rate} + self.control = self.log(args, metrics) + + eval_loss = None + if self.control.should_evaluate and eval_dataloader is not None: + eval_loss = self._evaluate_with_loss(model_body, eval_dataloader, args, loss_func) + learning_rate = scheduler_obj.get_last_lr()[0] + metrics = {"eval_embedding_loss": round(eval_loss, 4), "learning_rate": learning_rate} + self.control = self.log(args, metrics) + + self.control = self.callback_handler.on_evaluate(args, self.state, self.control, metrics) + + loss_func.zero_grad() + loss_func.train() + + if self.control.should_save: + checkpoint_dir = self._checkpoint(self.args.output_dir, args.save_total_limit, self.state.global_step) + self.control = self.callback_handler.on_save(self.args, self.state, self.control) + + if eval_loss is not None and (self.state.best_metric is None or eval_loss < self.state.best_metric): + self.state.best_metric = eval_loss + self.state.best_model_checkpoint = checkpoint_dir + def _evaluate_with_loss( self, model_body: SentenceTransformer, From 5edf540966cfd9edb6045ead18092c51ef3d073f Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 15:47:16 +0100 Subject: [PATCH 123/183] Update README in preparation for documentation --- README.md | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 34aa0a4a..42ee67d0 100644 --- a/README.md +++ b/README.md @@ -8,40 +8,43 @@ SetFit is an efficient and prompt-free framework for few-shot fine-tuning of [Sentence Transformers](https://sbert.net/). It achieves high accuracy with little labeled data - for instance, with only 8 labeled examples per class on the Customer Reviews sentiment dataset, SetFit is competitive with fine-tuning RoBERTa Large on the full training set of 3k examples 🤯! - Compared to other few-shot learning methods, SetFit has several unique features: * 🗣 **No prompts or verbalisers:** Current techniques for few-shot fine-tuning require handcrafted prompts or verbalisers to convert examples into a format that's suitable for the underlying language model. SetFit dispenses with prompts altogether by generating rich embeddings directly from text examples. * 🏎 **Fast to train:** SetFit doesn't require large-scale models like T0 or GPT-3 to achieve high accuracy. As a result, it is typically an order of magnitude (or more) faster to train and run inference with. * 🌎 **Multilingual support**: SetFit can be used with any [Sentence Transformer](https://huggingface.co/models?library=sentence-transformers&sort=downloads) on the Hub, which means you can classify text in multiple languages by simply fine-tuning a multilingual checkpoint. +Check out the [SetFit Documentation](https://huggingface.co/docs/setfit) for more information! + ## Installation Download and install `setfit` by running: ```bash -python -m pip install setfit +pip install setfit ``` -If you want the bleeding-edge version, install from source by running: +If you want the bleeding-edge version instead, install from source by running: ```bash -python -m pip install git+https://github.com/huggingface/setfit.git +pip install git+https://github.com/huggingface/setfit.git ``` ## Usage -The examples below provide a quick overview on the various features supported in `setfit`. For more examples, check out the [`notebooks`](https://github.com/huggingface/setfit/tree/main/notebooks) folder. +The [quickstart](https://huggingface.co/docs/setfit/quickstart) is a good place to learn about training, saving, loading, and performing inference with SetFit models. + +For more examples, check out the [`notebooks`](https://github.com/huggingface/setfit/tree/main/notebooks) directory, the [tutorials](https://huggingface.co/docs/setfit/tutorials/overview), or the [how-to guides](https://huggingface.co/docs/setfit/how_to/overview). ### Training a SetFit model `setfit` is integrated with the [Hugging Face Hub](https://huggingface.co/) and provides two main classes: -* `SetFitModel`: a wrapper that combines a pretrained body from `sentence_transformers` and a classification head from either [`scikit-learn`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) or [`SetFitHead`](https://github.com/huggingface/setfit/blob/main/src/setfit/modeling.py) (a differentiable head built upon `PyTorch` with similar APIs to `sentence_transformers`). -* `Trainer`: a helper class that wraps the fine-tuning process of SetFit. +* [`SetFitModel`](https://huggingface.co/docs/setfit/reference/main#setfit.SetFitModel): a wrapper that combines a pretrained body from `sentence_transformers` and a classification head from either [`scikit-learn`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html) or [`SetFitHead`](https://huggingface.co/docs/setfit/reference/main#setfit.SetFitHead) (a differentiable head built upon `PyTorch` with similar APIs to `sentence_transformers`). +* [`Trainer`](https://huggingface.co/docs/setfit/reference/trainer#setfit.Trainer): a helper class that wraps the fine-tuning process of SetFit. -Here is an end-to-end example using a classification head from `scikit-learn`: +Here is a simple end-to-end training example using the default classification head from `scikit-learn`: ```python @@ -54,15 +57,18 @@ dataset = load_dataset("sst2") # Simulate the few-shot regime by sampling 8 examples per class train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) -eval_dataset = dataset["validation"] +eval_dataset = dataset["validation"].select(range(100)) +test_dataset = dataset["validation"].select(range(100, len(dataset["validation"]))) # Load a SetFit model from Hub model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-mpnet-base-v2") args = TrainingArguments( batch_size=16, - num_iterations=20, # The number of text pairs to generate for contrastive learning - num_epochs=1 # The number of epochs to use for contrastive learning + num_epochs=4, + evaluation_strategy="epoch", + save_strategy="epoch", + load_best_model_at_end=True, ) trainer = Trainer( @@ -76,21 +82,25 @@ trainer = Trainer( # Train and evaluate trainer.train() -metrics = trainer.evaluate() +metrics = trainer.evaluate(test_dataset) +print(metrics) +# {'accuracy': 0.8691709844559585} # Push model to the Hub -trainer.push_to_hub("my-awesome-setfit-model") +trainer.push_to_hub("tomaarsen/setfit-paraphrase-mpnet-base-v2-sst2") # Download from Hub -model = SetFitModel.from_pretrained("lewtun/my-awesome-setfit-model") +model = SetFitModel.from_pretrained("tomaarsen/setfit-paraphrase-mpnet-base-v2-sst2") # Run inference -preds = model(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) +preds = model.predict(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) +print(preds) +# tensor([1, 0], dtype=torch.int32) ``` ## Reproducing the results from the paper -We provide scripts to reproduce the results for SetFit and various baselines presented in Table 2 of our paper. Check out the setup and training instructions in the `scripts/` directory. +We provide scripts to reproduce the results for SetFit and various baselines presented in Table 2 of our paper. Check out the setup and training instructions in the [`scripts/setfit/`](scripts/setfit) directory. ## Developer installation @@ -103,10 +113,10 @@ conda create -n setfit python=3.9 && conda activate setfit Then install the base requirements with: ```bash -python -m pip install -e '.[dev]' +pip install -e '.[dev]' ``` -This will install `datasets` and packages like `black` and `isort` that we use to ensure consistent code formatting. +This will install mandatory packages for SetFit like `datasets` as well as development packages like `black` and `isort` that we use to ensure consistent code formatting. ### Formatting your code @@ -122,6 +132,7 @@ make style && make quality ├── LICENSE ├── Makefile <- Makefile with commands like `make style` or `make tests` ├── README.md <- The top-level README for developers using this project. +├── docs <- Documentation source ├── notebooks <- Jupyter notebooks. ├── final_results <- Model predictions from the paper ├── scripts <- Scripts for training and inference @@ -139,7 +150,7 @@ make style && make quality ## Citation -``` +```bibtex @misc{https://doi.org/10.48550/arxiv.2209.11055, doi = {10.48550/ARXIV.2209.11055}, url = {https://arxiv.org/abs/2209.11055}, From c1b2f20e538aab46121d7599e5586f4630efe1d8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 16:07:49 +0100 Subject: [PATCH 124/183] Link to scripts rather than scripts/setfit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42ee67d0..a2931e85 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ print(preds) ## Reproducing the results from the paper -We provide scripts to reproduce the results for SetFit and various baselines presented in Table 2 of our paper. Check out the setup and training instructions in the [`scripts/setfit/`](scripts/setfit) directory. +We provide scripts to reproduce the results for SetFit and various baselines presented in Table 2 of our paper. Check out the setup and training instructions in the [`scripts/`](scripts/) directory. ## Developer installation From 70bd9357f2e4548f01680da619e4bfc83419ae58 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 18:22:46 +0100 Subject: [PATCH 125/183] Ensure correct device of "best model at the end" --- src/setfit/trainer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index bbe33f16..2c647be0 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -640,7 +640,8 @@ def _train_sentence_transformer( dir_name = Path(self.state.best_model_checkpoint).name if dir_name.startswith("step_"): logger.info(f"Loading best SentenceTransformer model from step {dir_name[5:]}.") - self.model.model_body = SentenceTransformer(self.state.best_model_checkpoint, device=model_body.device) + self.model.model_body = SentenceTransformer(self.state.best_model_checkpoint, device=model_body._target_device) + self.model.model_body.to(model_body._target_device) # Ensure logging the speed metrics num_train_samples = self.state.max_steps * args.embedding_batch_size # * args.gradient_accumulation_steps From c93b55a4d518d7fe4284e8f17c6dfc056d9a4893 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 28 Nov 2023 22:22:43 +0100 Subject: [PATCH 126/183] Add "labels" in a configuration file --- docs/source/en/quickstart.mdx | 31 ++++--- docs/source/en/reference/main.mdx | 2 + src/setfit/modeling.py | 145 ++++++++++++++++++++++++++---- src/setfit/span/modeling.py | 79 ++-------------- src/setfit/trainer.py | 6 +- tests/span/test_modeling.py | 12 +-- tests/test_modeling.py | 58 ++++++++++++ 7 files changed, 224 insertions(+), 109 deletions(-) diff --git a/docs/source/en/quickstart.mdx b/docs/source/en/quickstart.mdx index 4845055d..4779f745 100644 --- a/docs/source/en/quickstart.mdx +++ b/docs/source/en/quickstart.mdx @@ -25,7 +25,6 @@ SetFit is an efficient framework to train low-latency text classification models In this section, you'll load a [Sentence Transformer model](https://huggingface.co/models?library=sentence-transformers) and further finetune it for classifying movie reviews as positive or negative. To train a model, we will need to prepare the following three: 1) a **model**, 2) a **dataset**, and 3) **training arguments**. - **1**. Initialize a SetFit model using a Sentence Transformer model of our choice. Consider using the [MTEB Leaderboard](https://huggingface.co/spaces/mteb/leaderboard) to guide your decision on which Sentence Transformer model to choose. We will use [BAAI/bge-small-en-v1.5](https://huggingface.co/BAAI/bge-small-en-v1.5), a small but performant model. ```py @@ -79,6 +78,12 @@ Dataset({ }) ``` +**2c**. We can apply the labels from the dataset on the model, so the predictions output readable classes. You can also provide the labels directly to [`SetFitModel.from_pretrained`]. + +```py +>>> model.labels = ["negative", "positive"] +``` + **3**. Prepare the [`TrainingArguments`] for training. Note that training with 🤗 SetFit consists of two phases behind the scenes: **finetuning embeddings** and **training a classification head**. As a result, some of the training arguments can be tuples, where the two values are used for each of the two phases, respectively. The `num_epochs` and `max_steps` arguments are frequently used to increase and decrease the number of total training steps. Consider that with SetFit, better performance is reached with **more data, not more training**! Don't be afraid to train for less than 1 epoch if you have a lot of data. @@ -132,13 +137,13 @@ Feel free to experiment with increasing the number of samples per class to obser After training, you can save a 🤗 SetFit model to your local filesystem or to the Hugging Face Hub. Save a model to a local directory using [`SetFitModel.save_pretrained`] by providing a `save_directory`: ```py ->>> model.save_pretrained("setfit-8-shot-sst2") +>>> model.save_pretrained("setfit-bge-small-v1.5-sst2-8-shot") ``` Alternatively, push a model to the Hugging Face Hub using [`SetFitModel.push_to_hub`] by providing a `repo_id`: ```py ->>> model.push_to_hub("tomaarsen/setfit-8-shot-sst2") +>>> model.push_to_hub("tomaarsen/setfit-bge-small-v1.5-sst2-8-shot") ``` ### Loading a 🤗 SetFit model @@ -146,9 +151,9 @@ Alternatively, push a model to the Hugging Face Hub using [`SetFitModel.push_to_ A 🤗 SetFit model can be loaded using [`SetFitModel.from_pretrained`] by providing 1) a `repo_id` from the Hugging Face Hub or 2) a path to a local directory: ```py ->>> model = SetFitModel.from_pretrained("tomaarsen/setfit-8-shot-sst2") # Load from the Hugging Face Hub +>>> model = SetFitModel.from_pretrained("tomaarsen/setfit-bge-small-v1.5-sst2-8-shot") # Load from the Hugging Face Hub ->>> model = SetFitModel.from_pretrained("setfit-8-shot-sst2") # Load from a local directory +>>> model = SetFitModel.from_pretrained("setfit-bge-small-v1.5-sst2-8-shot") # Load from a local directory ``` ### Inference @@ -162,9 +167,9 @@ Once a 🤗 SetFit model has been trained, then it can be used for inference to ... "A sometimes tedious film.", ... ]) >>> preds -tensor([1, 0, 0], dtype=torch.int32) +['positive' 'negative' 'negative'] ``` -These predictions match the format of the input labels. If we had used string labels, then the outputs would have been `['positive' 'negative' 'negative']`. +These predictions rely on the `model.labels`. If not set, it will return predictions in the format that was used during training, e.g. `tensor([1, 0, 0])`. ## What's next? @@ -181,7 +186,7 @@ from setfit import SetFitModel, Trainer, TrainingArguments, sample_dataset from datasets import load_dataset # Initializing a new SetFit model -model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5") +model = SetFitModel.from_pretrained("BAAI/bge-small-en-v1.5", labels=["negative", "positive"]) # Preparing the dataset dataset = load_dataset("SetFit/sst2") @@ -208,14 +213,14 @@ print(metrics) # => {'accuracy': 0.8511806699615596} # Saving the trained model -model.save_pretrained("setfit-8-shot-sst2") +model.save_pretrained("setfit-bge-small-v1.5-sst2-8-shot") # or -model.push_to_hub("tomaarsen/setfit-8-shot-sst2") +model.push_to_hub("tomaarsen/setfit-bge-small-v1.5-sst2-8-shot") # Loading a trained model -model = SetFitModel.from_pretrained("tomaarsen/setfit-8-shot-sst2") # Load from the Hugging Face Hub +model = SetFitModel.from_pretrained("tomaarsen/setfit-bge-small-v1.5-sst2-8-shot") # Load from the Hugging Face Hub # or -model = SetFitModel.from_pretrained("setfit-8-shot-sst2") # Load from a local directory +model = SetFitModel.from_pretrained("setfit-bge-small-v1.5-sst2-8-shot") # Load from a local directory # Performing inference preds = model.predict([ @@ -224,5 +229,5 @@ preds = model.predict([ "A sometimes tedious film.", ]) print(preds) -# => tensor([1, 0, 0], dtype=torch.int32) +# => ["positive", "negative", "negative"] ``` diff --git a/docs/source/en/reference/main.mdx b/docs/source/en/reference/main.mdx index b746ebfe..6ceed9db 100644 --- a/docs/source/en/reference/main.mdx +++ b/docs/source/en/reference/main.mdx @@ -9,6 +9,8 @@ - save_pretrained - push_to_hub - __call__ + - label2id + - id2label ## SetFitHead diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index a1d1180e..6ff1ad9d 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -1,9 +1,10 @@ +import json import os import tempfile import warnings -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Set, Tuple, Union # For Python 3.7 compatibility @@ -89,6 +90,8 @@ ``` """ +CONFIG_NAME = "config_setfit.json" + class SetFitHead(models.Dense): """ @@ -244,18 +247,49 @@ def __repr__(self) -> str: @dataclass class SetFitModel(PyTorchModelHubMixin): - """A SetFit model with integration to the [Hugging Face Hub](https://huggingface.co).""" + """A SetFit model with integration to the [Hugging Face Hub](https://huggingface.co). + + Example:: + + >>> from setfit import SetFitModel + >>> model = SetFitModel.from_pretrained("tomaarsen/setfit-bge-small-v1.5-sst2-8-shot") + >>> model.predict([ + ... "It's a charming and often affecting journey.", + ... "It's slow -- very, very slow.", + ... "A sometimes tedious film.", + ... ]) + ['positive', 'negative', 'negative'] + """ - model_body: Optional[SentenceTransformer] = (None,) + model_body: Optional[SentenceTransformer] = None model_head: Optional[Union[SetFitHead, LogisticRegression]] = None multi_target_strategy: Optional[str] = None normalize_embeddings: bool = False + labels: Optional[List[str]] = None + + attributes_to_save: Set[str] = field( + init=False, repr=False, default_factory=lambda: {"normalize_embeddings", "labels"} + ) @property def has_differentiable_head(self) -> bool: # if False, sklearn is assumed to be used instead return isinstance(self.model_head, nn.Module) + @property + def id2label(self) -> Dict[int, str]: + """Return a mapping from integer IDs to string labels.""" + if self.labels is None: + return None + return dict(enumerate(self.labels)) + + @property + def label2id(self) -> Dict[str, int]: + """Return a mapping from string labels to integer IDs.""" + if self.labels is None: + return None + return {label: idx for idx, label in enumerate(self.labels)} + def fit( self, x_train: List[str], @@ -290,7 +324,6 @@ def fit( epochs and iterations. """ if self.has_differentiable_head: # train with pyTorch - device = self.model_body.device self.model_body.train() self.model_head.train() if not end_to_end: @@ -306,8 +339,8 @@ def fit( optimizer.zero_grad() # to model's device - features = {k: v.to(device) for k, v in features.items()} - labels = labels.to(device) + features = {k: v.to(self.device) for k, v in features.items()} + labels = labels.to(self.device) outputs = self.model_body(features) if self.normalize_embeddings: @@ -479,7 +512,12 @@ def _output_type_conversion( return outputs def predict( - self, inputs: List[str], batch_size: int = 32, as_numpy: bool = False, show_progress_bar: Optional[bool] = None + self, + inputs: List[str], + batch_size: int = 32, + as_numpy: bool = False, + use_labels: bool = True, + show_progress_bar: Optional[bool] = None, ) -> Union[torch.Tensor, np.ndarray]: """Predict the various classes. @@ -488,13 +526,14 @@ def predict( batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. Higher often means faster processing but higher memory usage. as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. + use_labels (`bool`, defaults to `True`): Whether to try and return elements of `SetFitModel.labels`. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. Example:: >>> model = SetFitModel.from_pretrained(...) >>> model.predict(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) - tensor([0, 1, 1], dtype=torch.int32) + ["negative", "positive", "positive"] Returns: `Union[torch.Tensor, np.ndarray]`: A vector with equal length to the inputs, denoting @@ -502,6 +541,14 @@ def predict( """ embeddings = self.encode(inputs, batch_size=batch_size, show_progress_bar=show_progress_bar) outputs = self.model_head.predict(embeddings) + # If labels are defined, we don't have multilabels & the output is not already strings, then we convert to string labels + if ( + use_labels + and self.labels + and self.multi_target_strategy is None + and (self.has_differentiable_head or outputs.dtype.char != "U") + ): + return [self.labels[output] for output in outputs] return self._output_type_conversion(outputs, as_numpy=as_numpy) def predict_proba( @@ -539,7 +586,7 @@ def device(self) -> torch.device: Returns: torch.device: The device that the model is on. """ - return self.model_body.device + return self.model_body._target_device def to(self, device: Union[str, torch.device]) -> "SetFitModel": """Move this SetFitModel to `device`, and then return `self`. This method does not copy. @@ -588,7 +635,12 @@ def create_model_card(self, path: str, model_name: Optional[str] = "SetFit Model f.write(model_card_content) def __call__( - self, inputs: List[str], batch_size: int = 32, as_numpy: bool = False, show_progress_bar: Optional[bool] = None + self, + inputs: List[str], + batch_size: int = 32, + as_numpy: bool = False, + use_labels: bool = True, + show_progress_bar: Optional[bool] = None, ) -> Union[torch.Tensor, np.ndarray]: """Predict the various classes. @@ -597,27 +649,49 @@ def __call__( batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. Higher often means faster processing but higher memory usage. as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. + use_labels (`bool`, defaults to `True`): Whether to try and return elements of `SetFitModel.labels`. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. Example:: >>> model = SetFitModel.from_pretrained(...) >>> model(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) - tensor([0, 1, 1], dtype=torch.int32) + ["negative", "positive", "positive"] Returns: `torch.Tensor`: A vector with equal length to the inputs, denoting to which class each input is predicted to belong. """ - return self.predict(inputs, batch_size=batch_size, as_numpy=as_numpy, show_progress_bar=show_progress_bar) + return self.predict( + inputs, + batch_size=batch_size, + as_numpy=as_numpy, + use_labels=use_labels, + show_progress_bar=show_progress_bar, + ) def _save_pretrained(self, save_directory: Union[Path, str]) -> None: save_directory = str(save_directory) + # Save the config + config_path = os.path.join(save_directory, CONFIG_NAME) + with open(config_path, "w") as f: + json.dump( + { + attr_name: getattr(self, attr_name) + for attr_name in self.attributes_to_save + if hasattr(self, attr_name) + }, + f, + indent=2, + ) + # Save the body self.model_body.save(path=save_directory, create_model_card=False) + # Save the README self.create_model_card(path=save_directory, model_name=save_directory) # Move the head to the CPU before saving if self.has_differentiable_head: self.model_head.to("cpu") + # Save the classification head joblib.dump(self.model_head, str(Path(save_directory) / MODEL_HEAD_NAME)) if self.has_differentiable_head: self.model_head.to(self.device) @@ -636,7 +710,6 @@ def _from_pretrained( token: Optional[Union[bool, str]] = None, multi_target_strategy: Optional[str] = None, use_differentiable_head: bool = False, - normalize_embeddings: bool = False, device: Optional[Union[torch.device, str]] = None, **model_kwargs, ) -> "SetFitModel": @@ -644,6 +717,33 @@ def _from_pretrained( device = model_body._target_device model_body.to(device) # put `model_body` on the target device + # Try to load a SetFit config file + config_file: Optional[str] = None + if os.path.isdir(model_id): + if CONFIG_NAME in os.listdir(model_id): + config_file = os.path.join(model_id, CONFIG_NAME) + else: + try: + config_file = hf_hub_download( + repo_id=model_id, + filename=CONFIG_NAME, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + token=token, + local_files_only=local_files_only, + ) + except requests.exceptions.RequestException: + pass + + if config_file is not None: + with open(config_file, "r", encoding="utf-8") as f: + config = json.load(f) + model_kwargs.update(config) + + # Try to load a model head file if os.path.isdir(model_id): if MODEL_HEAD_NAME in os.listdir(model_id): model_head_file = os.path.join(model_id, MODEL_HEAD_NAME) @@ -721,7 +821,6 @@ def _from_pretrained( model_body=model_body, model_head=model_head, multi_target_strategy=multi_target_strategy, - normalize_embeddings=normalize_embeddings, **model_kwargs, ) @@ -731,7 +830,10 @@ def _from_pretrained( if cut_index != -1: docstring = ( docstring[:cut_index] - + """multi_target_strategy (`str`, *optional*): + + """labels (`List[str]`, *optional*): + If the labels are integers ranging from `0` to `num_classes-1`, then these labels indicate + the corresponding labels. + multi_target_strategy (`str`, *optional*): The strategy to use with multi-label classification. One of "one-vs-rest", "multi-output", or "classifier-chain". use_differentiable_head (`bool`, *optional*): @@ -739,7 +841,16 @@ def _from_pretrained( normalize_embeddings (`bool`, *optional*): Whether to apply normalization on the embeddings produced by the Sentence Transformer body. device (`Union[torch.device, str]`, *optional*): - The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`.""" + The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`. + + Example:: + + >>> from setfit import SetFitModel + >>> model = SetFitModel.from_pretrained( + ... "sentence-transformers/paraphrase-mpnet-base-v2", + ... labels=["positive", "negative"], + ... ) + """ ) SetFitModel.from_pretrained = set_docstring(SetFitModel.from_pretrained, docstring) diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index db72d391..6fb86dbe 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -1,15 +1,12 @@ -import json import os import tempfile import types -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union -import requests import torch -from huggingface_hub import hf_hub_download -from huggingface_hub.utils import SoftTemporaryDirectory, validate_hf_hub_args +from huggingface_hub.utils import SoftTemporaryDirectory from jinja2 import Environment, FileSystemLoader from setfit.utils import set_docstring @@ -24,13 +21,15 @@ logger = logging.get_logger(__name__) -CONFIG_NAME = "config_span_setfit.json" - @dataclass class SpanSetFitModel(SetFitModel): span_context: int = 0 + attributes_to_save: Set[str] = field( + init=False, repr=False, default_factory=lambda: {"normalize_embeddings", "labels", "span_context"} + ) + def prepend_aspects(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[str]: for doc, aspects in zip(docs, aspects_list): for aspect_slice in aspects: @@ -44,68 +43,6 @@ def __call__(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[b iter_preds = iter(preds) return [[next(iter_preds) for _ in aspects] for aspects in aspects_list] - @classmethod - @validate_hf_hub_args - def _from_pretrained( - cls, - model_id: str, - span_context: Optional[int] = None, - revision: Optional[str] = None, - cache_dir: Optional[str] = None, - force_download: Optional[bool] = None, - proxies: Optional[Dict] = None, - resume_download: Optional[bool] = None, - local_files_only: Optional[bool] = None, - token: Optional[Union[bool, str]] = None, - **model_kwargs, - ) -> "SpanSetFitModel": - config_file: Optional[str] = None - if os.path.isdir(model_id): - if CONFIG_NAME in os.listdir(model_id): - config_file = os.path.join(model_id, CONFIG_NAME) - else: - try: - config_file = hf_hub_download( - repo_id=model_id, - filename=CONFIG_NAME, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - resume_download=resume_download, - token=token, - local_files_only=local_files_only, - ) - except requests.exceptions.RequestException: - pass - - if config_file is not None: - with open(config_file, "r", encoding="utf-8") as f: - config = json.load(f) - model_kwargs.update(config) - - if span_context is not None: - model_kwargs["span_context"] = span_context - - return super(SpanSetFitModel, cls)._from_pretrained( - model_id=model_id, - revision=revision, - cache_dir=cache_dir, - force_download=force_download, - proxies=proxies, - resume_download=resume_download, - local_files_only=local_files_only, - token=token, - **model_kwargs, - ) - - def _save_pretrained(self, save_directory: Union[Path, str]) -> None: - path = os.path.join(save_directory, CONFIG_NAME) - with open(path, "w") as f: - json.dump({"span_context": self.span_context}, f, indent=2) - - super()._save_pretrained(save_directory) - def create_model_card(self, path: str, model_name: Optional[str] = None) -> None: """Creates and saves a model card for a SetFit model. @@ -245,7 +182,7 @@ def from_pretrained( model_id: str, polarity_model_id: Optional[str] = None, spacy_model: Optional[str] = "en_core_web_lg", - span_contexts: Tuple[Optional[int], Optional[int]] = (None, None), + span_contexts: Tuple[Optional[int], Optional[int]] = (0, 3), force_download: bool = False, resume_download: bool = False, proxies: Optional[Dict] = None, diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 2c647be0..efe24535 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -640,7 +640,9 @@ def _train_sentence_transformer( dir_name = Path(self.state.best_model_checkpoint).name if dir_name.startswith("step_"): logger.info(f"Loading best SentenceTransformer model from step {dir_name[5:]}.") - self.model.model_body = SentenceTransformer(self.state.best_model_checkpoint, device=model_body._target_device) + self.model.model_body = SentenceTransformer( + self.state.best_model_checkpoint, device=model_body._target_device + ) self.model.model_body.to(model_body._target_device) # Ensure logging the speed metrics @@ -783,7 +785,7 @@ def evaluate(self, dataset: Optional[Dataset] = None) -> Dict[str, float]: y_test = eval_dataset["label"] logger.info("***** Running evaluation *****") - y_pred = self.model.predict(x_test) + y_pred = self.model.predict(x_test, use_labels=False) if isinstance(y_pred, torch.Tensor): y_pred = y_pred.cpu() diff --git a/tests/span/test_modeling.py b/tests/span/test_modeling.py index 81675aba..1260bb7b 100644 --- a/tests/span/test_modeling.py +++ b/tests/span/test_modeling.py @@ -48,10 +48,10 @@ def test_loading(): assert polarity_model.span_context == 12 model = AbsaModel.from_pretrained( - "sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm", span_contexts=(12, None) + "sentence-transformers/paraphrase-albert-small-v2", spacy_model="en_core_web_sm", span_contexts=(12, 4) ) assert model.aspect_model.span_context == 12 - assert model.polarity_model.span_context == 3 # <- default + assert model.polarity_model.span_context == 4 def test_save_load(absa_model: AbsaModel) -> None: @@ -60,8 +60,8 @@ def test_save_load(absa_model: AbsaModel) -> None: with TemporaryDirectory() as tmp_dir: tmp_dir = str(Path(tmp_dir) / "model") absa_model.save_pretrained(tmp_dir) - assert (Path(tmp_dir + "-aspect") / "config_span_setfit.json").exists() - assert (Path(tmp_dir + "-polarity") / "config_span_setfit.json").exists() + assert (Path(tmp_dir + "-aspect") / "config_setfit.json").exists() + assert (Path(tmp_dir + "-polarity") / "config_setfit.json").exists() fresh_model = AbsaModel.from_pretrained( tmp_dir + "-aspect", tmp_dir + "-polarity", spacy_model="en_core_web_sm" @@ -71,8 +71,8 @@ def test_save_load(absa_model: AbsaModel) -> None: with TemporaryDirectory() as aspect_tmp_dir: with TemporaryDirectory() as polarity_tmp_dir: absa_model.save_pretrained(aspect_tmp_dir, polarity_tmp_dir) - assert (Path(aspect_tmp_dir) / "config_span_setfit.json").exists() - assert (Path(polarity_tmp_dir) / "config_span_setfit.json").exists() + assert (Path(aspect_tmp_dir) / "config_setfit.json").exists() + assert (Path(polarity_tmp_dir) / "config_setfit.json").exists() fresh_model = AbsaModel.from_pretrained(aspect_tmp_dir, polarity_tmp_dir, spacy_model="en_core_web_sm") assert fresh_model.polarity_model.span_context == 5 diff --git a/tests/test_modeling.py b/tests/test_modeling.py index 71c683a1..6d244aff 100644 --- a/tests/test_modeling.py +++ b/tests/test_modeling.py @@ -1,3 +1,6 @@ +import json +from pathlib import Path +from tempfile import TemporaryDirectory from unittest import TestCase import numpy as np @@ -256,3 +259,58 @@ def test_load_model_on_device(device): assert model.model_body.device.type == device model.model_body.encode("This is a test sample to encode") + + +def test_save_load_config(model: SetFitModel) -> None: + with TemporaryDirectory() as tmp_dir: + tmp_dir = str(Path(tmp_dir) / "model") + model.save_pretrained(tmp_dir) + config_path = Path(tmp_dir) / "config_setfit.json" + assert config_path.exists() + with open(config_path, "r") as f: + config = json.load(f) + assert config == {"normalize_embeddings": False, "labels": None} + + with TemporaryDirectory() as tmp_dir: + tmp_dir = str(Path(tmp_dir) / "model") + model.normalize_embeddings = True + model.labels = ["negative", "positive"] + model.save_pretrained(tmp_dir) + config_path = Path(tmp_dir) / "config_setfit.json" + assert config_path.exists() + with open(config_path, "r") as f: + config = json.load(f) + assert config == {"normalize_embeddings": True, "labels": ["negative", "positive"]} + + fresh_model = model.from_pretrained(tmp_dir) + assert fresh_model.normalize_embeddings == True + assert fresh_model.labels == ["negative", "positive"] + + +def test_load_model() -> None: + model = SetFitModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", labels=["foo", "bar", "baz"] + ) + assert model.labels == ["foo", "bar", "baz"] + assert model.label2id == {"foo": 0, "bar": 1, "baz": 2} + assert model.id2label == {0: "foo", 1: "bar", 2: "baz"} + + +def test_inference_with_labels() -> None: + model = SetFitModel.from_pretrained("SetFit/test-setfit-sst2") + assert model.labels is None + assert model.predict(["Very good"]) == torch.tensor([1], dtype=torch.int32) + model.labels = ["negative", "positive"] + assert model.predict(["Very good"]) == ["positive"] + + model = SetFitModel.from_pretrained("SetFit/test-setfit-sst2-string-labels") + assert model.labels is None + assert model.predict(["Very good"]) == np.array(["positive"], dtype=" Date: Tue, 28 Nov 2023 22:43:06 +0100 Subject: [PATCH 127/183] Resolve flake issues --- src/setfit/modeling.py | 2 +- tests/test_modeling.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 6ff1ad9d..2708f711 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -842,7 +842,7 @@ def _from_pretrained( Whether to apply normalization on the embeddings produced by the Sentence Transformer body. device (`Union[torch.device, str]`, *optional*): The device on which to load the SetFit model, e.g. `"cuda:0"`, `"mps"` or `torch.device("cuda")`. - + Example:: >>> from setfit import SetFitModel diff --git a/tests/test_modeling.py b/tests/test_modeling.py index 6d244aff..4cba6307 100644 --- a/tests/test_modeling.py +++ b/tests/test_modeling.py @@ -283,7 +283,7 @@ def test_save_load_config(model: SetFitModel) -> None: assert config == {"normalize_embeddings": True, "labels": ["negative", "positive"]} fresh_model = model.from_pretrained(tmp_dir) - assert fresh_model.normalize_embeddings == True + assert fresh_model.normalize_embeddings is True assert fresh_model.labels == ["negative", "positive"] From 1af337f62119581ed0858faece7c176556d3dc58 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 29 Nov 2023 10:22:47 +0100 Subject: [PATCH 128/183] Add labels to migration guide --- docs/source/en/how_to/v1.0.0_migration_guide.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/en/how_to/v1.0.0_migration_guide.mdx b/docs/source/en/how_to/v1.0.0_migration_guide.mdx index 5051f930..2af3f3e3 100644 --- a/docs/source/en/how_to/v1.0.0_migration_guide.mdx +++ b/docs/source/en/how_to/v1.0.0_migration_guide.mdx @@ -29,6 +29,12 @@ To update your code to work with v1.0.0, the following changes must be made: This list contains new functionality that can be used starting from v1.0.0. +* [`SetFitModel.from_pretrained`] now accepts new arguments: + * `device`: Specifies the device on which to load the SetFit model. + * `labels`: Specify labels corresponding to the training labels - useful if the training labels are integers ranging from `0` to `num_classes - 1`. These are automatically applied on calling [`SetFitModel.predict`]. +* [`SetFitModel.predict`] now accepts new arguments: + * `batch_size` (defaults to `32`): The batch size to use in encoding the sentences to embeddings. Higher often means faster processing but higher memory usage. + * `use_labels` (defaults to `True`): Whether to use the `SetFitModel.labels` to convert integer labels to string labels. Not used if the training labels are already strings. * [`SetFitModel.encode`] has been introduce to convert input sentences to embeddings using the `SentenceTransformer` body. * [`SetFitModel.device`] has been introduced to determine the device of the model. * [`AbsaTrainer`] and [`AbsaModel`] have been introduced for applying SetFit for Aspect Based Sentiment Analysis. From 3876d624ee171f981064af29752556fa9e203221 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 29 Nov 2023 10:23:20 +0100 Subject: [PATCH 129/183] Update returns docstring for predict & __call__ --- src/setfit/modeling.py | 106 +++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 52 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 2708f711..a24386bc 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -511,6 +511,34 @@ def _output_type_conversion( outputs = torch.from_numpy(outputs) return outputs + def predict_proba( + self, inputs: List[str], batch_size: int = 32, as_numpy: bool = False, show_progress_bar: Optional[bool] = None + ) -> Union[torch.Tensor, np.ndarray]: + """Predict the probabilities of the various classes. + + Args: + inputs (`List[str]`): The input sentences to predict class probabilities for. + batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. + Higher often means faster processing but higher memory usage. + as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. + show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. + + Example:: + + >>> model = SetFitModel.from_pretrained(...) + >>> model.predict_proba(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) + tensor([[0.9367, 0.0633], + [0.0627, 0.9373], + [0.0890, 0.9110]], dtype=torch.float64) + + Returns: + `Union[torch.Tensor, np.ndarray]`: A matrix with shape [INPUT_LENGTH, NUM_CLASSES] denoting + probabilities of predicting an input as a class. + """ + embeddings = self.encode(inputs, batch_size=batch_size, show_progress_bar=show_progress_bar) + outputs = self.model_head.predict_proba(embeddings) + return self._output_type_conversion(outputs, as_numpy=as_numpy) + def predict( self, inputs: List[str], @@ -518,7 +546,7 @@ def predict( as_numpy: bool = False, use_labels: bool = True, show_progress_bar: Optional[bool] = None, - ) -> Union[torch.Tensor, np.ndarray]: + ) -> Union[torch.Tensor, np.ndarray, List[str]]: """Predict the various classes. Args: @@ -536,8 +564,9 @@ def predict( ["negative", "positive", "positive"] Returns: - `Union[torch.Tensor, np.ndarray]`: A vector with equal length to the inputs, denoting - to which class each input is predicted to belong. + `Union[torch.Tensor, np.ndarray, List[str]]`: A list of string labels with equal length to the inputs if + `use_labels` is `True` and `SetFitModel.labels` has been defined. Otherwise a vector with equal length + to the inputs, denoting to which class each input is predicted to belong. """ embeddings = self.encode(inputs, batch_size=batch_size, show_progress_bar=show_progress_bar) outputs = self.model_head.predict(embeddings) @@ -551,33 +580,42 @@ def predict( return [self.labels[output] for output in outputs] return self._output_type_conversion(outputs, as_numpy=as_numpy) - def predict_proba( - self, inputs: List[str], batch_size: int = 32, as_numpy: bool = False, show_progress_bar: Optional[bool] = None + def __call__( + self, + inputs: List[str], + batch_size: int = 32, + as_numpy: bool = False, + use_labels: bool = True, + show_progress_bar: Optional[bool] = None, ) -> Union[torch.Tensor, np.ndarray]: - """Predict the probabilities of the various classes. + """Predict the various classes. Args: - inputs (`List[str]`): The input sentences to predict class probabilities for. + inputs (`List[str]`): The input sentences to predict classes for. batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. Higher often means faster processing but higher memory usage. as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. + use_labels (`bool`, defaults to `True`): Whether to try and return elements of `SetFitModel.labels`. show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. Example:: >>> model = SetFitModel.from_pretrained(...) - >>> model.predict_proba(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) - tensor([[0.9367, 0.0633], - [0.0627, 0.9373], - [0.0890, 0.9110]], dtype=torch.float64) + >>> model(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) + ["negative", "positive", "positive"] Returns: - `Union[torch.Tensor, np.ndarray]`: A matrix with shape [INPUT_LENGTH, NUM_CLASSES] denoting - probabilities of predicting an input as a class. + `Union[torch.Tensor, np.ndarray, List[str]]`: A list of string labels with equal length to the inputs if + `use_labels` is `True` and `SetFitModel.labels` has been defined. Otherwise a vector with equal length + to the inputs, denoting to which class each input is predicted to belong. """ - embeddings = self.encode(inputs, batch_size=batch_size, show_progress_bar=show_progress_bar) - outputs = self.model_head.predict_proba(embeddings) - return self._output_type_conversion(outputs, as_numpy=as_numpy) + return self.predict( + inputs, + batch_size=batch_size, + as_numpy=as_numpy, + use_labels=use_labels, + show_progress_bar=show_progress_bar, + ) @property def device(self) -> torch.device: @@ -634,42 +672,6 @@ def create_model_card(self, path: str, model_name: Optional[str] = "SetFit Model with open(os.path.join(path, "README.md"), "w", encoding="utf-8") as f: f.write(model_card_content) - def __call__( - self, - inputs: List[str], - batch_size: int = 32, - as_numpy: bool = False, - use_labels: bool = True, - show_progress_bar: Optional[bool] = None, - ) -> Union[torch.Tensor, np.ndarray]: - """Predict the various classes. - - Args: - inputs (`List[str]`): The input sentences to predict classes for. - batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. - Higher often means faster processing but higher memory usage. - as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. - use_labels (`bool`, defaults to `True`): Whether to try and return elements of `SetFitModel.labels`. - show_progress_bar (`Optional[bool]`, defaults to `None`): Whether to show a progress bar while encoding. - - Example:: - - >>> model = SetFitModel.from_pretrained(...) - >>> model(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) - ["negative", "positive", "positive"] - - Returns: - `torch.Tensor`: A vector with equal length to the inputs, denoting to which class each - input is predicted to belong. - """ - return self.predict( - inputs, - batch_size=batch_size, - as_numpy=as_numpy, - use_labels=use_labels, - show_progress_bar=show_progress_bar, - ) - def _save_pretrained(self, save_directory: Union[Path, str]) -> None: save_directory = str(save_directory) # Save the config From 71be7a54de19908350994698d7692677eff48838 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 29 Nov 2023 10:43:40 +0100 Subject: [PATCH 130/183] Use ndim rather than "multi_target_strategy is None" --- src/setfit/modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index a24386bc..f265f911 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -574,7 +574,7 @@ def predict( if ( use_labels and self.labels - and self.multi_target_strategy is None + and outputs.ndim == 1 and (self.has_differentiable_head or outputs.dtype.char != "U") ): return [self.labels[output] for output in outputs] From cc97d108437897987eb9f76a42fb57e5b8d2a458 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 29 Nov 2023 11:06:25 +0100 Subject: [PATCH 131/183] Allow passing strings to model.predict --- src/setfit/modeling.py | 66 ++++++++++++++++++++++++++++-------------- tests/test_modeling.py | 10 +++++++ 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index f265f911..f1e12369 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -512,12 +512,16 @@ def _output_type_conversion( return outputs def predict_proba( - self, inputs: List[str], batch_size: int = 32, as_numpy: bool = False, show_progress_bar: Optional[bool] = None + self, + inputs: Union[str, List[str]], + batch_size: int = 32, + as_numpy: bool = False, + show_progress_bar: Optional[bool] = None, ) -> Union[torch.Tensor, np.ndarray]: """Predict the probabilities of the various classes. Args: - inputs (`List[str]`): The input sentences to predict class probabilities for. + inputs (`Union[str, List[str]]`): The input sentences to predict class probabilities for. batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. Higher often means faster processing but higher memory usage. as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. @@ -530,27 +534,34 @@ def predict_proba( tensor([[0.9367, 0.0633], [0.0627, 0.9373], [0.0890, 0.9110]], dtype=torch.float64) + >>> model.predict_proba("That was cool!") + tensor([0.8421, 0.1579], dtype=torch.float64) Returns: `Union[torch.Tensor, np.ndarray]`: A matrix with shape [INPUT_LENGTH, NUM_CLASSES] denoting - probabilities of predicting an input as a class. + probabilities of predicting an input as a class. If the input is a string, then the output + is a vector with shape [NUM_CLASSES,]. """ + is_singular = isinstance(inputs, str) + if is_singular: + inputs = [inputs] embeddings = self.encode(inputs, batch_size=batch_size, show_progress_bar=show_progress_bar) - outputs = self.model_head.predict_proba(embeddings) - return self._output_type_conversion(outputs, as_numpy=as_numpy) + probs = self.model_head.predict_proba(embeddings) + outputs = self._output_type_conversion(probs, as_numpy=as_numpy) + return outputs[0] if is_singular else outputs def predict( self, - inputs: List[str], + inputs: Union[str, List[str]], batch_size: int = 32, as_numpy: bool = False, use_labels: bool = True, show_progress_bar: Optional[bool] = None, - ) -> Union[torch.Tensor, np.ndarray, List[str]]: + ) -> Union[torch.Tensor, np.ndarray, List[str], int, str]: """Predict the various classes. Args: - inputs (`List[str]`): The input sentences to predict classes for. + inputs (`Union[str, List[str]]`): The input sentence or sentences to predict classes for. batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. Higher often means faster processing but higher memory usage. as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. @@ -562,36 +573,44 @@ def predict( >>> model = SetFitModel.from_pretrained(...) >>> model.predict(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) ["negative", "positive", "positive"] + >>> model.predict("That was cool!") + "positive" Returns: - `Union[torch.Tensor, np.ndarray, List[str]]`: A list of string labels with equal length to the inputs if - `use_labels` is `True` and `SetFitModel.labels` has been defined. Otherwise a vector with equal length - to the inputs, denoting to which class each input is predicted to belong. + `Union[torch.Tensor, np.ndarray, List[str], int, str]`: A list of string labels with equal length to the + inputs if `use_labels` is `True` and `SetFitModel.labels` has been defined. Otherwise a vector with + equal length to the inputs, denoting to which class each input is predicted to belong. If the inputs + is a single string, then the output is a single label as well. """ + is_singular = isinstance(inputs, str) + if is_singular: + inputs = [inputs] embeddings = self.encode(inputs, batch_size=batch_size, show_progress_bar=show_progress_bar) - outputs = self.model_head.predict(embeddings) + preds = self.model_head.predict(embeddings) # If labels are defined, we don't have multilabels & the output is not already strings, then we convert to string labels if ( use_labels and self.labels - and outputs.ndim == 1 - and (self.has_differentiable_head or outputs.dtype.char != "U") + and preds.ndim == 1 + and (self.has_differentiable_head or preds.dtype.char != "U") ): - return [self.labels[output] for output in outputs] - return self._output_type_conversion(outputs, as_numpy=as_numpy) + outputs = [self.labels[pred] for pred in preds] + else: + outputs = self._output_type_conversion(preds, as_numpy=as_numpy) + return outputs[0] if is_singular else outputs def __call__( self, - inputs: List[str], + inputs: Union[str, List[str]], batch_size: int = 32, as_numpy: bool = False, use_labels: bool = True, show_progress_bar: Optional[bool] = None, - ) -> Union[torch.Tensor, np.ndarray]: + ) -> Union[torch.Tensor, np.ndarray, List[str], int, str]: """Predict the various classes. Args: - inputs (`List[str]`): The input sentences to predict classes for. + inputs (`Union[str, List[str]]`): The input sentence or sentences to predict classes for. batch_size (`int`, defaults to `32`): The batch size to use in encoding the sentences to embeddings. Higher often means faster processing but higher memory usage. as_numpy (`bool`, defaults to `False`): Whether to output as numpy array instead. @@ -603,11 +622,14 @@ def __call__( >>> model = SetFitModel.from_pretrained(...) >>> model(["What a boring display", "Exhilarating through and through", "I'm wowed!"]) ["negative", "positive", "positive"] + >>> model("That was cool!") + "positive" Returns: - `Union[torch.Tensor, np.ndarray, List[str]]`: A list of string labels with equal length to the inputs if - `use_labels` is `True` and `SetFitModel.labels` has been defined. Otherwise a vector with equal length - to the inputs, denoting to which class each input is predicted to belong. + `Union[torch.Tensor, np.ndarray, List[str], int, str]`: A list of string labels with equal length to the + inputs if `use_labels` is `True` and `SetFitModel.labels` has been defined. Otherwise a vector with + equal length to the inputs, denoting to which class each input is predicted to belong. If the inputs + is a single string, then the output is a single label as well. """ return self.predict( inputs, diff --git a/tests/test_modeling.py b/tests/test_modeling.py index 4cba6307..5e75e9d9 100644 --- a/tests/test_modeling.py +++ b/tests/test_modeling.py @@ -314,3 +314,13 @@ def test_inference_with_labels() -> None: assert model.predict(["Very good"]) == torch.tensor([1], dtype=torch.int32, device=model.device) model.labels = ["negative", "positive"] assert model.predict(["Very good"]) == ["positive"] + + +def test_singular_predict() -> None: + model = SetFitModel.from_pretrained("SetFit/test-setfit-sst2") + assert model.predict("That was cool!") == torch.tensor(1, dtype=torch.int32) + probs = model.predict_proba("That was cool!") + assert probs.shape == (2,) + assert probs.argmax() == 1 + model.labels = ["negative", "positive"] + assert model("That was cool!") == "positive" From 62f7eea322fba19f04c040b6b2453ee58526b928 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 29 Nov 2023 11:25:57 +0100 Subject: [PATCH 132/183] Allow partial column mappings I.e. no longer require {"sentence": "text", "label": "label"}, you can just do {"sentence": "text"} --- src/setfit/trainer.py | 9 +++++++-- tests/test_trainer.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index efe24535..47b5cac0 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -97,10 +97,15 @@ def _validate_column_mapping(self, dataset: "Dataset") -> None: "Either make sure these columns are present, or specify which columns to use with column_mapping in Trainer." ) if self.column_mapping is not None: - missing_columns = self._REQUIRED_COLUMNS.difference(self.column_mapping.values()) + missing_columns = set(self._REQUIRED_COLUMNS) + # Remove columns that will be provided via the column mapping + missing_columns -= set(self.column_mapping.values()) + # Remove columns that will be provided because they are in the dataset & not mapped away + missing_columns -= set(dataset.column_names) - set(self.column_mapping.keys()) if missing_columns: raise ValueError( - f"The following columns are missing from the column mapping: {missing_columns}. Please provide a mapping for all required columns." + f"The following columns are missing from the column mapping: {missing_columns}. " + "Please provide a mapping for all required columns." ) if not set(self.column_mapping.keys()).issubset(column_names): raise ValueError( diff --git a/tests/test_trainer.py b/tests/test_trainer.py index c5654524..83ef1631 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -65,6 +65,19 @@ def test_trainer_works_with_column_mapping(self): metrics = trainer.evaluate() self.assertEqual(metrics["accuracy"], 1.0) + def test_trainer_works_with_partial_column_mapping(self): + dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) + trainer = Trainer( + model=self.model, + args=self.args, + train_dataset=dataset, + eval_dataset=dataset, + column_mapping={"text_new": "text"}, + ) + trainer.train() + metrics = trainer.evaluate() + self.assertEqual(metrics["accuracy"], 1.0) + def test_trainer_works_with_default_columns(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) From 6f226e5308fcad13b5d79c87511a1cb8609f7cf2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 29 Nov 2023 11:36:25 +0100 Subject: [PATCH 133/183] Allow normalize_embeddings with diff head See also #410 --- src/setfit/modeling.py | 4 +++- tests/test_trainer.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index f1e12369..9cf6c8e4 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -344,7 +344,9 @@ def fit( outputs = self.model_body(features) if self.normalize_embeddings: - outputs = nn.functional.normalize(outputs, p=2, dim=1) + outputs["sentence_embedding"] = nn.functional.normalize( + outputs["sentence_embedding"], p=2, dim=1 + ) outputs = self.model_head(outputs) logits = outputs["logits"] diff --git a/tests/test_trainer.py b/tests/test_trainer.py index c5654524..0dd857d6 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -231,6 +231,24 @@ def setUp(self): ) self.args = TrainingArguments(num_iterations=1) + def test_trainer_normalize(self): + self.model = SetFitModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", + use_differentiable_head=True, + head_params={"out_features": 3}, + normalize_embeddings=True, + ) + trainer = Trainer( + model=self.model, + args=self.args, + train_dataset=self.dataset, + eval_dataset=self.dataset, + column_mapping={"text_new": "text", "label_new": "label"}, + ) + trainer.train() + metrics = trainer.evaluate() + self.assertEqual(metrics, {"accuracy": 1.0}) + def test_trainer_max_length_exceeds_max_acceptable_length(self): trainer = Trainer( model=self.model, From 313bffc12012993e5b5b6e60f117a34b203cdc31 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 17:34:07 +0100 Subject: [PATCH 134/183] Update phrasing in SetFit intro --- README.md | 2 +- docs/source/en/index.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a2931e85..6de2902f 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ SetFit is an efficient and prompt-free framework for few-shot fine-tuning of [Se Compared to other few-shot learning methods, SetFit has several unique features: -* 🗣 **No prompts or verbalisers:** Current techniques for few-shot fine-tuning require handcrafted prompts or verbalisers to convert examples into a format that's suitable for the underlying language model. SetFit dispenses with prompts altogether by generating rich embeddings directly from text examples. +* 🗣 **No prompts or verbalizers:** Current techniques for few-shot fine-tuning require handcrafted prompts or verbalizers to convert examples into a format suitable for the underlying language model. SetFit dispenses with prompts altogether by generating rich embeddings directly from text examples. * 🏎 **Fast to train:** SetFit doesn't require large-scale models like T0 or GPT-3 to achieve high accuracy. As a result, it is typically an order of magnitude (or more) faster to train and run inference with. * 🌎 **Multilingual support**: SetFit can be used with any [Sentence Transformer](https://huggingface.co/models?library=sentence-transformers&sort=downloads) on the Hub, which means you can classify text in multiple languages by simply fine-tuning a multilingual checkpoint. diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 61253b47..35b87fa2 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -10,7 +10,7 @@ Compared to other few-shot learning methods, SetFit has several unique features: -* 🗣 **No prompts or verbalisers:** Current techniques for few-shot fine-tuning require handcrafted prompts or verbalisers to convert examples into a format that's suitable for the underlying language model. SetFit dispenses with prompts altogether by generating rich embeddings directly from text examples. +* 🗣 **No prompts or verbalizers:** Current techniques for few-shot fine-tuning require handcrafted prompts or verbalizers to convert examples into a format suitable for the underlying language model. SetFit dispenses with prompts altogether by generating rich embeddings directly from text examples. * 🏎 **Fast to train:** SetFit doesn't require large-scale models like T0, Llama or GPT-4 to achieve high accuracy. As a result, it is typically an order of magnitude (or more) faster to train and run inference with. * 🌎 **Multilingual support**: SetFit can be used with any [Sentence Transformer](https://huggingface.co/models?library=sentence-transformers&sort=downloads) on the Hub, which means you can classify text in multiple languages by simply fine-tuning a multilingual checkpoint. From 9976bb564f0771a8310c31965fc8c64004b90b91 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 29 Nov 2023 19:51:32 +0100 Subject: [PATCH 135/183] Heavily improve automatic model card generation --- MANIFEST.in | 3 +- src/setfit/__init__.py | 7 + src/setfit/model_card.py | 516 ++++++++++++++++++++++++++++++ src/setfit/model_card_template.md | 177 ++++++++++ src/setfit/modeling.py | 78 ++--- src/setfit/trainer.py | 73 +++-- 6 files changed, 771 insertions(+), 83 deletions(-) create mode 100644 src/setfit/model_card.py create mode 100644 src/setfit/model_card_template.md diff --git a/MANIFEST.in b/MANIFEST.in index 69617566..88bc0822 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include src/setfit/span/model_card_template.md \ No newline at end of file +include src/setfit/span/model_card_template.md +include src/setfit/model_card_template.md \ No newline at end of file diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index 9540a68a..e01e2164 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -1,5 +1,7 @@ __version__ = "1.0.0.dev0" +import importlib +import os import warnings from .data import get_templated_dataset, sample_dataset @@ -13,3 +15,8 @@ # Ensure that DeprecationWarnings are shown by default, as recommended by # https://docs.python.org/3/library/warnings.html#overriding-the-default-filter warnings.filterwarnings("default", category=DeprecationWarning) + +# If codecarbon is installed and the log level is not defined, +# automatically overwrite the default to "error" +if importlib.util.find_spec("codecarbon") and "CODECARBON_LOG_LEVEL" not in os.environ: + os.environ["CODECARBON_LOG_LEVEL"] = "error" diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py new file mode 100644 index 00000000..385fce67 --- /dev/null +++ b/src/setfit/model_card.py @@ -0,0 +1,516 @@ +from collections import Counter, defaultdict +import collections +import logging +import os +import random +from dataclasses import dataclass, field, fields +from pathlib import Path +from platform import python_version +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import datasets +import tokenizers +import torch +import transformers +from datasets import Dataset +from huggingface_hub import ( + CardData, + DatasetFilter, + ModelCard, + dataset_info, + list_datasets, + model_info, +) +from huggingface_hub.repocard_data import EvalResult, eval_results_to_model_index +from huggingface_hub.utils import yaml_dump +from transformers import TrainerCallback +from transformers.integrations import CodeCarbonCallback +from transformers.modelcard import ( + extract_hyperparameters_from_trainer, + make_markdown_table, +) +from transformers.trainer_callback import TrainerControl, TrainerState +from transformers.training_args import TrainingArguments +from transformers import PretrainedConfig + +logger = logging.getLogger(__name__) + +from setfit import __version__ as setfit_version +from sentence_transformers import __version__ as sentence_transformers_version +from sentence_transformers.models import Transformer + + +if TYPE_CHECKING: + from setfit.modeling import SetFitModel + from setfit.trainer import Trainer + + +class ModelCardCallback(TrainerCallback): + def __init__(self, trainer: "Trainer") -> None: + super().__init__() + self.trainer = trainer + + callbacks = [ + callback for callback in self.trainer.callback_handler.callbacks if isinstance(callback, CodeCarbonCallback) + ] + if callbacks: + trainer.model.model_card_data.code_carbon_callback = callbacks[0] + + def on_init_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, model: "SetFitModel", **kwargs): + if not model.model_card_data.dataset_id: + # Inferring is hacky - it may break in the future, so let's be safe + try: + model.model_card_data.infer_dataset_id(self.trainer.train_dataset) + except Exception: + pass + + dataset = self.trainer.eval_dataset or self.trainer.train_dataset + if dataset is not None: + if not model.model_card_data.widget: + model.model_card_data.set_widget_examples(dataset) + + if self.trainer.train_dataset: + model.model_card_data.num_classes = len(set(self.trainer.train_dataset["label"])) + model.model_card_data.set_train_set_metrics(self.trainer.train_dataset) + model.model_card_data.set_label_examples(self.trainer.train_dataset) + + def on_train_begin( + self, args: TrainingArguments, state: TrainerState, control: TrainerControl, model: "SetFitModel", **kwargs + ) -> None: + # model.model_card_data.hyperparameters = extract_hyperparameters_from_trainer(self.trainer) + ignore_keys = {"output_dir", "logging_dir", "logging_strategy", "logging_first_step", "logging_steps", "evaluation_strategy", "eval_steps", "eval_delay", "save_strategy", "save_steps", "save_total_limit", "metric_for_best_model", "greater_is_better", "report_to", "samples_per_label", "show_progress_bar"} + get_name_keys = {"loss", "distance_metric"} + args_dict = args.to_dict() + model.model_card_data.hyperparameters = {key: value.__name__ if key in get_name_keys else value for key, value in args_dict.items() if key not in ignore_keys and value is not None} + + def on_evaluate( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + model: "SetFitModel", + metrics: Dict[str, float], + **kwargs, + ) -> None: + # TODO: Highlight the loaded best step + if model.model_card_data.eval_lines_list and model.model_card_data.eval_lines_list[-1]["Step"] == state.global_step: + model.model_card_data.eval_lines_list[-1]["Validation Loss"] = metrics["eval_embedding_loss"] + else: + model.model_card_data.eval_lines_list.append( + { + # "Training Loss": self.state.log_history[-1]["loss"] if "loss" in self.state.log_history[-1] else "-", + "Epoch": state.epoch, + "Step": state.global_step, + "Training Loss": "-", + "Validation Loss": metrics["eval_embedding_loss"], + } + ) + + def on_log( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + model: "SetFitModel", + logs: Dict[str, float], + **kwargs, + ): + if "embedding_loss" in logs: + if model.model_card_data.eval_lines_list and model.model_card_data.eval_lines_list[-1]["Step"] == state.global_step: + model.model_card_data.eval_lines_list[-1]["Training Loss"] = logs["embedding_loss"] + else: + model.model_card_data.eval_lines_list.append( + { + # "Training Loss": self.state.log_history[-1]["loss"] if "loss" in self.state.log_history[-1] else "-", + "Epoch": state.epoch, + "Step": state.global_step, + "Training Loss": logs["embedding_loss"], + "Validation Loss": "-", + } + ) + + +YAML_FIELDS = [ + "language", + "license", + "library_name", + "tags", + "datasets", + "metrics", + "pipeline_tag", + "widget", + "model-index", + "co2_eq_emissions", + "base_model", +] +IGNORED_FIELDS = ["model"] + + +@dataclass +class SetFitModelCardData(CardData): + """A dataclass storing data used in the model card. + + Args: + language (`Optional[Union[str, List[str]]]`): The model language, either a string or a list, + e.g. "en" or ["en", "de", "nl"] + license: (`Optional[str]`): The license of the model, e.g. "apache-2.0", "mit" + or "cc-by-nc-sa-4.0" + model_name: (`Optional[str]`): The pretty name of the model, e.g. "SetFit with mBERT-base on CoNLL03". + If not defined, uses encoder_name/encoder_id and dataset_name/dataset_id to generate a model name. + model_id: (`Optional[str]`): The model ID when pushing the model to the Hub, + e.g. "tomaarsen/span-marker-mbert-base-multinerd". + dataset_name: (`Optional[str]`): The pretty name of the dataset, e.g. "CoNLL03". + dataset_id: (`Optional[str]`): The dataset ID of the dataset, e.g. "tner/bionlp2004". + dataset_revision: (`Optional[str]`): The dataset revision/commit that was for training/evaluation. + st_id: (`Optional[str]`): The Sentence Transformers model ID. + + Note: + + Install ``nltk`` to detokenize the examples used in the model card, i.e. attach punctuation and brackets. + Additionally, ``codecarbon`` can be installed to automatically track carbon emission usage. + + Example:: + + >>> model = SetFitModel.from_pretrained( + ... "sentence-transformers/paraphrase-mpnet-base-v2", + ... labels=["negative", "positive"], + ... # Model card variables + ... model_card_data=SetFitModelCardData( + ... model_id="tomaarsen/setfit-paraphrase-mpnet-base-v2-sst2", + ... dataset_name="SST2, + ... dataset_id="sst2", + ... license="apache-2.0", + ... language="en", + ... ), + ... ) + """ + + # Potentially provided by the user + language: Optional[Union[str, List[str]]] = None + license: Optional[str] = None + tags: Optional[List[str]] = field( + default_factory=lambda: [ + "setfit", + "sentence-transformers", + "text-classification", + "generated_from_setfit_trainer", + ] + ) + model_name: Optional[str] = None + model_id: Optional[str] = None + dataset_name: Optional[str] = None + dataset_id: Optional[str] = None + dataset_revision: Optional[str] = None + task_name: str = "Text Classification" + st_id: Optional[str] = None + + # Automatically filled by `ModelCardCallback` and the Trainer directly + hyperparameters: Dict[str, Any] = field(default_factory=dict, init=False) + eval_results_dict: Optional[Dict[str, Any]] = field(default_factory=dict, init=False) + eval_lines_list: List[Dict[str, float]] = field(default_factory=list, init=False) + metric_lines: List[Dict[str, float]] = field(default_factory=list, init=False) + widget: List[Dict[str, str]] = field(default_factory=list, init=False) + predict_example: Optional[str] = field(default=None, init=False) + label_example_list: List[Dict[str, str]] = field(default_factory=list, init=False) + tokenizer_warning: bool = field(default=False, init=False) + train_set_metrics_list: List[Dict[str, str]] = field(default_factory=list, init=False) + train_set_sentences_per_label_list: List[Dict[str, str]] = field(default_factory=list, init=False) + code_carbon_callback: Optional[CodeCarbonCallback] = field(default=None, init=False) + num_classes: Optional[int] = field(default=None, init=False) + best_model_step: Optional[int] = field(default=None, init=False) + + # Computed once, always unchanged + pipeline_tag: str = field(default="text-classification", init=False) + library_name: str = field(default="setfit", init=False) + version: Dict[str, str] = field( + default_factory=lambda: { + "python": python_version(), + "setfit": setfit_version, + "sentence_transformers": sentence_transformers_version, + "transformers": transformers.__version__, + "torch": torch.__version__, + "datasets": datasets.__version__, + "tokenizers": tokenizers.__version__, + }, + init=False, + ) + metrics: List[str] = field(default_factory=lambda: ["precision", "recall", "f1"], init=False) + + # Passed via `register_model` only + model: Optional["SetFitModel"] = field(default=None, init=False, repr=False) + head_class: Optional[str] = field(default=None, init=False, repr=False) + + def __post_init__(self): + # We don't want to save "ignore_metadata_errors" in our Model Card + if self.dataset_id: + if is_on_huggingface(self.dataset_id, is_model=False): + if self.language is None: + # if languages are not set, try to determine the language from the dataset on the Hub + try: + info = dataset_info(self.dataset_id) + except: + pass + else: + if info.cardData: + self.language = info.cardData.get("language", self.language) + else: + logger.warning( + f"The provided {self.dataset_id!r} dataset could not be found on the Hugging Face Hub." + " Setting `dataset_id` to None." + ) + self.dataset_id = None + + if self.model_id and self.model_id.count("/") != 1: + logger.warning( + f"The provided {self.model_id!r} model ID should include the organization or user," + ' such as "tomaarsen/setfit-bge-small-v1.5-sst2-8-shot". Setting `model_id` to None.' + ) + self.model_id = None + + def set_best_model_step(self, step: int) -> None: + self.best_model_step = step + + def set_widget_examples(self, dataset: Dataset) -> None: + samples = dataset.select(random.sample(range(len(dataset)), k=5))["text"] + self.widget = [{"text": sample} for sample in samples] + + samples.sort(key=len) + self.predict_example = samples[0] + + def set_train_set_metrics(self, dataset: Dataset) -> None: + def add_naive_word_count(sample: Dict[str, Any]) -> Dict[str, Any]: + sample["word_count"] = len(sample["text"].split(" ")) + return sample + + dataset = dataset.map(add_naive_word_count) + self.train_set_metrics_list = [ + { + "Training set": "Word count", + "Min": min(dataset["word_count"]), + "Median": sum(dataset["word_count"]) / len(dataset), + "Max": max(dataset["word_count"]), + }, + ] + # If not multi-label: + sample_label = dataset[0]["label"] + if isinstance(sample_label, collections.abc.Sequence) and not isinstance(sample_label, str): + return + try: + counter = Counter(dataset["label"]) + if self.model.labels: + self.train_set_sentences_per_label_list = [ + { + "Label": str_label, + "Training Sample Count": counter[str_label if isinstance(sample_label, str) else self.model.label2id[str_label]], + } + for str_label in self.model.labels + ] + else: + self.train_set_sentences_per_label_list = [ + { + "Label": self.model.labels[label] if self.model.labels and isinstance(label, int) else str(label), + "Training Sample Count": count, + } + for label, count in sorted(counter.items()) + ] + except: + # There are some tricky edge cases possible, e.g. if the user provided integer labels that do not fall + # between 0 to num_classes-1, so we make sure we never cause errors. + pass + + def set_label_examples(self, dataset: Dataset) -> None: + num_examples_per_label = 3 + examples = defaultdict(list) + finished_labels = set() + for sample in dataset: + text = sample["text"] + label = sample["label"] + if label not in finished_labels: + examples[label].append(f'
  • {repr(text)}
    • ') + if len(examples[label]) >= num_examples_per_label: + finished_labels.add(label) + if len(finished_labels) == self.num_classes: + break + self.label_example_list = [ + {"Label": self.model.labels[label] if self.model.labels and isinstance(label, int) else label, "Examples": "
      " + "".join(example_set) + "
    "} for label, example_set in examples.items() + ] + + def infer_dataset_id(self, dataset: Dataset) -> None: + def subtuple_finder(tuple: Tuple[str], subtuple: Tuple[str]) -> int: + for i, element in enumerate(tuple): + if element == subtuple[0] and tuple[i : i + len(subtuple)] == subtuple: + return i + return -1 + + def normalize(dataset_id: str) -> str: + for token in "/\\_-": + dataset_id = dataset_id.replace(token, "") + return dataset_id.lower() + + if (cache_files := dataset.cache_files) and "filename" in cache_files[0]: + cache_path_parts = Path(cache_files[0]["filename"]).parts + # Check if the cachefile is under "huggingface/datasets" + subtuple = ("huggingface", "datasets") + index = subtuple_finder(cache_path_parts, subtuple) + if index == -1: + return + + # Get the folder after "huggingface/datasets" + cache_dataset_name = cache_path_parts[index + len(subtuple)] + # If the dataset has an author: + if "___" in cache_dataset_name: + author, dataset_name = cache_dataset_name.split("___") + else: + author = None + dataset_name = cache_dataset_name + + # Make sure the normalized dataset IDs match + dataset_list = [ + dataset + for dataset in list_datasets(filter=DatasetFilter(author=author, dataset_name=dataset_name)) + if normalize(dataset.id) == normalize(cache_dataset_name) + ] + # If there's only one match, get the ID from it + if len(dataset_list) == 1: + self.dataset_id = dataset_list[0].id + + def register_model(self, model: "SetFitModel") -> None: + self.model = model + head_class = model.model_head.__class__.__name__ + self.head_class = { + "LogisticRegression": "[LogisticRegression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html)", + "SetFitHead": "[SetFitHead](huggingface.co/docs/setfit/reference/main#setfit.SetFitHead)", + }.get(head_class, head_class) + + if not self.model_name: + if self.st_id: + self.model_name = f"SetFit with {self.st_id}" + if self.dataset_name or self.dataset_id: + self.model_name += f" on {self.dataset_name or self.dataset_id}" + else: + self.model_name = "SetFit" + + def infer_st_id(self, setfit_model_id: str) -> None: + config_dict, _ = PretrainedConfig.get_config_dict(setfit_model_id) + st_id = config_dict.get("_name_or_path") + st_id_path = Path(st_id) + # Sometimes the name_or_path ends exactly with the model_id, e.g. + # "C:\\Users\\tom/.cache\\torch\\sentence_transformers\\BAAI_bge-small-en-v1.5\\" + candidate_model_ids = [ + "/".join(st_id_path.parts[-2:]) + ] + # Sometimes the name_or_path its final part contains the full model_id, with "/" replaced with a "_", e.g. + # "/root/.cache/torch/sentence_transformers/sentence-transformers_all-mpnet-base-v2/" + # In that case, we take the last part, split on _, and try all combinations + # e.g. "a_b_c_d" -> ['a/b_c_d', 'a_b/c_d', 'a_b_c/d'] + splits = st_id_path.name.split("_") + candidate_model_ids += ["_".join(splits[:idx]) + "/" + "_".join(splits[idx:]) for idx in range(1, len(splits))] + for model_id in candidate_model_ids: + if is_on_huggingface(model_id): + self.st_id = model_id + break + + def set_st_id(self, model_id: str) -> None: + if is_on_huggingface(model_id): + self.st_id = model_id + + def post_training_eval_results(self, results: Dict[str, float]) -> None: + self.eval_results_dict = results + + results_without_split = {key.split("_", maxsplit=1)[1].title(): value for key, value in results.items()} + self.metric_lines = [{ + "Label": "**all**", + **results_without_split + }] + + def _maybe_round(self, v, decimals=4): + if isinstance(v, float) and len(str(v).split(".")) > 1 and len(str(v).split(".")[1]) > decimals: + return f"{v:.{decimals}f}" + return str(v) + + def to_dict(self) -> Dict[str, Any]: + super_dict = {field.name: getattr(self, field.name) for field in fields(self)} + + # Compute required formats from the raw data + if self.eval_results_dict: + dataset_split = list(self.eval_results_dict.keys())[0].split("_")[0] + dataset_id = self.dataset_id or "unknown" + dataset_name = self.dataset_name or self.dataset_id or "Unknown" + eval_results = [ + EvalResult( + task_type="text-classification", + dataset_type=dataset_id, + dataset_name=dataset_name, + metric_type=metric_key.split("_", maxsplit=1)[1], + metric_value=metric_value, + task_name="Text Classification", + dataset_split=dataset_split, + dataset_revision=self.dataset_revision, + metric_name=metric_key.split("_", maxsplit=1)[1].title(), + ) + for metric_key, metric_value in self.eval_results_dict.items() + ] + super_dict["model-index"] = eval_results_to_model_index(self.model_name, eval_results) + eval_lines_list = [{key: f"**{self._maybe_round(value)}**" if line["Step"] == self.best_model_step else value for key, value in line.items()} for line in self.eval_lines_list] + super_dict["eval_lines"] = make_markdown_table(eval_lines_list) + super_dict["explain_bold_in_eval"] = "**" in super_dict["eval_lines"] + # Replace |:---:| with |:---| for left alignment + super_dict["label_examples"] = make_markdown_table(self.label_example_list).replace("-:|", "--|") + super_dict["train_set_metrics"] = make_markdown_table(self.train_set_metrics_list).replace("-:|", "--|") + super_dict["train_set_sentences_per_label_list"] = make_markdown_table(self.train_set_sentences_per_label_list).replace("-:|", "--|") + super_dict["metrics_table"] = make_markdown_table(self.metric_lines).replace("-:|", "--|") + if self.code_carbon_callback and self.code_carbon_callback.tracker: + emissions_data = self.code_carbon_callback.tracker._prepare_emissions_data() + super_dict["co2_eq_emissions"] = { + # * 1000 to convert kg to g + "emissions": float(emissions_data.emissions) * 1000, + "source": "codecarbon", + "training_type": "fine-tuning", + "on_cloud": emissions_data.on_cloud == "Y", + "cpu_model": emissions_data.cpu_model, + "ram_total_size": emissions_data.ram_total_size, + "hours_used": round(emissions_data.duration / 3600, 3), + } + if emissions_data.gpu_model: + super_dict["co2_eq_emissions"]["hardware_used"] = emissions_data.gpu_model + if self.dataset_id: + super_dict["datasets"] = [self.dataset_id] + if self.st_id: + super_dict["base_model"] = self.st_id + super_dict["model_max_length"] = self.model.model_body.get_max_seq_length() + if super_dict["num_classes"] is None: + if self.model.labels: + super_dict["num_classes"] = len(self.model.labels) + + for key in IGNORED_FIELDS: + super_dict.pop(key, None) + return super_dict + + def to_yaml(self, line_break=None) -> str: + return yaml_dump( + {key: value for key, value in self.to_dict().items() if key in YAML_FIELDS and value is not None}, + sort_keys=False, + line_break=line_break, + ).strip() + + +def is_on_huggingface(repo_id: str, is_model: bool = True) -> bool: + # Models with more than two 'sections' certainly are not public models + if len(repo_id.split("/")) > 2: + return False + + try: + if is_model: + model_info(repo_id) + else: + dataset_info(repo_id) + return True + except: + # Fetching models can fail for many reasons: Repository not existing, no internet access, HF down, etc. + return False + + +def generate_model_card(model: "SetFitModel") -> str: + template_path = Path(__file__).parent / "model_card_template.md" + model_card = ModelCard.from_template(card_data=model.model_card_data, template_path=template_path, hf_emoji="🤗") + return model_card.content diff --git a/src/setfit/model_card_template.md b/src/setfit/model_card_template.md new file mode 100644 index 00000000..c8ea09c3 --- /dev/null +++ b/src/setfit/model_card_template.md @@ -0,0 +1,177 @@ +--- +# For reference on model card metadata, see the spec: https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1 +# Doc / guide: https://huggingface.co/docs/hub/model-cards +{{ card_data }} +--- + +# {{ model_name | default("SetFit for Text Classification", true) }} + +This is a [SetFit](https://github.com/huggingface/setfit) model{% if dataset_id %} trained on the [{{ dataset_name if dataset_name else dataset_id }}](https://huggingface.co/datasets/{{ dataset_id }}) dataset{% endif %} that can be used for {{ task_name | default("Text Classification", true) }}.{% if st_id %} This SetFit model uses [{{ st_id }}](https://huggingface.co/{{ st_id }}) as the Sentence Transformer embedding model.{% endif %} For classification, it uses a {{ head_class }} instance. + +The model has been trained using an efficient few-shot learning technique that involves: + +1. Fine-tuning a [Sentence Transformer](https://www.sbert.net) with contrastive learning. +2. Training a classification head with features from the fine-tuned Sentence Transformer. + +## Model Details + +### Model Description +- **Model Type:** SetFit +{% if st_id -%} + - **Sentence Transformer body:** [{{ st_id }}](https://huggingface.co/{{ st_id }}) +{%- else -%} + +{%- endif %} +{% if head_class -%} + - **Classification head:** a {{ head_class }} instance. +{%- else -%} + +{%- endif %} +- **Maximum Sequence Length:** {{ model_max_length }} tokens +{% if num_classes -%} + - **Number of Classes:** {{ num_classes }} classes +{%- else -%} + +{%- endif %} +{% if dataset_id -%} + - **Training Dataset:** [{{ dataset_name if dataset_name else dataset_id }}](https://huggingface.co/datasets/{{ dataset_id }}) +{%- else -%} + +{%- endif %} +{% if language -%} + - **Language{{"s" if language is not string and language | length > 1 else ""}}:** + {%- if language is string %} {{ language }} + {%- else %} {% for lang in language -%} + {{ lang }}{{ ", " if not loop.last else "" }} + {%- endfor %} + {%- endif %} +{%- else -%} + +{%- endif %} +{% if license -%} + - **License:** {{ license }} +{%- else -%} + +{%- endif %} + +### Model Sources + +- **Repository:** [SetFit on GitHub](https://github.com/huggingface/setfit) +- **Paper:** [Efficient Few-Shot Learning Without Prompts](https://arxiv.org/abs/2209.11055) +- **Blogpost:** [SetFit: Efficient Few-Shot Learning Without Prompts](https://huggingface.co/blog/setfit) +{% if label_examples %} +### Model Labels +{{ label_examples }}{% endif -%} +{% if metrics_table %} +## Evaluation + +### Metrics +{{ metrics_table }}{% endif %} +## Uses + +### Direct Use for Inference + +First install the SetFit library: + +```bash +pip install setfit +``` + +Then you can load this model and run inference. + +```python +from setfit import SetFitModel + +# Download from {{ hf_emoji }} Hub +model = SetFitModel.from_pretrained("{{ model_id | default('setfit_model_id', true) }}") +# Run inference +preds = model("{{ predict_example | default("I loved the spiderman movie!", true) | replace('"', '\\"') }}") +``` + + + + + + + + +## Training Details +{% if train_set_metrics %} +### Training Set Metrics +{{ train_set_metrics }}{% if train_set_sentences_per_label_list %} +{{ train_set_sentences_per_label_list }}{% endif %}{% endif %}{% if hyperparameters %} +### Training Hyperparameters +{% for name, value in hyperparameters.items() %}- {{ name }}: {{ value }} +{% endfor %}{% endif %}{% if eval_lines %} +### Training Results +{{ eval_lines }}{% if explain_bold_in_eval %} +* The bold row denotes the saved checkpoint.{% endif %}{% endif %}{% if co2_eq_emissions %} +### Environmental Impact +Carbon emissions were measured using [CodeCarbon](https://github.com/mlco2/codecarbon). +- **Carbon Emitted**: {{ "%.3f"|format(co2_eq_emissions["emissions"] / 1000) }} kg of CO2 +- **Hours Used**: {{ co2_eq_emissions["hours_used"] }} hours + +### Training Hardware +- **On Cloud**: {{ "Yes" if co2_eq_emissions["on_cloud"] else "No" }} +- **GPU Model**: {{ co2_eq_emissions["hardware_used"] or "No GPU used" }} +- **CPU Model**: {{ co2_eq_emissions["cpu_model"] }} +- **RAM Size**: {{ "%.2f"|format(co2_eq_emissions["ram_total_size"]) }} GB +{% endif %} +### Framework Versions +- Python: {{ version["python"] }} +- SetFit: {{ version["setfit"] }} +- Sentence Transformers: {{ version["sentence_transformers"] }} +- Transformers: {{ version["transformers"] }} +- PyTorch: {{ version["torch"] }} +- Datasets: {{ version["datasets"] }} +- Tokenizers: {{ version["tokenizers"] }} + +## Citation + +### BibTeX +```bibtex +@article{https://doi.org/10.48550/arxiv.2209.11055, + doi = {10.48550/ARXIV.2209.11055}, + url = {https://arxiv.org/abs/2209.11055}, + author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}, + keywords = {Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {Efficient Few-Shot Learning Without Prompts}, + publisher = {arXiv}, + year = {2022}, + copyright = {Creative Commons Attribution 4.0 International} +} +``` + + + + + + \ No newline at end of file diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 9cf6c8e4..d249535c 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import Dict, List, Optional, Set, Tuple, Union +from setfit.model_card import SetFitModelCardData, generate_model_card + # For Python 3.7 compatibility try: @@ -37,59 +39,6 @@ logger = logging.get_logger(__name__) MODEL_HEAD_NAME = "model_head.pkl" - -MODEL_CARD_TEMPLATE = """--- -license: apache-2.0 -tags: -- setfit -- sentence-transformers -- text-classification -pipeline_tag: text-classification ---- - -# {model_name} - -This is a [SetFit model](https://github.com/huggingface/setfit) that can be used for text classification. \ -The model has been trained using an efficient few-shot learning technique that involves: - -1. Fine-tuning a [Sentence Transformer](https://www.sbert.net) with contrastive learning. -2. Training a classification head with features from the fine-tuned Sentence Transformer. - -## Usage - -To use this model for inference, first install the SetFit library: - -```bash -python -m pip install setfit -``` - -You can then run inference as follows: - -```python -from setfit import SetFitModel - -# Download from Hub and run inference -model = SetFitModel.from_pretrained("{model_name}") -# Run inference -preds = model(["i loved the spiderman movie!", "pineapple on pizza is the worst 🤮"]) -``` - -## BibTeX entry and citation info - -```bibtex -@article{{https://doi.org/10.48550/arxiv.2209.11055, - doi = {{10.48550/ARXIV.2209.11055}}, - url = {{https://arxiv.org/abs/2209.11055}}, - author = {{Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}}, - keywords = {{Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences}}, - title = {{Efficient Few-Shot Learning Without Prompts}}, - publisher = {{arXiv}}, - year = {{2022}}, - copyright = {{Creative Commons Attribution 4.0 International}} -}} -``` -""" - CONFIG_NAME = "config_setfit.json" @@ -266,11 +215,15 @@ class SetFitModel(PyTorchModelHubMixin): multi_target_strategy: Optional[str] = None normalize_embeddings: bool = False labels: Optional[List[str]] = None + model_card_data: Optional[SetFitModelCardData] = field(default_factory=SetFitModelCardData) attributes_to_save: Set[str] = field( init=False, repr=False, default_factory=lambda: {"normalize_embeddings", "labels"} ) + def __post_init__(self): + self.model_card_data.register_model(self) + @property def has_differentiable_head(self) -> bool: # if False, sklearn is assumed to be used instead @@ -690,11 +643,18 @@ def create_model_card(self, path: str, model_name: Optional[str] = "SetFit Model # directories model_path = Path(model_name) if model_path.exists() and Path(tempfile.gettempdir()) in model_path.resolve().parents: - model_name = "/".join(model_path.parts[-2:]) + self.model_card_data.model_id = "/".join(model_path.parts[-2:]) - model_card_content = MODEL_CARD_TEMPLATE.format(model_name=model_name) with open(os.path.join(path, "README.md"), "w", encoding="utf-8") as f: - f.write(model_card_content) + f.write(self.generate_model_card()) + + def generate_model_card(self) -> str: + """Generate and return a model card string based on the model card data. + + Returns: + str: The model card string. + """ + return generate_model_card(self) def _save_pretrained(self, save_directory: Union[Path, str]) -> None: save_directory = str(save_directory) @@ -800,10 +760,13 @@ def _from_pretrained( ) model_head_file = None + model_card_data: SetFitModelCardData = model_kwargs.pop("model_card_data", SetFitModelCardData()) + if model_head_file is not None: model_head = joblib.load(model_head_file) if isinstance(model_head, torch.nn.Module): model_head.to(device) + model_card_data.infer_st_id(model_id) else: head_params = model_kwargs.pop("head_params", {}) if use_differentiable_head: @@ -841,12 +804,15 @@ def _from_pretrained( else: model_head = clf + model_card_data.set_st_id(model_id if "/" in model_id else f"sentence-transformers/{model_id}") + # Remove the `transformers` config model_kwargs.pop("config", None) return cls( model_body=model_body, model_head=model_head, multi_target_strategy=multi_target_strategy, + model_card_data=model_card_data, **model_kwargs, ) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 47b5cac0..f6ce88d2 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -37,6 +37,8 @@ ) from transformers.utils.import_utils import is_in_notebook +from setfit.model_card import ModelCardCallback + from . import logging from .integrations import default_hp_search_backend, is_optuna_available, run_hp_search_optuna from .losses import SupConLoss @@ -186,12 +188,24 @@ def __init__( if args is not None and not isinstance(args, TrainingArguments): raise ValueError("`args` must be a `TrainingArguments` instance imported from `setfit`.") self.args = args or TrainingArguments() + self.column_mapping = column_mapping + if train_dataset: + self._validate_column_mapping(train_dataset) + if self.column_mapping is not None: + logger.info(f"Applying column mapping to the training dataset") + train_dataset = self._apply_column_mapping(train_dataset, self.column_mapping) self.train_dataset = train_dataset + + if eval_dataset: + self._validate_column_mapping(eval_dataset) + if self.column_mapping is not None: + logger.info(f"Applying column mapping to the evaluation dataset") + eval_dataset = self._apply_column_mapping(eval_dataset, self.column_mapping) self.eval_dataset = eval_dataset + self.model_init = model_init self.metric = metric self.metric_kwargs = metric_kwargs - self.column_mapping = column_mapping self.logs_mapper = {} # Seed must be set before instantiating the model when using model_init. @@ -217,7 +231,13 @@ def __init__( self.state = TrainerState() self.control = TrainerControl() self.add_callback(DEFAULT_PROGRESS_CALLBACK if self.args.show_progress_bar else PrinterCallback) - self.control = self.callback_handler.on_init_end(args, self.state, self.control) + self.control = self.callback_handler.on_init_end(self.args, self.state, self.control) + + # Add the callback for filling the model card data with hyperparameters + # and evaluation results + self.add_callback(ModelCardCallback(self)) + + self.callback_handler.on_init_end(args, self.state, self.control) def add_callback(self, callback: Union[type, TrainerCallback]) -> None: """ @@ -378,21 +398,11 @@ def train( f"Training requires a `train_dataset` given to the `{self.__class__.__name__}` initialization." ) - parameters = [] - for dataset, dataset_name in [(self.train_dataset, "training"), (self.eval_dataset, "evaluation")]: - if dataset is None: - continue - - self._validate_column_mapping(dataset) - if self.column_mapping is not None: - logger.info(f"Applying column mapping to {dataset_name} dataset") - dataset = self._apply_column_mapping(dataset, self.column_mapping) - - parameters.extend(self.dataset_to_parameters(dataset)) + train_parameters = self.dataset_to_parameters(self.train_dataset) + full_parameters = train_parameters + self.dataset_to_parameters(self.eval_dataset) if self.eval_dataset else train_parameters - self.train_embeddings(*parameters, args=args) - training_parameters = parameters[: len(parameters) // 2] if self.eval_dataset else parameters - self.train_classifier(*training_parameters, args=args) + self.train_embeddings(*full_parameters, args=args) + self.train_classifier(*train_parameters, args=args) def dataset_to_parameters(self, dataset: Dataset) -> List[Iterable]: return [dataset["text"], dataset["label"]] @@ -581,6 +591,8 @@ def _train_sentence_transformer( self.callback_handler.train_dataloader = train_dataloader self.callback_handler.eval_dataloader = eval_dataloader + self.callback_handler.on_train_begin(args, self.state, self.control) + data_iterator = iter(train_dataloader) skip_scheduler = False for epoch in range(args.embedding_num_epochs): @@ -644,7 +656,9 @@ def _train_sentence_transformer( if self.args.load_best_model_at_end and self.state.best_model_checkpoint: dir_name = Path(self.state.best_model_checkpoint).name if dir_name.startswith("step_"): - logger.info(f"Loading best SentenceTransformer model from step {dir_name[5:]}.") + step_to_load = dir_name[5:] + logger.info(f"Loading best SentenceTransformer model from step {step_to_load}.") + self.model.model_card_data.set_best_model_step(int(step_to_load)) self.model.model_body = SentenceTransformer( self.state.best_model_checkpoint, device=model_body._target_device ) @@ -764,7 +778,7 @@ def train_classifier( end_to_end=args.end_to_end, ) - def evaluate(self, dataset: Optional[Dataset] = None) -> Dict[str, float]: + def evaluate(self, dataset: Optional[Dataset] = None, metric_key_prefix: str = "test") -> Dict[str, float]: """ Computes the metrics for a given classifier. @@ -777,14 +791,18 @@ def evaluate(self, dataset: Optional[Dataset] = None) -> Dict[str, float]: `Dict[str, float]`: The evaluation metrics. """ - eval_dataset = dataset or self.eval_dataset + if dataset is not None: + self._validate_column_mapping(dataset) + if self.column_mapping is not None: + logger.info("Applying column mapping to the evaluation dataset") + eval_dataset = self._apply_column_mapping(dataset, self.column_mapping) + else: + eval_dataset = dataset + else: + eval_dataset = self.eval_dataset + if eval_dataset is None: raise ValueError("No evaluation dataset provided to `Trainer.evaluate` nor the `Trainer` initialzation.") - self._validate_column_mapping(eval_dataset) - - if self.column_mapping is not None: - logger.info("Applying column mapping to evaluation dataset") - eval_dataset = self._apply_column_mapping(eval_dataset, self.column_mapping) x_test = eval_dataset["text"] y_test = eval_dataset["label"] @@ -806,14 +824,17 @@ def evaluate(self, dataset: Optional[Dataset] = None) -> Dict[str, float]: metric_fn = evaluate.load(self.metric, config_name=metric_config) metric_kwargs = self.metric_kwargs or {} - return metric_fn.compute(predictions=y_pred, references=y_test, **metric_kwargs) + results = metric_fn.compute(predictions=y_pred, references=y_test, **metric_kwargs) elif callable(self.metric): - return self.metric(y_pred, y_test) + results = self.metric(y_pred, y_test) else: raise ValueError("metric must be a string or a callable") + self.model.model_card_data.post_training_eval_results({f"{metric_key_prefix}_{key}": value for key, value in results.items()}) + return results + def hyperparameter_search( self, hp_space: Optional[Callable[["optuna.Trial"], Dict[str, float]]] = None, From bbad20d4f7ebbd57166d7eca6b6ab6e2a16ded8c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 29 Nov 2023 19:53:10 +0100 Subject: [PATCH 136/183] Rewrite first paragraph somewhat --- src/setfit/model_card_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/model_card_template.md b/src/setfit/model_card_template.md index c8ea09c3..d49c4f52 100644 --- a/src/setfit/model_card_template.md +++ b/src/setfit/model_card_template.md @@ -6,7 +6,7 @@ # {{ model_name | default("SetFit for Text Classification", true) }} -This is a [SetFit](https://github.com/huggingface/setfit) model{% if dataset_id %} trained on the [{{ dataset_name if dataset_name else dataset_id }}](https://huggingface.co/datasets/{{ dataset_id }}) dataset{% endif %} that can be used for {{ task_name | default("Text Classification", true) }}.{% if st_id %} This SetFit model uses [{{ st_id }}](https://huggingface.co/{{ st_id }}) as the Sentence Transformer embedding model.{% endif %} For classification, it uses a {{ head_class }} instance. +This is a [SetFit](https://github.com/huggingface/setfit) model{% if dataset_id %} trained on the [{{ dataset_name if dataset_name else dataset_id }}](https://huggingface.co/datasets/{{ dataset_id }}) dataset{% endif %} that can be used for {{ task_name | default("Text Classification", true) }}.{% if st_id %} This SetFit model uses [{{ st_id }}](https://huggingface.co/{{ st_id }}) as the Sentence Transformer embedding model.{% endif %} A {{ head_class }} instance is used for classification. The model has been trained using an efficient few-shot learning technique that involves: From 6cd51ed6cd208604aa9900a342ca7ec8c6d7523c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 30 Nov 2023 14:59:59 +0100 Subject: [PATCH 137/183] Resolve issue with multi-label --- src/setfit/model_card.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index 385fce67..f1ac0227 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -70,9 +70,13 @@ def on_init_end(self, args: TrainingArguments, state: TrainerState, control: Tra model.model_card_data.set_widget_examples(dataset) if self.trainer.train_dataset: - model.model_card_data.num_classes = len(set(self.trainer.train_dataset["label"])) model.model_card_data.set_train_set_metrics(self.trainer.train_dataset) - model.model_card_data.set_label_examples(self.trainer.train_dataset) + # Does not work for multilabel + try: + model.model_card_data.num_classes = len(set(self.trainer.train_dataset["label"])) + model.model_card_data.set_label_examples(self.trainer.train_dataset) + except TypeError: + pass def on_train_begin( self, args: TrainingArguments, state: TrainerState, control: TrainerControl, model: "SetFitModel", **kwargs From 4a6852be9988bc52a6ac6cfa2c8188013ad4fa82 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 30 Nov 2023 15:13:15 +0100 Subject: [PATCH 138/183] Set inference=False for multilabel models --- src/setfit/model_card.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index f1ac0227..8e2b71d1 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -26,7 +26,6 @@ from transformers import TrainerCallback from transformers.integrations import CodeCarbonCallback from transformers.modelcard import ( - extract_hyperparameters_from_trainer, make_markdown_table, ) from transformers.trainer_callback import TrainerControl, TrainerState @@ -37,7 +36,6 @@ from setfit import __version__ as setfit_version from sentence_transformers import __version__ as sentence_transformers_version -from sentence_transformers.models import Transformer if TYPE_CHECKING: @@ -146,6 +144,7 @@ def on_log( "model-index", "co2_eq_emissions", "base_model", + "inference", ] IGNORED_FIELDS = ["model"] @@ -243,6 +242,7 @@ class SetFitModelCardData(CardData): # Passed via `register_model` only model: Optional["SetFitModel"] = field(default=None, init=False, repr=False) head_class: Optional[str] = field(default=None, init=False, repr=False) + inference: Optional[bool] = field(default=True, init=False, repr=False) def __post_init__(self): # We don't want to save "ignore_metadata_errors" in our Model Card @@ -394,6 +394,8 @@ def register_model(self, model: "SetFitModel") -> None: else: self.model_name = "SetFit" + self.inference = self.model.multi_target_strategy is None + def infer_st_id(self, setfit_model_id: str) -> None: config_dict, _ = PretrainedConfig.get_config_dict(setfit_model_id) st_id = config_dict.get("_name_or_path") From 671611e85e85996254b7d3d9599b81e75a9902f3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 30 Nov 2023 16:34:52 +0100 Subject: [PATCH 139/183] Add model card tests --- .github/workflows/tests.yml | 1 + docs/source/en/reference/main.mdx | 6 + setup.py | 2 + src/setfit/__init__.py | 1 + src/setfit/model_card.py | 11 +- tests/model_card_pattern.py | 227 ++++++++++++++++++++++++++++++ tests/test_model_card.py | 90 ++++++++++++ 7 files changed, 333 insertions(+), 5 deletions(-) create mode 100644 tests/model_card_pattern.py create mode 100644 tests/test_model_card.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 45dccb7f..8f4c8793 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,6 +43,7 @@ jobs: run: | python -m pip install --no-cache-dir --upgrade pip python -m pip install --no-cache-dir ${{ matrix.requirements }} + python -m pip install '.[codecarbon]' python -m spacy download en_core_web_lg python -m spacy download en_core_web_sm if: steps.restore-cache.outputs.cache-hit != 'true' diff --git a/docs/source/en/reference/main.mdx b/docs/source/en/reference/main.mdx index 6ceed9db..2138f9fe 100644 --- a/docs/source/en/reference/main.mdx +++ b/docs/source/en/reference/main.mdx @@ -16,6 +16,12 @@ [[autodoc]] SetFitHead +## SetFitModelCardData + +[[autodoc]] SetFitModelCardData + - to_dict + - to_yaml + ## AbsaModel [[autodoc]] AbsaModel diff --git a/setup.py b/setup.py index 16980b84..9437b96c 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ OPENVINO_REQUIRE = ["hummingbird-ml<0.4.9", "openvino==2022.3.0"] TESTS_REQUIRE = ["pytest", "pytest-cov"] + ONNX_REQUIRE + OPENVINO_REQUIRE + ABSA_REQUIRE DOCS_REQUIRE = ["hf-doc-builder>=0.3.0"] +CODECARBON_REQUIRE = ["codecarbon"] EXTRAS_REQUIRE = { "optuna": INTEGRATIONS_REQUIRE, "quality": QUALITY_REQUIRE, @@ -31,6 +32,7 @@ "openvino": ONNX_REQUIRE + OPENVINO_REQUIRE, "docs": DOCS_REQUIRE, "absa": ABSA_REQUIRE, + "codecarbon": CODECARBON_REQUIRE, } diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index e01e2164..e9f103d0 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -10,6 +10,7 @@ from .trainer import SetFitTrainer, Trainer from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer from .training_args import TrainingArguments +from .model_card import SetFitModelCardData # Ensure that DeprecationWarnings are shown by default, as recommended by diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index 8e2b71d1..fdb1fc45 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -1,7 +1,5 @@ from collections import Counter, defaultdict import collections -import logging -import os import random from dataclasses import dataclass, field, fields from pathlib import Path @@ -32,7 +30,9 @@ from transformers.training_args import TrainingArguments from transformers import PretrainedConfig -logger = logging.getLogger(__name__) +from . import logging + +logger = logging.get_logger(__name__) from setfit import __version__ as setfit_version from sentence_transformers import __version__ as sentence_transformers_version @@ -221,6 +221,7 @@ class SetFitModelCardData(CardData): code_carbon_callback: Optional[CodeCarbonCallback] = field(default=None, init=False) num_classes: Optional[int] = field(default=None, init=False) best_model_step: Optional[int] = field(default=None, init=False) + metrics: List[str] = field(default_factory=lambda: ["accuracy"], init=False) # Computed once, always unchanged pipeline_tag: str = field(default="text-classification", init=False) @@ -237,7 +238,6 @@ class SetFitModelCardData(CardData): }, init=False, ) - metrics: List[str] = field(default_factory=lambda: ["precision", "recall", "f1"], init=False) # Passed via `register_model` only model: Optional["SetFitModel"] = field(default=None, init=False, repr=False) @@ -275,7 +275,7 @@ def set_best_model_step(self, step: int) -> None: self.best_model_step = step def set_widget_examples(self, dataset: Dataset) -> None: - samples = dataset.select(random.sample(range(len(dataset)), k=5))["text"] + samples = dataset.select(random.sample(range(len(dataset)), k=min(len(dataset), 5)))["text"] self.widget = [{"text": sample} for sample in samples] samples.sort(key=len) @@ -456,6 +456,7 @@ def to_dict(self) -> Dict[str, Any]: ) for metric_key, metric_value in self.eval_results_dict.items() ] + super_dict["metrics"] = [metric_key.split("_", maxsplit=1)[1] for metric_key in self.eval_results_dict] super_dict["model-index"] = eval_results_to_model_index(self.model_name, eval_results) eval_lines_list = [{key: f"**{self._maybe_round(value)}**" if line["Step"] == self.best_model_step else value for key, value in line.items()} for line in self.eval_lines_list] super_dict["eval_lines"] = make_markdown_table(eval_lines_list) diff --git a/tests/model_card_pattern.py b/tests/model_card_pattern.py new file mode 100644 index 00000000..f0132fc6 --- /dev/null +++ b/tests/model_card_pattern.py @@ -0,0 +1,227 @@ +import re + +MODEL_CARD_PATTERN = re.compile( + """\ +--- +language: +- en +license: apache-2\.0 +library_name: setfit +tags: +- setfit +- sentence-transformers +- text-classification +- generated_from_setfit_trainer +datasets: +- sst2 +metrics: +- accuracy +widget: +- text: .* +pipeline_tag: text-classification +inference: true +co2_eq_emissions: + emissions: [\d\.\-e]+ + source: codecarbon + training_type: fine-tuning + on_cloud: (false|true) + cpu_model: .+ + ram_total_size: [\d\.]+ + hours_used: [\d\.]+ +( hardware_used: .+ +)?base_model: sentence-transformers/paraphrase-albert-small-v2 +model-index: +- name: SetFit with sentence-transformers\/paraphrase-albert-small-v2 on SST2 + results: + - task: + type: text-classification + name: Text Classification + dataset: + name: SST2 + type: sst2 + split: test + metrics: + - type: accuracy + value: [\d\.]+ + name: Accuracy +--- + +\# SetFit with sentence\-transformers/paraphrase\-albert\-small\-v2 on SST2 + +This is a \[SetFit\]\(https://github\.com/huggingface/setfit\) model trained on the \[SST2\]\(https://huggingface\.co/datasets/sst2\) dataset that can be used for Text Classification\. This SetFit model uses \[sentence\-transformers/paraphrase\-albert\-small\-v2\]\(https://huggingface\.co/sentence\-transformers/paraphrase\-albert\-small\-v2\) as the Sentence Transformer embedding model\. A \[LogisticRegression\]\(https://scikit\-learn\.org/stable/modules/generated/sklearn\.linear_model\.LogisticRegression\.html\) instance is used for classification\. + +The model has been trained using an efficient few\-shot learning technique that involves: + +1\. Fine\-tuning a \[Sentence Transformer\]\(https://www\.sbert\.net\) with contrastive learning\. +2\. Training a classification head with features from the fine\-tuned Sentence Transformer\. + +## Model Details + +### Model Description +- \*\*Model Type:\*\* SetFit +- \*\*Sentence Transformer body:\*\* \[sentence-transformers/paraphrase-albert-small-v2\]\(https://huggingface.co/sentence-transformers/paraphrase-albert-small-v2\) +- \*\*Classification head:\*\* a \[LogisticRegression\]\(https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\) instance. +- \*\*Maximum Sequence Length:\*\* 100 tokens +- \*\*Number of Classes:\*\* 2 classes +- \*\*Training Dataset:\*\* \[SST2\]\(https://huggingface.co/datasets/sst2\) +- \*\*Language:\*\* en +- \*\*License:\*\* apache-2.0 + +### Model Sources + +- \*\*Repository:\*\* \[SetFit on GitHub\]\(https://github.com/huggingface/setfit\) +- \*\*Paper:\*\* \[Efficient Few-Shot Learning Without Prompts\]\(https://arxiv.org/abs/2209.11055\) +- \*\*Blogpost:\*\* \[SetFit: Efficient Few-Shot Learning Without Prompts\]\(https://huggingface.co/blog/setfit\) + +### Model Labels +\| Label\s+\| Examples\s+\| +\|:-+\|:-+\| +\| negative\s+\| [^\|]+ \| +\| positive\s+\| [^\|]+ \| + +## Evaluation + +### Metrics +\| Label \| Accuracy \| +\|:--------\|:---------\| +\| \*\*all\*\* \| [\d\.]+\s+\| + +## Uses + +### Direct Use for Inference + +First install the SetFit library: + +```bash +pip install setfit +``` + +Then you can load this model and run inference. + +```python +from setfit import SetFitModel + +# Download from [^H]+ Hub +model = SetFitModel.from_pretrained\("tomaarsen/setfit-paraphrase-albert-small-v2-sst2"\) +# Run inference +preds = model\(".+"\) +``` + + + + + + + + +## Training Details + +### Training Set Metrics +\| Training set \| Min \| Median \| Max \| +\|:-------------\|:----\|:--------\|:----\| +\| Word count \| 2 \| 11.4375 \| 33 \| + +\| Label \| Training Sample Count \| +\|:---------\|:----------------------\| +\| negative \| 8 \| +\| positive \| 8 \| + +### Training Hyperparameters +- batch_size: \(1, 1\) +- num_epochs: \(1, 16\) +- max_steps: 2 +- sampling_strategy: oversampling +- body_learning_rate: \(2e-05, 1e-05\) +- head_learning_rate: 0.01 +- loss: CosineSimilarityLoss +- distance_metric: cosine_distance +- margin: 0.25 +- end_to_end: False +- use_amp: False +- warmup_proportion: 0.1 +- seed: 42 +- load_best_model_at_end: False + +### Training Results +\| Epoch \| Step \| Training Loss \| Validation Loss \| +\|:------:\|:----:\|:-------------:\|:---------------:\| +(\| [\d\.]+ +\| [\d\.]+ +\| [\d\.]+ +\| [\d\.]+ +\|\n)+ +### Environmental Impact +Carbon emissions were measured using \[CodeCarbon\]\(https://github.com/mlco2/codecarbon\)\. +- \*\*Carbon Emitted\*\*: [\d\.]+ kg of CO2 +- \*\*Hours Used\*\*: [\d\.]+ hours + +### Training Hardware +- \*\*On Cloud\*\*: (Yes|No) +- \*\*GPU Model\*\*: [^\n]+ +- \*\*CPU Model\*\*: [^\n]+ +- \*\*RAM Size\*\*: [\d\.]+ GB + +### Framework Versions +- Python: [^\n]+ +- SetFit: [^\n]+ +- Sentence Transformers: [^\n]+ +- Transformers: [^\n]+ +- PyTorch: [^\n]+ +- Datasets: [^\n]+ +- Tokenizers: [^\n]+ + +## Citation + +### BibTeX +```bibtex +@article{https://doi.org/10.48550/arxiv.2209.11055, + doi = {10.48550/ARXIV.2209.11055}, + url = {https://arxiv.org/abs/2209.11055}, + author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}, + keywords = {Computation and Language \(cs.CL\), FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {Efficient Few-Shot Learning Without Prompts}, + publisher = {arXiv}, + year = \{2022\}, + copyright = {Creative Commons Attribution 4.0 International} +} +``` + + + + + +""", + flags=re.DOTALL, +) + +# with open("readme", "r") as f: +# readme = f.read() + +# match = MODEL_CARD_PATTERN.match(readme) +# print(match) +# print(match.group(0)[-10:]) +# breakpoint() \ No newline at end of file diff --git a/tests/test_model_card.py b/tests/test_model_card.py new file mode 100644 index 00000000..a1470eb6 --- /dev/null +++ b/tests/test_model_card.py @@ -0,0 +1,90 @@ +from pathlib import Path + +import pytest +from datasets import Dataset, load_dataset + +from setfit import ( + SetFitModel, + SetFitModelCardData, + Trainer, + TrainingArguments, +) +from setfit.data import sample_dataset +from setfit.model_card import generate_model_card, is_on_huggingface + +from .model_card_pattern import MODEL_CARD_PATTERN + + +def test_model_card(tmp_path: Path) -> None: + dataset = load_dataset("sst2") + train_dataset = sample_dataset(dataset["train"], label_column="label", num_samples=8) + eval_dataset = dataset["validation"].select(range(10)) + model = SetFitModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", + labels=["negative", "positive"], + model_card_data=SetFitModelCardData( + model_id="tomaarsen/setfit-paraphrase-albert-small-v2-sst2", + dataset_id="sst2", + dataset_name="SST2", + language=["en"], + license="apache-2.0", + ), + ) + + args = TrainingArguments( + str(tmp_path), + report_to="codecarbon", + batch_size=1, + eval_steps=1, + logging_steps=1, + max_steps=2, + evaluation_strategy="steps", + ) + trainer = Trainer( + model=model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + column_mapping={"sentence": "text"} + ) + trainer.train() + trainer.evaluate() + model_card = generate_model_card(trainer.model) + assert MODEL_CARD_PATTERN.fullmatch(model_card) + + +def test_model_card_languages() -> None: + model = SetFitModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", + model_card_data=SetFitModelCardData( + language=["en", "nl", "de"], + ), + ) + model_card = model.generate_model_card() + assert "**Languages:** en, nl, de" in model_card + + +def test_is_on_huggingface_edge_case() -> None: + assert not is_on_huggingface("test_value") + assert not is_on_huggingface("a/test/value") + + +@pytest.mark.parametrize("dataset_id", ("SetFit/emotion", "SetFit/sst2")) +def test_infer_dataset_id(dataset_id: str) -> None: + model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + train_dataset = load_dataset(dataset_id, split="train") + + # This triggers inferring the dataset_id from train_dataset + Trainer(model=model, train_dataset=train_dataset) + assert model.model_card_data.dataset_id == dataset_id + + +def test_cant_infer_dataset_id(): + model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") + train_dataset = Dataset.from_dict( + {"text": ["a", "b", "c", "d"], "label": [0, 1, 1, 0]} + ) + + # This triggers inferring the dataset_id from train_dataset + Trainer(model=model, train_dataset=train_dataset) + assert model.model_card_data.dataset_id == None From 5f36d0efad3714e383d1be2d8b6b5432c0e4dd5d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 30 Nov 2023 16:35:27 +0100 Subject: [PATCH 140/183] Reformat --- src/setfit/__init__.py | 2 +- src/setfit/model_card.py | 108 ++++++++++++++++++++++++------------ src/setfit/trainer.py | 8 ++- tests/model_card_pattern.py | 9 +-- tests/test_model_card.py | 13 +---- 5 files changed, 82 insertions(+), 58 deletions(-) diff --git a/src/setfit/__init__.py b/src/setfit/__init__.py index e9f103d0..3c0ead98 100644 --- a/src/setfit/__init__.py +++ b/src/setfit/__init__.py @@ -5,12 +5,12 @@ import warnings from .data import get_templated_dataset, sample_dataset +from .model_card import SetFitModelCardData from .modeling import SetFitHead, SetFitModel from .span import AbsaModel, AbsaTrainer, AspectExtractor, AspectModel, PolarityModel from .trainer import SetFitTrainer, Trainer from .trainer_distillation import DistillationSetFitTrainer, DistillationTrainer from .training_args import TrainingArguments -from .model_card import SetFitModelCardData # Ensure that DeprecationWarnings are shown by default, as recommended by diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index fdb1fc45..65f22d5b 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -1,6 +1,6 @@ -from collections import Counter, defaultdict import collections import random +from collections import Counter, defaultdict from dataclasses import dataclass, field, fields from pathlib import Path from platform import python_version @@ -11,32 +11,24 @@ import torch import transformers from datasets import Dataset -from huggingface_hub import ( - CardData, - DatasetFilter, - ModelCard, - dataset_info, - list_datasets, - model_info, -) +from huggingface_hub import CardData, DatasetFilter, ModelCard, dataset_info, list_datasets, model_info from huggingface_hub.repocard_data import EvalResult, eval_results_to_model_index from huggingface_hub.utils import yaml_dump -from transformers import TrainerCallback +from transformers import PretrainedConfig, TrainerCallback from transformers.integrations import CodeCarbonCallback -from transformers.modelcard import ( - make_markdown_table, -) +from transformers.modelcard import make_markdown_table from transformers.trainer_callback import TrainerControl, TrainerState from transformers.training_args import TrainingArguments -from transformers import PretrainedConfig from . import logging + logger = logging.get_logger(__name__) -from setfit import __version__ as setfit_version from sentence_transformers import __version__ as sentence_transformers_version +from setfit import __version__ as setfit_version + if TYPE_CHECKING: from setfit.modeling import SetFitModel @@ -49,19 +41,23 @@ def __init__(self, trainer: "Trainer") -> None: self.trainer = trainer callbacks = [ - callback for callback in self.trainer.callback_handler.callbacks if isinstance(callback, CodeCarbonCallback) + callback + for callback in self.trainer.callback_handler.callbacks + if isinstance(callback, CodeCarbonCallback) ] if callbacks: trainer.model.model_card_data.code_carbon_callback = callbacks[0] - def on_init_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, model: "SetFitModel", **kwargs): + def on_init_end( + self, args: TrainingArguments, state: TrainerState, control: TrainerControl, model: "SetFitModel", **kwargs + ): if not model.model_card_data.dataset_id: # Inferring is hacky - it may break in the future, so let's be safe try: model.model_card_data.infer_dataset_id(self.trainer.train_dataset) except Exception: pass - + dataset = self.trainer.eval_dataset or self.trainer.train_dataset if dataset is not None: if not model.model_card_data.widget: @@ -80,10 +76,31 @@ def on_train_begin( self, args: TrainingArguments, state: TrainerState, control: TrainerControl, model: "SetFitModel", **kwargs ) -> None: # model.model_card_data.hyperparameters = extract_hyperparameters_from_trainer(self.trainer) - ignore_keys = {"output_dir", "logging_dir", "logging_strategy", "logging_first_step", "logging_steps", "evaluation_strategy", "eval_steps", "eval_delay", "save_strategy", "save_steps", "save_total_limit", "metric_for_best_model", "greater_is_better", "report_to", "samples_per_label", "show_progress_bar"} + ignore_keys = { + "output_dir", + "logging_dir", + "logging_strategy", + "logging_first_step", + "logging_steps", + "evaluation_strategy", + "eval_steps", + "eval_delay", + "save_strategy", + "save_steps", + "save_total_limit", + "metric_for_best_model", + "greater_is_better", + "report_to", + "samples_per_label", + "show_progress_bar", + } get_name_keys = {"loss", "distance_metric"} args_dict = args.to_dict() - model.model_card_data.hyperparameters = {key: value.__name__ if key in get_name_keys else value for key, value in args_dict.items() if key not in ignore_keys and value is not None} + model.model_card_data.hyperparameters = { + key: value.__name__ if key in get_name_keys else value + for key, value in args_dict.items() + if key not in ignore_keys and value is not None + } def on_evaluate( self, @@ -95,7 +112,10 @@ def on_evaluate( **kwargs, ) -> None: # TODO: Highlight the loaded best step - if model.model_card_data.eval_lines_list and model.model_card_data.eval_lines_list[-1]["Step"] == state.global_step: + if ( + model.model_card_data.eval_lines_list + and model.model_card_data.eval_lines_list[-1]["Step"] == state.global_step + ): model.model_card_data.eval_lines_list[-1]["Validation Loss"] = metrics["eval_embedding_loss"] else: model.model_card_data.eval_lines_list.append( @@ -118,7 +138,10 @@ def on_log( **kwargs, ): if "embedding_loss" in logs: - if model.model_card_data.eval_lines_list and model.model_card_data.eval_lines_list[-1]["Step"] == state.global_step: + if ( + model.model_card_data.eval_lines_list + and model.model_card_data.eval_lines_list[-1]["Step"] == state.global_step + ): model.model_card_data.eval_lines_list[-1]["Training Loss"] = logs["embedding_loss"] else: model.model_card_data.eval_lines_list.append( @@ -222,7 +245,7 @@ class SetFitModelCardData(CardData): num_classes: Optional[int] = field(default=None, init=False) best_model_step: Optional[int] = field(default=None, init=False) metrics: List[str] = field(default_factory=lambda: ["accuracy"], init=False) - + # Computed once, always unchanged pipeline_tag: str = field(default="text-classification", init=False) library_name: str = field(default="setfit", init=False) @@ -305,14 +328,18 @@ def add_naive_word_count(sample: Dict[str, Any]) -> Dict[str, Any]: self.train_set_sentences_per_label_list = [ { "Label": str_label, - "Training Sample Count": counter[str_label if isinstance(sample_label, str) else self.model.label2id[str_label]], + "Training Sample Count": counter[ + str_label if isinstance(sample_label, str) else self.model.label2id[str_label] + ], } for str_label in self.model.labels ] else: self.train_set_sentences_per_label_list = [ { - "Label": self.model.labels[label] if self.model.labels and isinstance(label, int) else str(label), + "Label": self.model.labels[label] + if self.model.labels and isinstance(label, int) + else str(label), "Training Sample Count": count, } for label, count in sorted(counter.items()) @@ -330,13 +357,17 @@ def set_label_examples(self, dataset: Dataset) -> None: text = sample["text"] label = sample["label"] if label not in finished_labels: - examples[label].append(f'
  • {repr(text)}
    • ') + examples[label].append(f"
  • {repr(text)}
    • ") if len(examples[label]) >= num_examples_per_label: finished_labels.add(label) if len(finished_labels) == self.num_classes: break self.label_example_list = [ - {"Label": self.model.labels[label] if self.model.labels and isinstance(label, int) else label, "Examples": "
      " + "".join(example_set) + "
    "} for label, example_set in examples.items() + { + "Label": self.model.labels[label] if self.model.labels and isinstance(label, int) else label, + "Examples": "
      " + "".join(example_set) + "
    ", + } + for label, example_set in examples.items() ] def infer_dataset_id(self, dataset: Dataset) -> None: @@ -402,9 +433,7 @@ def infer_st_id(self, setfit_model_id: str) -> None: st_id_path = Path(st_id) # Sometimes the name_or_path ends exactly with the model_id, e.g. # "C:\\Users\\tom/.cache\\torch\\sentence_transformers\\BAAI_bge-small-en-v1.5\\" - candidate_model_ids = [ - "/".join(st_id_path.parts[-2:]) - ] + candidate_model_ids = ["/".join(st_id_path.parts[-2:])] # Sometimes the name_or_path its final part contains the full model_id, with "/" replaced with a "_", e.g. # "/root/.cache/torch/sentence_transformers/sentence-transformers_all-mpnet-base-v2/" # In that case, we take the last part, split on _, and try all combinations @@ -418,16 +447,13 @@ def infer_st_id(self, setfit_model_id: str) -> None: def set_st_id(self, model_id: str) -> None: if is_on_huggingface(model_id): - self.st_id = model_id + self.st_id = model_id def post_training_eval_results(self, results: Dict[str, float]) -> None: self.eval_results_dict = results results_without_split = {key.split("_", maxsplit=1)[1].title(): value for key, value in results.items()} - self.metric_lines = [{ - "Label": "**all**", - **results_without_split - }] + self.metric_lines = [{"Label": "**all**", **results_without_split}] def _maybe_round(self, v, decimals=4): if isinstance(v, float) and len(str(v).split(".")) > 1 and len(str(v).split(".")[1]) > decimals: @@ -458,13 +484,21 @@ def to_dict(self) -> Dict[str, Any]: ] super_dict["metrics"] = [metric_key.split("_", maxsplit=1)[1] for metric_key in self.eval_results_dict] super_dict["model-index"] = eval_results_to_model_index(self.model_name, eval_results) - eval_lines_list = [{key: f"**{self._maybe_round(value)}**" if line["Step"] == self.best_model_step else value for key, value in line.items()} for line in self.eval_lines_list] + eval_lines_list = [ + { + key: f"**{self._maybe_round(value)}**" if line["Step"] == self.best_model_step else value + for key, value in line.items() + } + for line in self.eval_lines_list + ] super_dict["eval_lines"] = make_markdown_table(eval_lines_list) super_dict["explain_bold_in_eval"] = "**" in super_dict["eval_lines"] # Replace |:---:| with |:---| for left alignment super_dict["label_examples"] = make_markdown_table(self.label_example_list).replace("-:|", "--|") super_dict["train_set_metrics"] = make_markdown_table(self.train_set_metrics_list).replace("-:|", "--|") - super_dict["train_set_sentences_per_label_list"] = make_markdown_table(self.train_set_sentences_per_label_list).replace("-:|", "--|") + super_dict["train_set_sentences_per_label_list"] = make_markdown_table( + self.train_set_sentences_per_label_list + ).replace("-:|", "--|") super_dict["metrics_table"] = make_markdown_table(self.metric_lines).replace("-:|", "--|") if self.code_carbon_callback and self.code_carbon_callback.tracker: emissions_data = self.code_carbon_callback.tracker._prepare_emissions_data() diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index f6ce88d2..5d4219ad 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -399,7 +399,9 @@ def train( ) train_parameters = self.dataset_to_parameters(self.train_dataset) - full_parameters = train_parameters + self.dataset_to_parameters(self.eval_dataset) if self.eval_dataset else train_parameters + full_parameters = ( + train_parameters + self.dataset_to_parameters(self.eval_dataset) if self.eval_dataset else train_parameters + ) self.train_embeddings(*full_parameters, args=args) self.train_classifier(*train_parameters, args=args) @@ -832,7 +834,9 @@ def evaluate(self, dataset: Optional[Dataset] = None, metric_key_prefix: str = " else: raise ValueError("metric must be a string or a callable") - self.model.model_card_data.post_training_eval_results({f"{metric_key_prefix}_{key}": value for key, value in results.items()}) + self.model.model_card_data.post_training_eval_results( + {f"{metric_key_prefix}_{key}": value for key, value in results.items()} + ) return results def hyperparameter_search( diff --git a/tests/model_card_pattern.py b/tests/model_card_pattern.py index f0132fc6..5c2c0c81 100644 --- a/tests/model_card_pattern.py +++ b/tests/model_card_pattern.py @@ -1,5 +1,6 @@ import re + MODEL_CARD_PATTERN = re.compile( """\ --- @@ -217,11 +218,3 @@ -->""", flags=re.DOTALL, ) - -# with open("readme", "r") as f: -# readme = f.read() - -# match = MODEL_CARD_PATTERN.match(readme) -# print(match) -# print(match.group(0)[-10:]) -# breakpoint() \ No newline at end of file diff --git a/tests/test_model_card.py b/tests/test_model_card.py index a1470eb6..c22924ff 100644 --- a/tests/test_model_card.py +++ b/tests/test_model_card.py @@ -3,12 +3,7 @@ import pytest from datasets import Dataset, load_dataset -from setfit import ( - SetFitModel, - SetFitModelCardData, - Trainer, - TrainingArguments, -) +from setfit import SetFitModel, SetFitModelCardData, Trainer, TrainingArguments from setfit.data import sample_dataset from setfit.model_card import generate_model_card, is_on_huggingface @@ -45,7 +40,7 @@ def test_model_card(tmp_path: Path) -> None: args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, - column_mapping={"sentence": "text"} + column_mapping={"sentence": "text"}, ) trainer.train() trainer.evaluate() @@ -81,9 +76,7 @@ def test_infer_dataset_id(dataset_id: str) -> None: def test_cant_infer_dataset_id(): model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") - train_dataset = Dataset.from_dict( - {"text": ["a", "b", "c", "d"], "label": [0, 1, 1, 0]} - ) + train_dataset = Dataset.from_dict({"text": ["a", "b", "c", "d"], "label": [0, 1, 1, 0]}) # This triggers inferring the dataset_id from train_dataset Trainer(model=model, train_dataset=train_dataset) From 086ee02ea52eabc53f85cffe9531415b9ed98e65 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Thu, 30 Nov 2023 16:38:40 +0100 Subject: [PATCH 141/183] Satisfy flake8 --- src/setfit/model_card.py | 14 ++++++-------- src/setfit/trainer.py | 4 ++-- tests/model_card_pattern.py | 2 ++ tests/test_model_card.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index 65f22d5b..70777807 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -14,22 +14,20 @@ from huggingface_hub import CardData, DatasetFilter, ModelCard, dataset_info, list_datasets, model_info from huggingface_hub.repocard_data import EvalResult, eval_results_to_model_index from huggingface_hub.utils import yaml_dump +from sentence_transformers import __version__ as sentence_transformers_version from transformers import PretrainedConfig, TrainerCallback from transformers.integrations import CodeCarbonCallback from transformers.modelcard import make_markdown_table from transformers.trainer_callback import TrainerControl, TrainerState from transformers.training_args import TrainingArguments +from setfit import __version__ as setfit_version + from . import logging logger = logging.get_logger(__name__) -from sentence_transformers import __version__ as sentence_transformers_version - -from setfit import __version__ as setfit_version - - if TYPE_CHECKING: from setfit.modeling import SetFitModel from setfit.trainer import Trainer @@ -275,7 +273,7 @@ def __post_init__(self): # if languages are not set, try to determine the language from the dataset on the Hub try: info = dataset_info(self.dataset_id) - except: + except Exception: pass else: if info.cardData: @@ -344,7 +342,7 @@ def add_naive_word_count(sample: Dict[str, Any]) -> Dict[str, Any]: } for label, count in sorted(counter.items()) ] - except: + except Exception: # There are some tricky edge cases possible, e.g. if the user provided integer labels that do not fall # between 0 to num_classes-1, so we make sure we never cause errors. pass @@ -546,7 +544,7 @@ def is_on_huggingface(repo_id: str, is_model: bool = True) -> bool: else: dataset_info(repo_id) return True - except: + except Exception: # Fetching models can fail for many reasons: Repository not existing, no internet access, HF down, etc. return False diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 5d4219ad..6f62a4db 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -192,14 +192,14 @@ def __init__( if train_dataset: self._validate_column_mapping(train_dataset) if self.column_mapping is not None: - logger.info(f"Applying column mapping to the training dataset") + logger.info("Applying column mapping to the training dataset") train_dataset = self._apply_column_mapping(train_dataset, self.column_mapping) self.train_dataset = train_dataset if eval_dataset: self._validate_column_mapping(eval_dataset) if self.column_mapping is not None: - logger.info(f"Applying column mapping to the evaluation dataset") + logger.info("Applying column mapping to the evaluation dataset") eval_dataset = self._apply_column_mapping(eval_dataset, self.column_mapping) self.eval_dataset = eval_dataset diff --git a/tests/model_card_pattern.py b/tests/model_card_pattern.py index 5c2c0c81..4bed4c6d 100644 --- a/tests/model_card_pattern.py +++ b/tests/model_card_pattern.py @@ -1,3 +1,5 @@ +# flake8: noqa + import re diff --git a/tests/test_model_card.py b/tests/test_model_card.py index c22924ff..2e965cf7 100644 --- a/tests/test_model_card.py +++ b/tests/test_model_card.py @@ -80,4 +80,4 @@ def test_cant_infer_dataset_id(): # This triggers inferring the dataset_id from train_dataset Trainer(model=model, train_dataset=train_dataset) - assert model.model_card_data.dataset_id == None + assert model.model_card_data.dataset_id is None From 4990b09800ee9edcdabd682b6df8ccc5880c66d7 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 13:19:23 +0100 Subject: [PATCH 142/183] Make model card generation more robust E.g. if Distillation Trainer or no training samples --- src/setfit/model_card.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index 70777807..eb5dd637 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -67,7 +67,7 @@ def on_init_end( try: model.model_card_data.num_classes = len(set(self.trainer.train_dataset["label"])) model.model_card_data.set_label_examples(self.trainer.train_dataset) - except TypeError: + except: pass def on_train_begin( @@ -300,7 +300,8 @@ def set_widget_examples(self, dataset: Dataset) -> None: self.widget = [{"text": sample} for sample in samples] samples.sort(key=len) - self.predict_example = samples[0] + if samples: + self.predict_example = samples[0] def set_train_set_metrics(self, dataset: Dataset) -> None: def add_naive_word_count(sample: Dict[str, Any]) -> Dict[str, Any]: @@ -316,7 +317,10 @@ def add_naive_word_count(sample: Dict[str, Any]) -> Dict[str, Any]: "Max": max(dataset["word_count"]), }, ] - # If not multi-label: + # E.g. if unlabeled via DistillationTrainer + if "label" not in dataset.column_names: + return + sample_label = dataset[0]["label"] if isinstance(sample_label, collections.abc.Sequence) and not isinstance(sample_label, str): return From 58d58153031e60ede4729488d5d3bf06dde7c13c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 13:19:37 +0100 Subject: [PATCH 143/183] Allow compute_metric to return a non-dict --- src/setfit/trainer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 6f62a4db..dd3e01e2 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -834,6 +834,8 @@ def evaluate(self, dataset: Optional[Dataset] = None, metric_key_prefix: str = " else: raise ValueError("metric must be a string or a callable") + if not isinstance(results, dict): + results = {"metric": results} self.model.model_card_data.post_training_eval_results( {f"{metric_key_prefix}_{key}": value for key, value in results.items()} ) From 61cf947cd648061d3955900a2ab301d268c5bc7d Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 13:20:08 +0100 Subject: [PATCH 144/183] Update tests as datasets are now column-mapped at init --- tests/test_deprecated_trainer.py | 47 +++++++++---------- tests/test_deprecated_trainer_distillation.py | 36 +++++++------- tests/test_trainer.py | 33 ++++++------- tests/test_trainer_distillation.py | 36 +++++++------- 4 files changed, 69 insertions(+), 83 deletions(-) diff --git a/tests/test_deprecated_trainer.py b/tests/test_deprecated_trainer.py index 8e1ce1d5..771838d5 100644 --- a/tests/test_deprecated_trainer.py +++ b/tests/test_deprecated_trainer.py @@ -84,57 +84,53 @@ def test_trainer_works_with_alternate_dataset_for_evaluate(self): def test_trainer_raises_error_with_missing_label(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = SetFitTrainer( - model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations - ) with pytest.raises(ValueError): - trainer.train() + SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) def test_trainer_raises_error_with_missing_text(self): """If the required columns are missing from the dataset, the library should throw an error and list the columns found.""" dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) - trainer = SetFitTrainer( - model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations - ) expected_message = re.escape( "SetFit expected the dataset to have the columns ['label', 'text'], " "but only the columns ['extra_column', 'label'] were found. " "Either make sure these columns are present, or specify which columns to use with column_mapping in Trainer." ) with pytest.raises(ValueError, match=expected_message): - trainer._validate_column_mapping(trainer.train_dataset) + SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) def test_column_mapping_raises_error_when_mapped_columns_missing(self): """If the columns specified in the column mapping are missing from the dataset, the library should throw an error and list the columns found.""" dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = SetFitTrainer( - model=self.model, - train_dataset=dataset, - eval_dataset=dataset, - num_iterations=self.num_iterations, - column_mapping={"text_new": "text", "label_new": "label"}, - ) expected_message = re.escape( "The column mapping expected the columns ['label_new', 'text_new'] in the dataset, " "but the dataset had the columns ['extra_column', 'text'].", ) with pytest.raises(ValueError, match=expected_message): - trainer._validate_column_mapping(trainer.train_dataset) + SetFitTrainer( + model=self.model, + train_dataset=dataset, + eval_dataset=dataset, + num_iterations=self.num_iterations, + column_mapping={"text_new": "text", "label_new": "label"}, + ) def test_trainer_raises_error_when_dataset_not_split(self): """Verify that an error is raised if we pass an unsplit dataset to the trainer.""" dataset = Dataset.from_dict({"text": ["a", "b", "c", "d"], "label": [0, 0, 1, 1]}).train_test_split( test_size=0.5 ) - trainer = SetFitTrainer( - model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations - ) expected_message = re.escape( "SetFit expected a Dataset, but it got a DatasetDict with the splits ['test', 'train']. " "Did you mean to select one of these splits from the dataset?", ) with pytest.raises(ValueError, match=expected_message): - trainer._validate_column_mapping(trainer.train_dataset) + SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) def test_trainer_raises_error_when_dataset_is_dataset_dict_with_train(self): """Verify that a useful error is raised if we pass an unsplit dataset with only a `train` split to the trainer.""" @@ -142,15 +138,14 @@ def test_trainer_raises_error_when_dataset_is_dataset_dict_with_train(self): path = pathlib.Path(tmpdirname) / "test_dataset_dict_with_train.csv" path.write_text("label,text\n1,good\n0,terrible\n") dataset = load_dataset("csv", data_files=str(path)) - trainer = SetFitTrainer( - model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations - ) expected_message = re.escape( "SetFit expected a Dataset, but it got a DatasetDict with the split ['train']. " "Did you mean to select the training split with dataset['train']?", ) with pytest.raises(ValueError, match=expected_message): - trainer._validate_column_mapping(trainer.train_dataset) + SetFitTrainer( + model=self.model, train_dataset=dataset, eval_dataset=dataset, num_iterations=self.num_iterations + ) def test_column_mapping_multilabel(self): dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[0, 1], [1, 2], [2, 0]]}) @@ -163,8 +158,8 @@ def test_column_mapping_multilabel(self): column_mapping={"text_new": "text", "label_new": "label"}, ) - trainer._validate_column_mapping(trainer.train_dataset) - formatted_dataset = trainer._apply_column_mapping(trainer.train_dataset, trainer.column_mapping) + trainer._validate_column_mapping(dataset) + formatted_dataset = trainer._apply_column_mapping(dataset, trainer.column_mapping) assert formatted_dataset.column_names == ["text", "label"] diff --git a/tests/test_deprecated_trainer_distillation.py b/tests/test_deprecated_trainer_distillation.py index 5d59c4f5..d476e73b 100644 --- a/tests/test_deprecated_trainer_distillation.py +++ b/tests/test_deprecated_trainer_distillation.py @@ -73,28 +73,26 @@ def test_trainer_raises_error_with_missing_label(self): def test_trainer_raises_error_with_missing_text(self): dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) - trainer = DistillationSetFitTrainer( - teacher_model=self.teacher_model, - train_dataset=dataset, - student_model=self.student_model, - eval_dataset=dataset, - num_iterations=self.num_iterations, - ) with pytest.raises(ValueError): - trainer.train() + DistillationSetFitTrainer( + teacher_model=self.teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + num_iterations=self.num_iterations, + ) def test_column_mapping_with_missing_text(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = DistillationSetFitTrainer( - teacher_model=self.teacher_model, - train_dataset=dataset, - student_model=self.student_model, - eval_dataset=dataset, - num_iterations=self.num_iterations, - column_mapping={"label_new": "label"}, - ) with pytest.raises(ValueError): - trainer._validate_column_mapping(trainer.train_dataset) + DistillationSetFitTrainer( + teacher_model=self.teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + num_iterations=self.num_iterations, + column_mapping={"label_new": "label"}, + ) def test_column_mapping_multilabel(self): dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[0, 1], [1, 2], [2, 0]]}) @@ -108,8 +106,8 @@ def test_column_mapping_multilabel(self): column_mapping={"text_new": "text", "label_new": "label"}, ) - trainer._validate_column_mapping(trainer.train_dataset) - formatted_dataset = trainer._apply_column_mapping(trainer.train_dataset, trainer.column_mapping) + trainer._validate_column_mapping(dataset) + formatted_dataset = trainer._apply_column_mapping(dataset, trainer.column_mapping) assert formatted_dataset.column_names == ["text", "label"] assert formatted_dataset[0]["text"] == "a" diff --git a/tests/test_trainer.py b/tests/test_trainer.py index b0de64ed..0814c758 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -97,51 +97,47 @@ def test_trainer_works_with_alternate_dataset_for_evaluate(self): def test_trainer_raises_error_with_missing_label(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) with pytest.raises(ValueError): - trainer.train() + Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) def test_trainer_raises_error_with_missing_text(self): """If the required columns are missing from the dataset, the library should throw an error and list the columns found.""" dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) - trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) expected_message = re.escape( "SetFit expected the dataset to have the columns ['label', 'text'], " "but only the columns ['extra_column', 'label'] were found. " "Either make sure these columns are present, or specify which columns to use with column_mapping in Trainer." ) with pytest.raises(ValueError, match=expected_message): - trainer._validate_column_mapping(trainer.train_dataset) + Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) def test_column_mapping_raises_error_when_mapped_columns_missing(self): """If the columns specified in the column mapping are missing from the dataset, the library should throw an error and list the columns found.""" dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = Trainer( - model=self.model, - args=self.args, - train_dataset=dataset, - eval_dataset=dataset, - column_mapping={"text_new": "text", "label_new": "label"}, - ) expected_message = re.escape( "The column mapping expected the columns ['label_new', 'text_new'] in the dataset, " "but the dataset had the columns ['extra_column', 'text'].", ) with pytest.raises(ValueError, match=expected_message): - trainer._validate_column_mapping(trainer.train_dataset) + Trainer( + model=self.model, + args=self.args, + train_dataset=dataset, + eval_dataset=dataset, + column_mapping={"text_new": "text", "label_new": "label"}, + ) def test_trainer_raises_error_when_dataset_not_split(self): """Verify that an error is raised if we pass an unsplit dataset to the trainer.""" dataset = Dataset.from_dict({"text": ["a", "b", "c", "d"], "label": [0, 0, 1, 1]}).train_test_split( test_size=0.5 ) - trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) expected_message = re.escape( "SetFit expected a Dataset, but it got a DatasetDict with the splits ['test', 'train']. " "Did you mean to select one of these splits from the dataset?", ) with pytest.raises(ValueError, match=expected_message): - trainer._validate_column_mapping(trainer.train_dataset) + Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) def test_trainer_raises_error_when_dataset_is_dataset_dict_with_train(self): """Verify that a useful error is raised if we pass an unsplit dataset with only a `train` split to the trainer.""" @@ -149,13 +145,12 @@ def test_trainer_raises_error_when_dataset_is_dataset_dict_with_train(self): path = Path(tmpdirname) / "test_dataset_dict_with_train.csv" path.write_text("label,text\n1,good\n0,terrible\n") dataset = load_dataset("csv", data_files=str(path)) - trainer = Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) expected_message = re.escape( "SetFit expected a Dataset, but it got a DatasetDict with the split ['train']. " "Did you mean to select the training split with dataset['train']?", ) with pytest.raises(ValueError, match=expected_message): - trainer._validate_column_mapping(trainer.train_dataset) + Trainer(model=self.model, args=self.args, train_dataset=dataset, eval_dataset=dataset) def test_column_mapping_multilabel(self): dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[0, 1], [1, 2], [2, 0]]}) @@ -168,8 +163,8 @@ def test_column_mapping_multilabel(self): column_mapping={"text_new": "text", "label_new": "label"}, ) - trainer._validate_column_mapping(trainer.train_dataset) - formatted_dataset = trainer._apply_column_mapping(trainer.train_dataset, trainer.column_mapping) + trainer._validate_column_mapping(dataset) + formatted_dataset = trainer._apply_column_mapping(dataset, trainer.column_mapping) assert formatted_dataset.column_names == ["text", "label"] @@ -585,7 +580,7 @@ def test_train_amp_save(model: SetFitModel, tmp_path: Path) -> None: trainer = Trainer(model, args=args, train_dataset=dataset, eval_dataset=dataset) trainer.train() assert trainer.evaluate() == {"accuracy": 1.0} - assert os.listdir(tmp_path) == ["step_5"] + assert "step_5" in os.listdir(tmp_path) def test_train_load_best(model: SetFitModel, tmp_path: Path, caplog: LogCaptureFixture) -> None: diff --git a/tests/test_trainer_distillation.py b/tests/test_trainer_distillation.py index 7ddb4ba3..866c37ad 100644 --- a/tests/test_trainer_distillation.py +++ b/tests/test_trainer_distillation.py @@ -71,28 +71,26 @@ def test_trainer_raises_error_with_missing_label(self): def test_trainer_raises_error_with_missing_text(self): dataset = Dataset.from_dict({"label": [0, 1, 2], "extra_column": ["d", "e", "f"]}) - trainer = DistillationTrainer( - teacher_model=self.teacher_model, - train_dataset=dataset, - student_model=self.student_model, - eval_dataset=dataset, - args=self.args, - ) with pytest.raises(ValueError): - trainer.train() + DistillationTrainer( + teacher_model=self.teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + args=self.args, + ) def test_column_mapping_with_missing_text(self): dataset = Dataset.from_dict({"text": ["a", "b", "c"], "extra_column": ["d", "e", "f"]}) - trainer = DistillationTrainer( - teacher_model=self.teacher_model, - train_dataset=dataset, - student_model=self.student_model, - eval_dataset=dataset, - args=self.args, - column_mapping={"label_new": "label"}, - ) with pytest.raises(ValueError): - trainer._validate_column_mapping(trainer.train_dataset) + DistillationTrainer( + teacher_model=self.teacher_model, + train_dataset=dataset, + student_model=self.student_model, + eval_dataset=dataset, + args=self.args, + column_mapping={"label_new": "label"}, + ) def test_column_mapping_multilabel(self): dataset = Dataset.from_dict({"text_new": ["a", "b", "c"], "label_new": [[0, 1], [1, 2], [2, 0]]}) @@ -106,8 +104,8 @@ def test_column_mapping_multilabel(self): column_mapping={"text_new": "text", "label_new": "label"}, ) - trainer._validate_column_mapping(trainer.train_dataset) - formatted_dataset = trainer._apply_column_mapping(trainer.train_dataset, trainer.column_mapping) + trainer._validate_column_mapping(dataset) + formatted_dataset = trainer._apply_column_mapping(dataset, trainer.column_mapping) assert formatted_dataset.column_names == ["text", "label"] assert formatted_dataset[0]["text"] == "a" From 751ba807bdedd378bde0c03222b2ead7d466a7c4 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 13:36:05 +0100 Subject: [PATCH 145/183] Avoid bare except --- src/setfit/model_card.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index eb5dd637..cff99b3b 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -67,7 +67,7 @@ def on_init_end( try: model.model_card_data.num_classes = len(set(self.trainer.train_dataset["label"])) model.model_card_data.set_label_examples(self.trainer.train_dataset) - except: + except Exception: pass def on_train_begin( From 4a7255d235981d815caddcb951479beeade8ded8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 13:36:44 +0100 Subject: [PATCH 146/183] Avoid walrus operator for now for Python 3.7 compat --- src/setfit/model_card.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index cff99b3b..d7d9b737 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -384,7 +384,8 @@ def normalize(dataset_id: str) -> str: dataset_id = dataset_id.replace(token, "") return dataset_id.lower() - if (cache_files := dataset.cache_files) and "filename" in cache_files[0]: + cache_files = dataset.cache_files + if cache_files and "filename" in cache_files[0]: cache_path_parts = Path(cache_files[0]["filename"]).parts # Check if the cachefile is under "huggingface/datasets" subtuple = ("huggingface", "datasets") From 803213181985115872172d293d27ba4247f618d4 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 14:00:38 +0100 Subject: [PATCH 147/183] Increase minimal datasets version for dataset inferring --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9437b96c..1b2080bf 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ INTEGRATIONS_REQUIRE = ["optuna"] REQUIRED_PKGS = [ - "datasets>=2.3.0", + "datasets>=2.14.0", # Required for dataset inferring "sentence-transformers>=2.2.1", "evaluate>=0.3.0", "huggingface_hub>=0.13.0", From 54d71275813ccc744f86a5400328c7c09bf7b122 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 14:11:02 +0100 Subject: [PATCH 148/183] Keep datasets version low, but skip test if < 2.14 --- setup.py | 2 +- tests/test_model_card.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1b2080bf..9437b96c 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ INTEGRATIONS_REQUIRE = ["optuna"] REQUIRED_PKGS = [ - "datasets>=2.14.0", # Required for dataset inferring + "datasets>=2.3.0", "sentence-transformers>=2.2.1", "evaluate>=0.3.0", "huggingface_hub>=0.13.0", diff --git a/tests/test_model_card.py b/tests/test_model_card.py index 2e965cf7..c18b5e1a 100644 --- a/tests/test_model_card.py +++ b/tests/test_model_card.py @@ -1,7 +1,9 @@ from pathlib import Path +import datasets import pytest from datasets import Dataset, load_dataset +from packaging.version import Version, parse from setfit import SetFitModel, SetFitModelCardData, Trainer, TrainingArguments from setfit.data import sample_dataset @@ -64,6 +66,7 @@ def test_is_on_huggingface_edge_case() -> None: assert not is_on_huggingface("a/test/value") +@pytest.mark.skipif(parse(datasets.__version__) < Version("2.14.0")) @pytest.mark.parametrize("dataset_id", ("SetFit/emotion", "SetFit/sst2")) def test_infer_dataset_id(dataset_id: str) -> None: model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") From 87420b3407f3f85992602a6ae7a6855e50e62bbd Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Fri, 1 Dec 2023 14:20:14 +0100 Subject: [PATCH 149/183] Add reason to skipif --- tests/test_model_card.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_model_card.py b/tests/test_model_card.py index c18b5e1a..73cdc92d 100644 --- a/tests/test_model_card.py +++ b/tests/test_model_card.py @@ -66,7 +66,9 @@ def test_is_on_huggingface_edge_case() -> None: assert not is_on_huggingface("a/test/value") -@pytest.mark.skipif(parse(datasets.__version__) < Version("2.14.0")) +@pytest.mark.skipif( + parse(datasets.__version__) < Version("2.14.0"), reason="Inferring dataset_id only works from datasets >= 2.14.0" +) @pytest.mark.parametrize("dataset_id", ("SetFit/emotion", "SetFit/sst2")) def test_infer_dataset_id(dataset_id: str) -> None: model = SetFitModel.from_pretrained("sentence-transformers/paraphrase-albert-small-v2") From a73cb692c56ed21ff11e0188737508f4ad12673a Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 11:01:44 +0100 Subject: [PATCH 150/183] Always return dicts in id2label/label2id --- src/setfit/modeling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index d249535c..18d2079e 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -233,14 +233,14 @@ def has_differentiable_head(self) -> bool: def id2label(self) -> Dict[int, str]: """Return a mapping from integer IDs to string labels.""" if self.labels is None: - return None + return {} return dict(enumerate(self.labels)) @property def label2id(self) -> Dict[str, int]: """Return a mapping from string labels to integer IDs.""" if self.labels is None: - return None + return {} return {label: idx for idx, label in enumerate(self.labels)} def fit( From 859691bec32ebb78b097b628ec621680a918ac28 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 11:26:36 +0100 Subject: [PATCH 151/183] Introduce "no aspect", "aspect" labels for AspectModel --- src/setfit/span/modeling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index 6fb86dbe..08b01f63 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -101,11 +101,10 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None class AspectModel(SpanSetFitModel): - # TODO: Assumes binary SetFitModel with 0 == no aspect, 1 == aspect def __call__(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[bool]: sentence_preds = super().__call__(docs, aspects_list) return [ - [aspect for aspect, pred in zip(aspects, preds) if pred == 1] + [aspect for aspect, pred in zip(aspects, preds) if pred == "aspect"] for aspects, preds in zip(aspects_list, sentence_preds) ] @@ -208,6 +207,7 @@ def from_pretrained( local_files_only=local_files_only, use_differentiable_head=use_differentiable_head, normalize_embeddings=normalize_embeddings, + labels = ["no aspect", "aspect"], **model_kwargs, ) if polarity_model_id: From f9e6acb3c429a676e52080ee3bfe5d6969e9e1c9 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 11:36:15 +0100 Subject: [PATCH 152/183] Extend model card generation to ABSA + Tests --- src/setfit/model_card.py | 15 +- src/setfit/model_card_template.md | 36 +++- src/setfit/span/modeling.py | 30 ++- tests/model_card_pattern.py | 5 +- tests/span/__init__.py | 0 tests/span/aspect_model_card_pattern.py | 233 +++++++++++++++++++++ tests/span/polarity_model_card_pattern.py | 235 ++++++++++++++++++++++ tests/span/test_model_card.py | 59 ++++++ 8 files changed, 593 insertions(+), 20 deletions(-) create mode 100644 tests/span/__init__.py create mode 100644 tests/span/aspect_model_card_pattern.py create mode 100644 tests/span/polarity_model_card_pattern.py create mode 100644 tests/span/test_model_card.py diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index d7d9b737..3158757a 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -135,19 +135,19 @@ def on_log( logs: Dict[str, float], **kwargs, ): - if "embedding_loss" in logs: + keys = {"embedding_loss", "polarity_embedding_loss", "aspect_embedding_loss"} & set(logs) + if keys: if ( model.model_card_data.eval_lines_list and model.model_card_data.eval_lines_list[-1]["Step"] == state.global_step ): - model.model_card_data.eval_lines_list[-1]["Training Loss"] = logs["embedding_loss"] + model.model_card_data.eval_lines_list[-1]["Training Loss"] = logs[keys.pop()] else: model.model_card_data.eval_lines_list.append( { - # "Training Loss": self.state.log_history[-1]["loss"] if "loss" in self.state.log_history[-1] else "-", "Epoch": state.epoch, "Step": state.global_step, - "Training Loss": logs["embedding_loss"], + "Training Loss": logs[keys.pop()], "Validation Loss": "-", } ) @@ -225,7 +225,7 @@ class SetFitModelCardData(CardData): dataset_name: Optional[str] = None dataset_id: Optional[str] = None dataset_revision: Optional[str] = None - task_name: str = "Text Classification" + task_name: Optional[str] = None st_id: Optional[str] = None # Automatically filled by `ModelCardCallback` and the Trainer directly @@ -260,6 +260,9 @@ class SetFitModelCardData(CardData): init=False, ) + # ABSA-related arguments + absa: Dict[str, Any] = field(default=None, init=False, repr=False) + # Passed via `register_model` only model: Optional["SetFitModel"] = field(default=None, init=False, repr=False) head_class: Optional[str] = field(default=None, init=False, repr=False) @@ -525,6 +528,8 @@ def to_dict(self) -> Dict[str, Any]: if super_dict["num_classes"] is None: if self.model.labels: super_dict["num_classes"] = len(self.model.labels) + if super_dict["absa"]: + super_dict.update(super_dict.pop("absa")) for key in IGNORED_FIELDS: super_dict.pop(key, None) diff --git a/src/setfit/model_card_template.md b/src/setfit/model_card_template.md index d49c4f52..fabdadb0 100644 --- a/src/setfit/model_card_template.md +++ b/src/setfit/model_card_template.md @@ -4,15 +4,21 @@ {{ card_data }} --- -# {{ model_name | default("SetFit for Text Classification", true) }} +# {{ model_name if model_name else ( "SetFit Aspect Model for Aspect Based Sentiment Analysis" if is_aspect else ( "SetFit Polarity Model for Aspect Based Sentiment Analysis" if is_aspect is False else "SetFit Model for Text Classification"))}} -This is a [SetFit](https://github.com/huggingface/setfit) model{% if dataset_id %} trained on the [{{ dataset_name if dataset_name else dataset_id }}](https://huggingface.co/datasets/{{ dataset_id }}) dataset{% endif %} that can be used for {{ task_name | default("Text Classification", true) }}.{% if st_id %} This SetFit model uses [{{ st_id }}](https://huggingface.co/{{ st_id }}) as the Sentence Transformer embedding model.{% endif %} A {{ head_class }} instance is used for classification. +This is a [SetFit](https://github.com/huggingface/setfit) model{% if dataset_id %} trained on the [{{ dataset_name if dataset_name else dataset_id }}](https://huggingface.co/datasets/{{ dataset_id }}) dataset{% endif %} that can be used for {{ task_name | default("Text Classification", true) }}.{% if st_id %} This SetFit model uses [{{ st_id }}](https://huggingface.co/{{ st_id }}) as the Sentence Transformer embedding model.{% endif %} A {{ head_class }} instance is used for classification.{% if is_absa %} In particular, this model is in charge of {{ "filtering aspect span candidates" if is_aspect else "classifying aspect polarities"}}.{% endif %} The model has been trained using an efficient few-shot learning technique that involves: 1. Fine-tuning a [Sentence Transformer](https://www.sbert.net) with contrastive learning. 2. Training a classification head with features from the fine-tuned Sentence Transformer. +{% if is_absa %} +This model was trained within the context of a larger system for ABSA, which looks like so: +1. Use a spaCy model to select possible aspect span candidates. +2. {{ "**" if is_aspect else "" }}Use {{ "this" if is_aspect else "a" }} SetFit model to filter these possible aspect span candidates.{{ "**" if is_aspect else "" }} +3. {{ "**" if not is_aspect else "" }}Use {{ "this" if not is_aspect else "a" }} SetFit model to classify the filtered aspect span candidates.{{ "**" if not is_aspect else "" }} +{% endif %} ## Model Details ### Model Description @@ -23,9 +29,15 @@ The model has been trained using an efficient few-shot learning technique that i {%- endif %} {% if head_class -%} - - **Classification head:** a {{ head_class }} instance. + - **Classification head:** a {{ head_class }} instance {%- else -%} - + +{%- endif %} +{%- if aspect_model %} +- **SetFitABSA Aspect Model:** [{{ aspect_model }}](https://huggingface.co/{{ aspect_model }}) +{%- endif %} +{%- if polarity_model %} +- **SetFitABSA Polarity Model:** [{{ polarity_model }}](https://huggingface.co/{{ polarity_model }}) {%- endif %} - **Maximum Sequence Length:** {{ model_max_length }} tokens {% if num_classes -%} @@ -78,15 +90,29 @@ pip install setfit ``` Then you can load this model and run inference. +{% if is_absa %} +```python +from setfit import AbsaModel +# Download from the {{ hf_emoji }} Hub +model = AbsaModel.from_pretrained( + "{{ aspect_model }}", + "{{ polarity_model }}", +) +# Run inference +preds = model("The food was great, but the venue is just way too busy.") +``` +{%- else %} ```python from setfit import SetFitModel -# Download from {{ hf_emoji }} Hub +# Download from the {{ hf_emoji }} Hub model = SetFitModel.from_pretrained("{{ model_id | default('setfit_model_id', true) }}") # Run inference preds = model("{{ predict_example | default("I loved the spiderman movie!", true) | replace('"', '\\"') }}") ``` +{%- endif %} + +- \*\*Language:\*\* en +- \*\*License:\*\* apache-2.0 + +### Model Sources + +- \*\*Repository:\*\* \[SetFit on GitHub\]\(https://github.com/huggingface/setfit\) +- \*\*Paper:\*\* \[Efficient Few-Shot Learning Without Prompts\]\(https://arxiv.org/abs/2209.11055\) +- \*\*Blogpost:\*\* \[SetFit: Efficient Few-Shot Learning Without Prompts\]\(https://huggingface.co/blog/setfit\) + +### Model Labels +\| Label\s+\| Examples\s+\| +\|:-+\|:-+\| +\| no aspect\s+\| [^\|]+ \| +\| aspect\s+\| [^\|]+ \| + +## Evaluation + +### Metrics +\| Label \| Accuracy \| +\|:--------\|:---------\| +\| \*\*all\*\* \| [\d\.]+\s+\| + +## Uses + +### Direct Use for Inference + +First install the SetFit library: + +```bash +pip install setfit +``` + +Then you can load this model and run inference. + +```python +from setfit import AbsaModel + +# Download from the [^H]+ Hub +model = AbsaModel.from_pretrained\( + "[^\"]+", + "[^\"]+", +\) +# Run inference +preds = model\(".+"\) +``` + + + + + + + + + +## Training Details + +### Training Set Metrics +\| Training set \| Min \| Median \| Max \| +\|:-------------\|:----\|:--------\|:----\| +\| Word count \| 13 \| 29.6875 \| 36 \| + +\| Label \| Training Sample Count \| +\|:----------\|:----------------------\| +\| no aspect \| 10 \| +\| aspect \| 6 \| + +### Training Hyperparameters +- batch_size: \(1, 1\) +- num_epochs: \(1, 16\) +- max_steps: 2 +- sampling_strategy: oversampling +- body_learning_rate: \(2e-05, 1e-05\) +- head_learning_rate: 0.01 +- loss: CosineSimilarityLoss +- distance_metric: cosine_distance +- margin: 0.25 +- end_to_end: False +- use_amp: False +- warmup_proportion: 0.1 +- seed: 42 +- load_best_model_at_end: False + +### Training Results +\| Epoch \| Step \| Training Loss \| Validation Loss \| +\|:------:\|:----:\|:-------------:\|:---------------:\| +(\| [\d\.]+ +\| [\d\.]+ +\| [\d\.]+ +\| [\d\.]+ +\|\n)+ +### Environmental Impact +Carbon emissions were measured using \[CodeCarbon\]\(https://github.com/mlco2/codecarbon\)\. +- \*\*Carbon Emitted\*\*: [\d\.]+ kg of CO2 +- \*\*Hours Used\*\*: [\d\.]+ hours + +### Training Hardware +- \*\*On Cloud\*\*: (Yes|No) +- \*\*GPU Model\*\*: [^\n]+ +- \*\*CPU Model\*\*: [^\n]+ +- \*\*RAM Size\*\*: [\d\.]+ GB + +### Framework Versions +- Python: [^\n]+ +- SetFit: [^\n]+ +- Sentence Transformers: [^\n]+ +- Transformers: [^\n]+ +- PyTorch: [^\n]+ +- Datasets: [^\n]+ +- Tokenizers: [^\n]+ + +## Citation + +### BibTeX +```bibtex +@article{https://doi.org/10.48550/arxiv.2209.11055, + doi = {10.48550/ARXIV.2209.11055}, + url = {https://arxiv.org/abs/2209.11055}, + author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}, + keywords = {Computation and Language \(cs.CL\), FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {Efficient Few-Shot Learning Without Prompts}, + publisher = {arXiv}, + year = \{2022\}, + copyright = {Creative Commons Attribution 4.0 International} +} +``` + + + + + +""", + flags=re.DOTALL, +) diff --git a/tests/span/polarity_model_card_pattern.py b/tests/span/polarity_model_card_pattern.py new file mode 100644 index 00000000..3aafef61 --- /dev/null +++ b/tests/span/polarity_model_card_pattern.py @@ -0,0 +1,235 @@ +# flake8: noqa + +import re + + +POLARITY_MODEL_CARD_PATTERN = re.compile( + """\ +--- +language: +- en +license: apache-2\.0 +library_name: setfit +tags: +- setfit +- absa +- sentence-transformers +- text-classification +- generated_from_setfit_trainer +metrics: +- accuracy +widget: +- text: .* +pipeline_tag: text-classification +inference: false +co2_eq_emissions: + emissions: [\d\.\-e]+ + source: codecarbon + training_type: fine-tuning + on_cloud: (false|true) + cpu_model: .+ + ram_total_size: [\d\.]+ + hours_used: [\d\.]+ +( hardware_used: .+ +)?base_model: sentence-transformers/paraphrase-albert-small-v2 +model-index: +- name: SetFit Polarity Model with sentence-transformers\/paraphrase-albert-small-v2 + results: + - task: + type: text-classification + name: Text Classification + dataset: + name: Unknown + type: unknown + split: test + metrics: + - type: accuracy + value: [\d\.]+ + name: Accuracy +--- + +\# SetFit Polarity Model with sentence\-transformers/paraphrase\-albert\-small\-v2 + +This is a \[SetFit\]\(https://github\.com/huggingface/setfit\) model that can be used for Aspect Based Sentiment Analysis \(ABSA\)\. This SetFit model uses \[sentence\-transformers/paraphrase\-albert\-small\-v2\]\(https://huggingface\.co/sentence\-transformers/paraphrase\-albert\-small\-v2\) as the Sentence Transformer embedding model\. A \[LogisticRegression\]\(https://scikit\-learn\.org/stable/modules/generated/sklearn\.linear_model\.LogisticRegression\.html\) instance is used for classification\. In particular, this model is in charge of (filtering aspect span candidates|classifying aspect polarities)\. + +The model has been trained using an efficient few\-shot learning technique that involves: + +1\. Fine\-tuning a \[Sentence Transformer\]\(https://www\.sbert\.net\) with contrastive learning\. +2\. Training a classification head with features from the fine\-tuned Sentence Transformer\. + +This model was trained within the context of a larger system for ABSA, which looks like so\: + +1\. Use a spaCy model to select possible aspect span candidates\. +2\. Use a SetFit model to filter these possible aspect span candidates\. +3\. \*\*Use this SetFit model to classify the filtered aspect span candidates\.\*\* + +## Model Details + +### Model Description +- \*\*Model Type:\*\* SetFit +- \*\*Sentence Transformer body:\*\* \[sentence-transformers/paraphrase-albert-small-v2\]\(https://huggingface.co/sentence-transformers/paraphrase-albert-small-v2\) +- \*\*Classification head:\*\* a \[LogisticRegression\]\(https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\) instance +- \*\*SetFitABSA Aspect Model:\*\* \[\S+\]\(https:\/\/huggingface\.co/\S+\) +- \*\*SetFitABSA Polarity Model:\*\* \[\S+\]\(https:\/\/huggingface\.co/\S+\) +- \*\*Maximum Sequence Length:\*\* 100 tokens +- \*\*Number of Classes:\*\* 3 classes + +- \*\*Language:\*\* en +- \*\*License:\*\* apache-2.0 + +### Model Sources + +- \*\*Repository:\*\* \[SetFit on GitHub\]\(https://github.com/huggingface/setfit\) +- \*\*Paper:\*\* \[Efficient Few-Shot Learning Without Prompts\]\(https://arxiv.org/abs/2209.11055\) +- \*\*Blogpost:\*\* \[SetFit: Efficient Few-Shot Learning Without Prompts\]\(https://huggingface.co/blog/setfit\) + +### Model Labels +\| Label\s+\| Examples\s+\| +\|:-+\|:-+\| +\| neutral\s+\| [^\|]+ \| +\| positive\s+\| [^\|]+ \| +\| negative\s+\| [^\|]+ \| + +## Evaluation + +### Metrics +\| Label \| Accuracy \| +\|:--------\|:---------\| +\| \*\*all\*\* \| [\d\.]+\s+\| + +## Uses + +### Direct Use for Inference + +First install the SetFit library: + +```bash +pip install setfit +``` + +Then you can load this model and run inference. + +```python +from setfit import AbsaModel + +# Download from the [^H]+ Hub +model = AbsaModel.from_pretrained\( + "[^\"]+", + "[^\"]+", +\) +# Run inference +preds = model\(".+"\) +``` + + + + + + + + + +## Training Details + +### Training Set Metrics +\| Training set \| Min \| Median \| Max \| +\|:-------------\|:----\|:--------\|:----\| +\| Word count \| 22 \| 33.3333 \| 42 \| + +\| Label \| Training Sample Count \| +\|:---------\|:----------------------\| +\| negative \| 1 \| +\| neutral \| 2 \| +\| positive \| 3 \| + +### Training Hyperparameters +- batch_size: \(1, 1\) +- num_epochs: \(1, 16\) +- max_steps: 2 +- sampling_strategy: oversampling +- body_learning_rate: \(2e-05, 1e-05\) +- head_learning_rate: 0.01 +- loss: CosineSimilarityLoss +- distance_metric: cosine_distance +- margin: 0.25 +- end_to_end: False +- use_amp: False +- warmup_proportion: 0.1 +- seed: 42 +- load_best_model_at_end: False + +### Training Results +\| Epoch \| Step \| Training Loss \| Validation Loss \| +\|:------:\|:----:\|:-------------:\|:---------------:\| +(\| [\d\.]+ +\| [\d\.]+ +\| [\d\.]+ +\| [\d\.]+ +\|\n)+ +### Environmental Impact +Carbon emissions were measured using \[CodeCarbon\]\(https://github.com/mlco2/codecarbon\)\. +- \*\*Carbon Emitted\*\*: [\d\.]+ kg of CO2 +- \*\*Hours Used\*\*: [\d\.]+ hours + +### Training Hardware +- \*\*On Cloud\*\*: (Yes|No) +- \*\*GPU Model\*\*: [^\n]+ +- \*\*CPU Model\*\*: [^\n]+ +- \*\*RAM Size\*\*: [\d\.]+ GB + +### Framework Versions +- Python: [^\n]+ +- SetFit: [^\n]+ +- Sentence Transformers: [^\n]+ +- Transformers: [^\n]+ +- PyTorch: [^\n]+ +- Datasets: [^\n]+ +- Tokenizers: [^\n]+ + +## Citation + +### BibTeX +```bibtex +@article{https://doi.org/10.48550/arxiv.2209.11055, + doi = {10.48550/ARXIV.2209.11055}, + url = {https://arxiv.org/abs/2209.11055}, + author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}, + keywords = {Computation and Language \(cs.CL\), FOS: Computer and information sciences, FOS: Computer and information sciences}, + title = {Efficient Few-Shot Learning Without Prompts}, + publisher = {arXiv}, + year = \{2022\}, + copyright = {Creative Commons Attribution 4.0 International} +} +``` + + + + + +""", + flags=re.DOTALL, +) diff --git a/tests/span/test_model_card.py b/tests/span/test_model_card.py new file mode 100644 index 00000000..0cefe0ef --- /dev/null +++ b/tests/span/test_model_card.py @@ -0,0 +1,59 @@ +from pathlib import Path + +from datasets import load_dataset + +from setfit import AbsaModel, AbsaTrainer, SetFitModelCardData, TrainingArguments + +from .aspect_model_card_pattern import ASPECT_MODEL_CARD_PATTERN +from .polarity_model_card_pattern import POLARITY_MODEL_CARD_PATTERN + + +def test_model_card(tmp_path: Path) -> None: + dataset = load_dataset("tomaarsen/setfit-absa-semeval-laptops", split="train") + train_dataset = dataset.select(range(10)) + eval_dataset = dataset.select(range(10, 20)) + model = AbsaModel.from_pretrained( + "sentence-transformers/paraphrase-albert-small-v2", + model_card_data=SetFitModelCardData( + model_id="tomaarsen/setfit-absa-paraphrase-albert-small-v2-laptops", + language=["en"], + license="apache-2.0", + ), + ) + + args = TrainingArguments( + str(tmp_path), + report_to="codecarbon", + batch_size=1, + eval_steps=1, + logging_steps=1, + max_steps=2, + evaluation_strategy="steps", + ) + trainer = AbsaTrainer( + model=model, + args=args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + ) + trainer.train() + trainer.evaluate() + # for model_type, model in (("aspect", model.aspect_model), ("polarity", model.polarity_model)): + # model.create_model_card(tmp_path / model_type, tmp_path / model_type) + # with open(tmp_path / model_type / "README.md", "r", encoding="utf8") as f: + # model_card = f.read() + # with open(f"{model_type}_model_card.md", "w", encoding="utf8") as f: + # f.write(model_card) + # assert MODEL_CARD_PATTERN.fullmatch(model_card) + + path = tmp_path / "aspect" + model.aspect_model.create_model_card(path, path) + with open(path / "README.md", "r", encoding="utf8") as f: + model_card = f.read() + assert ASPECT_MODEL_CARD_PATTERN.fullmatch(model_card) + + path = tmp_path / "polarity" + model.polarity_model.create_model_card(path, path) + with open(path / "README.md", "r", encoding="utf8") as f: + model_card = f.read() + assert POLARITY_MODEL_CARD_PATTERN.fullmatch(model_card) From 4c4a9aac30bd8da0885ac797b6434fef68e13637 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 11:53:33 +0100 Subject: [PATCH 153/183] Correctly use create_model_card in ABSA test --- tests/span/test_model_card.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/span/test_model_card.py b/tests/span/test_model_card.py index 0cefe0ef..4c98d811 100644 --- a/tests/span/test_model_card.py +++ b/tests/span/test_model_card.py @@ -38,22 +38,15 @@ def test_model_card(tmp_path: Path) -> None: ) trainer.train() trainer.evaluate() - # for model_type, model in (("aspect", model.aspect_model), ("polarity", model.polarity_model)): - # model.create_model_card(tmp_path / model_type, tmp_path / model_type) - # with open(tmp_path / model_type / "README.md", "r", encoding="utf8") as f: - # model_card = f.read() - # with open(f"{model_type}_model_card.md", "w", encoding="utf8") as f: - # f.write(model_card) - # assert MODEL_CARD_PATTERN.fullmatch(model_card) path = tmp_path / "aspect" - model.aspect_model.create_model_card(path, path) + model.aspect_model.create_model_card(path, model_name=str(path)) with open(path / "README.md", "r", encoding="utf8") as f: model_card = f.read() assert ASPECT_MODEL_CARD_PATTERN.fullmatch(model_card) path = tmp_path / "polarity" - model.polarity_model.create_model_card(path, path) + model.polarity_model.create_model_card(path, model_name=str(path)) with open(path / "README.md", "r", encoding="utf8") as f: model_card = f.read() assert POLARITY_MODEL_CARD_PATTERN.fullmatch(model_card) From 0beedf2a4c8d94011a520c0d18d038d980f7a338 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 14:03:21 +0100 Subject: [PATCH 154/183] Speed up model card tests for ABSA --- tests/span/aspect_model_card_pattern.py | 15 ++++++++------- tests/span/polarity_model_card_pattern.py | 17 ++++++++--------- tests/span/test_model_card.py | 11 ++++------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/tests/span/aspect_model_card_pattern.py b/tests/span/aspect_model_card_pattern.py index 2113d899..782aa959 100644 --- a/tests/span/aspect_model_card_pattern.py +++ b/tests/span/aspect_model_card_pattern.py @@ -86,8 +86,8 @@ ### Model Labels \| Label\s+\| Examples\s+\| \|:-+\|:-+\| -\| no aspect\s+\| [^\|]+ \| \| aspect\s+\| [^\|]+ \| +\| no aspect\s+\| [^\|]+ \| ## Evaluation @@ -147,14 +147,14 @@ ## Training Details ### Training Set Metrics -\| Training set \| Min \| Median \| Max \| -\|:-------------\|:----\|:--------\|:----\| -\| Word count \| 13 \| 29.6875 \| 36 \| +\| Training set \| Min \| Median \| Max \| +\|:-------------\|:----\|:-------\|:----\| +\| Word count \| 5 \| 14.5 \| 23 \| \| Label \| Training Sample Count \| \|:----------\|:----------------------\| -\| no aspect \| 10 \| -\| aspect \| 6 \| +\| no aspect \| 1 \| +\| aspect \| 5 \| ### Training Hyperparameters - batch_size: \(1, 1\) @@ -228,6 +228,7 @@ ## Model Card Contact \*Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors\.\* --->""", +-->\ +""", flags=re.DOTALL, ) diff --git a/tests/span/polarity_model_card_pattern.py b/tests/span/polarity_model_card_pattern.py index 3aafef61..041bba25 100644 --- a/tests/span/polarity_model_card_pattern.py +++ b/tests/span/polarity_model_card_pattern.py @@ -72,7 +72,7 @@ - \*\*SetFitABSA Aspect Model:\*\* \[\S+\]\(https:\/\/huggingface\.co/\S+\) - \*\*SetFitABSA Polarity Model:\*\* \[\S+\]\(https:\/\/huggingface\.co/\S+\) - \*\*Maximum Sequence Length:\*\* 100 tokens -- \*\*Number of Classes:\*\* 3 classes +- \*\*Number of Classes:\*\* 2 classes - \*\*Language:\*\* en - \*\*License:\*\* apache-2.0 @@ -86,9 +86,8 @@ ### Model Labels \| Label\s+\| Examples\s+\| \|:-+\|:-+\| -\| neutral\s+\| [^\|]+ \| -\| positive\s+\| [^\|]+ \| \| negative\s+\| [^\|]+ \| +\| positive\s+\| [^\|]+ \| ## Evaluation @@ -148,14 +147,13 @@ ## Training Details ### Training Set Metrics -\| Training set \| Min \| Median \| Max \| -\|:-------------\|:----\|:--------\|:----\| -\| Word count \| 22 \| 33.3333 \| 42 \| +\| Training set \| Min \| Median \| Max \| +\|:-------------\|:----\|:-------\|:----\| +\| Word count \| 8 \| 16.8 \| 28 \| \| Label \| Training Sample Count \| \|:---------\|:----------------------\| -\| negative \| 1 \| -\| neutral \| 2 \| +\| negative \| 2 \| \| positive \| 3 \| ### Training Hyperparameters @@ -230,6 +228,7 @@ ## Model Card Contact \*Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors\.\* --->""", +-->\ +""", flags=re.DOTALL, ) diff --git a/tests/span/test_model_card.py b/tests/span/test_model_card.py index 4c98d811..4b636006 100644 --- a/tests/span/test_model_card.py +++ b/tests/span/test_model_card.py @@ -1,6 +1,6 @@ from pathlib import Path -from datasets import load_dataset +from datasets import Dataset from setfit import AbsaModel, AbsaTrainer, SetFitModelCardData, TrainingArguments @@ -8,10 +8,7 @@ from .polarity_model_card_pattern import POLARITY_MODEL_CARD_PATTERN -def test_model_card(tmp_path: Path) -> None: - dataset = load_dataset("tomaarsen/setfit-absa-semeval-laptops", split="train") - train_dataset = dataset.select(range(10)) - eval_dataset = dataset.select(range(10, 20)) +def test_model_card(absa_dataset: Dataset, tmp_path: Path) -> None: model = AbsaModel.from_pretrained( "sentence-transformers/paraphrase-albert-small-v2", model_card_data=SetFitModelCardData( @@ -33,8 +30,8 @@ def test_model_card(tmp_path: Path) -> None: trainer = AbsaTrainer( model=model, args=args, - train_dataset=train_dataset, - eval_dataset=eval_dataset, + train_dataset=absa_dataset, + eval_dataset=absa_dataset, ) trainer.train() trainer.evaluate() From 55e9380a208ee91ab3d7dbe51f1dcc64cddc1b74 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 14:12:02 +0100 Subject: [PATCH 155/183] Set default W&B project as "setfit" if not set via ENV var yet --- src/setfit/trainer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index dd3e01e2..1b968b62 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -1,4 +1,5 @@ import math +import os import shutil import time import warnings @@ -17,7 +18,7 @@ from torch.cuda.amp import autocast from torch.utils.data import DataLoader from tqdm.autonotebook import tqdm -from transformers.integrations import get_reporting_integration_callbacks +from transformers.integrations import get_reporting_integration_callbacks, WandbCallback from transformers.trainer_callback import ( CallbackHandler, DefaultFlowCallback, @@ -226,6 +227,9 @@ def __init__( # Setup the callbacks default_callbacks = DEFAULT_CALLBACKS + get_reporting_integration_callbacks(self.args.report_to) callbacks = default_callbacks if callbacks is None else default_callbacks + callbacks + if WandbCallback in callbacks: + # Set the W&B project via environment variables if it's not already set + os.environ.setdefault("WANDB_PROJECT", "setfit") # TODO: Observe optimizer and scheduler by wrapping SentenceTransformer._get_scheduler self.callback_handler = CallbackHandler(callbacks, self.model, self.model.model_body.tokenizer, None, None) self.state = TrainerState() From 8ad41a88368a2e5eddf32fd910300235f50f4044 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 14:18:59 +0100 Subject: [PATCH 156/183] Run formatting --- src/setfit/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 1b968b62..456f1e0d 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -18,7 +18,7 @@ from torch.cuda.amp import autocast from torch.utils.data import DataLoader from tqdm.autonotebook import tqdm -from transformers.integrations import get_reporting_integration_callbacks, WandbCallback +from transformers.integrations import WandbCallback, get_reporting_integration_callbacks from transformers.trainer_callback import ( CallbackHandler, DefaultFlowCallback, From a65a4e7900fbbb04705937e556f03167f59ac977 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 15:06:43 +0100 Subject: [PATCH 157/183] Remove the old ABSA model card template --- MANIFEST.in | 1 - src/setfit/span/model_card_template.md | 64 -------------------------- 2 files changed, 65 deletions(-) delete mode 100644 src/setfit/span/model_card_template.md diff --git a/MANIFEST.in b/MANIFEST.in index 88bc0822..5bb16aa4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1 @@ -include src/setfit/span/model_card_template.md include src/setfit/model_card_template.md \ No newline at end of file diff --git a/src/setfit/span/model_card_template.md b/src/setfit/span/model_card_template.md deleted file mode 100644 index 31ec618f..00000000 --- a/src/setfit/span/model_card_template.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -license: apache-2.0 -tags: -- setfit -- sentence-transformers -- absa -- token-classification -pipeline_tag: token-classification ---- - -# {{ model_name | default("SetFit ABSA Model", true) }} - -This is a [SetFit ABSA model](https://github.com/huggingface/setfit) that can be used for Aspect Based Sentiment Analysis (ABSA). \ -In particular, this model is in charge of {{ "filtering aspect span candidates" if is_aspect else "classifying aspect polarities"}}. -It has been trained using SetFit, an efficient few-shot learning technique that involves: - -1. Fine-tuning a [Sentence Transformer](https://www.sbert.net) with contrastive learning. -2. Training a classification head with features from the fine-tuned Sentence Transformer. - -This model was trained within the context of a larger system for ABSA, which looks like so: - -1. Use a spaCy model to select possible aspect span candidates. -2. {{ "**" if is_aspect else "" }}Use {{ "this" if is_aspect else "a" }} SetFit model to filter these possible aspect span candidates.{{ "**" if is_aspect else "" }} -3. {{ "**" if not is_aspect else "" }}Use {{ "this" if not is_aspect else "a" }} SetFit model to classify the filtered aspect span candidates.{{ "**" if not is_aspect else "" }} - -## Usage - -To use this model for inference, first install the SetFit library: - -```bash -pip install setfit -``` - -You can then run inference as follows: - -```python -from setfit import AbsaModel - -# Download from Hub and run inference -model = AbsaModel.from_pretrained( - "{{ aspect_model }}", - "{{ polarity_model }}", -) -# Run inference -preds = model([ - "The best pizza outside of Italy and really tasty.", - "The food here is great but the service is terrible", -]) -``` - -## BibTeX entry and citation info - -```bibtex -@article{https://doi.org/10.48550/arxiv.2209.11055, - doi = {10.48550/ARXIV.2209.11055}, - url = {https://arxiv.org/abs/2209.11055}, - author = {Tunstall, Lewis and Reimers, Nils and Jo, Unso Eun Seo and Bates, Luke and Korat, Daniel and Wasserblat, Moshe and Pereg, Oren}, - keywords = {Computation and Language (cs.CL), FOS: Computer and information sciences, FOS: Computer and information sciences}, - title = {Efficient Few-Shot Learning Without Prompts}, - publisher = {arXiv}, - year = {2022}, - copyright = {Creative Commons Attribution 4.0 International} -} -``` \ No newline at end of file From d0cda23423138a63a19e49da939a85912277beac Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 15:27:06 +0100 Subject: [PATCH 158/183] Set fsspec<2023.12.0 due to breakages with older datasets For the compatibility tests --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9437b96c..4aed657c 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,11 @@ def combine_requirements(base_keys): # For the combatibility tests we add pandas<2, as pandas 2.0.0 onwards is incompatible with old datasets versions, # and we assume few to no users would use old datasets versions with new pandas versions. # The only alternative is incrementing the minimum version for datasets, which seems unnecessary. +# Beyond that, fsspec is set to <2023.12.0 as that version is incompatible with datasets<=2.15.0 EXTRAS_REQUIRE["compat_tests"] = ( - [requirement.replace(">=", "==") for requirement in REQUIRED_PKGS] + TESTS_REQUIRE + ["pandas<2"] + [requirement.replace(">=", "==") for requirement in REQUIRED_PKGS] + + TESTS_REQUIRE + + ["pandas<2", "fsspec<2023.12.0"] ) setup( From 7dcc35e046ac9f0bdacee8f2c3a70d570beea3f3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 16:03:53 +0100 Subject: [PATCH 159/183] Make some model_card_data modifications for ABSA only once --- src/setfit/span/modeling.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index e8f59283..291f3165 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -73,6 +73,12 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None if model_name.endswith("-polarity"): aspect_model = model_name[: -len("-polarity")] + "-aspect" + # Only once: + if self.model_card_data.absa is None and self.model_card_data.model_name: + self.model_card_data.model_name = self.model_card_data.model_name.replace( + "SetFit", "SetFit Aspect Model" if is_aspect else "SetFit Polarity Model", 1 + ) + self.model_card_data.tags.insert(1, "absa") self.model_card_data.absa = { "is_absa": True, "is_aspect": is_aspect, @@ -81,11 +87,6 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None } if self.model_card_data.task_name is None: self.model_card_data.task_name = "Aspect Based Sentiment Analysis (ABSA)" - self.model_card_data.tags.insert(1, "absa") - if self.model_card_data.model_name: - self.model_card_data.model_name = self.model_card_data.model_name.replace( - "SetFit", "SetFit Aspect Model" if is_aspect else "SetFit Polarity Model", 1 - ) self.model_card_data.inference = False with open(os.path.join(path, "README.md"), "w", encoding="utf-8") as f: f.write(self.generate_model_card()) From 2cd004f5204fcc50d5b2029d7a73d5aa9d5b6c94 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 16:22:10 +0100 Subject: [PATCH 160/183] Reorder arguments --- src/setfit/model_card.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index 3158757a..266ff7ff 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -479,11 +479,11 @@ def to_dict(self) -> Dict[str, Any]: task_type="text-classification", dataset_type=dataset_id, dataset_name=dataset_name, + dataset_split=dataset_split, + dataset_revision=self.dataset_revision, metric_type=metric_key.split("_", maxsplit=1)[1], metric_value=metric_value, task_name="Text Classification", - dataset_split=dataset_split, - dataset_revision=self.dataset_revision, metric_name=metric_key.split("_", maxsplit=1)[1].title(), ) for metric_key, metric_value in self.eval_results_dict.items() From 3a5356e92a8a25457cf1f4856909f2a3135cef04 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 16:27:28 +0100 Subject: [PATCH 161/183] Update absa models in docs --- docs/source/en/how_to/absa.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/en/how_to/absa.mdx b/docs/source/en/how_to/absa.mdx index 20695d99..9f074d91 100644 --- a/docs/source/en/how_to/absa.mdx +++ b/docs/source/en/how_to/absa.mdx @@ -191,15 +191,15 @@ For example: from setfit import AbsaModel model = AbsaModel.from_pretrained( - "tomaarsen/setfit-absa-restaurants-aspect", - "tomaarsen/setfit-absa-restaurants-polarity", + "tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-aspect", + "tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-polarity", spacy_model="en_core_web_lg", ) ``` We've now successfully loaded the SetFitABSA model from: -* [tomaarsen/setfit-absa-restaurants-aspect](https://huggingface.co/tomaarsen/setfit-absa-restaurants-aspect) -* [tomaarsen/setfit-absa-restaurants-polarity](https://huggingface.co/tomaarsen/setfit-absa-restaurants-polarity) +* [tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-aspect](https://huggingface.co/tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-aspect) +* [tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-polarity](https://huggingface.co/tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-polarity) ## Inference with a SetFitABSA model From d513064bb5a3dffcac34bee70f661a8273006958 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 16:58:06 +0100 Subject: [PATCH 162/183] Move import --- src/setfit/modeling.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 18d2079e..d4f3256f 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -6,8 +6,6 @@ from pathlib import Path from typing import Dict, List, Optional, Set, Tuple, Union -from setfit.model_card import SetFitModelCardData, generate_model_card - # For Python 3.7 compatibility try: @@ -32,6 +30,7 @@ from . import logging from .data import SetFitDataset +from .model_card import SetFitModelCardData, generate_model_card from .utils import set_docstring From 1a60d093b28dca5beb10b8f86b09fe7f802f1e38 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 16:58:49 +0100 Subject: [PATCH 163/183] Add model_card_data to from_pretrained --- src/setfit/modeling.py | 3 +++ src/setfit/span/modeling.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index d4f3256f..d369849f 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -824,6 +824,9 @@ def _from_pretrained( + """labels (`List[str]`, *optional*): If the labels are integers ranging from `0` to `num_classes-1`, then these labels indicate the corresponding labels. + model_card_data (`SetFitModelCardData`, *optional*): + A [`SetFitModelCardData`] instance storing data such as model language, license, dataset name, + etc. to be used in the automatically generated model cards. multi_target_strategy (`str`, *optional*): The strategy to use with multi-label classification. One of "one-vs-rest", "multi-output", or "classifier-chain". diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index 291f3165..4384a357 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -97,7 +97,10 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None if cut_index != -1: docstring = ( docstring[:cut_index] - + """use_differentiable_head (`bool`, *optional*): + + """model_card_data (`SetFitModelCardData`, *optional*): + A [`SetFitModelCardData`] instance storing data such as model language, license, dataset name, + etc. to be used in the automatically generated model cards. + use_differentiable_head (`bool`, *optional*): Whether to load SetFit using a differentiable (i.e., Torch) head instead of Logistic Regression. normalize_embeddings (`bool`, *optional*): Whether to apply normalization on the embeddings produced by the Sentence Transformer body. From 681f8db1b63e3a16e0eda252295ad9c164203c6b Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 17:09:10 +0100 Subject: [PATCH 164/183] Remove useless brackets --- src/setfit/modeling.py | 2 +- src/setfit/span/modeling.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index d369849f..0dc7e7d4 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -825,7 +825,7 @@ def _from_pretrained( If the labels are integers ranging from `0` to `num_classes-1`, then these labels indicate the corresponding labels. model_card_data (`SetFitModelCardData`, *optional*): - A [`SetFitModelCardData`] instance storing data such as model language, license, dataset name, + A `SetFitModelCardData` instance storing data such as model language, license, dataset name, etc. to be used in the automatically generated model cards. multi_target_strategy (`str`, *optional*): The strategy to use with multi-label classification. One of "one-vs-rest", "multi-output", diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index 4384a357..4f69b12c 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -98,7 +98,7 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None docstring = ( docstring[:cut_index] + """model_card_data (`SetFitModelCardData`, *optional*): - A [`SetFitModelCardData`] instance storing data such as model language, license, dataset name, + A `SetFitModelCardData` instance storing data such as model language, license, dataset name, etc. to be used in the automatically generated model cards. use_differentiable_head (`bool`, *optional*): Whether to load SetFit using a differentiable (i.e., Torch) head instead of Logistic Regression. From d61ec69d862c08f4e2396dff003835835d7e00f9 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 17:16:06 +0100 Subject: [PATCH 165/183] Correct model_card docstring --- src/setfit/model_card.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index 266ff7ff..09994469 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -177,21 +177,23 @@ class SetFitModelCardData(CardData): Args: language (`Optional[Union[str, List[str]]]`): The model language, either a string or a list, e.g. "en" or ["en", "de", "nl"] - license: (`Optional[str]`): The license of the model, e.g. "apache-2.0", "mit" + license (`Optional[str]`): The license of the model, e.g. "apache-2.0", "mit", or "cc-by-nc-sa-4.0" - model_name: (`Optional[str]`): The pretty name of the model, e.g. "SetFit with mBERT-base on CoNLL03". + model_name (`Optional[str]`): The pretty name of the model, e.g. "SetFit with mBERT-base on CoNLL03". If not defined, uses encoder_name/encoder_id and dataset_name/dataset_id to generate a model name. - model_id: (`Optional[str]`): The model ID when pushing the model to the Hub, + model_id (`Optional[str]`): The model ID when pushing the model to the Hub, e.g. "tomaarsen/span-marker-mbert-base-multinerd". - dataset_name: (`Optional[str]`): The pretty name of the dataset, e.g. "CoNLL03". - dataset_id: (`Optional[str]`): The dataset ID of the dataset, e.g. "tner/bionlp2004". - dataset_revision: (`Optional[str]`): The dataset revision/commit that was for training/evaluation. - st_id: (`Optional[str]`): The Sentence Transformers model ID. + dataset_name (`Optional[str]`): The pretty name of the dataset, e.g. "CoNLL03". + dataset_id (`Optional[str]`): The dataset ID of the dataset, e.g. "tner/bionlp2004". + dataset_revision (`Optional[str]`): The dataset revision/commit that was for training/evaluation. + st_id (`Optional[str]`): The Sentence Transformers model ID. - Note: + - Install ``nltk`` to detokenize the examples used in the model card, i.e. attach punctuation and brackets. - Additionally, ``codecarbon`` can be installed to automatically track carbon emission usage. + Install [``codecarbon``](https://github.com/mlco2/codecarbon) to automatically track carbon emission usage and + include it in your model cards. + + Example:: From 77aff7c12c79c6e96bd1fd9762ad61a57dbd00f3 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 17:44:38 +0100 Subject: [PATCH 166/183] Only use the gold aspects/labels for training the polarity model --- src/setfit/span/trainer.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/setfit/span/trainer.py b/src/setfit/span/trainer.py index 2490605c..b67c9466 100644 --- a/src/setfit/span/trainer.py +++ b/src/setfit/span/trainer.py @@ -155,13 +155,9 @@ def index_ordinal(text: str, target: str, ordinal: int) -> Tuple[int, int]: # the predicted aspects are indeed true/gold aspects. aspect_labels.extend([aspect in gold_aspects for aspect in aspects]) - # The Polarity model uses the intersection of pred and gold aspects, with labels for the gold label. - intersected_aspects = [] - for gold_aspect, gold_label in zip(gold_aspects, gold_polarity_labels): - if gold_aspect in aspects: - intersected_aspects.append(gold_aspect) - polarity_labels.append(gold_label) - intersected_aspect_list.append(intersected_aspects) + # The Polarity model uses only the gold aspects and labels + polarity_labels.extend(gold_polarity_labels) + intersected_aspect_list.append(gold_aspects) aspect_texts = list(aspect_model.prepend_aspects(docs, aspects_list)) polarity_texts = list(polarity_model.prepend_aspects(docs, intersected_aspect_list)) From 9dbca0bbd8db8b4ce29947630be1c1edddda6333 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 17:54:25 +0100 Subject: [PATCH 167/183] Use text classification dataset examples --- src/setfit/model_card.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index 09994469..e453dee8 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -179,12 +179,12 @@ class SetFitModelCardData(CardData): e.g. "en" or ["en", "de", "nl"] license (`Optional[str]`): The license of the model, e.g. "apache-2.0", "mit", or "cc-by-nc-sa-4.0" - model_name (`Optional[str]`): The pretty name of the model, e.g. "SetFit with mBERT-base on CoNLL03". + model_name (`Optional[str]`): The pretty name of the model, e.g. "SetFit with mBERT-base on SST2". If not defined, uses encoder_name/encoder_id and dataset_name/dataset_id to generate a model name. model_id (`Optional[str]`): The model ID when pushing the model to the Hub, e.g. "tomaarsen/span-marker-mbert-base-multinerd". - dataset_name (`Optional[str]`): The pretty name of the dataset, e.g. "CoNLL03". - dataset_id (`Optional[str]`): The dataset ID of the dataset, e.g. "tner/bionlp2004". + dataset_name (`Optional[str]`): The pretty name of the dataset, e.g. "SST2". + dataset_id (`Optional[str]`): The dataset ID of the dataset, e.g. "dair-ai/emotion". dataset_revision (`Optional[str]`): The dataset revision/commit that was for training/evaluation. st_id (`Optional[str]`): The Sentence Transformers model ID. From 368155ccebf2042a7a0330390c5fce1f4261c505 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Mon, 4 Dec 2023 19:20:39 +0100 Subject: [PATCH 168/183] Add model card generation documentation --- docs/source/en/_toctree.yml | 2 + docs/source/en/how_to/model_cards.mdx | 100 ++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 docs/source/en/how_to/model_cards.mdx diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index bc53342f..24123729 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -21,6 +21,8 @@ title: Overview - local: how_to/callbacks title: Callbacks + - local: how_to/model_cards + title: Model Cards - local: how_to/classification_heads title: Classification Heads - local: how_to/multilabel diff --git a/docs/source/en/how_to/model_cards.mdx b/docs/source/en/how_to/model_cards.mdx new file mode 100644 index 00000000..96b0bcff --- /dev/null +++ b/docs/source/en/how_to/model_cards.mdx @@ -0,0 +1,100 @@ + +# Model Cards + +SetFit comes with extensive automatically generated model cards/READMEs. In this how-to guide, we will explore how to make the most of this automatic generation. + +As an example, the [tomaarsen/setfit-all-MiniLM-L6-v2-sst2-32-shot](https://huggingface.co/tomaarsen/setfit-all-MiniLM-L6-v2-sst2-32-shot) model has followed all steps from this guide to produce the most extensive automatically generated model card. + +## Specifying Metadata + +Although SetFit can infer a lot of information about your model through its training and configuration, some metadata can often not be (trivially) inferred. For example: + +* **language**: The model language, e.g. "en" for English. +* **license**: The model license, e.g. "mit" or "apache-2.0". +* **dataset_name**: The pretty name of a dataset, e.g. "Amazon Counterfactual". +* **dataset_id**: The dataset ID of the dataset, e.g. "dair-ai/emotion". + +It is recommended to specify this information to the [`SetFitModel`] upon calling [`SetFitModel.from_pretrained`], to allow this information to be included in the model card and its metadata. This can be done using an [`SetFitModelCardData`] instance and the `model_card_data` key-word argument, e.g. like so: + +```py +from setfit import SetFitModel + +model = SetFitModel.from_pretrained( + "BAAI/bge-small-en-v1.5", + model_card_data=SetFitModelCardData( + language="en", + license="apache-2.0", + dataset_id="sst2", + dataset_name="SST2", + ) +) +``` + +See the [`SetFitModelCardData`] documentation for more information that you can specify to be used in the README. + +## Labels + +If the labels from your training dataset are all integers, then you are recommended to provide your [`SetFitModel`] with labels. These labels can then 1) be used in inference and 2) be used in your model card. For example, if your training labels are `0` and `1` for negative and positive, respectively, then you can load your model like so: + +```py +model = SetFitModel.from_pretrained( + "BAAI/bge-small-en-v1.5", + labels=["negative", "positive"], + model_card_data=SetFitModelCardData( + language="en", + license="apache-2.0", + dataset_id="sst2", + dataset_name="SST2", + ) +) +``` + +When calling [`SetFitModel.predict`], the trained model will now output strings or lists of strings, rather than your integer labels: + +```py +model.predict([ + "It's a charming and often affecting journey.", + "It's slow -- very, very slow.", + "A sometimes tedious film.", +]) +# => ['positive', 'negative', 'negative'] +``` + +Additionally, the model card will include the labels, e.g. it will use the following table: + +| Label | Examples | +|:---------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| negative |
    • 'a tough pill to swallow and '
    • 'indignation '
    • 'that the typical hollywood disregard for historical truth and realism is at work here '
    | +| positive |
    • "a moving experience for people who have n't read the book "
    • 'in the best possible senses of both those words '
    • 'to serve the work especially well '
    | + +Rather than this one: + +| Label | Examples | +|:---------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0 |
    • 'a tough pill to swallow and '
    • 'indignation '
    • 'that the typical hollywood disregard for historical truth and realism is at work here '
    | +| 1 |
    • "a moving experience for people who have n't read the book "
    • 'in the best possible senses of both those words '
    • 'to serve the work especially well '
    | + +And the following table: + +| Label | Training Sample Count | +|:---------|:----------------------| +| negative | 32 | +| positive | 32 | + +Rather than this one: + +| Label | Training Sample Count | +|:------|:----------------------| +| 0 | 32 | +| 1 | 32 | + +## Emissions Tracking + +The [``codecarbon``](https://github.com/mlco2/codecarbon) Python package can be installed to automatically track carbon emissions during training. This information will be included in the model card, e.g. in a list [like so](https://huggingface.co/tomaarsen/setfit-all-MiniLM-L6-v2-sst2-32-shot#environmental-impact): + +

    Environmental Impact

    + +Carbon emissions were measured using [CodeCarbon](https://github.com/mlco2/codecarbon). + +- **Carbon Emitted**: 0.003 kg of CO2 +- **Hours Used**: 0.072 hours From 9c876856da7dc6f766ed1662f1acccc23ca23deb Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 10:00:52 +0100 Subject: [PATCH 169/183] Add spaCy version to ABSA model card --- src/setfit/model_card_template.md | 3 +++ src/setfit/span/modeling.py | 3 +++ tests/span/aspect_model_card_pattern.py | 1 + tests/span/polarity_model_card_pattern.py | 1 + 4 files changed, 8 insertions(+) diff --git a/src/setfit/model_card_template.md b/src/setfit/model_card_template.md index fabdadb0..c3916268 100644 --- a/src/setfit/model_card_template.md +++ b/src/setfit/model_card_template.md @@ -163,6 +163,9 @@ Carbon emissions were measured using [CodeCarbon](https://github.com/mlco2/codec - Python: {{ version["python"] }} - SetFit: {{ version["setfit"] }} - Sentence Transformers: {{ version["sentence_transformers"] }} +{%- if "spacy" in version %} +- spaCy: {{ version["spacy"] }} +{%- endif %} - Transformers: {{ version["transformers"] }} - PyTorch: {{ version["torch"] }} - Datasets: {{ version["datasets"] }} diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index 4f69b12c..9fbd9e52 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -75,10 +75,13 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None # Only once: if self.model_card_data.absa is None and self.model_card_data.model_name: + from spacy import __version__ as spacy_version + self.model_card_data.model_name = self.model_card_data.model_name.replace( "SetFit", "SetFit Aspect Model" if is_aspect else "SetFit Polarity Model", 1 ) self.model_card_data.tags.insert(1, "absa") + self.model_card_data.version["spacy"] = spacy_version self.model_card_data.absa = { "is_absa": True, "is_aspect": is_aspect, diff --git a/tests/span/aspect_model_card_pattern.py b/tests/span/aspect_model_card_pattern.py index 782aa959..a9d797af 100644 --- a/tests/span/aspect_model_card_pattern.py +++ b/tests/span/aspect_model_card_pattern.py @@ -191,6 +191,7 @@ - Python: [^\n]+ - SetFit: [^\n]+ - Sentence Transformers: [^\n]+ +- spaCy: [^\n]+ - Transformers: [^\n]+ - PyTorch: [^\n]+ - Datasets: [^\n]+ diff --git a/tests/span/polarity_model_card_pattern.py b/tests/span/polarity_model_card_pattern.py index 041bba25..da8bec39 100644 --- a/tests/span/polarity_model_card_pattern.py +++ b/tests/span/polarity_model_card_pattern.py @@ -191,6 +191,7 @@ - Python: [^\n]+ - SetFit: [^\n]+ - Sentence Transformers: [^\n]+ +- spaCy: [^\n]+ - Transformers: [^\n]+ - PyTorch: [^\n]+ - Datasets: [^\n]+ From b257e8240270860c99ced9dfe8465337bcf95c3f Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 11:26:29 +0100 Subject: [PATCH 170/183] Map to int to avoid potential warning --- src/setfit/modeling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 0dc7e7d4..95412981 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -548,7 +548,7 @@ def predict( and preds.ndim == 1 and (self.has_differentiable_head or preds.dtype.char != "U") ): - outputs = [self.labels[pred] for pred in preds] + outputs = [self.labels[int(pred)] for pred in preds] else: outputs = self._output_type_conversion(preds, as_numpy=as_numpy) return outputs[0] if is_singular else outputs From 3a6e23a06243f519f8f17b41efc6f181728f26a2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 11:31:56 +0100 Subject: [PATCH 171/183] Store used spaCy model configuration in aspect/polarity model --- src/setfit/model_card.py | 1 - src/setfit/model_card_template.md | 3 ++ src/setfit/modeling.py | 12 ++++- src/setfit/span/modeling.py | 31 +++++++++---- tests/span/aspect_model_card_pattern.py | 1 + tests/span/polarity_model_card_pattern.py | 1 + tests/span/test_modeling.py | 54 ++++++++++++++++++++++- 7 files changed, 90 insertions(+), 13 deletions(-) diff --git a/src/setfit/model_card.py b/src/setfit/model_card.py index e453dee8..1960791c 100644 --- a/src/setfit/model_card.py +++ b/src/setfit/model_card.py @@ -109,7 +109,6 @@ def on_evaluate( metrics: Dict[str, float], **kwargs, ) -> None: - # TODO: Highlight the loaded best step if ( model.model_card_data.eval_lines_list and model.model_card_data.eval_lines_list[-1]["Step"] == state.global_step diff --git a/src/setfit/model_card_template.md b/src/setfit/model_card_template.md index c3916268..41c73cba 100644 --- a/src/setfit/model_card_template.md +++ b/src/setfit/model_card_template.md @@ -33,6 +33,9 @@ This model was trained within the context of a larger system for ABSA, which loo {%- else -%} {%- endif %} +{%- if spacy_model %} +- **spaCy Model:** {{ spacy_model }} +{%- endif %} {%- if aspect_model %} - **SetFitABSA Aspect Model:** [{{ aspect_model }}](https://huggingface.co/{{ aspect_model }}) {%- endif %} diff --git a/src/setfit/modeling.py b/src/setfit/modeling.py index 95412981..263c3021 100644 --- a/src/setfit/modeling.py +++ b/src/setfit/modeling.py @@ -723,10 +723,20 @@ def _from_pretrained( except requests.exceptions.RequestException: pass + model_kwargs = {key: value for key, value in model_kwargs.items() if value is not None} + if config_file is not None: with open(config_file, "r", encoding="utf-8") as f: config = json.load(f) - model_kwargs.update(config) + # Update model_kwargs + warnings + for setting, value in config.items(): + if setting in model_kwargs: + if model_kwargs[setting] != value: + logger.warning( + f"Overriding {setting} in model configuration from {value} to {model_kwargs[setting]}." + ) + else: + model_kwargs[setting] = value # Try to load a model head file if os.path.isdir(model_id): diff --git a/src/setfit/span/modeling.py b/src/setfit/span/modeling.py index 9fbd9e52..efa95a64 100644 --- a/src/setfit/span/modeling.py +++ b/src/setfit/span/modeling.py @@ -24,10 +24,13 @@ @dataclass class SpanSetFitModel(SetFitModel): + spacy_model: str = "en_core_web_lg" span_context: int = 0 attributes_to_save: Set[str] = field( - init=False, repr=False, default_factory=lambda: {"normalize_embeddings", "labels", "span_context"} + init=False, + repr=False, + default_factory=lambda: {"normalize_embeddings", "labels", "span_context", "spacy_model"}, ) def prepend_aspects(self, docs: List["Doc"], aspects_list: List[List[slice]]) -> List[str]: @@ -85,6 +88,7 @@ def create_model_card(self, path: str, model_name: Optional[str] = None) -> None self.model_card_data.absa = { "is_absa": True, "is_aspect": is_aspect, + "spacy_model": self.spacy_model, "aspect_model": aspect_model, "polarity_model": polarity_model, } @@ -196,21 +200,23 @@ def from_pretrained( cls, model_id: str, polarity_model_id: Optional[str] = None, - spacy_model: Optional[str] = "en_core_web_lg", - span_contexts: Tuple[Optional[int], Optional[int]] = (0, 3), - force_download: bool = False, - resume_download: bool = False, + spacy_model: Optional[str] = None, + span_contexts: Tuple[Optional[int], Optional[int]] = (None, None), + force_download: bool = None, + resume_download: bool = None, proxies: Optional[Dict] = None, token: Optional[Union[str, bool]] = None, cache_dir: Optional[str] = None, - local_files_only: bool = False, - use_differentiable_head: bool = False, - normalize_embeddings: bool = False, + local_files_only: bool = None, + use_differentiable_head: bool = None, + normalize_embeddings: bool = None, **model_kwargs, ) -> "AbsaModel": revision = None if len(model_id.split("@")) == 2: model_id, revision = model_id.split("@") + if spacy_model: + model_kwargs["spacy_model"] = spacy_model aspect_model = AspectModel.from_pretrained( model_id, span_context=span_contexts[0], @@ -250,8 +256,15 @@ def from_pretrained( normalize_embeddings=normalize_embeddings, **model_kwargs, ) + if aspect_model.spacy_model != polarity_model.spacy_model: + logger.warning( + "The Aspect and Polarity models are configured to use different spaCy models:\n" + f"* {repr(aspect_model.spacy_model)} for the aspect model, and\n" + f"* {repr(polarity_model.spacy_model)} for the polarity model.\n" + f"This model will use {repr(aspect_model.spacy_model)}." + ) - aspect_extractor = AspectExtractor(spacy_model=spacy_model) + aspect_extractor = AspectExtractor(spacy_model=aspect_model.spacy_model) return cls(aspect_extractor, aspect_model, polarity_model) diff --git a/tests/span/aspect_model_card_pattern.py b/tests/span/aspect_model_card_pattern.py index a9d797af..db7fc2a8 100644 --- a/tests/span/aspect_model_card_pattern.py +++ b/tests/span/aspect_model_card_pattern.py @@ -69,6 +69,7 @@ - \*\*Model Type:\*\* SetFit - \*\*Sentence Transformer body:\*\* \[sentence-transformers/paraphrase-albert-small-v2\]\(https://huggingface.co/sentence-transformers/paraphrase-albert-small-v2\) - \*\*Classification head:\*\* a \[LogisticRegression\]\(https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\) instance +- \*\*spaCy Model:\*\* en_core_web_lg - \*\*SetFitABSA Aspect Model:\*\* \[\S+\]\(https:\/\/huggingface\.co/\S+\) - \*\*SetFitABSA Polarity Model:\*\* \[\S+\]\(https:\/\/huggingface\.co/\S+\) - \*\*Maximum Sequence Length:\*\* 100 tokens diff --git a/tests/span/polarity_model_card_pattern.py b/tests/span/polarity_model_card_pattern.py index da8bec39..91ac263e 100644 --- a/tests/span/polarity_model_card_pattern.py +++ b/tests/span/polarity_model_card_pattern.py @@ -69,6 +69,7 @@ - \*\*Model Type:\*\* SetFit - \*\*Sentence Transformer body:\*\* \[sentence-transformers/paraphrase-albert-small-v2\]\(https://huggingface.co/sentence-transformers/paraphrase-albert-small-v2\) - \*\*Classification head:\*\* a \[LogisticRegression\]\(https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html\) instance +- \*\*spaCy Model:\*\* en_core_web_lg - \*\*SetFitABSA Aspect Model:\*\* \[\S+\]\(https:\/\/huggingface\.co/\S+\) - \*\*SetFitABSA Polarity Model:\*\* \[\S+\]\(https:\/\/huggingface\.co/\S+\) - \*\*Maximum Sequence Length:\*\* 100 tokens diff --git a/tests/span/test_modeling.py b/tests/span/test_modeling.py index 1260bb7b..c92b3def 100644 --- a/tests/span/test_modeling.py +++ b/tests/span/test_modeling.py @@ -1,10 +1,13 @@ +import json from pathlib import Path from tempfile import TemporaryDirectory import pytest import torch +from pytest import LogCaptureFixture from setfit import AbsaModel +from setfit.logging import get_logger from setfit.span.aspect_extractor import AspectExtractor from setfit.span.modeling import AspectModel, PolarityModel from tests.test_modeling import torch_cuda_available @@ -54,7 +57,10 @@ def test_loading(): assert model.polarity_model.span_context == 4 -def test_save_load(absa_model: AbsaModel) -> None: +def test_save_load(absa_model: AbsaModel, caplog: LogCaptureFixture) -> None: + logger = get_logger("setfit") + logger.propagate = True + absa_model.polarity_model.span_context = 5 with TemporaryDirectory() as tmp_dir: @@ -68,14 +74,58 @@ def test_save_load(absa_model: AbsaModel) -> None: ) assert fresh_model.polarity_model.span_context == 5 + # We expect a warning if we override the configured data: + AbsaModel.from_pretrained(tmp_dir + "-aspect", tmp_dir + "-polarity", span_contexts=[4, 4]) + log_texts = [record[2] for record in caplog.record_tuples] + assert "Overriding span_context in model configuration from 0 to 4." in log_texts + assert "Overriding span_context in model configuration from 5 to 4." in log_texts + assert len(caplog.record_tuples) == 2 + caplog.clear() + + # Error because en_core_web_bla doesn't exist + with pytest.raises(OSError): + AbsaModel.from_pretrained(tmp_dir + "-aspect", tmp_dir + "-polarity", spacy_model="en_core_web_bla") + log_texts = [record[2] for record in caplog.record_tuples] + assert "Overriding spacy_model in model configuration from en_core_web_sm to en_core_web_bla." in log_texts + assert "Overriding spacy_model in model configuration from en_core_web_sm to en_core_web_bla." in log_texts + assert len(caplog.record_tuples) == 2 + caplog.clear() + with TemporaryDirectory() as aspect_tmp_dir: with TemporaryDirectory() as polarity_tmp_dir: absa_model.save_pretrained(aspect_tmp_dir, polarity_tmp_dir) assert (Path(aspect_tmp_dir) / "config_setfit.json").exists() assert (Path(polarity_tmp_dir) / "config_setfit.json").exists() - fresh_model = AbsaModel.from_pretrained(aspect_tmp_dir, polarity_tmp_dir, spacy_model="en_core_web_sm") + fresh_model = AbsaModel.from_pretrained(aspect_tmp_dir, polarity_tmp_dir) assert fresh_model.polarity_model.span_context == 5 + assert fresh_model.aspect_model.spacy_model == "en_core_web_sm" + assert fresh_model.polarity_model.spacy_model == "en_core_web_sm" + + # Loading a model with different spacy_model settings + polarity_config_path = str(Path(polarity_tmp_dir) / "config_setfit.json") + with open(polarity_config_path, "r") as f: + config = json.load(f) + assert config == { + "span_context": 5, + "normalize_embeddings": False, + "spacy_model": "en_core_web_sm", + "labels": None, + } + config["spacy_model"] = "en_core_web_bla" + with open(polarity_config_path, "w") as f: + json.dump(config, f) + # Load a model with the updated config, there should be a warning + fresh_model = AbsaModel.from_pretrained(aspect_tmp_dir, polarity_tmp_dir) + assert len(caplog.record_tuples) == 1 + assert caplog.record_tuples[0][2] == ( + "The Aspect and Polarity models are configured to use different spaCy models:\n" + "* 'en_core_web_sm' for the aspect model, and\n" + "* 'en_core_web_bla' for the polarity model.\n" + "This model will use 'en_core_web_sm'." + ) + + logger.propagate = False @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA must be available to move a model between devices") From 3bc0125e08c1bf93ef423fb0017d0c97d70ca5d2 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 11:35:16 +0100 Subject: [PATCH 172/183] Correctly test against log --- tests/span/test_trainer.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/span/test_trainer.py b/tests/span/test_trainer.py index f89044dc..57c2cc9f 100644 --- a/tests/span/test_trainer.py +++ b/tests/span/test_trainer.py @@ -1,9 +1,10 @@ from datasets import Dataset from transformers import TrainerCallback +from pytest import LogCaptureFixture from setfit import AbsaTrainer from setfit.span.modeling import AbsaModel - +from setfit.logging import get_logger def test_trainer(absa_model: AbsaModel, absa_dataset: Dataset) -> None: trainer = AbsaTrainer(absa_model, train_dataset=absa_dataset, eval_dataset=absa_dataset) @@ -52,7 +53,10 @@ class TestCallback(TrainerCallback): assert callback not in trainer.polarity_trainer.callback_handler.callbacks -def test_train_ordinal_too_high(absa_model: AbsaModel) -> None: +def test_train_ordinal_too_high(absa_model: AbsaModel, caplog: LogCaptureFixture) -> None: + logger = get_logger("setfit") + logger.propagate = True + absa_dataset = Dataset.from_dict( { "text": [ @@ -64,7 +68,13 @@ def test_train_ordinal_too_high(absa_model: AbsaModel) -> None: } ) AbsaTrainer(absa_model, train_dataset=absa_dataset) - # TODO: Capture warning and test against it. + assert len(caplog.record_tuples) == 1 + assert caplog.record_tuples[0][2] == ( + "The ordinal of 1 for span 'food' in 'It is about food and ambiance, and imagine how dreadful it will be " + "it we only had to listen to an idle engine.' is too high. Skipping this sample." + ) + + logger.propagate = False def test_train_column_mapping(absa_model: AbsaModel, absa_dataset: Dataset) -> None: From 35c7461a72cb6053366f5a4bd32e294d12c080a5 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 11:38:00 +0100 Subject: [PATCH 173/183] Reformat test imports --- tests/span/test_trainer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/span/test_trainer.py b/tests/span/test_trainer.py index 57c2cc9f..38f562e6 100644 --- a/tests/span/test_trainer.py +++ b/tests/span/test_trainer.py @@ -1,10 +1,11 @@ from datasets import Dataset -from transformers import TrainerCallback from pytest import LogCaptureFixture +from transformers import TrainerCallback from setfit import AbsaTrainer -from setfit.span.modeling import AbsaModel from setfit.logging import get_logger +from setfit.span.modeling import AbsaModel + def test_trainer(absa_model: AbsaModel, absa_dataset: Dataset) -> None: trainer = AbsaTrainer(absa_model, train_dataset=absa_dataset, eval_dataset=absa_dataset) From 5d04965a580faa9c2dd6f43768da235b9b3f9cae Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 11:53:44 +0100 Subject: [PATCH 174/183] Try to resolve failing test on CI It passes locally --- tests/span/test_trainer.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/span/test_trainer.py b/tests/span/test_trainer.py index 38f562e6..0a8ca3d6 100644 --- a/tests/span/test_trainer.py +++ b/tests/span/test_trainer.py @@ -1,3 +1,5 @@ +import logging + from datasets import Dataset from pytest import LogCaptureFixture from transformers import TrainerCallback @@ -68,12 +70,14 @@ def test_train_ordinal_too_high(absa_model: AbsaModel, caplog: LogCaptureFixture "ordinal": [1], } ) - AbsaTrainer(absa_model, train_dataset=absa_dataset) - assert len(caplog.record_tuples) == 1 - assert caplog.record_tuples[0][2] == ( - "The ordinal of 1 for span 'food' in 'It is about food and ambiance, and imagine how dreadful it will be " - "it we only had to listen to an idle engine.' is too high. Skipping this sample." - ) + with caplog.at_level(logging.INFO): + AbsaTrainer(absa_model, train_dataset=absa_dataset) + assert len(caplog.record_tuples) == 1 + assert caplog.record_tuples[0][2] == ( + "The ordinal of 1 for span 'food' in 'It is about food and ambiance, and imagine how dreadful it will be " + "it we only had to listen to an idle engine.' is too high. Skipping this sample." + ) + assert caplog.record_tuples[0][1] == logging.INFO logger.propagate = False From 815e45ab80275b2dfa8554f3f0bf9df63bed76e8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 12:16:48 +0100 Subject: [PATCH 175/183] debugging: test against trainer dataset size This passes locally, not the entire test fails on CI --- tests/span/test_trainer.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/span/test_trainer.py b/tests/span/test_trainer.py index 0a8ca3d6..bb695058 100644 --- a/tests/span/test_trainer.py +++ b/tests/span/test_trainer.py @@ -71,13 +71,15 @@ def test_train_ordinal_too_high(absa_model: AbsaModel, caplog: LogCaptureFixture } ) with caplog.at_level(logging.INFO): - AbsaTrainer(absa_model, train_dataset=absa_dataset) - assert len(caplog.record_tuples) == 1 - assert caplog.record_tuples[0][2] == ( - "The ordinal of 1 for span 'food' in 'It is about food and ambiance, and imagine how dreadful it will be " - "it we only had to listen to an idle engine.' is too high. Skipping this sample." - ) - assert caplog.record_tuples[0][1] == logging.INFO + trainer = AbsaTrainer(absa_model, train_dataset=absa_dataset) + assert len(trainer.aspect_trainer.train_dataset) == 3 + assert len(trainer.polarity_trainer.train_dataset) == 0 + assert len(caplog.record_tuples) == 1 + assert caplog.record_tuples[0][2] == ( + "The ordinal of 1 for span 'food' in 'It is about food and ambiance, and imagine how dreadful it will be " + "it we only had to listen to an idle engine.' is too high. Skipping this sample." + ) + assert caplog.record_tuples[0][1] == logging.INFO logger.propagate = False From 267e21dd5dd061df12a6fa4ddabec3ab56ced6d8 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 12:41:01 +0100 Subject: [PATCH 176/183] Ignore log tests --- tests/span/test_trainer.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/span/test_trainer.py b/tests/span/test_trainer.py index bb695058..86610928 100644 --- a/tests/span/test_trainer.py +++ b/tests/span/test_trainer.py @@ -74,12 +74,13 @@ def test_train_ordinal_too_high(absa_model: AbsaModel, caplog: LogCaptureFixture trainer = AbsaTrainer(absa_model, train_dataset=absa_dataset) assert len(trainer.aspect_trainer.train_dataset) == 3 assert len(trainer.polarity_trainer.train_dataset) == 0 - assert len(caplog.record_tuples) == 1 - assert caplog.record_tuples[0][2] == ( - "The ordinal of 1 for span 'food' in 'It is about food and ambiance, and imagine how dreadful it will be " - "it we only had to listen to an idle engine.' is too high. Skipping this sample." - ) - assert caplog.record_tuples[0][1] == logging.INFO + # These tests are ignored as the caplog is inconsistent: + # assert len(caplog.record_tuples) == 1 + # assert caplog.record_tuples[0][2] == ( + # "The ordinal of 1 for span 'food' in 'It is about food and ambiance, and imagine how dreadful it will be " + # "it we only had to listen to an idle engine.' is too high. Skipping this sample." + # ) + # assert caplog.record_tuples[0][1] == logging.INFO logger.propagate = False From 243fcb2366e5c1d7b0b512d088e693635debf552 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 14:36:52 +0100 Subject: [PATCH 177/183] Add 'eval_max_steps', reduce load time before train This uses max_steps and eval_max_steps to limit the number of pairs that are generated. --- src/setfit/sampler.py | 12 +++++++++++ src/setfit/trainer.py | 26 +++++++++++++++++------ src/setfit/trainer_distillation.py | 8 +++++-- src/setfit/training_args.py | 4 ++++ tests/model_card_pattern.py | 1 + tests/span/aspect_model_card_pattern.py | 1 + tests/span/polarity_model_card_pattern.py | 1 + 7 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/setfit/sampler.py b/src/setfit/sampler.py index 0eceba1e..d9045ed5 100644 --- a/src/setfit/sampler.py +++ b/src/setfit/sampler.py @@ -39,6 +39,7 @@ def __init__( multilabel: bool, num_iterations: Optional[None] = None, sampling_strategy: str = "oversampling", + max_pairs: int = -1, ) -> None: """Generates positive and negative text pairs for contrastive learning. @@ -48,6 +49,8 @@ def __init__( sampling_strategy: "unique", "oversampling", or "undersampling" num_iterations: if provided explicitly sets the number of pairs to be generated where n_pairs = n_iterations * n_sentences * 2 (for pos & neg pairs) + max_pairs: If not -1, then we only sample pairs until we have certainly reached + max_pairs pairs. """ super().__init__() self.pos_index = 0 @@ -57,6 +60,7 @@ def __init__( self.sentences = np.array([s.texts[0] for s in examples]) self.labels = np.array([s.label for s in examples]) self.sentence_labels = list(zip(self.sentences, self.labels)) + self.max_pairs = max_pairs if multilabel: self.generate_multilabel_pairs() @@ -88,6 +92,8 @@ def generate_pairs(self) -> None: self.pos_pairs.append(InputExample(texts=[_text, text], label=1.0)) else: self.neg_pairs.append(InputExample(texts=[_text, text], label=0.0)) + if self.max_pairs != -1 and len(self.pos_pairs) > self.max_pairs and len(self.neg_pairs) > self.max_pairs: + break def generate_multilabel_pairs(self) -> None: for (_text, _label), (text, label) in shuffle_combinations(self.sentence_labels): @@ -96,6 +102,8 @@ def generate_multilabel_pairs(self) -> None: self.pos_pairs.append(InputExample(texts=[_text, text], label=1.0)) else: self.neg_pairs.append(InputExample(texts=[_text, text], label=0.0)) + if self.max_pairs != -1 and len(self.pos_pairs) > self.max_pairs and len(self.neg_pairs) > self.max_pairs: + break def get_positive_pairs(self) -> List[InputExample]: pairs = [] @@ -133,6 +141,7 @@ def __init__( cos_sim_matrix: torch.Tensor, num_iterations: Optional[None] = None, sampling_strategy: str = "oversampling", + max_pairs: int = -1, ) -> None: self.cos_sim_matrix = cos_sim_matrix super().__init__( @@ -140,6 +149,7 @@ def __init__( multilabel=False, num_iterations=num_iterations, sampling_strategy=sampling_strategy, + max_pairs=max_pairs, ) # Internally we store all pairs in pos_pairs, regardless of sampling strategy. # After all, without labels, there isn't much of a strategy. @@ -154,3 +164,5 @@ def __init__( def generate_pairs(self) -> None: for (text_one, id_one), (text_two, id_two) in shuffle_combinations(self.sentence_labels): self.pos_pairs.append(InputExample(texts=[text_one, text_two], label=self.cos_sim_matrix[id_one][id_two])) + if self.max_pairs != -1 and len(self.pos_pairs) > self.max_pairs: + break diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index 456f1e0d..c557753f 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -439,9 +439,13 @@ def train_embeddings( self.state.global_step = 0 self.state.total_flos = 0 - train_dataloader, loss_func, batch_size = self.get_dataloader(x_train, y_train, args=args) + train_max_pairs = -1 if args.max_steps == -1 else args.max_steps * args.embedding_batch_size + train_dataloader, loss_func, batch_size = self.get_dataloader( + x_train, y_train, args=args, max_pairs=train_max_pairs + ) if x_eval is not None and args.evaluation_strategy != IntervalStrategy.NO: - eval_dataloader, _, _ = self.get_dataloader(x_eval, y_eval, args=args) + eval_max_pairs = -1 if args.eval_max_steps == -1 else args.eval_max_steps * args.embedding_batch_size + eval_dataloader, _, _ = self.get_dataloader(x_eval, y_eval, args=args, max_pairs=eval_max_pairs) else: eval_dataloader = None @@ -466,7 +470,7 @@ def train_embeddings( ) def get_dataloader( - self, x: List[str], y: Union[List[int], List[List[int]]], args: TrainingArguments + self, x: List[str], y: Union[List[int], List[List[int]]], args: TrainingArguments, max_pairs: int = -1 ) -> Tuple[DataLoader, nn.Module, int]: # sentence-transformers adaptation input_data = [InputExample(texts=[text], label=label) for text, label in zip(x, y)] @@ -497,7 +501,11 @@ def get_dataloader( ) else: data_sampler = ContrastiveDataset( - input_data, self.model.multi_target_strategy, args.num_iterations, args.sampling_strategy + input_data, + self.model.multi_target_strategy, + args.num_iterations, + args.sampling_strategy, + max_pairs=max_pairs, ) # shuffle_sampler = True can be dropped in for further 'randomising' shuffle_sampler = True if args.sampling_strategy == "unique" else False @@ -721,8 +729,11 @@ def _evaluate_with_loss( ) -> float: model_body.eval() losses = [] - for data in tqdm( - iter(eval_dataloader), total=len(eval_dataloader), leave=False, disable=not args.show_progress_bar + eval_steps = ( + min(len(eval_dataloader), args.eval_max_steps) if args.eval_max_steps != -1 else len(eval_dataloader) + ) + for step, data in enumerate( + tqdm(iter(eval_dataloader), total=eval_steps, leave=False, disable=not args.show_progress_bar), start=1 ): features, labels = data labels = labels.to(model_body._target_device) @@ -736,6 +747,9 @@ def _evaluate_with_loss( else: losses.append(loss_func(features, labels).item()) + if step >= eval_steps: + break + model_body.train() return sum(losses) / len(losses) diff --git a/src/setfit/trainer_distillation.py b/src/setfit/trainer_distillation.py index da5ec4b8..d3675051 100644 --- a/src/setfit/trainer_distillation.py +++ b/src/setfit/trainer_distillation.py @@ -78,7 +78,11 @@ def dataset_to_parameters(self, dataset: Dataset) -> List[Iterable]: return [dataset["text"]] def get_dataloader( - self, x: List[str], y: Optional[Union[List[int], List[List[int]]]], args: TrainingArguments + self, + x: List[str], + y: Optional[Union[List[int], List[List[int]]]], + args: TrainingArguments, + max_pairs: int = -1, ) -> Tuple[DataLoader, nn.Module, int]: x_embd_student = self.teacher_model.model_body.encode( x, convert_to_tensor=self.teacher_model.has_differentiable_head @@ -87,7 +91,7 @@ def get_dataloader( input_data = [InputExample(texts=[text]) for text in x] data_sampler = ContrastiveDistillationDataset( - input_data, cos_sim_matrix, args.num_iterations, args.sampling_strategy + input_data, cos_sim_matrix, args.num_iterations, args.sampling_strategy, max_pairs=max_pairs ) # shuffle_sampler = True can be dropped in for further 'randomising' shuffle_sampler = True if args.sampling_strategy == "unique" else False diff --git a/src/setfit/training_args.py b/src/setfit/training_args.py index 389dfec9..9669c460 100644 --- a/src/setfit/training_args.py +++ b/src/setfit/training_args.py @@ -137,6 +137,9 @@ class TrainingArguments: eval_delay (`float`, *optional*): Number of epochs or steps to wait for before the first evaluation can be performed, depending on the evaluation_strategy. + eval_max_steps (`int`, defaults to `-1`): + If set to a positive number, the total number of evaluation steps to perform. The evaluation may stop + before reaching the set number of steps when all data is exhausted. save_strategy (`str` or [`~transformers.trainer_utils.IntervalStrategy`], *optional*, defaults to `"steps"`): The checkpoint save strategy to adopt during training. Possible values are: @@ -208,6 +211,7 @@ class TrainingArguments: evaluation_strategy: str = "no" eval_steps: Optional[int] = None eval_delay: int = 0 + eval_max_steps: int = -1 save_strategy: str = "steps" save_steps: int = 500 diff --git a/tests/model_card_pattern.py b/tests/model_card_pattern.py index dab082e4..8a058238 100644 --- a/tests/model_card_pattern.py +++ b/tests/model_card_pattern.py @@ -160,6 +160,7 @@ - use_amp: False - warmup_proportion: 0.1 - seed: 42 +- eval_max_steps: -1 - load_best_model_at_end: False ### Training Results diff --git a/tests/span/aspect_model_card_pattern.py b/tests/span/aspect_model_card_pattern.py index db7fc2a8..8295a25a 100644 --- a/tests/span/aspect_model_card_pattern.py +++ b/tests/span/aspect_model_card_pattern.py @@ -171,6 +171,7 @@ - use_amp: False - warmup_proportion: 0.1 - seed: 42 +- eval_max_steps: -1 - load_best_model_at_end: False ### Training Results diff --git a/tests/span/polarity_model_card_pattern.py b/tests/span/polarity_model_card_pattern.py index 91ac263e..921ad2bb 100644 --- a/tests/span/polarity_model_card_pattern.py +++ b/tests/span/polarity_model_card_pattern.py @@ -171,6 +171,7 @@ - use_amp: False - warmup_proportion: 0.1 - seed: 42 +- eval_max_steps: -1 - load_best_model_at_end: False ### Training Results From c039e17088713797b280bb27a885a187b5a25116 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Tue, 5 Dec 2023 15:05:33 +0100 Subject: [PATCH 178/183] Also pass metric_kwargs to custom metric callable --- src/setfit/trainer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setfit/trainer.py b/src/setfit/trainer.py index c557753f..29fb2962 100644 --- a/src/setfit/trainer.py +++ b/src/setfit/trainer.py @@ -839,15 +839,15 @@ def evaluate(self, dataset: Optional[Dataset] = None, metric_key_prefix: str = " y_test = encoder.transform(y_test) y_pred = encoder.transform(y_pred) + metric_kwargs = self.metric_kwargs or {} if isinstance(self.metric, str): metric_config = "multilabel" if self.model.multi_target_strategy is not None else None metric_fn = evaluate.load(self.metric, config_name=metric_config) - metric_kwargs = self.metric_kwargs or {} results = metric_fn.compute(predictions=y_pred, references=y_test, **metric_kwargs) elif callable(self.metric): - results = self.metric(y_pred, y_test) + results = self.metric(y_pred, y_test, **metric_kwargs) else: raise ValueError("metric must be a string or a callable") From 37592eb21bf76403e3460e3898796f0777cb897c Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 6 Dec 2023 10:18:46 +0100 Subject: [PATCH 179/183] Use gold aspects as True, and non-overlapping pred aspects as False For the Aspect model training set --- src/setfit/span/trainer.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/setfit/span/trainer.py b/src/setfit/span/trainer.py index b67c9466..846a945a 100644 --- a/src/setfit/span/trainer.py +++ b/src/setfit/span/trainer.py @@ -128,11 +128,22 @@ def index_ordinal(text: str, target: str, ordinal: int) -> Tuple[int, int]: find_from = start_idx + 1 return start_idx, start_idx + len(target) + def overlaps(aspect: slice, aspects: List[slice]) -> bool: + for test_aspect in aspects: + overlapping_indices = set(range(aspect.start, aspect.stop + 1)) & set( + range(test_aspect.start, test_aspect.stop + 1) + ) + if overlapping_indices: + return True + return False + docs, aspects_list = self.aspect_extractor(grouped_data.keys()) - intersected_aspect_list = [] - polarity_labels = [] + aspect_aspect_list = [] aspect_labels = [] + polarity_aspect_list = [] + polarity_labels = [] for doc, aspects, text in zip(docs, aspects_list, grouped_data): + # Collect all of the gold aspects gold_aspects = [] gold_polarity_labels = [] for annotation in grouped_data[text]: @@ -151,16 +162,21 @@ def index_ordinal(text: str, target: str, ordinal: int) -> Tuple[int, int]: gold_aspects.append(slice(gold_aspect_span.start, gold_aspect_span.end)) gold_polarity_labels.append(annotation["label"]) - # The Aspect model uses all predicted aspects, with labels depending on whether - # the predicted aspects are indeed true/gold aspects. - aspect_labels.extend([aspect in gold_aspects for aspect in aspects]) + # The Aspect model uses all gold aspects as "True", and all non-overlapping predicted + # aspects as "False" + aspect_labels.extend([True] * len(gold_aspects)) + aspect_aspect_list.append(gold_aspects[:]) + for aspect in aspects: + if not overlaps(aspect, gold_aspects): + aspect_labels.append(False) + aspect_aspect_list[-1].append(aspect) # The Polarity model uses only the gold aspects and labels polarity_labels.extend(gold_polarity_labels) - intersected_aspect_list.append(gold_aspects) + polarity_aspect_list.append(gold_aspects) - aspect_texts = list(aspect_model.prepend_aspects(docs, aspects_list)) - polarity_texts = list(polarity_model.prepend_aspects(docs, intersected_aspect_list)) + aspect_texts = list(aspect_model.prepend_aspects(docs, aspect_aspect_list)) + polarity_texts = list(polarity_model.prepend_aspects(docs, polarity_aspect_list)) return Dataset.from_dict({"text": aspect_texts, "label": aspect_labels}), Dataset.from_dict( {"text": polarity_texts, "label": polarity_labels} ) From 522a420cda662aba4961c433b9c9dca48260713e Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 6 Dec 2023 11:37:13 +0100 Subject: [PATCH 180/183] Add missing +1 on edge case in Aspect Extractor --- src/setfit/span/aspect_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setfit/span/aspect_extractor.py b/src/setfit/span/aspect_extractor.py index 096b9bb6..aea39426 100644 --- a/src/setfit/span/aspect_extractor.py +++ b/src/setfit/span/aspect_extractor.py @@ -23,7 +23,7 @@ def find_groups(self, aspect_mask: List[bool]): yield slice(start, idx) start = None if start is not None: - yield slice(start, idx) + yield slice(start, idx + 1) def __call__(self, texts: List[str]) -> Tuple[List["Doc"], List[slice]]: aspects_list = [] From ebaf5a2e7df7820299e38651991a883cc73f807b Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 6 Dec 2023 13:50:10 +0100 Subject: [PATCH 181/183] Update ABSA documentation slightly --- docs/source/en/how_to/absa.mdx | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/source/en/how_to/absa.mdx b/docs/source/en/how_to/absa.mdx index 9f074d91..68992827 100644 --- a/docs/source/en/how_to/absa.mdx +++ b/docs/source/en/how_to/absa.mdx @@ -68,7 +68,7 @@ Two datasets that already match this format are these datasets of reviews from t ```py # The training/eval dataset must have `text`, `span`, `label`, and `ordinal` columns -dataset = load_dataset("tomaarsen/setfit-absa-semeval-laptops", split="train") +dataset = load_dataset("tomaarsen/setfit-absa-semeval-restaurants", split="train") train_dataset = dataset.select(range(128)) eval_dataset = dataset.select(range(128, 256)) ``` @@ -162,19 +162,25 @@ Note that the aspect accuracy refers to the accuracy of classifying aspect candi ## Saving a SetFitABSA model -Once trained, we can use familiar [`AbsaTrainer.save_pretrained`]/[`AbsaModel.save_pretrained`] and [`AbsaTrainer.push_to_hub`]/[`AbsaModel.push_to_hub`] methods to save the model. However, unlike normally, saving an [`AbsaModel`] involves saving two separate models: the **aspect** SetFit model and the **polarity** SetFit model. Consequently, we can provide two directories or `repo_id`'s: +Once trained, we can use familiar [`AbsaModel.save_pretrained`] and [`AbsaTrainer.push_to_hub`]/[`AbsaModel.push_to_hub`] methods to save the model. However, unlike normally, saving an [`AbsaModel`] involves saving two separate models: the **aspect** SetFit model and the **polarity** SetFit model. Consequently, we can provide two directories or `repo_id`'s: ```py -model.save_pretrained("models/setfit-absa-model-aspect", "models/setfit-absa-model-polarity") +model.save_pretrained( + "models/setfit-absa-model-aspect", + "models/setfit-absa-model-polarity", +) # or -model.push_to_hub("tomaarsen/setfit-absa-model-aspect", "tomaarsen/setfit-absa-model-polarity") +model.push_to_hub( + "tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-aspect", + "tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-polarity", +) ``` However, you can also provide just one directory or `repo_id`, and `-aspect` and `-polarity` will be automatically added. So, the following code is equivalent to the previous snippet: ```py model.save_pretrained("models/setfit-absa-model") # or -model.push_to_hub("tomaarsen/setfit-absa-model") +model.push_to_hub("tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants") ``` ## Loading a SetFitABSA model @@ -218,3 +224,7 @@ print(preds) # [{'span': 'waiting number', 'polarity': 'negative'}] # ] ``` + +## Challenge + +If you're up for it, then I challenge you to train and upload a SetFitABSA model for [laptop reviews](https://huggingface.co/datasets/tomaarsen/setfit-absa-semeval-laptops) based on this documentation. From 937c491642fa721bee006c5531ceb0e99390c271 Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 6 Dec 2023 13:50:19 +0100 Subject: [PATCH 182/183] Specify AbsaTrainer methods --- docs/source/en/reference/trainer.mdx | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/en/reference/trainer.mdx b/docs/source/en/reference/trainer.mdx index 5b23f545..ddc26da8 100644 --- a/docs/source/en/reference/trainer.mdx +++ b/docs/source/en/reference/trainer.mdx @@ -41,4 +41,12 @@ ## AbsaTrainer -[[autodoc]] AbsaTrainer \ No newline at end of file +[[autodoc]] AbsaTrainer + - add_callback + - evaluate + - pop_callback + - push_to_hub + - remove_callback + - train + - train_aspect + - train_polarity From 3152e499c50a21ad4d14add86fccc25f9a9cfbbf Mon Sep 17 00:00:00 2001 From: Tom Aarsen Date: Wed, 6 Dec 2023 13:51:10 +0100 Subject: [PATCH 183/183] Update v1.0.0 migration; expand changelog --- .../en/how_to/v1.0.0_migration_guide.mdx | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/source/en/how_to/v1.0.0_migration_guide.mdx b/docs/source/en/how_to/v1.0.0_migration_guide.mdx index 2af3f3e3..daa8a87f 100644 --- a/docs/source/en/how_to/v1.0.0_migration_guide.mdx +++ b/docs/source/en/how_to/v1.0.0_migration_guide.mdx @@ -12,34 +12,36 @@ To update your code to work with v1.0.0, the following changes must be made: ## Training Migration Guide 1. Replace all uses of `SetFitTrainer` with [`Trainer`], and all uses of `DistillationSetFitTrainer` with [`DistillationTrainer`]. -2. Remove `num_iterations`, `num_epochs`, `learning_rate`, `batch_size`, `seed`, `use_amp`, `warmup_proportion`, `distance_metric`, `margin`, `samples_per_label` and `loss_class` from a `Trainer` initialisation, and move them to a `TrainerArguments` initialisation instead. This instance should then be passed to the trainer via the `args` argument. +2. Remove `num_iterations`, `num_epochs`, `learning_rate`, `batch_size`, `seed`, `use_amp`, `warmup_proportion`, `distance_metric`, `margin`, `samples_per_label` and `loss_class` from a `Trainer` initialization, and move them to a `TrainerArguments` initialization instead. This instance should then be passed to the trainer via the `args` argument. - * `num_iterations` has been deprecated, the number of training steps should now be controlled exclusively via `num_epochs`, `max_steps` or [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback). + * `num_iterations` has been deprecated, the number of training steps should now be controlled via `num_epochs`, `max_steps` or [`EarlyStoppingCallback`](https://huggingface.co/docs/transformers/main_classes/callback#transformers.EarlyStoppingCallback). * `learning_rate` has been split up into `body_learning_rate` and `head_learning_rate`. * `loss_class` has been renamed to `loss`. -3. Stop providing training arguments like `num_epochs` directly to `Trainer.train`: pass a `TrainingArguments` instance instead. -4. Refactor multiple `trainer.train()`, `trainer.freeze()` and `trainer.unfreeze()` calls that were previously necessary to train the differentiable head into just one `trainer.train()` call by setting `batch_size` and `num_epochs` on the `TrainingArguments` dataclass with Tuples. The first value is for training the embeddings, and the second is for training the classifier. +3. Stop providing training arguments like `num_epochs` directly to `Trainer.train`: pass a `TrainingArguments` instance via the `args` argument instead. +4. Refactor multiple `trainer.train()`, `trainer.freeze()` and `trainer.unfreeze()` calls that were previously necessary to train the differentiable head into just one `trainer.train()` call by setting `batch_size` and `num_epochs` on the `TrainingArguments` dataclass with tuples. The first value in the tuple is for training the embeddings, and the second is for training the classifier. ## Hard deprecations * `SetFitBaseModel`, `SKLearnWrapper` and `SetFitPipeline` have been removed. These can no longer be used starting from v1.0.0. -## New functionality +## v1.0.0 Changelog This list contains new functionality that can be used starting from v1.0.0. * [`SetFitModel.from_pretrained`] now accepts new arguments: * `device`: Specifies the device on which to load the SetFit model. * `labels`: Specify labels corresponding to the training labels - useful if the training labels are integers ranging from `0` to `num_classes - 1`. These are automatically applied on calling [`SetFitModel.predict`]. + * `model_card_data`: Provide a [`SetFitModelCardData`] instance storing data such as model language, license, dataset name, etc. to be used in the automatically generated model cards. +* Certain SetFit configuration options, such as the new `labels` argument from [`SetFitModel.from_pretrained`], now get saved in `config_setfit.json` files when a model is saved. This allows `labels` to be automatically fetched when a model is loaded. * [`SetFitModel.predict`] now accepts new arguments: * `batch_size` (defaults to `32`): The batch size to use in encoding the sentences to embeddings. Higher often means faster processing but higher memory usage. * `use_labels` (defaults to `True`): Whether to use the `SetFitModel.labels` to convert integer labels to string labels. Not used if the training labels are already strings. * [`SetFitModel.encode`] has been introduce to convert input sentences to embeddings using the `SentenceTransformer` body. * [`SetFitModel.device`] has been introduced to determine the device of the model. -* [`AbsaTrainer`] and [`AbsaModel`] have been introduced for applying SetFit for Aspect Based Sentiment Analysis. +* [`AbsaTrainer`] and [`AbsaModel`] have been introduced for applying [SetFit for Aspect Based Sentiment Analysis](absa). * [`Trainer`] now supports a `callbacks` argument for a list of [`transformers` `TrainerCallback` instances](https://huggingface.co/docs/transformers/main/en/main_classes/callback). - * By default, all installed callbacks integrated with `transformers` are supported, including [`TensorBoardCallback`](https://huggingface.co/docs/transformers/main/en/main_classes/callback#transformers.integrations.TensorBoardCallback), [`WandbCallback`](https://huggingface.co/docs/transformers/main/en/main_classes/callback#transformers.integrations.WandbCallback) to log training logs to TensorBoard and W&B, respectively. + * By default, all installed callbacks integrated with `transformers` are supported, including [`TensorBoardCallback`](https://huggingface.co/docs/transformers/main/en/main_classes/callback#transformers.integrations.TensorBoardCallback), [`WandbCallback`](https://huggingface.co/docs/transformers/main/en/main_classes/callback#transformers.integrations.WandbCallback) to log training logs to [TensorBoard](https://www.tensorflow.org/tensorboard) and [W&B](https://wandb.ai), respectively. * The [`Trainer`] will now print `embedding_loss` in the terminal, as well as `eval_embedding_loss` if `evaluation_strategy` is set to `"epoch"` or `"steps"` in [`TrainingArguments`]. * [`Trainer.evaluate`] now works with string labels. * An updated contrastive pair sampler increases the variety of training pairs. @@ -71,6 +73,7 @@ This list contains new functionality that can be used starting from v1.0.0. * `eval_steps`: Number of update steps between two evaluations if `evaluation_strategy="steps"`. Will default to the same as `logging_steps` if not set. * `eval_delay`: Number of epochs or steps to wait for before the first evaluation can be performed, depending on the `evaluation_strategy`. + * `eval_max_steps`: If set to a positive number, the total number of evaluation steps to perform. The evaluation may stop before reaching the set number of steps when all data is exhausted. * `save_strategy`: The checkpoint save strategy to adopt during training. Possible values are: - `"no"`: No save is done during training. @@ -86,4 +89,5 @@ This list contains new functionality that can be used starting from v1.0.0. When set to `True`, the parameters `save_strategy` needs to be the same as `evaluation_strategy`, and in the case it is "steps", `save_steps` must be a round multiple of `eval_steps`. - \ No newline at end of file + +* Pushing SetFit or SetFitABSA models to the Hub with [`SetFitModel.push_to_hub`] or [`AbsaModel.push_to_hub`] now results in a detailed model card. As an example, see [this SetFitModel](https://huggingface.co/tomaarsen/setfit-paraphrase-mpnet-base-v2-sst2-8-shot) or [this SetFitABSA polarity model](https://huggingface.co/tomaarsen/setfit-absa-bge-small-en-v1.5-restaurants-polarity). \ No newline at end of file