diff --git a/.circleci/config.yml b/.circleci/config.yml index 64f027221d8c..30e71945fd4f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,6 +79,7 @@ jobs: - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install .[sklearn,tf-cpu,torch,testing,sentencepiece] + - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html - save_cache: key: v0.4-{{ checksum "setup.py" }} paths: @@ -105,6 +106,7 @@ jobs: - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install .[sklearn,torch,testing,sentencepiece] + - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html - save_cache: key: v0.4-torch-{{ checksum "setup.py" }} paths: @@ -183,6 +185,7 @@ jobs: - v0.4-{{ checksum "setup.py" }} - run: pip install --upgrade pip - run: pip install .[sklearn,torch,testing,sentencepiece] + - run: pip install tapas torch-scatter -f https://pytorch-geometric.com/whl/torch-1.7.0+cpu.html - save_cache: key: v0.4-torch-{{ checksum "setup.py" }} paths: diff --git a/.github/workflows/self-push.yml b/.github/workflows/self-push.yml index 7080c6aa78d1..b497c159950d 100644 --- a/.github/workflows/self-push.yml +++ b/.github/workflows/self-push.yml @@ -50,6 +50,7 @@ jobs: pip install --upgrade pip pip install .[torch,sklearn,testing,onnxruntime,sentencepiece] pip install git+https://github.com/huggingface/datasets + pip install pandas torch-scatter -f https://pytorch-geometric.com/whl/torch-$(python -c "import torch; print(''.join(torch.__version__)")+$(python -c "import torch; print(''.join(torch.version.cuda.split('.')))").html - name: Are GPUs recognized by our DL frameworks run: | @@ -187,6 +188,7 @@ jobs: pip install --upgrade pip pip install .[torch,sklearn,testing,onnxruntime,sentencepiece] pip install git+https://github.com/huggingface/datasets + pip install pandas torch-scatter -f https://pytorch-geometric.com/whl/torch-$(python -c "import torch; print(''.join(torch.__version__)")+$(python -c "import torch; print(''.join(torch.version.cuda.split('.')))").html - name: Are GPUs recognized by our DL frameworks run: | diff --git a/README.md b/README.md index ccdfbd1ae084..88d57ec4b38b 100644 --- a/README.md +++ b/README.md @@ -222,6 +222,7 @@ Min, Patrick Lewis, Ledell Wu, Sergey Edunov, Danqi Chen, and Wen-tau Yih. ultilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT. 1. **[SqueezeBert](https://huggingface.co/transformers/model_doc/squeezebert.html)** released with the paper [SqueezeBERT: What can computer vision teach NLP about efficient neural networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W. Keutzer. 1. **[T5](https://huggingface.co/transformers/model_doc/t5.html)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. +1. **[TAPAS](https://huggingface.co/transformers/master/model_doc/tapas.html)** released with the paper [TAPAS: Weakly Supervised Table Parsing via Pre-training](https://arxiv.org/abs/2004.02349) by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos. 1. **[Transformer-XL](https://huggingface.co/transformers/model_doc/transformerxl.html)** (from Google/CMU) released with the paper [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context](https://arxiv.org/abs/1901.02860) by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov. 1. **[XLM](https://huggingface.co/transformers/model_doc/xlm.html)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 1. **[XLM-ProphetNet](https://huggingface.co/transformers/model_doc/xlmprophetnet.html)** (from Microsoft Research) released with the paper [ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training](https://arxiv.org/abs/2001.04063) by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. diff --git a/docs/source/index.rst b/docs/source/index.rst index cc99825d7f87..7f7f004449e0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -176,19 +176,22 @@ and conversion utilities for the following models: 30. :doc:`T5 ` (from Google AI) released with the paper `Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer `__ by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. -31. :doc:`Transformer-XL ` (from Google/CMU) released with the paper `Transformer-XL: +31. `TAPAS `__ released with the paper `TAPAS: Weakly + Supervised Table Parsing via Pre-training `__ by Jonathan Herzig, Paweł Krzysztof + Nowak, Thomas Müller, Francesco Piccinno and Julian Martin Eisenschlos. +32. :doc:`Transformer-XL ` (from Google/CMU) released with the paper `Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context `__ by Zihang Dai*, Zhilin Yang*, Yiming Yang, Jaime Carbonell, Quoc V. Le, Ruslan Salakhutdinov. -32. :doc:`XLM ` (from Facebook) released together with the paper `Cross-lingual Language Model +33. :doc:`XLM ` (from Facebook) released together with the paper `Cross-lingual Language Model Pretraining `__ by Guillaume Lample and Alexis Conneau. -33. :doc:`XLM-ProphetNet ` (from Microsoft Research) released with the paper `ProphetNet: +34. :doc:`XLM-ProphetNet ` (from Microsoft Research) released with the paper `ProphetNet: Predicting Future N-gram for Sequence-to-Sequence Pre-training `__ by Yu Yan, Weizhen Qi, Yeyun Gong, Dayiheng Liu, Nan Duan, Jiusheng Chen, Ruofei Zhang and Ming Zhou. -34. :doc:`XLM-RoBERTa ` (from Facebook AI), released together with the paper `Unsupervised +35. :doc:`XLM-RoBERTa ` (from Facebook AI), released together with the paper `Unsupervised Cross-lingual Representation Learning at Scale `__ by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. -35. :doc:`XLNet ` (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive +36. :doc:`XLNet ` (from Google/CMU) released with the paper `​XLNet: Generalized Autoregressive Pretraining for Language Understanding `__ by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. @@ -269,6 +272,8 @@ TensorFlow and/or Flax. +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+ | T5 | ✅ | ✅ | ✅ | ✅ | ❌ | +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+ +| TAPAS | ✅ | ❌ | ✅ | ❌ | ❌ | ++-----------------------------+----------------+----------------+-----------------+--------------------+--------------+ | Transformer-XL | ✅ | ❌ | ✅ | ✅ | ❌ | +-----------------------------+----------------+----------------+-----------------+--------------------+--------------+ | XLM | ✅ | ❌ | ✅ | ✅ | ❌ | @@ -382,6 +387,7 @@ TensorFlow and/or Flax. model_doc/roberta model_doc/squeezebert model_doc/t5 + model_doc/tapas model_doc/transformerxl model_doc/xlm model_doc/xlmprophetnet diff --git a/docs/source/model_doc/tapas.rst b/docs/source/model_doc/tapas.rst new file mode 100644 index 000000000000..3355ab72c1e7 --- /dev/null +++ b/docs/source/model_doc/tapas.rst @@ -0,0 +1,424 @@ +TAPAS +----------------------------------------------------------------------------------------------------------------------- + +Overview +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The TAPAS model was proposed in `TAPAS: Weakly Supervised Table Parsing via Pre-training +`__ by Jonathan Herzig, Paweł Krzysztof Nowak, Thomas Müller, +Francesco Piccinno and Julian Martin Eisenschlos. It's a BERT-based model specifically designed (and pre-trained) for +answering questions about tabular data. Compared to BERT, TAPAS uses relative position embeddings and has 7 token types +that encode tabular structure. TAPAS is pre-trained on the masked language modeling (MLM) objective on a large dataset +comprising millions of tables from English Wikipedia and corresponding texts. For question answering, TAPAS has 2 heads +on top: a cell selection head and an aggregation head, for (optionally) performing aggregations (such as counting or +summing) among selected cells. TAPAS has been fine-tuned on several datasets: `SQA +`__ (Sequential Question Answering by Microsoft), `WTQ +`__ (Wiki Table Questions by Stanford University) and `WikiSQL +`__ (by Salesforce). It achieves state-of-the-art on both SQA and WTQ, while +having comparable performance to SOTA on WikiSQL, with a much simpler architecture. + +The abstract from the paper is the following: + +*Answering natural language questions over tables is usually seen as a semantic parsing task. To alleviate the +collection cost of full logical forms, one popular approach focuses on weak supervision consisting of denotations +instead of logical forms. However, training semantic parsers from weak supervision poses difficulties, and in addition, +the generated logical forms are only used as an intermediate step prior to retrieving the denotation. In this paper, we +present TAPAS, an approach to question answering over tables without generating logical forms. TAPAS trains from weak +supervision, and predicts the denotation by selecting table cells and optionally applying a corresponding aggregation +operator to such selection. TAPAS extends BERT's architecture to encode tables as input, initializes from an effective +joint pre-training of text segments and tables crawled from Wikipedia, and is trained end-to-end. We experiment with +three different semantic parsing datasets, and find that TAPAS outperforms or rivals semantic parsing models by +improving state-of-the-art accuracy on SQA from 55.1 to 67.2 and performing on par with the state-of-the-art on WIKISQL +and WIKITQ, but with a simpler model architecture. We additionally find that transfer learning, which is trivial in our +setting, from WIKISQL to WIKITQ, yields 48.7 accuracy, 4.2 points above the state-of-the-art.* + +In addition, the authors have further pre-trained TAPAS to recognize **table entailment**, by creating a balanced +dataset of millions of automatically created training examples which are learned in an intermediate step prior to +fine-tuning. The authors of TAPAS call this further pre-training intermediate pre-training (since TAPAS is first +pre-trained on MLM, and then on another dataset). They found that intermediate pre-training further improves +performance on SQA, achieving a new state-of-the-art as well as state-of-the-art on `TabFact +`__, a large-scale dataset with 16k Wikipedia tables for table +entailment (a binary classification task). For more details, see their follow-up paper: `Understanding tables with +intermediate pre-training `__ by Julian Martin Eisenschlos, +Syrine Krichene and Thomas Müller. + +The original code can be found `here `__. + +Tips: + +- TAPAS is a model that uses relative position embeddings by default (restarting the position embeddings at every cell + of the table). Note that this is something that was added after the publication of the original TAPAS paper. + According to the authors, this usually results in a slightly better performance, and allows you to encode longer + sequences without running out of embeddings. This is reflected in the ``reset_position_index_per_cell`` parameter of + :class:`~transformers.TapasConfig`, which is set to ``True`` by default. The latest versions of the models available + in the `model hub `_ all use relative position embeddings. You can still + use the ones with absolute position embeddings by passing in a certain version when calling the + ``.from_pretrained()`` method as explained in the model cards. Note that it's usually advised to pad the inputs on + the right rather than the left. +- TAPAS is based on BERT, so ``TAPAS-base`` for example corresponds to a ``BERT-base`` architecture. Of course, + TAPAS-large will result in the best performance (the results reported in the paper are from TAPAS-large). Results of + the various sized models are shown on the `original Github repository `_. +- TAPAS has checkpoints fine-tuned on SQA, which are capable of answering questions related to a table in a + conversational set-up. This means that you can ask follow-up questions such as "what is his age?" related to the + previous question. Note that the forward pass of TAPAS is a bit different in case of a conversational set-up: in that + case, you have to feed every table-question pair one by one to the model, such that the `prev_labels` token type ids + can be overwritten by the predicted `labels` of the model to the previous question. See "Usage" section for more + info. +- TAPAS is similar to BERT and therefore relies on the masked language modeling (MLM) objective. It is therefore + efficient at predicting masked tokens and at NLU in general, but is not optimal for text generation. Models trained + with a causal language modeling (CLM) objective are better in that regard. + + +Usage: fine-tuning +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here we explain how you can fine-tune :class:`~transformers.TapasForQuestionAnswering` on your own dataset. + +**STEP 1: Choose one of the 3 ways in which you can use TAPAS - or experiment** + +Basically, there are 3 different ways in which one can fine-tune :class:`~transformers.TapasForQuestionAnswering`, +corresponding to the different datasets on which Tapas was fine-tuned: + +1. SQA: if you're interested in asking follow-up questions related to a table, in a conversational set-up. For example + if you first ask "what's the name of the first actor?" then you can ask a follow-up question such as "how old is + he?". Here, questions do not involve any aggregation (all questions are cell selection questions). +2. WTQ: if you're not interested in asking questions in a conversational set-up, but rather just asking questions + related to a table, which might involve aggregation, such as counting a number of rows, summing up cell values or + averaging cell values. You can then for example ask "what's the total number of goals Cristiano Ronaldo made in his + career?". This case is also called **weak supervision**, since the model itself must learn the appropriate + aggregation operator (SUM/COUNT/AVERAGE/NONE) given only the answer to the question as supervision. +3. WikiSQL-supervised: this dataset is based on WikiSQL with the model being given the ground truth aggregation + operator during training. This is also called **strong supervision**. Here, learning the appropriate aggregation + operator is much easier. + +To summarize: + ++------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+ +| **Task** | **Example dataset** | **Description** | ++------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+ +| Conversational | SQA | Conversational, only cell selection questions | ++------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+ +| Weak supervision for aggregation | WTQ | Questions might involve aggregation, and the model must learn this given only the answer as supervision | ++------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+ +| Strong supervision for aggregation | WikiSQL-supervised | Questions might involve aggregation, and the model must learn this given the gold aggregation operator | ++------------------------------------+----------------------+-------------------------------------------------------------------------------------------------------------------+ + +Initializing a model with a pre-trained base and randomly initialized classification heads from the model hub can be +done as follows (be sure to have installed the `torch-scatter dependency `_ +for your environment): + +.. code-block:: + + >>> from transformers import TapasConfig, TapasForQuestionAnswering + + >>> # for example, the base sized model with default SQA configuration + >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base') + + >>> # or, the base sized model with WTQ configuration + >>> config = TapasConfig.from_pretrained('google/tapas-base-finetuned-wtq') + >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config) + + >>> # or, the base sized model with WikiSQL configuration + >>> config = TapasConfig('google-base-finetuned-wikisql-supervised') + >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config) + + +Of course, you don't necessarily have to follow one of these three ways in which TAPAS was fine-tuned. You can also +experiment by defining any hyperparameters you want when initializing :class:`~transformers.TapasConfig`, and then +create a :class:`~transformers.TapasForQuestionAnswering` based on that configuration. For example, if you have a +dataset that has both conversational questions and questions that might involve aggregation, then you can do it this +way. Here's an example: + +.. code-block:: + + >>> from transformers import TapasConfig, TapasForQuestionAnswering + + >>> # you can initialize the classification heads any way you want (see docs of TapasConfig) + >>> config = TapasConfig(num_aggregation_labels=3, average_logits_per_cell=True, select_one_column=False) + >>> # initializing the pre-trained base sized model with our custom classification heads + >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base', config=config) + +What you can also do is start from an already fine-tuned checkpoint. A note here is that the already fine-tuned +checkpoint on WTQ has some issues due to the L2-loss which is somewhat brittle. See `here +`__ for more info. + +For a list of all pre-trained and fine-tuned TAPAS checkpoints available in the HuggingFace model hub, see `here +`__. + +**STEP 2: Prepare your data in the SQA format** + +Second, no matter what you picked above, you should prepare your dataset in the `SQA format +`__. This format is a TSV/CSV file with the following +columns: + +- ``id``: optional, id of the table-question pair, for bookkeeping purposes. +- ``annotator``: optional, id of the person who annotated the table-question pair, for bookkeeping purposes. +- ``position``: integer indicating if the question is the first, second, third,... related to the table. Only required + in case of conversational setup (SQA). You don't need this column in case you're going for WTQ/WikiSQL-supervised. +- ``question``: string +- ``table_file``: string, name of a csv file containing the tabular data +- ``answer_coordinates``: list of one or more tuples (each tuple being a cell coordinate, i.e. row, column pair that is + part of the answer) +- ``answer_text``: list of one or more strings (each string being a cell value that is part of the answer) +- ``aggregation_label``: index of the aggregation operator. Only required in case of strong supervision for aggregation + (the WikiSQL-supervised case) +- ``float_answer``: the float answer to the question, if there is one (np.nan if there isn't). Only required in case of + weak supervision for aggregation (such as WTQ and WikiSQL) + +The tables themselves should be present in a folder, each table being a separate csv file. Note that the authors of the +TAPAS algorithm used conversion scripts with some automated logic to convert the other datasets (WTQ, WikiSQL) into the +SQA format. The author explains this `here +`__. Interestingly, these conversion scripts +are not perfect (the ``answer_coordinates`` and ``float_answer`` fields are populated based on the ``answer_text``), +meaning that WTQ and WikiSQL results could actually be improved. + +**STEP 3: Convert your data into PyTorch tensors using TapasTokenizer** + +Third, given that you've prepared your data in this TSV/CSV format (and corresponding CSV files containing the tabular +data), you can then use :class:`~transformers.TapasTokenizer` to convert table-question pairs into :obj:`input_ids`, +:obj:`attention_mask`, :obj:`token_type_ids` and so on. Again, based on which of the three cases you picked above, +:class:`~transformers.TapasForQuestionAnswering` requires different inputs to be fine-tuned: + ++------------------------------------+----------------------------------------------------------------------------------------------+ +| **Task** | **Required inputs** | ++------------------------------------+----------------------------------------------------------------------------------------------+ +| Conversational | ``input_ids``, ``attention_mask``, ``token_type_ids``, ``labels`` | ++------------------------------------+----------------------------------------------------------------------------------------------+ +| Weak supervision for aggregation | ``input_ids``, ``attention_mask``, ``token_type_ids``, ``labels``, ``numeric_values``, | +| | ``numeric_values_scale``, ``float_answer`` | ++------------------------------------+----------------------------------------------------------------------------------------------+ +| Strong supervision for aggregation | ``input ids``, ``attention mask``, ``token type ids``, ``labels``, ``aggregation_labels`` | ++------------------------------------+----------------------------------------------------------------------------------------------+ + +:class:`~transformers.TapasTokenizer` creates the ``labels``, ``numeric_values`` and ``numeric_values_scale`` based on +the ``answer_coordinates`` and ``answer_text`` columns of the TSV file. The ``float_answer`` and ``aggregation_labels`` +are already in the TSV file of step 2. Here's an example: + +.. code-block:: + + >>> from transformers import TapasTokenizer + >>> import pandas as pd + + >>> model_name = 'google/tapas-base' + >>> tokenizer = TapasTokenizer.from_pretrained(model_name) + + >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], 'Number of movies': ["87", "53", "69"]} + >>> queries = ["What is the name of the first actor?", "How many movies has George Clooney played in?", "What is the total number of movies?"] + >>> answer_coordinates = [[(0, 0)], [(2, 1)], [(0, 1), (1, 1), (2, 1)]] + >>> answer_text = [["Brad Pitt"], ["69"], ["209"]] + >>> table = pd.DataFrame.from_dict(data) + >>> inputs = tokenizer(table=table, queries=queries, answer_coordinates=answer_coordinates, answer_text=answer_text, padding='max_length', return_tensors='pt') + >>> inputs + {'input_ids': tensor([[ ... ]]), 'attention_mask': tensor([[...]]), 'token_type_ids': tensor([[[...]]]), + 'numeric_values': tensor([[ ... ]]), 'numeric_values_scale: tensor([[ ... ]]), labels: tensor([[ ... ]])} + +Note that :class:`~transformers.TapasTokenizer` expects the data of the table to be **text-only**. You can use +``.astype(str)`` on a dataframe to turn it into text-only data. Of course, this only shows how to encode a single +training example. It is advised to create a PyTorch dataset and a corresponding dataloader: + +.. code-block:: + + >>> import torch + >>> import pandas as pd + + >>> tsv_path = "your_path_to_the_tsv_file" + >>> table_csv_path = "your_path_to_a_directory_containing_all_csv_files" + + >>> class TableDataset(torch.utils.data.Dataset): + ... def __init__(self, data, tokenizer): + ... self.data = data + ... self.tokenizer = tokenizer + ... + ... def __getitem__(self, idx): + ... item = data.iloc[idx] + ... table = pd.read_csv(table_csv_path + item.table_file).astype(str) # be sure to make your table data text only + ... encoding = self.tokenizer(table=table, + ... queries=item.question, + ... answer_coordinates=item.answer_coordinates, + ... answer_text=item.answer_text, + ... truncation=True, + ... padding="max_length", + ... return_tensors="pt" + ... ) + ... # remove the batch dimension which the tokenizer adds by default + ... encoding = {key: val.squeeze(0) for key, val in encoding.items()} + ... # add the float_answer which is also required (weak supervision for aggregation case) + ... encoding["float_answer"] = torch.tensor(item.float_answer) + ... return encoding + ... + ... def __len__(self): + ... return len(self.data) + + >>> data = pd.read_csv(tsv_path, sep='\t') + >>> train_dataset = TableDataset(data, tokenizer) + >>> train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=32) + +Note that here, we encode each table-question pair independently. This is fine as long as your dataset is **not +conversational**. In case your dataset involves conversational questions (such as in SQA), then you should first group +together the ``queries``, ``answer_coordinates`` and ``answer_text`` per table (in the order of their ``position`` +index) and batch encode each table with its questions. This will make sure that the ``prev_labels`` token types (see +docs of :class:`~transformers.TapasTokenizer`) are set correctly. + +**STEP 4: Train (fine-tune) TapasForQuestionAnswering** + +You can then fine-tune :class:`~transformers.TapasForQuestionAnswering` using native PyTorch as follows (shown here for +the weak supervision for aggregation case): + +.. code-block:: + + >>> from transformers import TapasConfig, TapasForQuestionAnswering, AdamW + + >>> # this is the default WTQ configuration + >>> config = TapasConfig( + ... num_aggregation_labels = 4, + ... use_answer_as_supervision = True, + ... answer_loss_cutoff = 0.664694, + ... cell_selection_preference = 0.207951, + ... huber_loss_delta = 0.121194, + ... init_cell_selection_weights_to_zero = True, + ... select_one_column = True, + ... allow_empty_column_selection = False, + ... temperature = 0.0352513, + ... ) + >>> model = TapasForQuestionAnswering.from_pretrained("google/tapas-base", config=config) + + >>> optimizer = AdamW(model.parameters(), lr=5e-5) + + >>> for epoch in range(2): # loop over the dataset multiple times + ... for idx, batch in enumerate(train_dataloader): + ... # get the inputs; + ... input_ids = batch["input_ids"] + ... attention_mask = batch["attention_mask"] + ... token_type_ids = batch["token_type_ids"] + ... labels = batch["labels"] + ... numeric_values = batch["numeric_values"] + ... numeric_values_scale = batch["numeric_values_scale"] + ... float_answer = batch["float_answer"] + + ... # zero the parameter gradients + ... optimizer.zero_grad() + + ... # forward + backward + optimize + ... outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, + ... labels=labels, numeric_values=numeric_values, numeric_values_scale=numeric_values_scale, + ... float_answer=float_answer) + ... loss = outputs.loss + ... loss.backward() + ... optimizer.step() + +Usage: inference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here we explain how you can use :class:`~transformers.TapasForQuestionAnswering` for inference (i.e. making predictions +on new data). For inference, only ``input_ids``, ``attention_mask`` and ``token_type_ids`` (which you can obtain using +:class:`~transformers.TapasTokenizer`) have to be provided to the model to obtain the logits. Next, you can use the +handy ``convert_logits_to_predictions`` method of :class:`~transformers.TapasTokenizer` to convert these into predicted +coordinates and optional aggregation indices. + +However, note that inference is **different** depending on whether or not the setup is conversational. In a +non-conversational set-up, inference can be done in parallel on all table-question pairs of a batch. Here's an example +of that: + +.. code-block:: + + >>> from transformers import TapasTokenizer, TapasForQuestionAnswering + >>> import pandas as pd + + >>> model_name = 'google/tapas-base-finetuned-wtq' + >>> model = TapasForQuestionAnswering.from_pretrained(model_name) + >>> tokenizer = TapasTokenizer.from_pretrained(model_name) + + >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], 'Number of movies': ["87", "53", "69"]} + >>> queries = ["What is the name of the first actor?", "How many movies has George Clooney played in?", "What is the total number of movies?"] + >>> table = pd.DataFrame.from_dict(data) + >>> inputs = tokenizer(table=table, queries=queries, padding='max_length', return_tensors="pt") + >>> outputs = model(**inputs) + >>> predicted_answer_coordinates, predicted_aggregation_indices = tokenizer.convert_logits_to_predictions( + ... inputs, + ... outputs.logits.detach(), + ... outputs.logits_aggregation.detach() + ...) + + >>> # let's print out the results: + >>> id2aggregation = {0: "NONE", 1: "SUM", 2: "AVERAGE", 3:"COUNT"} + >>> aggregation_predictions_string = [id2aggregation[x] for x in predicted_aggregation_indices] + + >>> answers = [] + >>> for coordinates in predicted_answer_coordinates: + ... if len(coordinates) == 1: + ... # only a single cell: + ... answers.append(table.iat[coordinates[0]]) + ... else: + ... # multiple cells + ... cell_values = [] + ... for coordinate in coordinates: + ... cell_values.append(table.iat[coordinate]) + ... answers.append(", ".join(cell_values)) + + >>> display(table) + >>> print("") + >>> for query, answer, predicted_agg in zip(queries, answers, aggregation_predictions_string): + ... print(query) + ... if predicted_agg == "NONE": + ... print("Predicted answer: " + answer) + ... else: + ... print("Predicted answer: " + predicted_agg + " > " + answer) + What is the name of the first actor? + Predicted answer: Brad Pitt + How many movies has George Clooney played in? + Predicted answer: COUNT > 69 + What is the total number of movies? + Predicted answer: SUM > 87, 53, 69 + +In case of a conversational set-up, then each table-question pair must be provided **sequentially** to the model, such +that the ``prev_labels`` token types can be overwritten by the predicted ``labels`` of the previous table-question +pair. + + +Tapas specific outputs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.models.tapas.modeling_tapas.TableQuestionAnsweringOutput + :members: + + +TapasConfig +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TapasConfig + :members: + + +TapasTokenizer +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TapasTokenizer + :members: __call__, convert_logits_to_predictions, save_vocabulary + + +TapasModel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TapasModel + :members: forward + + +TapasForMaskedLM +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TapasForMaskedLM + :members: forward + + +TapasForSequenceClassification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TapasForSequenceClassification + :members: forward + + +TapasForQuestionAnswering +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TapasForQuestionAnswering + :members: forward diff --git a/docs/source/model_sharing.rst b/docs/source/model_sharing.rst index d4b512d247bf..2adbd0c4da04 100644 --- a/docs/source/model_sharing.rst +++ b/docs/source/model_sharing.rst @@ -60,7 +60,7 @@ Basic steps In order to upload a model, you'll need to first create a git repo. This repo will live on the model hub, allowing users to clone it and you (and your organization members) to push to it. -You can create a model repo directly from the website, `here `. +You can create a model repo **directly from `the /new page on the website `__.** Alternatively, you can use the ``transformers-cli``. The next steps describe that process: @@ -82,12 +82,12 @@ This creates a repo on the model hub, which can be cloned. .. code-block:: bash - git clone https://huggingface.co/username/your-model-name - # Make sure you have git-lfs installed # (https://git-lfs.github.com/) git lfs install + git clone https://huggingface.co/username/your-model-name + When you have your local clone of your repo and lfs installed, you can then add/remove from that clone as you would with any other git repo. @@ -98,8 +98,12 @@ with any other git repo. echo "hello" >> README.md git add . && git commit -m "Update from $USER" -We are intentionally not wrapping git too much, so as to stay intuitive and easy-to-use. +We are intentionally not wrapping git too much, so that you can go on with the workflow you're used to and the tools +you already know. +The only learning curve you might have compared to regular git is the one for git-lfs. The documentation at +`git-lfs.github.com `__ is decent, but we'll work on a tutorial with some tips and tricks +in the coming weeks! Make your model work on all frameworks ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -110,7 +114,7 @@ Make your model work on all frameworks You probably have your favorite framework, but so will other users! That's why it's best to upload your model with both PyTorch `and` TensorFlow checkpoints to make it easier to use (if you skip this step, users will still be able to load your model in another framework, but it will be slower, as it will have to be converted on the fly). Don't worry, it's -super easy to do (and in a future version, it will all be automatic). You will need to install both PyTorch and +super easy to do (and in a future version, it might all be automatic). You will need to install both PyTorch and TensorFlow for this step, but you don't need to worry about the GPU, so it should be very easy. Check the `TensorFlow installation page `__ and/or the `PyTorch installation page `__ to see how. @@ -192,7 +196,7 @@ status`` command: git add --all git status -Finally, the files should be comitted: +Finally, the files should be committed: .. code-block:: bash @@ -210,23 +214,20 @@ This will upload the folder containing the weights, tokenizer and configuration Add a model card ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To make sure everyone knows what your model can do, what its limitations and potential bias or ethetical -considerations, please add a README.md model card to the 🤗 Transformers repo under `model_cards/`. It should then be -placed in a subfolder with your username or organization, then another subfolder named like your model -(`awesome-name-you-picked`). Or just click on the "Create a model card on GitHub" button on the model page, it will get -you directly to the right location. If you need one, `here `__ is a model -card template (meta-suggestions are welcome). +To make sure everyone knows what your model can do, what its limitations, potential bias or ethical considerations are, +please add a README.md model card to your model repo. You can just create it, or there's also a convenient button +titled "Add a README.md" on your model page. A model card template can be found `here +`__ (meta-suggestions are welcome). model card template (meta-suggestions +are welcome). -If your model is fine-tuned from another model coming from the model hub (all 🤗 Transformers pretrained models do), -don't forget to link to its model card so that people can fully trace how your model was built. +.. note:: -If you have never made a pull request to the 🤗 Transformers repo, look at the :doc:`contributing guide ` -to see the steps to follow. + Model cards used to live in the 🤗 Transformers repo under `model_cards/`, but for consistency and scalability we + migrated every model card from the repo to its corresponding huggingface.co model repo. -.. note:: +If your model is fine-tuned from another model coming from the model hub (all 🤗 Transformers pretrained models do), +don't forget to link to its model card so that people can fully trace how your model was built. - You can also send your model card in the folder you uploaded with the CLI by placing it in a `README.md` file - inside `path/to/awesome-name-you-picked/`. Using your model ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -262,7 +263,8 @@ First you need to install `git-lfs` in the environment used by the notebook: sudo apt-get install git-lfs -Then you can use the :obj:`transformers-cli` to create your new repo: +Then you can use either create a repo directly from `huggingface.co `__ , or use the +:obj:`transformers-cli` to create it: .. code-block:: bash @@ -274,13 +276,14 @@ Once it's created, you can clone it and configure it (replace username by your u .. code-block:: bash + git lfs install + git clone https://username:password@huggingface.co/username/your-model-name # Alternatively if you have a token, # you can use it instead of your password git clone https://username:token@huggingface.co/username/your-model-name cd your-model-name - git lfs install git config --global user.email "email@example.com" # Tip: using the same email than for your huggingface.co account will link your commits to your profile git config --global user.name "Your name" diff --git a/examples/question-answering/utils_qa.py b/examples/question-answering/utils_qa.py index 13dda6da3c04..c26d37f9dbb0 100644 --- a/examples/question-answering/utils_qa.py +++ b/examples/question-answering/utils_qa.py @@ -76,9 +76,7 @@ def postprocess_qa_predictions( assert len(predictions) == 2, "`predictions` should be a tuple with two elements (start_logits, end_logits)." all_start_logits, all_end_logits = predictions - assert len(predictions[0]) == len( - features - ), f"Got {len(predictions[0])} predicitions and {len(features)} features." + assert len(predictions[0]) == len(features), f"Got {len(predictions[0])} predictions and {len(features)} features." # Build a map example to its corresponding features. example_id_to_index = {k: i for i, k in enumerate(examples["id"])} @@ -118,7 +116,7 @@ def postprocess_qa_predictions( # Update minimum null prediction. feature_null_score = start_logits[0] + end_logits[0] - if min_null_prediction is None or min_null_prediction["score"] < feature_null_score: + if min_null_prediction is None or min_null_prediction["score"] > feature_null_score: min_null_prediction = { "offsets": (0, 0), "score": feature_null_score, diff --git a/model_cards/Cinnamon/electra-small-japanese-discriminator/README.md b/model_cards/Cinnamon/electra-small-japanese-discriminator/README.md deleted file mode 100644 index 6efb8cf4b8fc..000000000000 --- a/model_cards/Cinnamon/electra-small-japanese-discriminator/README.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -language: ja -license: apache-2.0 ---- - -## Japanese ELECTRA-small - -We provide a Japanese **ELECTRA-Small** model, as described in [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -Our pretraining process employs subword units derived from the [Japanese Wikipedia](https://dumps.wikimedia.org/jawiki/latest), using the [Byte-Pair Encoding](https://www.aclweb.org/anthology/P16-1162.pdf) method and building on an initial tokenization with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd). For optimal performance, please take care to set your MeCab dictionary appropriately. - -## How to use the discriminator in `transformers` - -``` -from transformers import BertJapaneseTokenizer, ElectraForPreTraining - -tokenizer = BertJapaneseTokenizer.from_pretrained('Cinnamon/electra-small-japanese-discriminator', mecab_kwargs={"mecab_option": "-d /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd"}) - -model = ElectraForPreTraining.from_pretrained('Cinnamon/electra-small-japanese-discriminator') -``` diff --git a/model_cards/Cinnamon/electra-small-japanese-generator/README.md b/model_cards/Cinnamon/electra-small-japanese-generator/README.md deleted file mode 100644 index 0e014328e77f..000000000000 --- a/model_cards/Cinnamon/electra-small-japanese-generator/README.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -language: ja ---- -## Japanese ELECTRA-small - -We provide a Japanese **ELECTRA-Small** model, as described in [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -Our pretraining process employs subword units derived from the [Japanese Wikipedia](https://dumps.wikimedia.org/jawiki/latest), using the [Byte-Pair Encoding](https://www.aclweb.org/anthology/P16-1162.pdf) method and building on an initial tokenization with [mecab-ipadic-NEologd](https://github.com/neologd/mecab-ipadic-neologd). For optimal performance, please take care to set your MeCab dictionary appropriately. - -``` -# ELECTRA-small generator usage - -from transformers import BertJapaneseTokenizer, ElectraForMaskedLM - -tokenizer = BertJapaneseTokenizer.from_pretrained('Cinnamon/electra-small-japanese-generator', mecab_kwargs={"mecab_option": "-d /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd"}) - -model = ElectraForMaskedLM.from_pretrained('Cinnamon/electra-small-japanese-generator') -``` diff --git a/model_cards/DJSammy/bert-base-danish-uncased_BotXO,ai/README.md b/model_cards/DJSammy/bert-base-danish-uncased_BotXO,ai/README.md deleted file mode 100644 index 58d8a49f5b02..000000000000 --- a/model_cards/DJSammy/bert-base-danish-uncased_BotXO,ai/README.md +++ /dev/null @@ -1,142 +0,0 @@ ---- -language: da -tags: -- bert -- masked-lm -license: cc-by-4.0 -datasets: -- common_crawl -- wikipedia -pipeline_tag: fill-mask -widget: -- text: "København er [MASK] i Danmark." ---- - -# Danish BERT (uncased) model - -[BotXO.ai](https://www.botxo.ai/) developed this model. For data and training details see their [GitHub repository](https://github.com/botxo/nordic_bert). - -The original model was trained in TensorFlow then I converted it to Pytorch using [transformers-cli](https://huggingface.co/transformers/converting_tensorflow_models.html?highlight=cli). - -For TensorFlow version download here: https://www.dropbox.com/s/19cjaoqvv2jicq9/danish_bert_uncased_v2.zip?dl=1 - - -## Architecture - -```python -from transformers import AutoModelForPreTraining - -model = AutoModelForPreTraining.from_pretrained("DJSammy/bert-base-danish-uncased_BotXO,ai") - -params = list(model.named_parameters()) -print('danish_bert_uncased_v2 has {:} different named parameters.\n'.format(len(params))) - -print('==== Embedding Layer ====\n') -for p in params[0:5]: - print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size())))) - -print('\n==== First Transformer ====\n') -for p in params[5:21]: - print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size())))) - -print('\n==== Last Transformer ====\n') -for p in params[181:197]: - print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size())))) - -print('\n==== Output Layer ====\n') -for p in params[197:]: - print("{:<55} {:>12}".format(p[0], str(tuple(p[1].size())))) - -# danish_bert_uncased_v2 has 206 different named parameters. - -# ==== Embedding Layer ==== - -# bert.embeddings.word_embeddings.weight (32000, 768) -# bert.embeddings.position_embeddings.weight (512, 768) -# bert.embeddings.token_type_embeddings.weight (2, 768) -# bert.embeddings.LayerNorm.weight (768,) -# bert.embeddings.LayerNorm.bias (768,) - -# ==== First Transformer ==== - -# bert.encoder.layer.0.attention.self.query.weight (768, 768) -# bert.encoder.layer.0.attention.self.query.bias (768,) -# bert.encoder.layer.0.attention.self.key.weight (768, 768) -# bert.encoder.layer.0.attention.self.key.bias (768,) -# bert.encoder.layer.0.attention.self.value.weight (768, 768) -# bert.encoder.layer.0.attention.self.value.bias (768,) -# bert.encoder.layer.0.attention.output.dense.weight (768, 768) -# bert.encoder.layer.0.attention.output.dense.bias (768,) -# bert.encoder.layer.0.attention.output.LayerNorm.weight (768,) -# bert.encoder.layer.0.attention.output.LayerNorm.bias (768,) -# bert.encoder.layer.0.intermediate.dense.weight (3072, 768) -# bert.encoder.layer.0.intermediate.dense.bias (3072,) -# bert.encoder.layer.0.output.dense.weight (768, 3072) -# bert.encoder.layer.0.output.dense.bias (768,) -# bert.encoder.layer.0.output.LayerNorm.weight (768,) -# bert.encoder.layer.0.output.LayerNorm.bias (768,) - -# ==== Last Transformer ==== - -# bert.encoder.layer.11.attention.self.query.weight (768, 768) -# bert.encoder.layer.11.attention.self.query.bias (768,) -# bert.encoder.layer.11.attention.self.key.weight (768, 768) -# bert.encoder.layer.11.attention.self.key.bias (768,) -# bert.encoder.layer.11.attention.self.value.weight (768, 768) -# bert.encoder.layer.11.attention.self.value.bias (768,) -# bert.encoder.layer.11.attention.output.dense.weight (768, 768) -# bert.encoder.layer.11.attention.output.dense.bias (768,) -# bert.encoder.layer.11.attention.output.LayerNorm.weight (768,) -# bert.encoder.layer.11.attention.output.LayerNorm.bias (768,) -# bert.encoder.layer.11.intermediate.dense.weight (3072, 768) -# bert.encoder.layer.11.intermediate.dense.bias (3072,) -# bert.encoder.layer.11.output.dense.weight (768, 3072) -# bert.encoder.layer.11.output.dense.bias (768,) -# bert.encoder.layer.11.output.LayerNorm.weight (768,) -# bert.encoder.layer.11.output.LayerNorm.bias (768,) - -# ==== Output Layer ==== - -# bert.pooler.dense.weight (768, 768) -# bert.pooler.dense.bias (768,) -# cls.predictions.bias (32000,) -# cls.predictions.transform.dense.weight (768, 768) -# cls.predictions.transform.dense.bias (768,) -# cls.predictions.transform.LayerNorm.weight (768,) -# cls.predictions.transform.LayerNorm.bias (768,) -# cls.seq_relationship.weight (2, 768) -# cls.seq_relationship.bias (2,) -``` - -## Example Pipeline - -```python -from transformers import pipeline -unmasker = pipeline('fill-mask', model='DJSammy/bert-base-danish-uncased_BotXO,ai') - -unmasker('København er [MASK] i Danmark.') - -# Copenhagen is the [MASK] of Denmark. -# => - -# [{'score': 0.788068950176239, -# 'sequence': '[CLS] københavn er hovedstad i danmark. [SEP]', -# 'token': 12610, -# 'token_str': 'hovedstad'}, -# {'score': 0.07606703042984009, -# 'sequence': '[CLS] københavn er hovedstaden i danmark. [SEP]', -# 'token': 8108, -# 'token_str': 'hovedstaden'}, -# {'score': 0.04299738258123398, -# 'sequence': '[CLS] københavn er metropol i danmark. [SEP]', -# 'token': 23305, -# 'token_str': 'metropol'}, -# {'score': 0.008163209073245525, -# 'sequence': '[CLS] københavn er ikke i danmark. [SEP]', -# 'token': 89, -# 'token_str': 'ikke'}, -# {'score': 0.006238455418497324, -# 'sequence': '[CLS] københavn er ogsa i danmark. [SEP]', -# 'token': 25253, -# 'token_str': 'ogsa'}] -``` diff --git a/model_cards/DeepPavlov/bert-base-bg-cs-pl-ru-cased/README.md b/model_cards/DeepPavlov/bert-base-bg-cs-pl-ru-cased/README.md deleted file mode 100644 index b4ea3db9f60d..000000000000 --- a/model_cards/DeepPavlov/bert-base-bg-cs-pl-ru-cased/README.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -language: -- bg -- cs -- pl -- ru ---- - -# bert-base-bg-cs-pl-ru-cased - -SlavicBERT\[1\] \(Slavic \(bg, cs, pl, ru\), cased, 12‑layer, 768‑hidden, 12‑heads, 180M parameters\) was trained on Russian News and four Wikipedias: Bulgarian, Czech, Polish, and Russian. Subtoken vocabulary was built using this data. Multilingual BERT was used as an initialization for SlavicBERT. - - -\[1\]: Arkhipov M., Trofimova M., Kuratov Y., Sorokin A. \(2019\). [Tuning Multilingual Transformers for Language-Specific Named Entity Recognition](https://www.aclweb.org/anthology/W19-3712/). ACL anthology W19-3712. diff --git a/model_cards/DeepPavlov/bert-base-cased-conversational/README.md b/model_cards/DeepPavlov/bert-base-cased-conversational/README.md deleted file mode 100644 index 7bb9231d86b3..000000000000 --- a/model_cards/DeepPavlov/bert-base-cased-conversational/README.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -language: en ---- - -# bert-base-cased-conversational - -Conversational BERT \(English, cased, 12‑layer, 768‑hidden, 12‑heads, 110M parameters\) was trained on the English part of Twitter, Reddit, DailyDialogues\[1\], OpenSubtitles\[2\], Debates\[3\], Blogs\[4\], Facebook News Comments. We used this training data to build the vocabulary of English subtokens and took English cased version of BERT‑base as an initialization for English Conversational BERT. - - -\[1\]: Yanran Li, Hui Su, Xiaoyu Shen, Wenjie Li, Ziqiang Cao, and Shuzi Niu. DailyDialog: A Manually Labelled Multi-turn Dialogue Dataset. IJCNLP 2017. - -\[2\]: P. Lison and J. Tiedemann, 2016, OpenSubtitles2016: Extracting Large Parallel Corpora from Movie and TV Subtitles. In Proceedings of the 10th International Conference on Language Resources and Evaluation \(LREC 2016\) - -\[3\]: Justine Zhang, Ravi Kumar, Sujith Ravi, Cristian Danescu-Niculescu-Mizil. Proceedings of NAACL, 2016. - -\[4\]: J. Schler, M. Koppel, S. Argamon and J. Pennebaker \(2006\). Effects of Age and Gender on Blogging in Proceedings of 2006 AAAI Spring Symposium on Computational Approaches for Analyzing Weblogs. diff --git a/model_cards/DeepPavlov/bert-base-multilingual-cased-sentence/README.md b/model_cards/DeepPavlov/bert-base-multilingual-cased-sentence/README.md deleted file mode 100644 index e8d22dff30d2..000000000000 --- a/model_cards/DeepPavlov/bert-base-multilingual-cased-sentence/README.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -language: -- multilingual ---- - -# bert-base-multilingual-cased-sentence - -Sentence Multilingual BERT \(101 languages, cased, 12‑layer, 768‑hidden, 12‑heads, 180M parameters\) is a representation‑based sentence encoder for 101 languages of Multilingual BERT. It is initialized with Multilingual BERT and then fine‑tuned on english MultiNLI\[1\] and on dev set of multilingual XNLI\[2\]. Sentence representations are mean pooled token embeddings in the same manner as in Sentence‑BERT\[3\]. - - -\[1\]: Williams A., Nangia N. & Bowman S. \(2017\) A Broad-Coverage Challenge Corpus for Sentence Understanding through Inference. arXiv preprint [arXiv:1704.05426](https://arxiv.org/abs/1704.05426) - -\[2\]: Williams A., Bowman S. \(2018\) XNLI: Evaluating Cross-lingual Sentence Representations. arXiv preprint [arXiv:1809.05053](https://arxiv.org/abs/1809.05053) - -\[3\]: N. Reimers, I. Gurevych \(2019\) Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks. arXiv preprint [arXiv:1908.10084](https://arxiv.org/abs/1908.10084) diff --git a/model_cards/DeepPavlov/rubert-base-cased-conversational/README.md b/model_cards/DeepPavlov/rubert-base-cased-conversational/README.md deleted file mode 100644 index ac0fe252699c..000000000000 --- a/model_cards/DeepPavlov/rubert-base-cased-conversational/README.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -language: -- ru ---- - -# rubert-base-cased-conversational - -Conversational RuBERT \(Russian, cased, 12‑layer, 768‑hidden, 12‑heads, 180M parameters\) was trained on OpenSubtitles\[1\], [Dirty](https://d3.ru/), [Pikabu](https://pikabu.ru/), and a Social Media segment of Taiga corpus\[2\]. We assembled a new vocabulary for Conversational RuBERT model on this data and initialized the model with [RuBERT](../rubert-base-cased). - - -\[1\]: P. Lison and J. Tiedemann, 2016, OpenSubtitles2016: Extracting Large Parallel Corpora from Movie and TV Subtitles. In Proceedings of the 10th International Conference on Language Resources and Evaluation \(LREC 2016\) - -\[2\]: Shavrina T., Shapovalova O. \(2017\) TO THE METHODOLOGY OF CORPUS CONSTRUCTION FOR MACHINE LEARNING: «TAIGA» SYNTAX TREE CORPUS AND PARSER. in proc. of “CORPORA2017”, international conference , Saint-Petersbourg, 2017. diff --git a/model_cards/DeepPavlov/rubert-base-cased-sentence/README.md b/model_cards/DeepPavlov/rubert-base-cased-sentence/README.md deleted file mode 100644 index f57d8715f061..000000000000 --- a/model_cards/DeepPavlov/rubert-base-cased-sentence/README.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -language: -- ru ---- - -# rubert-base-cased-sentence - -Sentence RuBERT \(Russian, cased, 12-layer, 768-hidden, 12-heads, 180M parameters\) is a representation‑based sentence encoder for Russian. It is initialized with RuBERT and fine‑tuned on SNLI\[1\] google-translated to russian and on russian part of XNLI dev set\[2\]. Sentence representations are mean pooled token embeddings in the same manner as in Sentence‑BERT\[3\]. - - -\[1\]: S. R. Bowman, G. Angeli, C. Potts, and C. D. Manning. \(2015\) A large annotated corpus for learning natural language inference. arXiv preprint [arXiv:1508.05326](https://arxiv.org/abs/1508.05326) - -\[2\]: Williams A., Bowman S. \(2018\) XNLI: Evaluating Cross-lingual Sentence Representations. arXiv preprint [arXiv:1809.05053](https://arxiv.org/abs/1809.05053) - -\[3\]: N. Reimers, I. Gurevych \(2019\) Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks. arXiv preprint [arXiv:1908.10084](https://arxiv.org/abs/1908.10084) diff --git a/model_cards/DeepPavlov/rubert-base-cased/README.md b/model_cards/DeepPavlov/rubert-base-cased/README.md deleted file mode 100644 index a686a92da422..000000000000 --- a/model_cards/DeepPavlov/rubert-base-cased/README.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -language: -- ru ---- - -# rubert-base-cased - -RuBERT \(Russian, cased, 12‑layer, 768‑hidden, 12‑heads, 180M parameters\) was trained on the Russian part of Wikipedia and news data. We used this training data to build a vocabulary of Russian subtokens and took a multilingual version of BERT‑base as an initialization for RuBERT\[1\]. - - -\[1\]: Kuratov, Y., Arkhipov, M. \(2019\). Adaptation of Deep Bidirectional Multilingual Transformers for Russian Language. arXiv preprint [arXiv:1905.07213](https://arxiv.org/abs/1905.07213). diff --git a/model_cards/Geotrend/bert-base-15lang-cased/README.md b/model_cards/Geotrend/bert-base-15lang-cased/README.md deleted file mode 100644 index c9cbd4d742ed..000000000000 --- a/model_cards/Geotrend/bert-base-15lang-cased/README.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." -- text: "Paris est la [MASK] de la France." -- text: "Paris est la capitale de la [MASK]." -- text: "L'élection américaine a eu [MASK] en novembre 2020." -- text: "تقع سويسرا في [MASK] أوروبا" -- text: "إسمي محمد وأسكن في [MASK]." ---- - -# bert-base-15lang-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -The measurements below have been computed on a [Google Cloud n1-standard-1 machine (1 vCPU, 3.75 GB)](https://cloud.google.com/compute/docs/machine-types\#n1_machine_type): - -| Model | Num parameters | Size | Memory | Loading time | -| ------------------------------- | -------------- | -------- | -------- | ------------ | -| bert-base-multilingual-cased | 178 million | 714 MB | 1400 MB | 4.2 sec | -| Geotrend/bert-base-15lang-cased | 141 million | 564 MB | 1098 MB | 3.1 sec | - -Handled languages: en, fr, es, de, zh, ar, ru, vi, el, bg, th, tr, hi, ur and sw. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-15lang-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-15lang-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-ar-cased/README.md b/model_cards/Geotrend/bert-base-ar-cased/README.md deleted file mode 100644 index 2fe73d3fa34e..000000000000 --- a/model_cards/Geotrend/bert-base-ar-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: ar - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "تقع سويسرا في [MASK] أوروبا" -- text: "إسمي محمد وأسكن في [MASK]." ---- - -# bert-base-ar-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-ar-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-ar-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-bg-cased/README.md b/model_cards/Geotrend/bert-base-bg-cased/README.md deleted file mode 100644 index 3c75e3cbf233..000000000000 --- a/model_cards/Geotrend/bert-base-bg-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: bg - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-bg-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-bg-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-bg-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-de-cased/README.md b/model_cards/Geotrend/bert-base-de-cased/README.md deleted file mode 100644 index a8b5d7d21fc1..000000000000 --- a/model_cards/Geotrend/bert-base-de-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: de - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-de-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-de-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-de-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-el-cased/README.md b/model_cards/Geotrend/bert-base-el-cased/README.md deleted file mode 100644 index f567e0f323a8..000000000000 --- a/model_cards/Geotrend/bert-base-el-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: el - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-el-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-el-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-el-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-ar-cased/README.md b/model_cards/Geotrend/bert-base-en-ar-cased/README.md deleted file mode 100644 index cb1054581076..000000000000 --- a/model_cards/Geotrend/bert-base-en-ar-cased/README.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." -- text: "تقع سويسرا في [MASK] أوروبا" -- text: "إسمي محمد وأسكن في [MASK]." ---- - -# bert-base-en-ar-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-ar-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-ar-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-bg-cased/README.md b/model_cards/Geotrend/bert-base-en-bg-cased/README.md deleted file mode 100644 index cb248bea4a8d..000000000000 --- a/model_cards/Geotrend/bert-base-en-bg-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-bg-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-bg-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-bg-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-cased/README.md b/model_cards/Geotrend/bert-base-en-cased/README.md deleted file mode 100644 index 5121ad5fec4d..000000000000 --- a/model_cards/Geotrend/bert-base-en-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: en - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-de-cased/README.md b/model_cards/Geotrend/bert-base-en-de-cased/README.md deleted file mode 100644 index 32eced8b1315..000000000000 --- a/model_cards/Geotrend/bert-base-en-de-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-de-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-de-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-de-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-el-cased/README.md b/model_cards/Geotrend/bert-base-en-el-cased/README.md deleted file mode 100644 index d87f088f96ce..000000000000 --- a/model_cards/Geotrend/bert-base-en-el-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-el-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-el-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-el-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-es-cased/README.md b/model_cards/Geotrend/bert-base-en-es-cased/README.md deleted file mode 100644 index faae4283e8aa..000000000000 --- a/model_cards/Geotrend/bert-base-en-es-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-es-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-es-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-es-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-fr-cased/README.md b/model_cards/Geotrend/bert-base-en-fr-cased/README.md deleted file mode 100644 index 43d32d407697..000000000000 --- a/model_cards/Geotrend/bert-base-en-fr-cased/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." -- text: "Paris est la [MASK] de la France." -- text: "Paris est la capitale de la [MASK]." -- text: "L'élection américaine a eu [MASK] en novembre 2020." ---- - -# bert-base-en-fr-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-fr-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-fr-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-hi-cased/README.md b/model_cards/Geotrend/bert-base-en-hi-cased/README.md deleted file mode 100644 index b9d1c0fe661c..000000000000 --- a/model_cards/Geotrend/bert-base-en-hi-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-hi-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-hi-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-hi-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-ru-cased/README.md b/model_cards/Geotrend/bert-base-en-ru-cased/README.md deleted file mode 100644 index 1c36b7d4be5a..000000000000 --- a/model_cards/Geotrend/bert-base-en-ru-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-ru-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-ru-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-ru-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-sw-cased/README.md b/model_cards/Geotrend/bert-base-en-sw-cased/README.md deleted file mode 100644 index b17f65b33025..000000000000 --- a/model_cards/Geotrend/bert-base-en-sw-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-sw-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-sw-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-sw-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-th-cased/README.md b/model_cards/Geotrend/bert-base-en-th-cased/README.md deleted file mode 100644 index 7a358458a98c..000000000000 --- a/model_cards/Geotrend/bert-base-en-th-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-th-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-th-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-th-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-tr-cased/README.md b/model_cards/Geotrend/bert-base-en-tr-cased/README.md deleted file mode 100644 index 4c640a7761b1..000000000000 --- a/model_cards/Geotrend/bert-base-en-tr-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-tr-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-tr-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-tr-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-ur-cased/README.md b/model_cards/Geotrend/bert-base-en-ur-cased/README.md deleted file mode 100644 index 938ce86861e8..000000000000 --- a/model_cards/Geotrend/bert-base-en-ur-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-ur-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-ur-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-ur-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-vi-cased/README.md b/model_cards/Geotrend/bert-base-en-vi-cased/README.md deleted file mode 100644 index 64eb6bdf0cbc..000000000000 --- a/model_cards/Geotrend/bert-base-en-vi-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-vi-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-vi-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-vi-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-en-zh-cased/README.md b/model_cards/Geotrend/bert-base-en-zh-cased/README.md deleted file mode 100644 index 12c0b2b17192..000000000000 --- a/model_cards/Geotrend/bert-base-en-zh-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: multilingual - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Google generated 46 billion [MASK] in revenue." -- text: "Paris is the capital of [MASK]." -- text: "Algiers is the largest city in [MASK]." ---- - -# bert-base-en-zh-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-en-zh-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-en-zh-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-es-cased/README.md b/model_cards/Geotrend/bert-base-es-cased/README.md deleted file mode 100644 index 90bc84de9664..000000000000 --- a/model_cards/Geotrend/bert-base-es-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: es - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-es-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-es-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-es-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-fr-cased/README.md b/model_cards/Geotrend/bert-base-fr-cased/README.md deleted file mode 100644 index 7f1b30f77c9c..000000000000 --- a/model_cards/Geotrend/bert-base-fr-cased/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: fr - -datasets: wikipedia - -license: apache-2.0 - -widget: -- text: "Paris est la [MASK] de la France." -- text: "Paris est la capitale de la [MASK]." -- text: "L'élection américaine a eu [MASK] en novembre 2020." ---- - -# bert-base-fr-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-fr-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-fr-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-hi-cased/README.md b/model_cards/Geotrend/bert-base-hi-cased/README.md deleted file mode 100644 index 03f83e747605..000000000000 --- a/model_cards/Geotrend/bert-base-hi-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: hi - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-hi-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-hi-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-hi-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-ru-cased/README.md b/model_cards/Geotrend/bert-base-ru-cased/README.md deleted file mode 100644 index 9e19b5659657..000000000000 --- a/model_cards/Geotrend/bert-base-ru-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: ru - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-ru-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-ru-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-ru-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-sw-cased/README.md b/model_cards/Geotrend/bert-base-sw-cased/README.md deleted file mode 100644 index 03f4d48d6357..000000000000 --- a/model_cards/Geotrend/bert-base-sw-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: sw - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-sw-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-sw-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-sw-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-th-cased/README.md b/model_cards/Geotrend/bert-base-th-cased/README.md deleted file mode 100644 index 822bbcdd69c9..000000000000 --- a/model_cards/Geotrend/bert-base-th-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: th - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-th-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-th-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-th-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-tr-cased/README.md b/model_cards/Geotrend/bert-base-tr-cased/README.md deleted file mode 100644 index 4aad67379ec1..000000000000 --- a/model_cards/Geotrend/bert-base-tr-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: tr - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-tr-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-tr-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-tr-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-ur-cased/README.md b/model_cards/Geotrend/bert-base-ur-cased/README.md deleted file mode 100644 index bdbdb0b18e92..000000000000 --- a/model_cards/Geotrend/bert-base-ur-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: ur - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-ur-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-ur-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-ur-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-vi-cased/README.md b/model_cards/Geotrend/bert-base-vi-cased/README.md deleted file mode 100644 index 3bdcad5cc1b5..000000000000 --- a/model_cards/Geotrend/bert-base-vi-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: vi - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-vi-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-vi-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-vi-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Geotrend/bert-base-zh-cased/README.md b/model_cards/Geotrend/bert-base-zh-cased/README.md deleted file mode 100644 index 0f620c1d7fec..000000000000 --- a/model_cards/Geotrend/bert-base-zh-cased/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: zh - -datasets: wikipedia - -license: apache-2.0 ---- - -# bert-base-zh-cased - -We are sharing smaller versions of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handle a custom number of languages. - -Unlike [distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased), our versions give exactly the same representations produced by the original model which preserves the original accuracy. - -For more information please visit our paper: [Load What You Need: Smaller Versions of Multilingual BERT](https://www.aclweb.org/anthology/2020.sustainlp-1.16.pdf). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Geotrend/bert-base-zh-cased") -model = AutoModel.from_pretrained("Geotrend/bert-base-zh-cased") - -``` - -To generate other smaller versions of multilingual transformers please visit [our Github repo](https://github.com/Geotrend-research/smaller-transformers). - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-arabic/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-arabic/README.md deleted file mode 100644 index 4234735d79a2..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-arabic/README.md +++ /dev/null @@ -1,18 +0,0 @@ -This model is used detecting **hatespeech** in **Arabic language**. The mono in the name refers to the monolingual setting, where the model is trained using only Arabic language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.877609 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-english/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-english/README.md deleted file mode 100644 index eff753889235..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-english/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This model is used detecting **hatespeech** in **English language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.726030 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - - - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-french/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-french/README.md deleted file mode 100644 index 703b3acaf7bc..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-french/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This model is used detecting **hatespeech** in **French language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.692094 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - - - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-german/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-german/README.md deleted file mode 100644 index 306bfdd42bf0..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-german/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This model is used detecting **hatespeech** in **German language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.649794 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - - - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-indonesian/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-indonesian/README.md deleted file mode 100644 index d40ac4c47550..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-indonesian/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This model is used detecting **hatespeech** in **Indonesian language**. The mono in the name refers to the monolingual setting, where the model is trained using only Arabic language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.844494 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - - - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-italian/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-italian/README.md deleted file mode 100644 index 0ffe017e6e8b..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-italian/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This model is used detecting **hatespeech** in **Italian language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.837288 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - - - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-polish/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-polish/README.md deleted file mode 100644 index fd0852b0e5db..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-polish/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This model is used detecting **hatespeech** in **Polish language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.723254 for a learning rate of 2e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - - - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-portugese/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-portugese/README.md deleted file mode 100644 index 4859c10c62a9..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-portugese/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This model is used detecting **hatespeech** in **Portuguese language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.716119 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - - - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/Hate-speech-CNERG/dehatebert-mono-spanish/README.md b/model_cards/Hate-speech-CNERG/dehatebert-mono-spanish/README.md deleted file mode 100644 index 8a8e69713106..000000000000 --- a/model_cards/Hate-speech-CNERG/dehatebert-mono-spanish/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This model is used detecting **hatespeech** in **Spanish language**. The mono in the name refers to the monolingual setting, where the model is trained using only English language data. It is finetuned on multilingual bert model. -The model is trained with different learning rates and the best validation score achieved is 0.740287 for a learning rate of 3e-5. Training code can be found at this [url](https://github.com/punyajoy/DE-LIMIT) - - - -### For more details about our paper - -Sai Saketh Aluru, Binny Mathew, Punyajoy Saha and Animesh Mukherjee. "[Deep Learning Models for Multilingual Hate Speech Detection](https://arxiv.org/abs/2004.06465)". Accepted at ECML-PKDD 2020. - -***Please cite our paper in any published work that uses any of these resources.*** - -~~~ -@article{aluru2020deep, - title={Deep Learning Models for Multilingual Hate Speech Detection}, - author={Aluru, Sai Saket and Mathew, Binny and Saha, Punyajoy and Mukherjee, Animesh}, - journal={arXiv preprint arXiv:2004.06465}, - year={2020} -} - -~~~ diff --git a/model_cards/HooshvareLab/bert-base-parsbert-armanner-uncased/README.md b/model_cards/HooshvareLab/bert-base-parsbert-armanner-uncased/README.md deleted file mode 100644 index 80112e431dfe..000000000000 --- a/model_cards/HooshvareLab/bert-base-parsbert-armanner-uncased/README.md +++ /dev/null @@ -1,124 +0,0 @@ -## ParsBERT: Transformer-based Model for Persian Language Understanding - -ParsBERT is a monolingual language model based on Google’s BERT architecture with the same configurations as BERT-Base. - -Paper presenting ParsBERT: [arXiv:2005.12515](https://arxiv.org/abs/2005.12515) - -All the models (downstream tasks) are uncased and trained with whole word masking. (coming soon stay tuned) - - -## Persian NER [ARMAN, PEYMA, ARMAN+PEYMA] - -This task aims to extract named entities in the text, such as names and label with appropriate `NER` classes such as locations, organizations, etc. The datasets used for this task contain sentences that are marked with `IOB` format. In this format, tokens that are not part of an entity are tagged as `”O”` the `”B”`tag corresponds to the first word of an object, and the `”I”` tag corresponds to the rest of the terms of the same entity. Both `”B”` and `”I”` tags are followed by a hyphen (or underscore), followed by the entity category. Therefore, the NER task is a multi-class token classification problem that labels the tokens upon being fed a raw text. There are two primary datasets used in Persian NER, `ARMAN`, and `PEYMA`. In ParsBERT, we prepared ner for both datasets as well as a combination of both datasets. - - - -### PEYMA - -PEYMA dataset includes 7,145 sentences with a total of 302,530 tokens from which 41,148 tokens are tagged with seven different classes. - -1. Organization -2. Money -3. Location -4. Date -5. Time -6. Person -7. Percent - - -| Label | # | -|:------------:|:-----:| -| Organization | 16964 | -| Money | 2037 | -| Location | 8782 | -| Date | 4259 | -| Time | 732 | -| Person | 7675 | -| Percent | 699 | - - - -**Download** -You can download the dataset from [here](http://nsurl.org/tasks/task-7-named-entity-recognition-ner-for-farsi/) - ---- - -### ARMAN - -ARMAN dataset holds 7,682 sentences with 250,015 sentences tagged over six different classes. - -1. Organization -2. Location -3. Facility -4. Event -5. Product -6. Person - - -| Label | # | -|:------------:|:-----:| -| Organization | 30108 | -| Location | 12924 | -| Facility | 4458 | -| Event | 7557 | -| Product | 4389 | -| Person | 15645 | - - - -**Download** -You can download the dataset from [here](https://github.com/HaniehP/PersianNER) - - - -## Results - -The following table summarizes the F1 score obtained by ParsBERT as compared to other models and architectures. - -| Dataset | ParsBERT | MorphoBERT | Beheshti-NER | LSTM-CRF | Rule-Based CRF | BiLSTM-CRF | -|:---------------:|:--------:|:----------:|:--------------:|:----------:|:----------------:|:------------:| -| ARMAN + PEYMA | 95.13* | - | - | - | - | - | -| PEYMA | 98.79* | - | 90.59 | - | 84.00 | - | -| ARMAN | 93.10* | 89.9 | 84.03 | 86.55 | - | 77.45 | - - -## How to use :hugs: -| Notebook | Description | | -|:----------|:-------------|------:| -| [How to use Pipelines](https://github.com/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb) | Simple and efficient way to use State-of-the-Art models on downstream tasks through transformers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb) | - - -## Cite - -Please cite the following paper in your publication if you are using [ParsBERT](https://arxiv.org/abs/2005.12515) in your research: - -```markdown -@article{ParsBERT, - title={ParsBERT: Transformer-based Model for Persian Language Understanding}, - author={Mehrdad Farahani, Mohammad Gharachorloo, Marzieh Farahani, Mohammad Manthouri}, - journal={ArXiv}, - year={2020}, - volume={abs/2005.12515} -} -``` - - -## Acknowledgments - -We hereby, express our gratitude to the [Tensorflow Research Cloud (TFRC) program](https://tensorflow.org/tfrc) for providing us with the necessary computation resources. We also thank [Hooshvare](https://hooshvare.com) Research Group for facilitating dataset gathering and scraping online text resources. - - -## Contributors - -- Mehrdad Farahani: [Linkedin](https://www.linkedin.com/in/m3hrdadfi/), [Twitter](https://twitter.com/m3hrdadfi), [Github](https://github.com/m3hrdadfi) -- Mohammad Gharachorloo: [Linkedin](https://www.linkedin.com/in/mohammad-gharachorloo/), [Twitter](https://twitter.com/MGharachorloo), [Github](https://github.com/baarsaam) -- Marzieh Farahani: [Linkedin](https://www.linkedin.com/in/marziehphi/), [Twitter](https://twitter.com/marziehphi), [Github](https://github.com/marziehphi) -- Mohammad Manthouri: [Linkedin](https://www.linkedin.com/in/mohammad-manthouri-aka-mansouri-07030766/), [Twitter](https://twitter.com/mmanthouri), [Github](https://github.com/mmanthouri) -- Hooshvare Team: [Official Website](https://hooshvare.com/), [Linkedin](https://www.linkedin.com/company/hooshvare), [Twitter](https://twitter.com/hooshvare), [Github](https://github.com/hooshvare), [Instagram](https://www.instagram.com/hooshvare/) - -+ And a special thanks to Sara Tabrizi for her fantastic poster design. Follow her on: [Linkedin](https://www.linkedin.com/in/sara-tabrizi-64548b79/), [Behance](https://www.behance.net/saratabrizi), [Instagram](https://www.instagram.com/sara_b_tabrizi/) - -## Releases - -### Release v0.1 (May 29, 2019) -This is the first version of our ParsBERT NER! diff --git a/model_cards/HooshvareLab/bert-base-parsbert-ner-uncased/README.md b/model_cards/HooshvareLab/bert-base-parsbert-ner-uncased/README.md deleted file mode 100644 index 80112e431dfe..000000000000 --- a/model_cards/HooshvareLab/bert-base-parsbert-ner-uncased/README.md +++ /dev/null @@ -1,124 +0,0 @@ -## ParsBERT: Transformer-based Model for Persian Language Understanding - -ParsBERT is a monolingual language model based on Google’s BERT architecture with the same configurations as BERT-Base. - -Paper presenting ParsBERT: [arXiv:2005.12515](https://arxiv.org/abs/2005.12515) - -All the models (downstream tasks) are uncased and trained with whole word masking. (coming soon stay tuned) - - -## Persian NER [ARMAN, PEYMA, ARMAN+PEYMA] - -This task aims to extract named entities in the text, such as names and label with appropriate `NER` classes such as locations, organizations, etc. The datasets used for this task contain sentences that are marked with `IOB` format. In this format, tokens that are not part of an entity are tagged as `”O”` the `”B”`tag corresponds to the first word of an object, and the `”I”` tag corresponds to the rest of the terms of the same entity. Both `”B”` and `”I”` tags are followed by a hyphen (or underscore), followed by the entity category. Therefore, the NER task is a multi-class token classification problem that labels the tokens upon being fed a raw text. There are two primary datasets used in Persian NER, `ARMAN`, and `PEYMA`. In ParsBERT, we prepared ner for both datasets as well as a combination of both datasets. - - - -### PEYMA - -PEYMA dataset includes 7,145 sentences with a total of 302,530 tokens from which 41,148 tokens are tagged with seven different classes. - -1. Organization -2. Money -3. Location -4. Date -5. Time -6. Person -7. Percent - - -| Label | # | -|:------------:|:-----:| -| Organization | 16964 | -| Money | 2037 | -| Location | 8782 | -| Date | 4259 | -| Time | 732 | -| Person | 7675 | -| Percent | 699 | - - - -**Download** -You can download the dataset from [here](http://nsurl.org/tasks/task-7-named-entity-recognition-ner-for-farsi/) - ---- - -### ARMAN - -ARMAN dataset holds 7,682 sentences with 250,015 sentences tagged over six different classes. - -1. Organization -2. Location -3. Facility -4. Event -5. Product -6. Person - - -| Label | # | -|:------------:|:-----:| -| Organization | 30108 | -| Location | 12924 | -| Facility | 4458 | -| Event | 7557 | -| Product | 4389 | -| Person | 15645 | - - - -**Download** -You can download the dataset from [here](https://github.com/HaniehP/PersianNER) - - - -## Results - -The following table summarizes the F1 score obtained by ParsBERT as compared to other models and architectures. - -| Dataset | ParsBERT | MorphoBERT | Beheshti-NER | LSTM-CRF | Rule-Based CRF | BiLSTM-CRF | -|:---------------:|:--------:|:----------:|:--------------:|:----------:|:----------------:|:------------:| -| ARMAN + PEYMA | 95.13* | - | - | - | - | - | -| PEYMA | 98.79* | - | 90.59 | - | 84.00 | - | -| ARMAN | 93.10* | 89.9 | 84.03 | 86.55 | - | 77.45 | - - -## How to use :hugs: -| Notebook | Description | | -|:----------|:-------------|------:| -| [How to use Pipelines](https://github.com/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb) | Simple and efficient way to use State-of-the-Art models on downstream tasks through transformers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb) | - - -## Cite - -Please cite the following paper in your publication if you are using [ParsBERT](https://arxiv.org/abs/2005.12515) in your research: - -```markdown -@article{ParsBERT, - title={ParsBERT: Transformer-based Model for Persian Language Understanding}, - author={Mehrdad Farahani, Mohammad Gharachorloo, Marzieh Farahani, Mohammad Manthouri}, - journal={ArXiv}, - year={2020}, - volume={abs/2005.12515} -} -``` - - -## Acknowledgments - -We hereby, express our gratitude to the [Tensorflow Research Cloud (TFRC) program](https://tensorflow.org/tfrc) for providing us with the necessary computation resources. We also thank [Hooshvare](https://hooshvare.com) Research Group for facilitating dataset gathering and scraping online text resources. - - -## Contributors - -- Mehrdad Farahani: [Linkedin](https://www.linkedin.com/in/m3hrdadfi/), [Twitter](https://twitter.com/m3hrdadfi), [Github](https://github.com/m3hrdadfi) -- Mohammad Gharachorloo: [Linkedin](https://www.linkedin.com/in/mohammad-gharachorloo/), [Twitter](https://twitter.com/MGharachorloo), [Github](https://github.com/baarsaam) -- Marzieh Farahani: [Linkedin](https://www.linkedin.com/in/marziehphi/), [Twitter](https://twitter.com/marziehphi), [Github](https://github.com/marziehphi) -- Mohammad Manthouri: [Linkedin](https://www.linkedin.com/in/mohammad-manthouri-aka-mansouri-07030766/), [Twitter](https://twitter.com/mmanthouri), [Github](https://github.com/mmanthouri) -- Hooshvare Team: [Official Website](https://hooshvare.com/), [Linkedin](https://www.linkedin.com/company/hooshvare), [Twitter](https://twitter.com/hooshvare), [Github](https://github.com/hooshvare), [Instagram](https://www.instagram.com/hooshvare/) - -+ And a special thanks to Sara Tabrizi for her fantastic poster design. Follow her on: [Linkedin](https://www.linkedin.com/in/sara-tabrizi-64548b79/), [Behance](https://www.behance.net/saratabrizi), [Instagram](https://www.instagram.com/sara_b_tabrizi/) - -## Releases - -### Release v0.1 (May 29, 2019) -This is the first version of our ParsBERT NER! diff --git a/model_cards/HooshvareLab/bert-base-parsbert-peymaner-uncased/README.md b/model_cards/HooshvareLab/bert-base-parsbert-peymaner-uncased/README.md deleted file mode 100644 index 80112e431dfe..000000000000 --- a/model_cards/HooshvareLab/bert-base-parsbert-peymaner-uncased/README.md +++ /dev/null @@ -1,124 +0,0 @@ -## ParsBERT: Transformer-based Model for Persian Language Understanding - -ParsBERT is a monolingual language model based on Google’s BERT architecture with the same configurations as BERT-Base. - -Paper presenting ParsBERT: [arXiv:2005.12515](https://arxiv.org/abs/2005.12515) - -All the models (downstream tasks) are uncased and trained with whole word masking. (coming soon stay tuned) - - -## Persian NER [ARMAN, PEYMA, ARMAN+PEYMA] - -This task aims to extract named entities in the text, such as names and label with appropriate `NER` classes such as locations, organizations, etc. The datasets used for this task contain sentences that are marked with `IOB` format. In this format, tokens that are not part of an entity are tagged as `”O”` the `”B”`tag corresponds to the first word of an object, and the `”I”` tag corresponds to the rest of the terms of the same entity. Both `”B”` and `”I”` tags are followed by a hyphen (or underscore), followed by the entity category. Therefore, the NER task is a multi-class token classification problem that labels the tokens upon being fed a raw text. There are two primary datasets used in Persian NER, `ARMAN`, and `PEYMA`. In ParsBERT, we prepared ner for both datasets as well as a combination of both datasets. - - - -### PEYMA - -PEYMA dataset includes 7,145 sentences with a total of 302,530 tokens from which 41,148 tokens are tagged with seven different classes. - -1. Organization -2. Money -3. Location -4. Date -5. Time -6. Person -7. Percent - - -| Label | # | -|:------------:|:-----:| -| Organization | 16964 | -| Money | 2037 | -| Location | 8782 | -| Date | 4259 | -| Time | 732 | -| Person | 7675 | -| Percent | 699 | - - - -**Download** -You can download the dataset from [here](http://nsurl.org/tasks/task-7-named-entity-recognition-ner-for-farsi/) - ---- - -### ARMAN - -ARMAN dataset holds 7,682 sentences with 250,015 sentences tagged over six different classes. - -1. Organization -2. Location -3. Facility -4. Event -5. Product -6. Person - - -| Label | # | -|:------------:|:-----:| -| Organization | 30108 | -| Location | 12924 | -| Facility | 4458 | -| Event | 7557 | -| Product | 4389 | -| Person | 15645 | - - - -**Download** -You can download the dataset from [here](https://github.com/HaniehP/PersianNER) - - - -## Results - -The following table summarizes the F1 score obtained by ParsBERT as compared to other models and architectures. - -| Dataset | ParsBERT | MorphoBERT | Beheshti-NER | LSTM-CRF | Rule-Based CRF | BiLSTM-CRF | -|:---------------:|:--------:|:----------:|:--------------:|:----------:|:----------------:|:------------:| -| ARMAN + PEYMA | 95.13* | - | - | - | - | - | -| PEYMA | 98.79* | - | 90.59 | - | 84.00 | - | -| ARMAN | 93.10* | 89.9 | 84.03 | 86.55 | - | 77.45 | - - -## How to use :hugs: -| Notebook | Description | | -|:----------|:-------------|------:| -| [How to use Pipelines](https://github.com/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb) | Simple and efficient way to use State-of-the-Art models on downstream tasks through transformers | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/hooshvare/parsbert-ner/blob/master/persian-ner-pipeline.ipynb) | - - -## Cite - -Please cite the following paper in your publication if you are using [ParsBERT](https://arxiv.org/abs/2005.12515) in your research: - -```markdown -@article{ParsBERT, - title={ParsBERT: Transformer-based Model for Persian Language Understanding}, - author={Mehrdad Farahani, Mohammad Gharachorloo, Marzieh Farahani, Mohammad Manthouri}, - journal={ArXiv}, - year={2020}, - volume={abs/2005.12515} -} -``` - - -## Acknowledgments - -We hereby, express our gratitude to the [Tensorflow Research Cloud (TFRC) program](https://tensorflow.org/tfrc) for providing us with the necessary computation resources. We also thank [Hooshvare](https://hooshvare.com) Research Group for facilitating dataset gathering and scraping online text resources. - - -## Contributors - -- Mehrdad Farahani: [Linkedin](https://www.linkedin.com/in/m3hrdadfi/), [Twitter](https://twitter.com/m3hrdadfi), [Github](https://github.com/m3hrdadfi) -- Mohammad Gharachorloo: [Linkedin](https://www.linkedin.com/in/mohammad-gharachorloo/), [Twitter](https://twitter.com/MGharachorloo), [Github](https://github.com/baarsaam) -- Marzieh Farahani: [Linkedin](https://www.linkedin.com/in/marziehphi/), [Twitter](https://twitter.com/marziehphi), [Github](https://github.com/marziehphi) -- Mohammad Manthouri: [Linkedin](https://www.linkedin.com/in/mohammad-manthouri-aka-mansouri-07030766/), [Twitter](https://twitter.com/mmanthouri), [Github](https://github.com/mmanthouri) -- Hooshvare Team: [Official Website](https://hooshvare.com/), [Linkedin](https://www.linkedin.com/company/hooshvare), [Twitter](https://twitter.com/hooshvare), [Github](https://github.com/hooshvare), [Instagram](https://www.instagram.com/hooshvare/) - -+ And a special thanks to Sara Tabrizi for her fantastic poster design. Follow her on: [Linkedin](https://www.linkedin.com/in/sara-tabrizi-64548b79/), [Behance](https://www.behance.net/saratabrizi), [Instagram](https://www.instagram.com/sara_b_tabrizi/) - -## Releases - -### Release v0.1 (May 29, 2019) -This is the first version of our ParsBERT NER! diff --git a/model_cards/HooshvareLab/bert-base-parsbert-uncased/README.md b/model_cards/HooshvareLab/bert-base-parsbert-uncased/README.md deleted file mode 100644 index c931a2e29d8e..000000000000 --- a/model_cards/HooshvareLab/bert-base-parsbert-uncased/README.md +++ /dev/null @@ -1,124 +0,0 @@ -## ParsBERT: Transformer-based Model for Persian Language Understanding - -ParsBERT is a monolingual language model based on Google’s BERT architecture with the same configurations as BERT-Base. - -Paper presenting ParsBERT: [arXiv:2005.12515](https://arxiv.org/abs/2005.12515) - -All the models (downstream tasks) are uncased and trained with whole word masking. (coming soon stay tuned) - - ---- - -## Introduction - -This model is pre-trained on a large Persian corpus with various writing styles from numerous subjects (e.g., scientific, novels, news) with more than 2M documents. A large subset of this corpus was crawled manually. - -As a part of ParsBERT methodology, an extensive pre-processing combining POS tagging and WordPiece segmentation was carried out to bring the corpus into a proper format. This process produces more than 40M true sentences. - - -## Evaluation - -ParsBERT is evaluated on three NLP downstream tasks: Sentiment Analysis (SA), Text Classification, and Named Entity Recognition (NER). For this matter and due to insufficient resources, two large datasets for SA and two for text classification were manually composed, which are available for public use and benchmarking. ParsBERT outperformed all other language models, including multilingual BERT and other hybrid deep learning models for all tasks, improving the state-of-the-art performance in Persian language modeling. - -## Results - -The following table summarizes the F1 score obtained by ParsBERT as compared to other models and architectures. - - - -### Sentiment Analysis (SA) task - -| Dataset | ParsBERT | mBERT | DeepSentiPers | -|:--------------------------:|:---------:|:-----:|:-------------:| -| Digikala User Comments | 81.74* | 80.74 | - | -| SnappFood User Comments | 88.12* | 87.87 | - | -| SentiPers (Multi Class) | 71.11* | - | 69.33 | -| SentiPers (Binary Class) | 92.13* | - | 91.98 | - - - -### Text Classification (TC) task - -| Dataset | ParsBERT | mBERT | -|:-----------------:|:--------:|:-----:| -| Digikala Magazine | 93.59* | 90.72 | -| Persian News | 97.19* | 95.79 | - - -### Named Entity Recognition (NER) task - -| Dataset | ParsBERT | mBERT | MorphoBERT | Beheshti-NER | LSTM-CRF | Rule-Based CRF | BiLSTM-CRF | -|:-------:|:--------:|:--------:|:----------:|:--------------:|:----------:|:----------------:|:------------:| -| PEYMA | 93.10* | 86.64 | - | 90.59 | - | 84.00 | - | -| ARMAN | 98.79* | 95.89 | 89.9 | 84.03 | 86.55 | - | 77.45 | - - -**If you tested ParsBERT on a public dataset and you want to add your results to the table above, open a pull request or contact us. Also make sure to have your code available online so we can add it as a reference** - -## How to use - -### TensorFlow 2.0 - -```python -from transformers import AutoConfig, AutoTokenizer, TFAutoModel - -config = AutoConfig.from_pretrained("HooshvareLab/bert-base-parsbert-uncased") -tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-base-parsbert-uncased") -model = AutoModel.from_pretrained("HooshvareLab/bert-base-parsbert-uncased") - -text = "ما در هوشواره معتقدیم با انتقال صحیح دانش و آگاهی، همه افراد می‌توانند از ابزارهای هوشمند استفاده کنند. شعار ما هوش مصنوعی برای همه است." -tokenizer.tokenize(text) - ->>> ['ما', 'در', 'هوش', '##واره', 'معتقدیم', 'با', 'انتقال', 'صحیح', 'دانش', 'و', 'اگاهی', '،', 'همه', 'افراد', 'میتوانند', 'از', 'ابزارهای', 'هوشمند', 'استفاده', 'کنند', '.', 'شعار', 'ما', 'هوش', 'مصنوعی', 'برای', 'همه', 'است', '.'] - -``` - -### Pytorch - -```python -from transformers import AutoConfig, AutoTokenizer, AutoModel - -config = AutoConfig.from_pretrained("HooshvareLab/bert-base-parsbert-uncased") -tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-base-parsbert-uncased") -model = AutoModel.from_pretrained("HooshvareLab/bert-base-parsbert-uncased") -``` - - -## NLP Tasks Tutorial - -Coming soon stay tuned - - -## Cite - -Please cite the following paper in your publication if you are using [ParsBERT](https://arxiv.org/abs/2005.12515) in your research: - -```markdown -@article{ParsBERT, - title={ParsBERT: Transformer-based Model for Persian Language Understanding}, - author={Mehrdad Farahani, Mohammad Gharachorloo, Marzieh Farahani, Mohammad Manthouri}, - journal={ArXiv}, - year={2020}, - volume={abs/2005.12515} -} -``` - - -## Acknowledgments - -We hereby, express our gratitude to the [Tensorflow Research Cloud (TFRC) program](https://tensorflow.org/tfrc) for providing us with the necessary computation resources. We also thank [Hooshvare](https://hooshvare.com) Research Group for facilitating dataset gathering and scraping online text resources. - - -## Contributors - -- Mehrdad Farahani: [Linkedin](https://www.linkedin.com/in/m3hrdadfi/), [Twitter](https://twitter.com/m3hrdadfi), [Github](https://github.com/m3hrdadfi) -- Mohammad Gharachorloo: [Linkedin](https://www.linkedin.com/in/mohammad-gharachorloo/), [Twitter](https://twitter.com/MGharachorloo), [Github](https://github.com/baarsaam) -- Marzieh Farahani: [Linkedin](https://www.linkedin.com/in/marziehphi/), [Twitter](https://twitter.com/marziehphi), [Github](https://github.com/marziehphi) -- Mohammad Manthouri: [Linkedin](https://www.linkedin.com/in/mohammad-manthouri-aka-mansouri-07030766/), [Twitter](https://twitter.com/mmanthouri), [Github](https://github.com/mmanthouri) -- Hooshvare Team: [Official Website](https://hooshvare.com/), [Linkedin](https://www.linkedin.com/company/hooshvare), [Twitter](https://twitter.com/hooshvare), [Github](https://github.com/hooshvare), [Instagram](https://www.instagram.com/hooshvare/) - - -## Releases - -### Release v0.1 (May 27, 2019) -This is the first version of our ParsBERT based on BERTBASE diff --git a/model_cards/HooshvareLab/bert-fa-base-uncased/README.md b/model_cards/HooshvareLab/bert-fa-base-uncased/README.md deleted file mode 100644 index 1516c7adcb6e..000000000000 --- a/model_cards/HooshvareLab/bert-fa-base-uncased/README.md +++ /dev/null @@ -1,147 +0,0 @@ ---- -language: fa -tags: -- bert-fa -- bert-persian -- persian-lm -license: apache-2.0 ---- - -# ParsBERT (v2.0) -A Transformer-based Model for Persian Language Understanding - - -We reconstructed the vocabulary and fine-tuned the ParsBERT v1.1 on the new Persian corpora in order to provide some functionalities for using ParsBERT in other scopes! -Please follow the [ParsBERT](https://github.com/hooshvare/parsbert) repo for the latest information about previous and current models. - -## Introduction - -ParsBERT is a monolingual language model based on Google’s BERT architecture. This model is pre-trained on large Persian corpora with various writing styles from numerous subjects (e.g., scientific, novels, news) with more than `3.9M` documents, `73M` sentences, and `1.3B` words. - -Paper presenting ParsBERT: [arXiv:2005.12515](https://arxiv.org/abs/2005.12515) - -## Intended uses & limitations - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?search=bert-fa) to look for -fine-tuned versions on a task that interests you. - - -### How to use - -#### TensorFlow 2.0 - -```python -from transformers import AutoConfig, AutoTokenizer, TFAutoModel - -config = AutoConfig.from_pretrained("HooshvareLab/bert-fa-base-uncased") -tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-fa-base-uncased") -model = TFAutoModel.from_pretrained("HooshvareLab/bert-fa-base-uncased") - -text = "ما در هوشواره معتقدیم با انتقال صحیح دانش و آگاهی، همه افراد میتوانند از ابزارهای هوشمند استفاده کنند. شعار ما هوش مصنوعی برای همه است." -tokenizer.tokenize(text) - ->>> ['ما', 'در', 'هوش', '##واره', 'معتقدیم', 'با', 'انتقال', 'صحیح', 'دانش', 'و', 'اگاهی', '،', 'همه', 'افراد', 'میتوانند', 'از', 'ابزارهای', 'هوشمند', 'استفاده', 'کنند', '.', 'شعار', 'ما', 'هوش', 'مصنوعی', 'برای', 'همه', 'است', '.'] -``` - -#### Pytorch - -```python -from transformers import AutoConfig, AutoTokenizer, AutoModel - -config = AutoConfig.from_pretrained("HooshvareLab/bert-fa-base-uncased") -tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-fa-base-uncased") -model = AutoModel.from_pretrained("HooshvareLab/bert-fa-base-uncased") -``` - -## Training - -ParsBERT trained on a massive amount of public corpora ([Persian Wikidumps](https://dumps.wikimedia.org/fawiki/), [MirasText](https://github.com/miras-tech/MirasText)) and six other manually crawled text data from a various type of websites ([BigBang Page](https://bigbangpage.com/) `scientific`, [Chetor](https://www.chetor.com/) `lifestyle`, [Eligasht](https://www.eligasht.com/Blog/) `itinerary`, [Digikala](https://www.digikala.com/mag/) `digital magazine`, [Ted Talks](https://www.ted.com/talks) `general conversational`, Books `novels, storybooks, short stories from old to the contemporary era`). - -As a part of ParsBERT methodology, an extensive pre-processing combining POS tagging and WordPiece segmentation was carried out to bring the corpora into a proper format. - -## Goals -Objective goals during training are as below (after 300k steps). - -``` bash -***** Eval results ***** -global_step = 300000 -loss = 1.4392426 -masked_lm_accuracy = 0.6865794 -masked_lm_loss = 1.4469004 -next_sentence_accuracy = 1.0 -next_sentence_loss = 6.534152e-05 -``` - - -## Derivative models - -### Base Config - -#### ParsBERT v2.0 Model -- [HooshvareLab/bert-fa-base-uncased](https://huggingface.co/HooshvareLab/bert-fa-base-uncased) - -#### ParsBERT v2.0 Sentiment Analysis -- [HooshvareLab/bert-fa-base-uncased-sentiment-digikala](https://huggingface.co/HooshvareLab/bert-fa-base-uncased-sentiment-digikala) -- [HooshvareLab/bert-fa-base-uncased-sentiment-snappfood](https://huggingface.co/HooshvareLab/bert-fa-base-uncased-sentiment-snappfood) -- [HooshvareLab/bert-fa-base-uncased-sentiment-deepsentipers-binary](https://huggingface.co/HooshvareLab/bert-fa-base-uncased-sentiment-deepsentipers-binary) -- [HooshvareLab/bert-fa-base-uncased-sentiment-deepsentipers-multi](https://huggingface.co/HooshvareLab/bert-fa-base-uncased-sentiment-deepsentipers-multi) - -#### ParsBERT v2.0 Text Classification -- [HooshvareLab/bert-fa-base-uncased-clf-digimag](https://huggingface.co/HooshvareLab/bert-fa-base-uncased-clf-digimag) -- [HooshvareLab/bert-fa-base-uncased-clf-persiannews](https://huggingface.co/HooshvareLab/bert-fa-base-uncased-clf-persiannews) - -#### ParsBERT v2.0 NER -- [HooshvareLab/bert-fa-base-uncased-ner-peyma](https://huggingface.co/HooshvareLab/bert-fa-base-uncased-ner-peyma) -- [HooshvareLab/bert-fa-base-uncased-ner-arman](https://huggingface.co/HooshvareLab/bert-fa-base-uncased-ner-arman) - - -## Eval results - -ParsBERT is evaluated on three NLP downstream tasks: Sentiment Analysis (SA), Text Classification, and Named Entity Recognition (NER). For this matter and due to insufficient resources, two large datasets for SA and two for text classification were manually composed, which are available for public use and benchmarking. ParsBERT outperformed all other language models, including multilingual BERT and other hybrid deep learning models for all tasks, improving the state-of-the-art performance in Persian language modeling. - - -### Sentiment Analysis (SA) Task - -| Dataset | ParsBERT v2 | ParsBERT v1 | mBERT | DeepSentiPers | -|:------------------------:|:-----------:|:-----------:|:-----:|:-------------:| -| Digikala User Comments | 81.72 | 81.74* | 80.74 | - | -| SnappFood User Comments | 87.98 | 88.12* | 87.87 | - | -| SentiPers (Multi Class) | 71.31* | 71.11 | - | 69.33 | -| SentiPers (Binary Class) | 92.42* | 92.13 | - | 91.98 | - - -### Text Classification (TC) Task - -| Dataset | ParsBERT v2 | ParsBERT v1 | mBERT | -|:-----------------:|:-----------:|:-----------:|:-----:| -| Digikala Magazine | 93.65* | 93.59 | 90.72 | -| Persian News | 97.44* | 97.19 | 95.79 | - - -### Named Entity Recognition (NER) Task - -| Dataset | ParsBERT v2 | ParsBERT v1 | mBERT | MorphoBERT | Beheshti-NER | LSTM-CRF | Rule-Based CRF | BiLSTM-CRF | -|:-------:|:-----------:|:-----------:|:-----:|:----------:|:------------:|:--------:|:--------------:|:----------:| -| PEYMA | 93.40* | 93.10 | 86.64 | - | 90.59 | - | 84.00 | - | -| ARMAN | 99.84* | 98.79 | 95.89 | 89.9 | 84.03 | 86.55 | - | 77.45 | - - - - -### BibTeX entry and citation info - -Please cite in publications as the following: - -```bibtex -@article{ParsBERT, - title={ParsBERT: Transformer-based Model for Persian Language Understanding}, - author={Mehrdad Farahani, Mohammad Gharachorloo, Marzieh Farahani, Mohammad Manthouri}, - journal={ArXiv}, - year={2020}, - volume={abs/2005.12515} -} -``` - -## Questions? -Post a Github issue on the [ParsBERT Issues](https://github.com/hooshvare/parsbert/issues) repo. diff --git a/model_cards/KB/albert-base-swedish-cased-alpha/README.md b/model_cards/KB/albert-base-swedish-cased-alpha/README.md deleted file mode 100644 index 46bf3c700f7e..000000000000 --- a/model_cards/KB/albert-base-swedish-cased-alpha/README.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -language: sv ---- - -# Swedish BERT Models - -The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on approximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. A more complete description will be published later on. - -The following three models are currently available: - -- **bert-base-swedish-cased** (*v1*) - A BERT trained with the same hyperparameters as first published by Google. -- **bert-base-swedish-cased-ner** (*experimental*) - a BERT fine-tuned for NER using SUC 3.0. -- **albert-base-swedish-cased-alpha** (*alpha*) - A first attempt at an ALBERT for Swedish. - -All models are cased and trained with whole word masking. - -## Files - -| **name** | **files** | -|---------------------------------|-----------| -| bert-base-swedish-cased | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/config.json), [vocab](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/vocab.txt), [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/pytorch_model.bin) | -| bert-base-swedish-cased-ner | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/config.json), [vocab](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/vocab.txt) [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/pytorch_model.bin) | -| albert-base-swedish-cased-alpha | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/config.json), [sentencepiece model](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/spiece.model), [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/pytorch_model.bin) | - -TensorFlow model weights will be released soon. - -## Usage requirements / installation instructions - -The examples below require Huggingface Transformers 2.4.1 and Pytorch 1.3.1 or greater. For Transformers<2.4.0 the tokenizer must be instantiated manually and the `do_lower_case` flag parameter set to `False` and `keep_accents` to `True` (for ALBERT). - -To create an environment where the examples can be run, run the following in an terminal on your OS of choice. - -``` -# git clone https://github.com/Kungbib/swedish-bert-models -# cd swedish-bert-models -# python3 -m venv venv -# source venv/bin/activate -# pip install --upgrade pip -# pip install -r requirements.txt -``` - -### BERT Base Swedish - -A standard BERT base for Swedish trained on a variety of sources. Vocabulary size is ~50k. Using Huggingface Transformers the model can be loaded in Python as follows: - -```python -from transformers import AutoModel,AutoTokenizer - -tok = AutoTokenizer.from_pretrained('KB/bert-base-swedish-cased') -model = AutoModel.from_pretrained('KB/bert-base-swedish-cased') -``` - - -### BERT base fine-tuned for Swedish NER - -This model is fine-tuned on the SUC 3.0 dataset. Using the Huggingface pipeline the model can be easily instantiated. For Transformer<2.4.1 it seems the tokenizer must be loaded separately to disable lower-casing of input strings: - -```python -from transformers import pipeline - -nlp = pipeline('ner', model='KB/bert-base-swedish-cased-ner', tokenizer='KB/bert-base-swedish-cased-ner') - -nlp('Idag släpper KB tre språkmodeller.') -``` - -Running the Python code above should produce in something like the result below. Entity types used are `TME` for time, `PRS` for personal names, `LOC` for locations, `EVN` for events and `ORG` for organisations. These labels are subject to change. - -```python -[ { 'word': 'Idag', 'score': 0.9998126029968262, 'entity': 'TME' }, - { 'word': 'KB', 'score': 0.9814832210540771, 'entity': 'ORG' } ] -``` - -The BERT tokenizer often splits words into multiple tokens, with the subparts starting with `##`, for example the string `Engelbert kör Volvo till Herrängens fotbollsklubb` gets tokenized as `Engel ##bert kör Volvo till Herr ##ängens fotbolls ##klubb`. To glue parts back together one can use something like this: - -```python -text = 'Engelbert tar Volvon till Tele2 Arena för att titta på Djurgården IF ' +\ - 'som spelar fotboll i VM klockan två på kvällen.' - -l = [] -for token in nlp(text): - if token['word'].startswith('##'): - l[-1]['word'] += token['word'][2:] - else: - l += [ token ] - -print(l) -``` - -Which should result in the following (though less cleanly formatted): - -```python -[ { 'word': 'Engelbert', 'score': 0.99..., 'entity': 'PRS'}, - { 'word': 'Volvon', 'score': 0.99..., 'entity': 'OBJ'}, - { 'word': 'Tele2', 'score': 0.99..., 'entity': 'LOC'}, - { 'word': 'Arena', 'score': 0.99..., 'entity': 'LOC'}, - { 'word': 'Djurgården', 'score': 0.99..., 'entity': 'ORG'}, - { 'word': 'IF', 'score': 0.99..., 'entity': 'ORG'}, - { 'word': 'VM', 'score': 0.99..., 'entity': 'EVN'}, - { 'word': 'klockan', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'två', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'på', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'kvällen', 'score': 0.54..., 'entity': 'TME'} ] -``` - -### ALBERT base - -The easiest way to do this is, again, using Huggingface Transformers: - -```python -from transformers import AutoModel,AutoTokenizer - -tok = AutoTokenizer.from_pretrained('KB/albert-base-swedish-cased-alpha'), -model = AutoModel.from_pretrained('KB/albert-base-swedish-cased-alpha') -``` - -## Acknowledgements ❤️ - -- Resources from Stockholms University, Umeå University and Swedish Language Bank at Gothenburg University were used when fine-tuning BERT for NER. -- Model pretraining was made partly in-house at the KBLab and partly (for material without active copyright) with the support of Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -- Models are hosted on S3 by Huggingface 🤗 - diff --git a/model_cards/KB/bert-base-swedish-cased-ner/README.md b/model_cards/KB/bert-base-swedish-cased-ner/README.md deleted file mode 100644 index 46bf3c700f7e..000000000000 --- a/model_cards/KB/bert-base-swedish-cased-ner/README.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -language: sv ---- - -# Swedish BERT Models - -The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on approximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. A more complete description will be published later on. - -The following three models are currently available: - -- **bert-base-swedish-cased** (*v1*) - A BERT trained with the same hyperparameters as first published by Google. -- **bert-base-swedish-cased-ner** (*experimental*) - a BERT fine-tuned for NER using SUC 3.0. -- **albert-base-swedish-cased-alpha** (*alpha*) - A first attempt at an ALBERT for Swedish. - -All models are cased and trained with whole word masking. - -## Files - -| **name** | **files** | -|---------------------------------|-----------| -| bert-base-swedish-cased | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/config.json), [vocab](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/vocab.txt), [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/pytorch_model.bin) | -| bert-base-swedish-cased-ner | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/config.json), [vocab](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/vocab.txt) [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/pytorch_model.bin) | -| albert-base-swedish-cased-alpha | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/config.json), [sentencepiece model](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/spiece.model), [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/pytorch_model.bin) | - -TensorFlow model weights will be released soon. - -## Usage requirements / installation instructions - -The examples below require Huggingface Transformers 2.4.1 and Pytorch 1.3.1 or greater. For Transformers<2.4.0 the tokenizer must be instantiated manually and the `do_lower_case` flag parameter set to `False` and `keep_accents` to `True` (for ALBERT). - -To create an environment where the examples can be run, run the following in an terminal on your OS of choice. - -``` -# git clone https://github.com/Kungbib/swedish-bert-models -# cd swedish-bert-models -# python3 -m venv venv -# source venv/bin/activate -# pip install --upgrade pip -# pip install -r requirements.txt -``` - -### BERT Base Swedish - -A standard BERT base for Swedish trained on a variety of sources. Vocabulary size is ~50k. Using Huggingface Transformers the model can be loaded in Python as follows: - -```python -from transformers import AutoModel,AutoTokenizer - -tok = AutoTokenizer.from_pretrained('KB/bert-base-swedish-cased') -model = AutoModel.from_pretrained('KB/bert-base-swedish-cased') -``` - - -### BERT base fine-tuned for Swedish NER - -This model is fine-tuned on the SUC 3.0 dataset. Using the Huggingface pipeline the model can be easily instantiated. For Transformer<2.4.1 it seems the tokenizer must be loaded separately to disable lower-casing of input strings: - -```python -from transformers import pipeline - -nlp = pipeline('ner', model='KB/bert-base-swedish-cased-ner', tokenizer='KB/bert-base-swedish-cased-ner') - -nlp('Idag släpper KB tre språkmodeller.') -``` - -Running the Python code above should produce in something like the result below. Entity types used are `TME` for time, `PRS` for personal names, `LOC` for locations, `EVN` for events and `ORG` for organisations. These labels are subject to change. - -```python -[ { 'word': 'Idag', 'score': 0.9998126029968262, 'entity': 'TME' }, - { 'word': 'KB', 'score': 0.9814832210540771, 'entity': 'ORG' } ] -``` - -The BERT tokenizer often splits words into multiple tokens, with the subparts starting with `##`, for example the string `Engelbert kör Volvo till Herrängens fotbollsklubb` gets tokenized as `Engel ##bert kör Volvo till Herr ##ängens fotbolls ##klubb`. To glue parts back together one can use something like this: - -```python -text = 'Engelbert tar Volvon till Tele2 Arena för att titta på Djurgården IF ' +\ - 'som spelar fotboll i VM klockan två på kvällen.' - -l = [] -for token in nlp(text): - if token['word'].startswith('##'): - l[-1]['word'] += token['word'][2:] - else: - l += [ token ] - -print(l) -``` - -Which should result in the following (though less cleanly formatted): - -```python -[ { 'word': 'Engelbert', 'score': 0.99..., 'entity': 'PRS'}, - { 'word': 'Volvon', 'score': 0.99..., 'entity': 'OBJ'}, - { 'word': 'Tele2', 'score': 0.99..., 'entity': 'LOC'}, - { 'word': 'Arena', 'score': 0.99..., 'entity': 'LOC'}, - { 'word': 'Djurgården', 'score': 0.99..., 'entity': 'ORG'}, - { 'word': 'IF', 'score': 0.99..., 'entity': 'ORG'}, - { 'word': 'VM', 'score': 0.99..., 'entity': 'EVN'}, - { 'word': 'klockan', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'två', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'på', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'kvällen', 'score': 0.54..., 'entity': 'TME'} ] -``` - -### ALBERT base - -The easiest way to do this is, again, using Huggingface Transformers: - -```python -from transformers import AutoModel,AutoTokenizer - -tok = AutoTokenizer.from_pretrained('KB/albert-base-swedish-cased-alpha'), -model = AutoModel.from_pretrained('KB/albert-base-swedish-cased-alpha') -``` - -## Acknowledgements ❤️ - -- Resources from Stockholms University, Umeå University and Swedish Language Bank at Gothenburg University were used when fine-tuning BERT for NER. -- Model pretraining was made partly in-house at the KBLab and partly (for material without active copyright) with the support of Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -- Models are hosted on S3 by Huggingface 🤗 - diff --git a/model_cards/KB/bert-base-swedish-cased/README.md b/model_cards/KB/bert-base-swedish-cased/README.md deleted file mode 100644 index ca87015b8467..000000000000 --- a/model_cards/KB/bert-base-swedish-cased/README.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -language: sv ---- - -# Swedish BERT Models - -The National Library of Sweden / KBLab releases three pretrained language models based on BERT and ALBERT. The models are trained on aproximately 15-20GB of text (200M sentences, 3000M tokens) from various sources (books, news, government publications, swedish wikipedia and internet forums) aiming to provide a representative BERT model for Swedish text. A more complete description will be published later on. - -The following three models are currently available: - -- **bert-base-swedish-cased** (*v1*) - A BERT trained with the same hyperparameters as first published by Google. -- **bert-base-swedish-cased-ner** (*experimental*) - a BERT fine-tuned for NER using SUC 3.0. -- **albert-base-swedish-cased-alpha** (*alpha*) - A first attempt at an ALBERT for Swedish. - -All models are cased and trained with whole word masking. - -## Files - -| **name** | **files** | -|---------------------------------|-----------| -| bert-base-swedish-cased | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/config.json), [vocab](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/vocab.txt), [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased/pytorch_model.bin) | -| bert-base-swedish-cased-ner | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/config.json), [vocab](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/vocab.txt) [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/bert-base-swedish-cased-ner/pytorch_model.bin) | -| albert-base-swedish-cased-alpha | [config](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/config.json), [sentencepiece model](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/spiece.model), [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/KB/albert-base-swedish-cased-alpha/pytorch_model.bin) | - -TensorFlow model weights will be released soon. - -## Usage requirements / installation instructions - -The examples below require Huggingface Transformers 2.4.1 and Pytorch 1.3.1 or greater. For Transformers<2.4.0 the tokenizer must be instantiated manually and the `do_lower_case` flag parameter set to `False` and `keep_accents` to `True` (for ALBERT). - -To create an environment where the examples can be run, run the following in an terminal on your OS of choice. - -``` -# git clone https://github.com/Kungbib/swedish-bert-models -# cd swedish-bert-models -# python3 -m venv venv -# source venv/bin/activate -# pip install --upgrade pip -# pip install -r requirements.txt -``` - -### BERT Base Swedish - -A standard BERT base for Swedish trained on a variety of sources. Vocabulary size is ~50k. Using Huggingface Transformers the model can be loaded in Python as follows: - -```python -from transformers import AutoModel,AutoTokenizer - -tok = AutoTokenizer.from_pretrained('KB/bert-base-swedish-cased') -model = AutoModel.from_pretrained('KB/bert-base-swedish-cased') -``` - - -### BERT base fine-tuned for Swedish NER - -This model is fine-tuned on the SUC 3.0 dataset. Using the Huggingface pipeline the model can be easily instantiated. For Transformer<2.4.1 it seems the tokenizer must be loaded separately to disable lower-casing of input strings: - -```python -from transformers import pipeline - -nlp = pipeline('ner', model='KB/bert-base-swedish-cased-ner', tokenizer='KB/bert-base-swedish-cased-ner') - -nlp('Idag släpper KB tre språkmodeller.') -``` - -Running the Python code above should produce in something like the result below. Entity types used are `TME` for time, `PRS` for personal names, `LOC` for locations, `EVN` for events and `ORG` for organisations. These labels are subject to change. - -```python -[ { 'word': 'Idag', 'score': 0.9998126029968262, 'entity': 'TME' }, - { 'word': 'KB', 'score': 0.9814832210540771, 'entity': 'ORG' } ] -``` - -The BERT tokenizer often splits words into multiple tokens, with the subparts starting with `##`, for example the string `Engelbert kör Volvo till Herrängens fotbollsklubb` gets tokenized as `Engel ##bert kör Volvo till Herr ##ängens fotbolls ##klubb`. To glue parts back together one can use something like this: - -```python -text = 'Engelbert tar Volvon till Tele2 Arena för att titta på Djurgården IF ' +\ - 'som spelar fotboll i VM klockan två på kvällen.' - -l = [] -for token in nlp(text): - if token['word'].startswith('##'): - l[-1]['word'] += token['word'][2:] - else: - l += [ token ] - -print(l) -``` - -Which should result in the following (though less cleanly formated): - -```python -[ { 'word': 'Engelbert', 'score': 0.99..., 'entity': 'PRS'}, - { 'word': 'Volvon', 'score': 0.99..., 'entity': 'OBJ'}, - { 'word': 'Tele2', 'score': 0.99..., 'entity': 'LOC'}, - { 'word': 'Arena', 'score': 0.99..., 'entity': 'LOC'}, - { 'word': 'Djurgården', 'score': 0.99..., 'entity': 'ORG'}, - { 'word': 'IF', 'score': 0.99..., 'entity': 'ORG'}, - { 'word': 'VM', 'score': 0.99..., 'entity': 'EVN'}, - { 'word': 'klockan', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'två', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'på', 'score': 0.99..., 'entity': 'TME'}, - { 'word': 'kvällen', 'score': 0.54..., 'entity': 'TME'} ] -``` - -### ALBERT base - -The easisest way to do this is, again, using Huggingface Transformers: - -```python -from transformers import AutoModel,AutoTokenizer - -tok = AutoTokenizer.from_pretrained('KB/albert-base-swedish-cased-alpha'), -model = AutoModel.from_pretrained('KB/albert-base-swedish-cased-alpha') -``` - -## Acknowledgements ❤️ - -- Resources from Stockholms University, Umeå University and Swedish Language Bank at Gothenburg University were used when fine-tuning BERT for NER. -- Model pretraining was made partly in-house at the KBLab and partly (for material without active copyright) with the support of Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -- Models are hosted on S3 by Huggingface 🤗 - diff --git a/model_cards/LorenzoDeMattei/GePpeTto/README.md b/model_cards/LorenzoDeMattei/GePpeTto/README.md deleted file mode 100644 index d6dcafb226d2..000000000000 --- a/model_cards/LorenzoDeMattei/GePpeTto/README.md +++ /dev/null @@ -1,141 +0,0 @@ ---- -language: it ---- - -# GePpeTto GPT2 Model 🇮🇹 - -Pretrained GPT2 117M model for Italian. - -You can find further details in the paper: - -Lorenzo De Mattei, Michele Cafagna, Felice Dell’Orletta, Malvina Nissim, Marco Guerini "GePpeTto Carves Italian into a Language Model", arXiv preprint. Pdf available at: https://arxiv.org/abs/2004.14253 - -## Pretraining Corpus - -The pretraining set comprises two main sources. The first one is a dump of Italian Wikipedia (November 2019), -consisting of 2.8GB of text. The second one is the ItWac corpus (Baroni et al., 2009), which amounts to 11GB of web -texts. This collection provides a mix of standard and less standard Italian, on a rather wide chronological span, -with older texts than the Wikipedia dump (the latter stretches only to the late 2000s). - -## Pretraining details - -This model was trained using GPT2's Hugging Face implemenation on 4 NVIDIA Tesla T4 GPU for 620k steps. - -Training parameters: - -- GPT-2 small configuration -- vocabulary size: 30k -- Batch size: 32 -- Block size: 100 -- Adam Optimizer -- Initial learning rate: 5e-5 -- Warm up steps: 10k - -## Perplexity scores - -| Domain | Perplexity | -|---|---| -| Wikipedia | 26.1052 | -| ItWac | 30.3965 | -| Legal | 37.2197 | -| News | 45.3859 | -| Social Media | 84.6408 | - -For further details, qualitative analysis and human evaluation check out: https://arxiv.org/abs/2004.14253 - -## Load Pretrained Model - -You can use this model by installing Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import GPT2Tokenizer, GPT2Model - -model = GPT2Model.from_pretrained('LorenzoDeMattei/GePpeTto') -tokenizer = GPT2Tokenizer.from_pretrained( - 'LorenzoDeMattei/GePpeTto', -) -``` - -## Example using GPT2LMHeadModel - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead, pipeline, GPT2Tokenizer - -tokenizer = AutoTokenizer.from_pretrained("LorenzoDeMattei/GePpeTto") -model = AutoModelWithLMHead.from_pretrained("LorenzoDeMattei/GePpeTto") - -text_generator = pipeline('text-generation', model=model, tokenizer=tokenizer) -prompts = [ - "Wikipedia Geppetto", - "Maestro Ciliegia regala il pezzo di legno al suo amico Geppetto, il quale lo prende per fabbricarsi un burattino maraviglioso"] - - -samples_outputs = text_generator( - prompts, - do_sample=True, - max_length=50, - top_k=50, - top_p=0.95, - num_return_sequences=3 -) - - -for i, sample_outputs in enumerate(samples_outputs): - print(100 * '-') - print("Prompt:", prompts[i]) - for sample_output in sample_outputs: - print("Sample:", sample_output['generated_text']) - print() - -``` - -Output is, - -``` ----------------------------------------------------------------------------------------------------- -Prompt: Wikipedia Geppetto -Sample: Wikipedia Geppetto rosso (film 1920) - -Geppetto rosso ("The Smokes in the Black") è un film muto del 1920 diretto da Henry H. Leonard. - -Il film fu prodotto dalla Selig Poly - -Sample: Wikipedia Geppetto - -Geppetto ("Geppetto" in piemontese) è un comune italiano di 978 abitanti della provincia di Cuneo in Piemonte. - -L'abitato, che si trova nel versante valtellinese, si sviluppa nella - -Sample: Wikipedia Geppetto di Natale (romanzo) - -Geppetto di Natale è un romanzo di Mario Caiano, pubblicato nel 2012. - ----------------------------------------------------------------------------------------------------- -Prompt: Maestro Ciliegia regala il pezzo di legno al suo amico Geppetto, il quale lo prende per fabbricarsi un burattino maraviglioso -Sample: Maestro Ciliegia regala il pezzo di legno al suo amico Geppetto, il quale lo prende per fabbricarsi un burattino maraviglioso. Il burattino riesce a scappare. Dopo aver trovato un prezioso sacchetto si reca - -Sample: Maestro Ciliegia regala il pezzo di legno al suo amico Geppetto, il quale lo prende per fabbricarsi un burattino maraviglioso, e l'unico che lo possiede, ma, di fronte a tutte queste prove - -Sample: Maestro Ciliegia regala il pezzo di legno al suo amico Geppetto, il quale lo prende per fabbricarsi un burattino maraviglioso: - A voi gli occhi, le guance! A voi il mio pezzo! -``` - -## Citation - -Please use the following bibtex entry: - -``` -@misc{mattei2020geppetto, - title={GePpeTto Carves Italian into a Language Model}, - author={Lorenzo De Mattei and Michele Cafagna and Felice Dell'Orletta and Malvina Nissim and Marco Guerini}, - year={2020}, - eprint={2004.14253}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - -## References - -Marco Baroni, Silvia Bernardini, Adriano Ferraresi, -and Eros Zanchetta. 2009. The WaCky wide web: a -collection of very large linguistically processed webcrawled corpora. Language resources and evaluation, 43(3):209–226. diff --git a/model_cards/Michau/t5-base-en-generate-headline/README.md b/model_cards/Michau/t5-base-en-generate-headline/README.md deleted file mode 100644 index 5e9add0d4850..000000000000 --- a/model_cards/Michau/t5-base-en-generate-headline/README.md +++ /dev/null @@ -1,47 +0,0 @@ -## About the model - -The model has been trained on a collection of 500k articles with headings. Its purpose is to create a one-line heading suitable for the given article. - -Sample code with a WikiNews article: - -```python -import torch -from transformers import T5ForConditionalGeneration,T5Tokenizer - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -model = T5ForConditionalGeneration.from_pretrained("Michau/t5-base-en-generate-headline") -tokenizer = T5Tokenizer.from_pretrained("Michau/t5-base-en-generate-headline") -model = model.to(device) - -article = ''' -Very early yesterday morning, the United States President Donald Trump reported he and his wife First Lady Melania Trump tested positive for COVID-19. Officials said the Trumps' 14-year-old son Barron tested negative as did First Family and Senior Advisors Jared Kushner and Ivanka Trump. -Trump took to social media, posting at 12:54 am local time (0454 UTC) on Twitter, "Tonight, [Melania] and I tested positive for COVID-19. We will begin our quarantine and recovery process immediately. We will get through this TOGETHER!" Yesterday afternoon Marine One landed on the White House's South Lawn flying Trump to Walter Reed National Military Medical Center (WRNMMC) in Bethesda, Maryland. -Reports said both were showing "mild symptoms". Senior administration officials were tested as people were informed of the positive test. Senior advisor Hope Hicks had tested positive on Thursday. -Presidential physician Sean Conley issued a statement saying Trump has been given zinc, vitamin D, Pepcid and a daily Aspirin. Conley also gave a single dose of the experimental polyclonal antibodies drug from Regeneron Pharmaceuticals. -According to official statements, Trump, now operating from the WRNMMC, is to continue performing his duties as president during a 14-day quarantine. In the event of Trump becoming incapacitated, Vice President Mike Pence could take over the duties of president via the 25th Amendment of the US Constitution. The Pence family all tested negative as of yesterday and there were no changes regarding Pence's campaign events. -''' - -text = "headline: " + article - -max_len = 256 - -encoding = tokenizer.encode_plus(text, return_tensors = "pt") -input_ids = encoding["input_ids"].to(device) -attention_masks = encoding["attention_mask"].to(device) - -beam_outputs = model.generate( - input_ids = input_ids, - attention_mask = attention_masks, - max_length = 64, - num_beams = 3, - early_stopping = True, -) - -result = tokenizer.decode(beam_outputs[0]) -print(result) -``` - -Result: - -```Trump and First Lady Melania Test Positive for COVID-19``` diff --git a/model_cards/MoseliMotsoehli/TswanaBert/README.md b/model_cards/MoseliMotsoehli/TswanaBert/README.md deleted file mode 100644 index 5a1a483dd449..000000000000 --- a/model_cards/MoseliMotsoehli/TswanaBert/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: tn ---- - -# TswanaBert -Pretrained model on the Tswana language using a masked language modeling (MLM) objective. - -## Model Description. -TswanaBERT is a transformer model pre-trained on a corpus of Setswana in a self-supervised fashion by masking part of the input words and training to predict the masks by using byte-level tokens. - -## Intended uses & limitations -The model can be used for either masked language modeling or next word prediction. It can also be fine-tuned on a specific down-stream NLP application. - -#### How to use - -```python ->>> from transformers import pipeline ->>> from transformers import AutoTokenizer, AutoModelWithLMHead - ->>> tokenizer = AutoTokenizer.from_pretrained("MoseliMotsoehli/TswanaBert") ->>> model = AutoModelWithLMHead.from_pretrained("MoseliMotsoehli/TswanaBert") ->>> unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer) ->>> unmasker("Ntshopotse e godile.") - -[{'score': 0.32749542593955994, - 'sequence': 'Ntshopotse setse e godile.', - 'token': 538, - 'token_str': 'Ġsetse'}, - {'score': 0.060260992497205734, - 'sequence': 'Ntshopotse le e godile.', - 'token': 270, - 'token_str': 'Ġle'}, - {'score': 0.058460816740989685, - 'sequence': 'Ntshopotse bone e godile.', - 'token': 364, - 'token_str': 'Ġbone'}, - {'score': 0.05694682151079178, - 'sequence': 'Ntshopotse ga e godile.', - 'token': 298, - 'token_str': 'Ġga'}, - {'score': 0.0565204992890358, - 'sequence': 'Ntshopotse, e godile.', - 'token': 16, - 'token_str': ','}] -``` - -#### Limitations and bias -The model is trained on a relatively small collection of setwana, mostly from news articles and creative writtings, and so is not representative enough of the language as yet. - -## Training data - -1. The largest portion of this dataset (10k) sentences of text, comes from the [Leipzig Corpora Collection](https://wortschatz.uni-leipzig.de/en/download) - -2. I Then added SABC news headlines collected by Marivate Vukosi, & Sefara Tshephisho, (2020) that is generously made available on [zenoodo](http://doi.org/10.5281/zenodo.3668495 ). This added 185 tswana sentences to my corpus. - -3. I went on to add 300 more sentences by scrapping following news sites and blogs that mosty originate in Botswana. I actively continue to expand the dataset. - -* http://setswana.blogspot.com/ -* https://omniglot.com/writing/tswana.php -* http://www.dailynews.gov.bw/ -* http://www.mmegi.bw/index.php -* https://tsena.co.bw -* http://www.botswana.co.za/Cultural_Issues-travel/botswana-country-guide-en-route.html -* https://www.poemhunter.com/poem/2013-setswana/ -https://www.poemhunter.com/poem/ngwana-wa-mosetsana/ - - -### BibTeX entry and citation info - -```bibtex -@inproceedings{author = {Moseli Motsoehli}, - year={2020} -} -``` diff --git a/model_cards/MoseliMotsoehli/zuBERTa/README.md b/model_cards/MoseliMotsoehli/zuBERTa/README.md deleted file mode 100644 index 7e8fe6a844d7..000000000000 --- a/model_cards/MoseliMotsoehli/zuBERTa/README.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -language: zu ---- - -# zuBERTa -zuBERTa is a RoBERTa style transformer language model trained on zulu text. - -## Intended uses & limitations -The model can be used for getting embeddings to use on a down-stream task such as question answering. - -#### How to use - -```python ->>> from transformers import pipeline ->>> from transformers import AutoTokenizer, AutoModelWithLMHead - ->>> tokenizer = AutoTokenizer.from_pretrained("MoseliMotsoehli/zuBERTa") ->>> model = AutoModelWithLMHead.from_pretrained("MoseliMotsoehli/zuBERTa") ->>> unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer) ->>> unmasker("Abafika eNkandla bafika sebeholwa uMpongo kaZingelwayo.") - -[ - { - "sequence": "Abafika eNkandla bafika sebeholwa khona uMpongo kaZingelwayo.", - "score": 0.050459690392017365, - "token": 555, - "token_str": "Ġkhona" - }, - { - "sequence": "Abafika eNkandla bafika sebeholwa inkosi uMpongo kaZingelwayo.", - "score": 0.03668094798922539, - "token": 2321, - "token_str": "Ġinkosi" - }, - { - "sequence": "Abafika eNkandla bafika sebeholwa ubukhosi uMpongo kaZingelwayo.", - "score": 0.028774697333574295, - "token": 5101, - "token_str": "Ġubukhosi" - } -] -``` - -## Training data - -1. 30k sentences of text, came from the [Leipzig Corpora Collection](https://wortschatz.uni-leipzig.de/en/download) of zulu 2018. These were collected from news articles and creative writtings. -2. ~7500 articles of human generated translations were scraped from the zulu [wikipedia](https://zu.wikipedia.org/wiki/Special:AllPages). - -### BibTeX entry and citation info - -```bibtex -@inproceedings{author = {Moseli Motsoehli}, - title = {Towards transformation of Southern African language models through transformers.}, - year={2020} -} -``` diff --git a/model_cards/Musixmatch/umberto-commoncrawl-cased-v1/README.md b/model_cards/Musixmatch/umberto-commoncrawl-cased-v1/README.md deleted file mode 100644 index c48ce71a2aea..000000000000 --- a/model_cards/Musixmatch/umberto-commoncrawl-cased-v1/README.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -language: it ---- - -# UmBERTo Commoncrawl Cased - -[UmBERTo](https://github.com/musixmatchresearch/umberto) is a Roberta-based Language Model trained on large Italian Corpora and uses two innovative approaches: SentencePiece and Whole Word Masking. Now available at [github.com/huggingface/transformers](https://huggingface.co/Musixmatch/umberto-commoncrawl-cased-v1) - -

-
- Marco Lodola, Monument to Umberto Eco, Alessandria 2019 -

- -## Dataset -UmBERTo-Commoncrawl-Cased utilizes the Italian subcorpus of [OSCAR](https://traces1.inria.fr/oscar/) as training set of the language model. We used deduplicated version of the Italian corpus that consists in 70 GB of plain text data, 210M sentences with 11B words where the sentences have been filtered and shuffled at line level in order to be used for NLP research. - -## Pre-trained model - -| Model | WWM | Cased | Tokenizer | Vocab Size | Train Steps | Download | -| ------ | ------ | ------ | ------ | ------ |------ | ------ | -| `umberto-commoncrawl-cased-v1` | YES | YES | SPM | 32K | 125k | [Link](http://bit.ly/35zO7GH) | - -This model was trained with [SentencePiece](https://github.com/google/sentencepiece) and Whole Word Masking. - -## Downstream Tasks -These results refers to umberto-commoncrawl-cased model. All details are at [Umberto](https://github.com/musixmatchresearch/umberto) Official Page. - -#### Named Entity Recognition (NER) - -| Dataset | F1 | Precision | Recall | Accuracy | -| ------ | ------ | ------ | ------ | ------ | -| **ICAB-EvalITA07** | **87.565** | 86.596 | 88.556 | 98.690 | -| **WikiNER-ITA** | **92.531** | 92.509 | 92.553 | 99.136 | - -#### Part of Speech (POS) - -| Dataset | F1 | Precision | Recall | Accuracy | -| ------ | ------ | ------ | ------ | ------ | -| **UD_Italian-ISDT** | 98.870 | 98.861 | 98.879 | **98.977** | -| **UD_Italian-ParTUT** | 98.786 | 98.812 | 98.760 | **98.903** | - - - -## Usage - -##### Load UmBERTo with AutoModel, Autotokenizer: - -```python - -import torch -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Musixmatch/umberto-commoncrawl-cased-v1") -umberto = AutoModel.from_pretrained("Musixmatch/umberto-commoncrawl-cased-v1") - -encoded_input = tokenizer.encode("Umberto Eco è stato un grande scrittore") -input_ids = torch.tensor(encoded_input).unsqueeze(0) # Batch size 1 -outputs = umberto(input_ids) -last_hidden_states = outputs[0] # The last hidden-state is the first element of the output -``` - -##### Predict masked token: - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="Musixmatch/umberto-commoncrawl-cased-v1", - tokenizer="Musixmatch/umberto-commoncrawl-cased-v1" -) - -result = fill_mask("Umberto Eco è un grande scrittore") -# {'sequence': ' Umberto Eco è considerato un grande scrittore', 'score': 0.18599839508533478, 'token': 5032} -# {'sequence': ' Umberto Eco è stato un grande scrittore', 'score': 0.17816807329654694, 'token': 471} -# {'sequence': ' Umberto Eco è sicuramente un grande scrittore', 'score': 0.16565583646297455, 'token': 2654} -# {'sequence': ' Umberto Eco è indubbiamente un grande scrittore', 'score': 0.0932890921831131, 'token': 17908} -# {'sequence': ' Umberto Eco è certamente un grande scrittore', 'score': 0.054701317101716995, 'token': 5269} -``` - - -## Citation -All of the original datasets are publicly available or were released with the owners' grant. The datasets are all released under a CC0 or CCBY license. - -* UD Italian-ISDT Dataset [Github](https://github.com/UniversalDependencies/UD_Italian-ISDT) -* UD Italian-ParTUT Dataset [Github](https://github.com/UniversalDependencies/UD_Italian-ParTUT) -* I-CAB (Italian Content Annotation Bank), EvalITA [Page](http://www.evalita.it/) -* WIKINER [Page](https://figshare.com/articles/Learning_multilingual_named_entity_recognition_from_Wikipedia/5462500) , [Paper](https://www.sciencedirect.com/science/article/pii/S0004370212000276?via%3Dihub) - -``` -@inproceedings {magnini2006annotazione, - title = {Annotazione di contenuti concettuali in un corpus italiano: I - CAB}, - author = {Magnini,Bernardo and Cappelli,Amedeo and Pianta,Emanuele and Speranza,Manuela and Bartalesi Lenzi,V and Sprugnoli,Rachele and Romano,Lorenza and Girardi,Christian and Negri,Matteo}, - booktitle = {Proc.of SILFI 2006}, - year = {2006} -} -@inproceedings {magnini2006cab, - title = {I - CAB: the Italian Content Annotation Bank.}, - author = {Magnini,Bernardo and Pianta,Emanuele and Girardi,Christian and Negri,Matteo and Romano,Lorenza and Speranza,Manuela and Lenzi,Valentina Bartalesi and Sprugnoli,Rachele}, - booktitle = {LREC}, - pages = {963--968}, - year = {2006}, - organization = {Citeseer} -} -``` - -## Authors - -**Loreto Parisi**: `loreto at musixmatch dot com`, [loretoparisi](https://github.com/loretoparisi) -**Simone Francia**: `simone.francia at musixmatch dot com`, [simonefrancia](https://github.com/simonefrancia) -**Paolo Magnani**: `paul.magnani95 at gmail dot com`, [paulthemagno](https://github.com/paulthemagno) - -## About Musixmatch AI -![Musxmatch Ai mac app icon-128](https://user-images.githubusercontent.com/163333/72244273-396aa380-35ee-11ea-894b-4ea48230c02b.png) -We do Machine Learning and Artificial Intelligence @[musixmatch](https://twitter.com/Musixmatch) -Follow us on [Twitter](https://twitter.com/musixmatchai) [Github](https://github.com/musixmatchresearch) - - diff --git a/model_cards/Musixmatch/umberto-wikipedia-uncased-v1/README.md b/model_cards/Musixmatch/umberto-wikipedia-uncased-v1/README.md deleted file mode 100644 index bbb559bf02e8..000000000000 --- a/model_cards/Musixmatch/umberto-wikipedia-uncased-v1/README.md +++ /dev/null @@ -1,117 +0,0 @@ ---- -language: it ---- - -# UmBERTo Wikipedia Uncased - -[UmBERTo](https://github.com/musixmatchresearch/umberto) is a Roberta-based Language Model trained on large Italian Corpora and uses two innovative approaches: SentencePiece and Whole Word Masking. Now available at [github.com/huggingface/transformers](https://huggingface.co/Musixmatch/umberto-commoncrawl-cased-v1) - -

-
- Marco Lodola, Monument to Umberto Eco, Alessandria 2019 -

- -## Dataset -UmBERTo-Wikipedia-Uncased Training is trained on a relative small corpus (~7GB) extracted from [Wikipedia-ITA](https://linguatools.org/tools/corpora/wikipedia-monolingual-corpora/). - -## Pre-trained model - -| Model | WWM | Cased | Tokenizer | Vocab Size | Train Steps | Download | -| ------ | ------ | ------ | ------ | ------ |------ | ------ | -| `umberto-wikipedia-uncased-v1` | YES | YES | SPM | 32K | 100k | [Link](http://bit.ly/35wbSj6) | - -This model was trained with [SentencePiece](https://github.com/google/sentencepiece) and Whole Word Masking. - -## Downstream Tasks -These results refers to umberto-wikipedia-uncased model. All details are at [Umberto](https://github.com/musixmatchresearch/umberto) Official Page. - -#### Named Entity Recognition (NER) - -| Dataset | F1 | Precision | Recall | Accuracy | -| ------ | ------ | ------ | ------ | ----- | -| **ICAB-EvalITA07** | **86.240** | 85.939 | 86.544 | 98.534 | -| **WikiNER-ITA** | **90.483** | 90.328 | 90.638 | 98.661 | - -#### Part of Speech (POS) - -| Dataset | F1 | Precision | Recall | Accuracy | -| ------ | ------ | ------ | ------ | ------ | -| **UD_Italian-ISDT** | 98.563 | 98.508 | 98.618 | **98.717** | -| **UD_Italian-ParTUT** | 97.810 | 97.835 | 97.784 | **98.060** | - - - -## Usage - -##### Load UmBERTo Wikipedia Uncased with AutoModel, Autotokenizer: - -```python - -import torch -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("Musixmatch/umberto-wikipedia-uncased-v1") -umberto = AutoModel.from_pretrained("Musixmatch/umberto-wikipedia-uncased-v1") - -encoded_input = tokenizer.encode("Umberto Eco è stato un grande scrittore") -input_ids = torch.tensor(encoded_input).unsqueeze(0) # Batch size 1 -outputs = umberto(input_ids) -last_hidden_states = outputs[0] # The last hidden-state is the first element of the output -``` - -##### Predict masked token: - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="Musixmatch/umberto-wikipedia-uncased-v1", - tokenizer="Musixmatch/umberto-wikipedia-uncased-v1" -) - -result = fill_mask("Umberto Eco è un grande scrittore") -# {'sequence': ' umberto eco è stato un grande scrittore', 'score': 0.5784581303596497, 'token': 361} -# {'sequence': ' umberto eco è anche un grande scrittore', 'score': 0.33813193440437317, 'token': 269} -# {'sequence': ' umberto eco è considerato un grande scrittore', 'score': 0.027196012437343597, 'token': 3236} -# {'sequence': ' umberto eco è diventato un grande scrittore', 'score': 0.013716378249228, 'token': 5742} -# {'sequence': ' umberto eco è inoltre un grande scrittore', 'score': 0.010662357322871685, 'token': 1030} -``` - - -## Citation -All of the original datasets are publicly available or were released with the owners' grant. The datasets are all released under a CC0 or CCBY license. - -* UD Italian-ISDT Dataset [Github](https://github.com/UniversalDependencies/UD_Italian-ISDT) -* UD Italian-ParTUT Dataset [Github](https://github.com/UniversalDependencies/UD_Italian-ParTUT) -* I-CAB (Italian Content Annotation Bank), EvalITA [Page](http://www.evalita.it/) -* WIKINER [Page](https://figshare.com/articles/Learning_multilingual_named_entity_recognition_from_Wikipedia/5462500) , [Paper](https://www.sciencedirect.com/science/article/pii/S0004370212000276?via%3Dihub) - -``` -@inproceedings {magnini2006annotazione, - title = {Annotazione di contenuti concettuali in un corpus italiano: I - CAB}, - author = {Magnini,Bernardo and Cappelli,Amedeo and Pianta,Emanuele and Speranza,Manuela and Bartalesi Lenzi,V and Sprugnoli,Rachele and Romano,Lorenza and Girardi,Christian and Negri,Matteo}, - booktitle = {Proc.of SILFI 2006}, - year = {2006} -} -@inproceedings {magnini2006cab, - title = {I - CAB: the Italian Content Annotation Bank.}, - author = {Magnini,Bernardo and Pianta,Emanuele and Girardi,Christian and Negri,Matteo and Romano,Lorenza and Speranza,Manuela and Lenzi,Valentina Bartalesi and Sprugnoli,Rachele}, - booktitle = {LREC}, - pages = {963--968}, - year = {2006}, - organization = {Citeseer} -} -``` - -## Authors - -**Loreto Parisi**: `loreto at musixmatch dot com`, [loretoparisi](https://github.com/loretoparisi) -**Simone Francia**: `simone.francia at musixmatch dot com`, [simonefrancia](https://github.com/simonefrancia) -**Paolo Magnani**: `paul.magnani95 at gmail dot com`, [paulthemagno](https://github.com/paulthemagno) - -## About Musixmatch AI -![Musxmatch Ai mac app icon-128](https://user-images.githubusercontent.com/163333/72244273-396aa380-35ee-11ea-894b-4ea48230c02b.png) -We do Machine Learning and Artificial Intelligence @[musixmatch](https://twitter.com/Musixmatch) -Follow us on [Twitter](https://twitter.com/musixmatchai) [Github](https://github.com/musixmatchresearch) - diff --git a/model_cards/NLP4H/ms_bert/README.md b/model_cards/NLP4H/ms_bert/README.md deleted file mode 100644 index c5a1513455bc..000000000000 --- a/model_cards/NLP4H/ms_bert/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# MS-BERT - -## Introduction - -This repository provides codes and models of MS-BERT. -MS-BERT was pre-trained on notes from neurological examination for Multiple Sclerosis (MS) patients at St. Michael's Hospital in Toronto, Canada. - -## Data - -The dataset contained approximately 75,000 clinical notes, for about 5000 patients, totaling to over 35.7 million words. -These notes were collected from patients who visited St. Michael's Hospital MS Clinic between 2015 to 2019. -The notes contained a variety of information pertaining to a neurological exam. -For example, a note can contain information on the patient's condition, their progress over time and diagnosis. -The gender split within the dataset was observed to be 72% female and 28% male ([which reflects the natural discrepancy seen in MS][1]). -Further sections will describe how MS-BERT was pre trained through the use of these clinically relevant and rich neurological notes. - -## Data pre-processing - -The data was pre-processed to remove any identifying information. This includes information on: patient names, doctor names, hospital names, patient identification numbers, phone numbers, addresses, and time. In order to de-identify the information, we used a curated database that contained patient and doctor information. This curated database was paired with regular expressions to find and remove any identifying pieces of information. Each of these identifiers were replaced with a specific token. These tokens were chosen based on three criteria: (1) they belong to the current BERT vocab, (2), they have relatively the same semantic meaning as the word they are replacing, and (3), the token is not found in the original unprocessed dataset. The replacements that met the criteria above were as follows: - -Female first names -> Lucie - -Male first names -> Ezekiel - -Last/family names -> Salamanca. - -Dates -> 2010s - -Patient IDs -> 999 - -Phone numbers -> 1718 - -Addresses -> Silesia - -Time -> 1610 - -Locations/Hospital/Clinic names -> Troy - -## Pre-training - -The starting point for our model is the already pre-trained and fine-tuned BLUE-BERT base. We further pre-train it using the masked language modelling task from the huggingface transformers [library](https://github.com/huggingface). - -The hyperparameters can be found in the config file in this repository or [here](https://s3.amazonaws.com/models.huggingface.co/bert/NLP4H/ms_bert/config.json) - -## Acknowledgements - -We would like to thank the researchers and staff at the Data Science and Advanced Analytics (DSAA) department, St. Michael’s Hospital, for providing consistent support and guidance throughout this project. -We would also like to thank Dr. Marzyeh Ghassemi, Taylor Killan, Nathan Ng and Haoran Zhang for providing us the opportunity to work on this exciting project. - -## Disclaimer - -MS-BERT shows the results of research conducted at the Data Science and Advanced Analytics (DSAA) department, St. Michael’s Hospital. The results produced by MS-BERT are not intended for direct diagnostic use or medical decision-making without review and oversight by a clinical professional. Individuals should not make decisions about their health solely on the basis of the results produced by MS-BERT. St. Michael’s Hospital does not independently verify the validity or utility of the results produced by MS-BERT. If you have questions about the results produced by MS-BERT please consult a healthcare professional. If you would like more information about the research conducted at DSAA please contact [Zhen Yang](mailto:zhen.yang@unityhealth.to). If you would like more information on neurological examination notes please contact [Dr. Tony Antoniou](mailto:tony.antoniou@unityhealth.to) or [Dr. Jiwon Oh](mailto:jiwon.oh@unityhealth.to) from the MS clinic at St. Michael's Hospital. - -[1]: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3707353/ diff --git a/model_cards/Naveen-k/KanBERTo/README.md b/model_cards/Naveen-k/KanBERTo/README.md deleted file mode 100644 index 5dd913682f1c..000000000000 --- a/model_cards/Naveen-k/KanBERTo/README.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -language: kn ---- - -# Welcome to KanBERTo (ಕನ್ಬರ್ಟೋ) - -## Model Description - -> This is a small language model for [Kannada](https://en.wikipedia.org/wiki/Kannada) language with 1M data samples taken from - [OSCAR page](https://traces1.inria.fr/oscar/files/compressed-orig/kn.txt.gz) - -## Training params - -- **Dataset** - 1M data samples are used to train this model from OSCAR page(https://traces1.inria.fr/oscar/) eventhough data set is of 1.7 GB due to resource constraint to train -I have picked only 1M data from the total 1.7GB data set. If you are interested in collaboration and have computational resources to train on you are most welcome to do so. - -- **Preprocessing** - ByteLevelBPETokenizer is used to tokenize the sentences at character level and vocabulary size is set to 52k as per standard values given by 🤗 -- **Hyperparameters** - __ByteLevelBPETokenizer__ : vocabulary size = 52_000 and min_frequency = 2 - __Trainer__ : num_train_epochs=12 - trained for 12 epochs - per_gpu_train_batch_size=64 - batch size for the datasamples is 64 - save_steps=10_000 - save model for every 10k steps - save_total_limit=2 - save limit is set for 2 - -**Intended uses & limitations** - this is for anyone who wants to make use of kannada language models for various tasks like language generation, translation and many more use cases. - -**Whatever else is helpful!** - If you are intersted in collaboration feel free to reach me [Naveen](mailto:naveen.maltesh@gmail.com) diff --git a/model_cards/NeuML/bert-small-cord19-squad2/README.md b/model_cards/NeuML/bert-small-cord19-squad2/README.md deleted file mode 100644 index 94e86be7e9f7..000000000000 --- a/model_cards/NeuML/bert-small-cord19-squad2/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# BERT-Small CORD-19 fine-tuned on SQuAD 2.0 - -[bert-small-cord19 model](https://huggingface.co/NeuML/bert-small-cord19) fine-tuned on SQuAD 2.0 - -## Building the model - -```bash -python run_squad.py - --model_type bert - --model_name_or_path bert-small-cord19 - --do_train - --do_eval - --do_lower_case - --version_2_with_negative - --train_file train-v2.0.json - --predict_file dev-v2.0.json - --per_gpu_train_batch_size 8 - --learning_rate 3e-5 - --num_train_epochs 3.0 - --max_seq_length 384 - --doc_stride 128 - --output_dir bert-small-cord19-squad2 - --save_steps 0 - --threads 8 - --overwrite_cache - --overwrite_output_dir diff --git a/model_cards/NeuML/bert-small-cord19/README.md b/model_cards/NeuML/bert-small-cord19/README.md deleted file mode 100644 index 60486535e860..000000000000 --- a/model_cards/NeuML/bert-small-cord19/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# BERT-Small fine-tuned on CORD-19 dataset - -[BERT L6_H-512_A-8 model](https://huggingface.co/google/bert_uncased_L-6_H-512_A-8) fine-tuned on the [CORD-19 dataset](https://www.semanticscholar.org/cord19). - -## CORD-19 data subset -The training data for this dataset is stored as a [Kaggle dataset](https://www.kaggle.com/davidmezzetti/cord19-qa?select=cord19.txt). The training -data is a subset of the full corpus, focusing on high-quality, study-design detected articles. - -## Building the model - -```bash -python run_language_modeling.py - --model_type bert - --model_name_or_path google/bert_uncased_L-6_H-512_A-8 - --do_train - --mlm - --line_by_line - --block_size 512 - --train_data_file cord19.txt - --per_gpu_train_batch_size 4 - --learning_rate 3e-5 - --num_train_epochs 3.0 - --output_dir bert-small-cord19 - --save_steps 0 - --overwrite_output_dir diff --git a/model_cards/NeuML/bert-small-cord19qa/README.md b/model_cards/NeuML/bert-small-cord19qa/README.md deleted file mode 100644 index f9f3c492a2eb..000000000000 --- a/model_cards/NeuML/bert-small-cord19qa/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# BERT-Small fine-tuned on CORD-19 QA dataset - -[bert-small-cord19-squad model](https://huggingface.co/NeuML/bert-small-cord19-squad2) fine-tuned on the [CORD-19 QA dataset](https://www.kaggle.com/davidmezzetti/cord19-qa?select=cord19-qa.json). - -## CORD-19 QA dataset -The CORD-19 QA dataset is a SQuAD 2.0 formatted list of question, context, answer combinations covering the [CORD-19 dataset](https://www.semanticscholar.org/cord19). - -## Building the model - -```bash -python run_squad.py \ - --model_type bert \ - --model_name_or_path bert-small-cord19-squad \ - --do_train \ - --do_lower_case \ - --version_2_with_negative \ - --train_file cord19-qa.json \ - --per_gpu_train_batch_size 8 \ - --learning_rate 5e-5 \ - --num_train_epochs 10.0 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir bert-small-cord19qa \ - --save_steps 0 \ - --threads 8 \ - --overwrite_cache \ - --overwrite_output_dir -``` - -## Testing the model - -Example usage below: - -```python -from transformers import pipeline - -qa = pipeline( - "question-answering", - model="NeuML/bert-small-cord19qa", - tokenizer="NeuML/bert-small-cord19qa" -) - -qa({ - "question": "What is the median incubation period?", - "context": "The incubation period is around 5 days (range: 4-7 days) with a maximum of 12-13 day" -}) - -qa({ - "question": "What is the incubation period range?", - "context": "The incubation period is around 5 days (range: 4-7 days) with a maximum of 12-13 day" -}) - -qa({ - "question": "What type of surfaces does it persist?", - "context": "The virus can survive on surfaces for up to 72 hours such as plastic and stainless steel ." -}) -``` - -```json -{"score": 0.5970273583242793, "start": 32, "end": 38, "answer": "5 days"} -{"score": 0.999555868193891, "start": 39, "end": 56, "answer": "(range: 4-7 days)"} -{"score": 0.9992726505196998, "start": 61, "end": 88, "answer": "plastic and stainless steel"} -``` diff --git a/model_cards/NlpHUST/vibert4news-base-cased/README.md b/model_cards/NlpHUST/vibert4news-base-cased/README.md deleted file mode 100644 index e0c0177dc9dc..000000000000 --- a/model_cards/NlpHUST/vibert4news-base-cased/README.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -language: vn ---- -# BERT for Vietnamese is trained on more 20 GB news dataset - -Apply for task sentiment analysis on using [AIViVN's comments dataset](https://www.aivivn.com/contests/6) - -The model achieved 0.90268 on the public leaderboard, (winner's score is 0.90087) -Bert4news is used for a toolkit Vietnames(segmentation and Named Entity Recognition) at ViNLPtoolkit(https://github.com/bino282/ViNLP) - -***************New Mar 11 , 2020 *************** - -**[BERT](https://github.com/google-research/bert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805). - -We use word sentencepiece, use basic bert tokenization and same config with bert base with lowercase = False. - -You can download trained model: -- [tensorflow](https://drive.google.com/file/d/1X-sRDYf7moS_h61J3L79NkMVGHP-P-k5/view?usp=sharing). -- [pytorch](https://drive.google.com/file/d/11aFSTpYIurn-oI2XpAmcCTccB_AonMOu/view?usp=sharing). - -Use with huggingface/transformers -``` bash -import torch -from transformers import AutoTokenizer,AutoModel -tokenizer= AutoTokenizer.from_pretrained("NlpHUST/vibert4news-base-cased") -bert_model = AutoModel.from_pretrained("NlpHUST/vibert4news-base-cased") - -line = "Tôi là sinh viên trường Bách Khoa Hà Nội ." -input_id = tokenizer.encode(line,add_special_tokens = True) -att_mask = [int(token_id > 0) for token_id in input_id] -input_ids = torch.tensor([input_id]) -att_masks = torch.tensor([att_mask]) -with torch.no_grad(): - features = bert_model(input_ids,att_masks) - -print(features) - -``` - -Run training with base config - -``` bash - -python train_pytorch.py \ - --model_path=bert4news.pytorch \ - --max_len=200 \ - --batch_size=16 \ - --epochs=6 \ - --lr=2e-5 - -``` - -### Contact information -For personal communication related to this project, please contact Nha Nguyen Van (nha282@gmail.com). - diff --git a/model_cards/Norod78/hewiki-articles-distilGPT2py-il/README.md b/model_cards/Norod78/hewiki-articles-distilGPT2py-il/README.md deleted file mode 100644 index 3cf994eba6b4..000000000000 --- a/model_cards/Norod78/hewiki-articles-distilGPT2py-il/README.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -language: he - -thumbnail: https://avatars1.githubusercontent.com/u/3617152?norod.jpg -widget: -- text: "<|startoftext|>החוק השני של מועדון קרב הוא" -- text: "<|startoftext|>ראש הממשלה בן גוריון" -- text: "<|startoftext|>למידת מכונה (סרט)" -- text: "<|startoftext|>מנשה פומפרניקל" -- text: "<|startoftext|>אי שוויון " - -license: mit ---- - - -# hewiki-articles-distilGPT2py-il - -## A tiny GPT2 model for generating Hebrew text - -A distilGPT2 sized model.
-Training data was hewiki-20200701-pages-articles-multistream.xml.bz2 from https://dumps.wikimedia.org/hewiki/20200701/
-XML has been converted to plain text using Wikipedia Extractor http://medialab.di.unipi.it/wiki/Wikipedia_Extractor
-I then added <|startoftext|> and <|endoftext|> markers and deleted empty lines.
- -#### How to use - -```python -import torch -import torch.nn as nn -from transformers import GPT2Tokenizer, GPT2LMHeadModel - -tokenizer = GPT2Tokenizer.from_pretrained("Norod78/hewiki-articles-distilGPT2py-il") -model = GPT2LMHeadModel.from_pretrained("Norod78/hewiki-articles-distilGPT2py-il").eval() - -bos_token = tokenizer.bos_token #Beginning of sentace -eos_token = tokenizer.eos_token #End of sentence - -def generate_word(model, tokens_tensor, temperature=1.0): - """ - Sample a word given a tensor of tokens of previous words from a model. Given - the words we have, sample a plausible word. Temperature is used for - controlling randomness. If using temperature==0 we simply use a greedy arg max. - Else, we sample from a multinomial distribution using a lower inverse - temperature to allow for more randomness to escape repetitions. - """ - with torch.no_grad(): - outputs = model(tokens_tensor) - predictions = outputs[0] - if temperature>0: - # Make the distribution more or less skewed based on the temperature - predictions = outputs[0]/temperature - # Sample from the distribution - softmax = nn.Softmax(dim=0) - predicted_index = torch.multinomial(softmax(predictions[0,-1,:]),1).item() - # Simply take the arg-max of the distribution - else: - predicted_index = torch.argmax(predictions[0, -1, :]).item() - # Decode the encoding to the corresponding word - predicted_text = tokenizer.decode([predicted_index]) - return predicted_text - -def generate_sentence(model, tokenizer, initial_text, temperature=1.0): - """ Generate a sentence given some initial text using a model and a tokenizer. - Returns the new sentence. """ - - # Encode a text inputs - text = "" - sentence = text - - # We avoid an infinite loop by setting a maximum range - for i in range(0,84): - indexed_tokens = tokenizer.encode(initial_text + text) - - # Convert indexed tokens in a PyTorch tensor - tokens_tensor = torch.tensor([indexed_tokens]) - - new_word = generate_word(model, tokens_tensor, temperature=temperature) - - # Here the temperature is slowly decreased with each generated word, - # this ensures that the sentence (ending) makes more sense. - # We don't decrease to a temperature of 0.0 to leave some randomness in. - if temperature<(1-0.008): - temperature += 0.008 - else: - temperature = 0.996 - - text = text+new_word - - # Stop generating new words when we have reached the end of the line or the poem - if eos_token in new_word: - # returns new sentence and whether poem is done - return (text.replace(eos_token,"").strip(), True) - elif '/' in new_word: - return (text.strip(), False) - elif bos_token in new_word: - return (text.replace(bos_token,"").strip(), False) - - return (text, True) - -for output_num in range(1,5): - init_text = "בוקר טוב" - text = bos_token + init_text - for i in range(0,84): - sentence = generate_sentence(model, tokenizer, text, temperature=0.9) - text = init_text + sentence[0] - print(text) - if (sentence[1] == True): - break -``` diff --git a/model_cards/Ogayo/Hel-ach-en/README.md b/model_cards/Ogayo/Hel-ach-en/README.md deleted file mode 100644 index bd38761483d9..000000000000 --- a/model_cards/Ogayo/Hel-ach-en/README.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -language: -- ach -- en -tags: -- translation -license: cc-by-4.0 -datasets: -- JW300 -metrics: -- bleu ---- - -# HEL-ACH-EN - -## Model description - -MT model translating Acholi to English initialized with weights from [opus-mt-luo-en](https://huggingface.co/Helsinki-NLP/opus-mt-luo-en) on HuggingFace. - -## Intended uses & limitations -Machine Translation experiments. Do not use for sensitive tasks. -#### How to use - -```python -# You can include sample code which will be formatted -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - -tokenizer = AutoTokenizer.from_pretrained("Ogayo/Hel-ach-en") - -model = AutoModelForSeq2SeqLM.from_pretrained("Ogayo/Hel-ach-en") - -``` - -#### Limitations and bias - -Trained on Jehovah Witnesses data so contains theirs and Christian views. - -## Training data -Trained on OPUS JW300 data. -Initialized with weights from [opus-mt-luo-en](https://huggingface.co/Helsinki-NLP/opus-mt-luo-en?text=Bed+gi+nyasi+mar+chieng%27+nyuol+mopong%27+gi+mor%21#model_card) - -## Training procedure - -Remove duplicates and rows with no alphabetic characters. Used GPU -## Eval results -testset | BLEU ---- | --- -JW300.luo.en| 46.1 diff --git a/model_cards/Primer/bart-squad2/README.md b/model_cards/Primer/bart-squad2/README.md deleted file mode 100644 index df586039993f..000000000000 --- a/model_cards/Primer/bart-squad2/README.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -language: "en" ---- - -# BART-Squad2 - -## Model description - -BART for extractive (span-based) question answering, trained on Squad 2.0. - -F1 score of 87.4. - -## Intended uses & limitations - -Unfortunately, the Huggingface auto-inference API won't run this model, so if you're attempting to try it through the input box above and it complains, don't be discouraged! - -#### How to use - -Here's a quick way to get question answering running locally: - -```python -from transformers import AutoTokenizer, AutoModelForQuestionAnswering - -tokenizer = AutoTokenizer.from_pretrained("Primer/bart-squad2") -model = AutoModelForQuestionAnswering.from_pretrained("Primer/bart-squad2") -model.to('cuda'); model.eval() - -def answer(question, text): - seq = '' + question + ' ' + text + ' ' - tokens = tokenizer.encode_plus(seq, return_tensors='pt', padding='max_length', max_length=1024) - input_ids = tokens['input_ids'].to('cuda') - attention_mask = tokens['attention_mask'].to('cuda') - start, end, _ = model(input_ids, attention_mask=attention_mask) - start_idx = int(start.argmax().int()) - end_idx = int(end.argmax().int()) - print(tokenizer.decode(input_ids[0, start_idx:end_idx]).strip()) - # ^^ it will be an empty string if the model decided "unanswerable" - ->>> question = "Where does Tom live?" ->>> context = "Tom is an engineer in San Francisco." ->>> answer(question, context) -San Francisco -``` - -(Just drop the `.to('cuda')` stuff if running on CPU). - -#### Limitations and bias - -Unknown, no further evaluation has been performed. In a technical sense one big limitation is that it's 1.6G 😬 - -## Training procedure - -`run_squad.py` with: - -|param|value| -|---|---| -|batch size|8| -|max_seq_length|1024| -|learning rate|1e-5| -|epochs|2| - -Modified to freeze shared parameters and encoder embeddings. - diff --git a/model_cards/README.md b/model_cards/README.md new file mode 100644 index 000000000000..4bf6ac6186f3 --- /dev/null +++ b/model_cards/README.md @@ -0,0 +1,26 @@ +## 🔥 Model cards now live inside each huggingface.co model repo 🔥 + + +For consistency, ease of use and scalability, `README.md` model cards now live directly inside each model repo on the HuggingFace model hub. + +### How to update a model card + +You can directly update a model card inside any model repo you have **write access** to, i.e.: +- a model under your username namespace +- a model under any organization you are a part of. + +You can either: +- update it, commit and push using your usual git workflow (command line, GUI, etc.) +- or edit it directly from the website's UI. + +**What if you want to create or update a model card for a model you don't have write access to?** + +In that case, given that we don't have a Pull request system yet on huggingface.co (🤯), +you can open an issue here, post the card's content, and tag the model author(s) and/or the Hugging Face team. + +We might implement a more seamless process at some point, so your early feedback is precious! +Please let us know of any suggestion. + +### What happened to the model cards here? + +We migrated every model card from the repo to its corresponding huggingface.co model repo. Individual commits were preserved, and they link back to the original commit on GitHub. diff --git a/model_cards/Rostlab/prot_bert/README.md b/model_cards/Rostlab/prot_bert/README.md deleted file mode 100644 index 75d576c5e053..000000000000 --- a/model_cards/Rostlab/prot_bert/README.md +++ /dev/null @@ -1,141 +0,0 @@ ---- -language: protein -tags: -- protein language model -datasets: -- Uniref100 ---- - -# ProtBert model - -Pretrained model on protein sequences using a masked language modeling (MLM) objective. It was introduced in -[this paper](https://doi.org/10.1101/2020.07.12.199554) and first released in -[this repository](https://github.com/agemagician/ProtTrans). This model is trained on uppercase amino acids: it only works with capital letter amino acids. - - -## Model description - -ProtBert is based on Bert model which pretrained on a large corpus of protein sequences in a self-supervised fashion. -This means it was pretrained on the raw protein sequences only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those protein sequences. - -One important difference between our Bert model and the original Bert version is the way of dealing with sequences as separate documents. -This means the Next sentence prediction is not used, as each sequence is treated as a complete document. -The masking follows the original Bert training with randomly masks 15% of the amino acids in the input. - -At the end, the feature extracted from this model revealed that the LM-embeddings from unlabeled data (only protein sequences) captured important biophysical properties governing protein -shape. -This implied learning some of the grammar of the language of life realized in protein sequences. - -## Intended uses & limitations - -The model could be used for protein feature extraction or to be fine-tuned on downstream tasks. -We have noticed in some tasks you could gain more accuracy by fine-tuning the model rather than using it as a feature extractor. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import BertForMaskedLM, BertTokenizer, pipeline ->>> tokenizer = BertTokenizer.from_pretrained("Rostlab/prot_bert", do_lower_case=False ) ->>> model = BertForMaskedLM.from_pretrained("Rostlab/prot_bert") ->>> unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer) ->>> unmasker('D L I P T S S K L V V [MASK] D T S L Q V K K A F F A L V T') - -[{'score': 0.11088453233242035, - 'sequence': '[CLS] D L I P T S S K L V V L D T S L Q V K K A F F A L V T [SEP]', - 'token': 5, - 'token_str': 'L'}, - {'score': 0.08402521163225174, - 'sequence': '[CLS] D L I P T S S K L V V S D T S L Q V K K A F F A L V T [SEP]', - 'token': 10, - 'token_str': 'S'}, - {'score': 0.07328339666128159, - 'sequence': '[CLS] D L I P T S S K L V V V D T S L Q V K K A F F A L V T [SEP]', - 'token': 8, - 'token_str': 'V'}, - {'score': 0.06921856850385666, - 'sequence': '[CLS] D L I P T S S K L V V K D T S L Q V K K A F F A L V T [SEP]', - 'token': 12, - 'token_str': 'K'}, - {'score': 0.06382402777671814, - 'sequence': '[CLS] D L I P T S S K L V V I D T S L Q V K K A F F A L V T [SEP]', - 'token': 11, - 'token_str': 'I'}] -``` - -Here is how to use this model to get the features of a given protein sequence in PyTorch: - -```python -from transformers import BertModel, BertTokenizer -import re -tokenizer = BertTokenizer.from_pretrained("Rostlab/prot_bert", do_lower_case=False ) -model = BertModel.from_pretrained("Rostlab/prot_bert") -sequence_Example = "A E T C Z A O" -sequence_Example = re.sub(r"[UZOB]", "X", sequence_Example) -encoded_input = tokenizer(sequence_Example, return_tensors='pt') -output = model(**encoded_input) -``` - -## Training data - -The ProtBert model was pretrained on [Uniref100](https://www.uniprot.org/downloads), a dataset consisting of 217 million protein sequences. - -## Training procedure - -### Preprocessing - -The protein sequences are uppercased and tokenized using a single space and a vocabulary size of 21. The rare amino acids "U,Z,O,B" were mapped to "X". -The inputs of the model are then of the form: - -``` -[CLS] Protein Sequence A [SEP] Protein Sequence B [SEP] -``` - -Furthermore, each protein sequence was treated as a separate document. -The preprocessing step was performed twice, once for a combined length (2 sequences) of less than 512 amino acids, and another time using a combined length (2 sequences) of less than 2048 amino acids. - -The details of the masking procedure for each sequence followed the original Bert model as following: -- 15% of the amino acids are masked. -- In 80% of the cases, the masked amino acids are replaced by `[MASK]`. -- In 10% of the cases, the masked amino acids are replaced by a random amino acid (different) from the one they replace. -- In the 10% remaining cases, the masked amino acids are left as is. - -### Pretraining - -The model was trained on a single TPU Pod V3-512 for 400k steps in total. -300K steps using sequence length 512 (batch size 15k), and 100K steps using sequence length 2048 (batch size 2.5k). -The optimizer used is Lamb with a learning rate of 0.002, a weight decay of 0.01, learning rate warmup for 40k steps and linear decay of the learning rate after. - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Test results : - -| Task/Dataset | secondary structure (3-states) | secondary structure (8-states) | Localization | Membrane | -|:-----:|:-----:|:-----:|:-----:|:-----:| -| CASP12 | 75 | 63 | | | -| TS115 | 83 | 72 | | | -| CB513 | 81 | 66 | | | -| DeepLoc | | | 79 | 91 | - -### BibTeX entry and citation info - -```bibtex -@article {Elnaggar2020.07.12.199554, - author = {Elnaggar, Ahmed and Heinzinger, Michael and Dallago, Christian and Rehawi, Ghalia and Wang, Yu and Jones, Llion and Gibbs, Tom and Feher, Tamas and Angerer, Christoph and Steinegger, Martin and BHOWMIK, DEBSINDHU and Rost, Burkhard}, - title = {ProtTrans: Towards Cracking the Language of Life{\textquoteright}s Code Through Self-Supervised Deep Learning and High Performance Computing}, - elocation-id = {2020.07.12.199554}, - year = {2020}, - doi = {10.1101/2020.07.12.199554}, - publisher = {Cold Spring Harbor Laboratory}, - abstract = {Computational biology and bioinformatics provide vast data gold-mines from protein sequences, ideal for Language Models (LMs) taken from Natural Language Processing (NLP). These LMs reach for new prediction frontiers at low inference costs. Here, we trained two auto-regressive language models (Transformer-XL, XLNet) and two auto-encoder models (Bert, Albert) on data from UniRef and BFD containing up to 393 billion amino acids (words) from 2.1 billion protein sequences (22- and 112 times the entire English Wikipedia). The LMs were trained on the Summit supercomputer at Oak Ridge National Laboratory (ORNL), using 936 nodes (total 5616 GPUs) and one TPU Pod (V3-512 or V3-1024). We validated the advantage of up-scaling LMs to larger models supported by bigger data by predicting secondary structure (3-states: Q3=76-84, 8 states: Q8=65-73), sub-cellular localization for 10 cellular compartments (Q10=74) and whether a protein is membrane-bound or water-soluble (Q2=89). Dimensionality reduction revealed that the LM-embeddings from unlabeled data (only protein sequences) captured important biophysical properties governing protein shape. This implied learning some of the grammar of the language of life realized in protein sequences. The successful up-scaling of protein LMs through HPC to larger data sets slightly reduced the gap between models trained on evolutionary information and LMs. Availability ProtTrans: \<a href="https://github.com/agemagician/ProtTrans"\>https://github.com/agemagician/ProtTrans\</a\>Competing Interest StatementThe authors have declared no competing interest.}, - URL = {https://www.biorxiv.org/content/early/2020/07/21/2020.07.12.199554}, - eprint = {https://www.biorxiv.org/content/early/2020/07/21/2020.07.12.199554.full.pdf}, - journal = {bioRxiv} -} -``` - -> Created by [Ahmed Elnaggar/@Elnaggar_AI](https://twitter.com/Elnaggar_AI) | [LinkedIn](https://www.linkedin.com/in/prof-ahmed-elnaggar/) diff --git a/model_cards/Rostlab/prot_bert_bfd/README.md b/model_cards/Rostlab/prot_bert_bfd/README.md deleted file mode 100644 index 0d2079241215..000000000000 --- a/model_cards/Rostlab/prot_bert_bfd/README.md +++ /dev/null @@ -1,141 +0,0 @@ ---- -language: protein -tags: -- protein language model -datasets: -- BFD ---- - -# ProtBert-BFD model - -Pretrained model on protein sequences using a masked language modeling (MLM) objective. It was introduced in -[this paper](https://doi.org/10.1101/2020.07.12.199554) and first released in -[this repository](https://github.com/agemagician/ProtTrans). This model is trained on uppercase amino acids: it only works with capital letter amino acids. - - -## Model description - -ProtBert-BFD is based on Bert model which pretrained on a large corpus of protein sequences in a self-supervised fashion. -This means it was pretrained on the raw protein sequences only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those protein sequences. - -One important difference between our Bert model and the original Bert version is the way of dealing with sequences as separate documents -This means the Next sentence prediction is not used, as each sequence is treated as a complete document. -The masking follows the original Bert training with randomly masks 15% of the amino acids in the input. - -At the end, the feature extracted from this model revealed that the LM-embeddings from unlabeled data (only protein sequences) captured important biophysical properties governing protein -shape. -This implied learning some of the grammar of the language of life realized in protein sequences. - -## Intended uses & limitations - -The model could be used for protein feature extraction or to be fine-tuned on downstream tasks. -We have noticed in some tasks you could gain more accuracy by fine-tuning the model rather than using it as a feature extractor. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import BertForMaskedLM, BertTokenizer, pipeline ->>> tokenizer = BertTokenizer.from_pretrained('Rostlab/prot_bert_bfd', do_lower_case=False ) ->>> model = BertForMaskedLM.from_pretrained("Rostlab/prot_bert_bfd") ->>> unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer) ->>> unmasker('D L I P T S S K L V V [MASK] D T S L Q V K K A F F A L V T') - -[{'score': 0.1165614128112793, - 'sequence': '[CLS] D L I P T S S K L V V L D T S L Q V K K A F F A L V T [SEP]', - 'token': 5, - 'token_str': 'L'}, - {'score': 0.08976086974143982, - 'sequence': '[CLS] D L I P T S S K L V V V D T S L Q V K K A F F A L V T [SEP]', - 'token': 8, - 'token_str': 'V'}, - {'score': 0.08864385634660721, - 'sequence': '[CLS] D L I P T S S K L V V S D T S L Q V K K A F F A L V T [SEP]', - 'token': 10, - 'token_str': 'S'}, - {'score': 0.06227643042802811, - 'sequence': '[CLS] D L I P T S S K L V V A D T S L Q V K K A F F A L V T [SEP]', - 'token': 6, - 'token_str': 'A'}, - {'score': 0.06194969266653061, - 'sequence': '[CLS] D L I P T S S K L V V T D T S L Q V K K A F F A L V T [SEP]', - 'token': 15, - 'token_str': 'T'}] -``` - -Here is how to use this model to get the features of a given protein sequence in PyTorch: - -```python -from transformers import BertModel, BertTokenizer -import re -tokenizer = BertTokenizer.from_pretrained('Rostlab/prot_bert_bfd', do_lower_case=False ) -model = BertModel.from_pretrained("Rostlab/prot_bert_bfd") -sequence_Example = "A E T C Z A O" -sequence_Example = re.sub(r"[UZOB]", "X", sequence_Example) -encoded_input = tokenizer(sequence_Example, return_tensors='pt') -output = model(**encoded_input) -``` - -## Training data - -The ProtBert-BFD model was pretrained on [BFD](https://bfd.mmseqs.com/), a dataset consisting of 2.1 billion protein sequences. - -## Training procedure - -### Preprocessing - -The protein sequences are uppercased and tokenized using a single space and a vocabulary size of 21. -The inputs of the model are then of the form: - -``` -[CLS] Protein Sequence A [SEP] Protein Sequence B [SEP] -``` - -Furthermore, each protein sequence was treated as a separate document. -The preprocessing step was performed twice, once for a combined length (2 sequences) of less than 512 amino acids, and another time using a combined length (2 sequences) of less than 2048 amino acids. - -The details of the masking procedure for each sequence followed the original Bert model as following: -- 15% of the amino acids are masked. -- In 80% of the cases, the masked amino acids are replaced by `[MASK]`. -- In 10% of the cases, the masked amino acids are replaced by a random amino acid (different) from the one they replace. -- In the 10% remaining cases, the masked amino acids are left as is. - -### Pretraining - -The model was trained on a single TPU Pod V3-1024 for one million steps in total. -800k steps using sequence length 512 (batch size 32k), and 200K steps using sequence length 2048 (batch size 6k). -The optimizer used is Lamb with a learning rate of 0.002, a weight decay of 0.01, learning rate warmup for 140k steps and linear decay of the learning rate after. - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Test results : - -| Task/Dataset | secondary structure (3-states) | secondary structure (8-states) | Localization | Membrane | -|:-----:|:-----:|:-----:|:-----:|:-----:| -| CASP12 | 76 | 65 | | | -| TS115 | 84 | 73 | | | -| CB513 | 83 | 70 | | | -| DeepLoc | | | 78 | 91 | - -### BibTeX entry and citation info - -```bibtex -@article {Elnaggar2020.07.12.199554, - author = {Elnaggar, Ahmed and Heinzinger, Michael and Dallago, Christian and Rehawi, Ghalia and Wang, Yu and Jones, Llion and Gibbs, Tom and Feher, Tamas and Angerer, Christoph and Steinegger, Martin and BHOWMIK, DEBSINDHU and Rost, Burkhard}, - title = {ProtTrans: Towards Cracking the Language of Life{\textquoteright}s Code Through Self-Supervised Deep Learning and High Performance Computing}, - elocation-id = {2020.07.12.199554}, - year = {2020}, - doi = {10.1101/2020.07.12.199554}, - publisher = {Cold Spring Harbor Laboratory}, - abstract = {Computational biology and bioinformatics provide vast data gold-mines from protein sequences, ideal for Language Models (LMs) taken from Natural Language Processing (NLP). These LMs reach for new prediction frontiers at low inference costs. Here, we trained two auto-regressive language models (Transformer-XL, XLNet) and two auto-encoder models (Bert, Albert) on data from UniRef and BFD containing up to 393 billion amino acids (words) from 2.1 billion protein sequences (22- and 112 times the entire English Wikipedia). The LMs were trained on the Summit supercomputer at Oak Ridge National Laboratory (ORNL), using 936 nodes (total 5616 GPUs) and one TPU Pod (V3-512 or V3-1024). We validated the advantage of up-scaling LMs to larger models supported by bigger data by predicting secondary structure (3-states: Q3=76-84, 8 states: Q8=65-73), sub-cellular localization for 10 cellular compartments (Q10=74) and whether a protein is membrane-bound or water-soluble (Q2=89). Dimensionality reduction revealed that the LM-embeddings from unlabeled data (only protein sequences) captured important biophysical properties governing protein shape. This implied learning some of the grammar of the language of life realized in protein sequences. The successful up-scaling of protein LMs through HPC to larger data sets slightly reduced the gap between models trained on evolutionary information and LMs. Availability ProtTrans: \<a href="https://github.com/agemagician/ProtTrans"\>https://github.com/agemagician/ProtTrans\</a\>Competing Interest StatementThe authors have declared no competing interest.}, - URL = {https://www.biorxiv.org/content/early/2020/07/21/2020.07.12.199554}, - eprint = {https://www.biorxiv.org/content/early/2020/07/21/2020.07.12.199554.full.pdf}, - journal = {bioRxiv} -} -``` - -> Created by [Ahmed Elnaggar/@Elnaggar_AI](https://twitter.com/Elnaggar_AI) | [LinkedIn](https://www.linkedin.com/in/prof-ahmed-elnaggar/) diff --git a/model_cards/Rostlab/prot_t5_xl_bfd/README.md b/model_cards/Rostlab/prot_t5_xl_bfd/README.md deleted file mode 100644 index 418d4c32d235..000000000000 --- a/model_cards/Rostlab/prot_t5_xl_bfd/README.md +++ /dev/null @@ -1,125 +0,0 @@ ---- -language: protein -tags: -- protein language model -datasets: -- BFD ---- - -# ProtT5-XL-BFD model - -Pretrained model on protein sequences using a masked language modeling (MLM) objective. It was introduced in -[this paper](https://doi.org/10.1101/2020.07.12.199554) and first released in -[this repository](https://github.com/agemagician/ProtTrans). This model is trained on uppercase amino acids: it only works with capital letter amino acids. - - -## Model description - -ProtT5-XL-BFD is based on the `t5-3b` model and was pretrained on a large corpus of protein sequences in a self-supervised fashion. -This means it was pretrained on the raw protein sequences only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those protein sequences. - -One important difference between this T5 model and the original T5 version is the denosing objective. -The original T5-3B model was pretrained using a span denosing objective, while this model was pre-trained with a Bart-like MLM denosing objective. -The masking probability is consistent with the original T5 training by randomly masking 15% of the amino acids in the input. - -It has been shown that the features extracted from this self-supervised model (LM-embeddings) captured important biophysical properties governing protein shape. -shape. -This implied learning some of the grammar of the language of life realized in protein sequences. - -## Intended uses & limitations - -The model could be used for protein feature extraction or to be fine-tuned on downstream tasks. -We have noticed in some tasks on can gain more accuracy by fine-tuning the model rather than using it as a feature extractor. -We have also noticed that for feature extraction, its better to use the feature extracted from the encoder not from the decoder. - -### How to use - -Here is how to use this model to extract the features of a given protein sequence in PyTorch: - -```python -from transformers import T5Tokenizer, T5Model -import re -import torch - -tokenizer = T5Tokenizer.from_pretrained('Rostlab/prot_t5_xl_bfd', do_lower_case=False) - -model = T5Model.from_pretrained("Rostlab/prot_t5_xl_bfd") - -sequences_Example = ["A E T C Z A O","S K T Z P"] - -sequences_Example = [re.sub(r"[UZOB]", "X", sequence) for sequence in sequences_Example] - -ids = tokenizer.batch_encode_plus(sequences_Example, add_special_tokens=True, padding=True) - -input_ids = torch.tensor(ids['input_ids']) -attention_mask = torch.tensor(ids['attention_mask']) - -with torch.no_grad(): - embedding = model(input_ids=input_ids,attention_mask=attention_mask,decoder_input_ids=None) - -# For feature extraction we recommend to use the encoder embedding -encoder_embedding = embedding[2].cpu().numpy() -decoder_embedding = embedding[0].cpu().numpy() -``` - -## Training data - -The ProtT5-XL-BFD model was pretrained on [BFD](https://bfd.mmseqs.com/), a dataset consisting of 2.1 billion protein sequences. - -## Training procedure - -### Preprocessing - -The protein sequences are uppercased and tokenized using a single space and a vocabulary size of 21. The rare amino acids "U,Z,O,B" were mapped to "X". -The inputs of the model are then of the form: - -``` -Protein Sequence [EOS] -``` - -The preprocessing step was performed on the fly, by cutting and padding the protein sequences up to 512 tokens. - -The details of the masking procedure for each sequence are as follows: -- 15% of the amino acids are masked. -- In 90% of the cases, the masked amino acids are replaced by `[MASK]` token. -- In 10% of the cases, the masked amino acids are replaced by a random amino acid (different) from the one they replace. - -### Pretraining - -The model was trained on a single TPU Pod V3-1024 for 1.2 million steps in total, using sequence length 512 (batch size 4k). -It has a total of approximately 3B parameters and was trained using the encoder-decoder architecture. -The optimizer used is AdaFactor with inverse square root learning rate schedule for pre-training. - - -## Evaluation results - -When the model is used for feature etraction, this model achieves the following results: - -Test results : - -| Task/Dataset | secondary structure (3-states) | secondary structure (8-states) | Localization | Membrane | -|:-----:|:-----:|:-----:|:-----:|:-----:| -| CASP12 | 77 | 66 | | | -| TS115 | 85 | 74 | | | -| CB513 | 84 | 71 | | | -| DeepLoc | | | 77 | 91 | - -### BibTeX entry and citation info - -```bibtex -@article {Elnaggar2020.07.12.199554, - author = {Elnaggar, Ahmed and Heinzinger, Michael and Dallago, Christian and Rehawi, Ghalia and Wang, Yu and Jones, Llion and Gibbs, Tom and Feher, Tamas and Angerer, Christoph and Steinegger, Martin and BHOWMIK, DEBSINDHU and Rost, Burkhard}, - title = {ProtTrans: Towards Cracking the Language of Life{\textquoteright}s Code Through Self-Supervised Deep Learning and High Performance Computing}, - elocation-id = {2020.07.12.199554}, - year = {2020}, - doi = {10.1101/2020.07.12.199554}, - publisher = {Cold Spring Harbor Laboratory}, - abstract = {Computational biology and bioinformatics provide vast data gold-mines from protein sequences, ideal for Language Models (LMs) taken from Natural Language Processing (NLP). These LMs reach for new prediction frontiers at low inference costs. Here, we trained two auto-regressive language models (Transformer-XL, XLNet) and two auto-encoder models (Bert, Albert) on data from UniRef and BFD containing up to 393 billion amino acids (words) from 2.1 billion protein sequences (22- and 112 times the entire English Wikipedia). The LMs were trained on the Summit supercomputer at Oak Ridge National Laboratory (ORNL), using 936 nodes (total 5616 GPUs) and one TPU Pod (V3-512 or V3-1024). We validated the advantage of up-scaling LMs to larger models supported by bigger data by predicting secondary structure (3-states: Q3=76-84, 8 states: Q8=65-73), sub-cellular localization for 10 cellular compartments (Q10=74) and whether a protein is membrane-bound or water-soluble (Q2=89). Dimensionality reduction revealed that the LM-embeddings from unlabeled data (only protein sequences) captured important biophysical properties governing protein shape. This implied learning some of the grammar of the language of life realized in protein sequences. The successful up-scaling of protein LMs through HPC to larger data sets slightly reduced the gap between models trained on evolutionary information and LMs. Availability ProtTrans: \<a href="https://github.com/agemagician/ProtTrans"\>https://github.com/agemagician/ProtTrans\</a\>Competing Interest StatementThe authors have declared no competing interest.}, - URL = {https://www.biorxiv.org/content/early/2020/07/21/2020.07.12.199554}, - eprint = {https://www.biorxiv.org/content/early/2020/07/21/2020.07.12.199554.full.pdf}, - journal = {bioRxiv} -} -``` - -> Created by [Ahmed Elnaggar/@Elnaggar_AI](https://twitter.com/Elnaggar_AI) | [LinkedIn](https://www.linkedin.com/in/prof-ahmed-elnaggar/) diff --git a/model_cards/SZTAKI-HLT/hubert-base-cc/README.md b/model_cards/SZTAKI-HLT/hubert-base-cc/README.md deleted file mode 100644 index 96fe00f640cb..000000000000 --- a/model_cards/SZTAKI-HLT/hubert-base-cc/README.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -language: hu -license: apache-2.0 -datasets: -- common_crawl -- wikipedia ---- - -# huBERT base model (cased) - -## Model description - -Cased BERT model for Hungarian, trained on the (filtered, deduplicated) Hungarian subset of the Common Crawl and a snapshot of the Hungarian Wikipedia. - -## Intended uses & limitations - -The model can be used as any other (cased) BERT model. It has been tested on the chunking and -named entity recognition tasks and set a new state-of-the-art on the former. - -## Training - -Details of the training data and procedure can be found in the PhD thesis linked below. (With the caveat that it only contains preliminary results -based on the Wikipedia subcorpus. Evaluation of the full model will appear in a future paper.) - -## Eval results - -When fine-tuned (via `BertForTokenClassification`) on chunking and NER, the model outperforms multilingual BERT, achieves state-of-the-art results on the -former task and comes within 0.5% F1 to the SotA on the latter. The exact scores are - -| NER | Minimal NP | Maximal NP | -|-----|------------|------------| -| 97.62% | **97.14%** | **96.97%** | - -### BibTeX entry and citation info - -The training corpus, parameters and the evaluation methods are discussed in the -[following PhD thesis](https://hlt.bme.hu/en/publ/nemeskey_2020): - -```bibtex -@PhDThesis{ Nemeskey:2020, - author = {Nemeskey, Dávid Márk}, - title = {Natural Language Processing Methods for Language Modeling}, - year = {2020}, - school = {E\"otv\"os Lor\'and University} -} -``` diff --git a/model_cards/SparkBeyond/roberta-large-sts-b/README.md b/model_cards/SparkBeyond/roberta-large-sts-b/README.md deleted file mode 100644 index a32cb57486e2..000000000000 --- a/model_cards/SparkBeyond/roberta-large-sts-b/README.md +++ /dev/null @@ -1,50 +0,0 @@ - - -# Roberta Large STS-B - -This model is a fine tuned RoBERTA model over STS-B. -It was trained with these params: -!python /content/transformers/examples/text-classification/run_glue.py \ - --model_type roberta \ - --model_name_or_path roberta-large \ - --task_name STS-B \ - --do_train \ - --do_eval \ - --do_lower_case \ - --data_dir /content/glue_data/STS-B/ \ - --max_seq_length 128 \ - --per_gpu_eval_batch_size=8 \ - --per_gpu_train_batch_size=8 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --output_dir /content/roberta-sts-b - - -## How to run - -```python - - - -import toolz -import torch -batch_size = 6 - -def roberta_similarity_batches(to_predict): - batches = toolz.partition(batch_size, to_predict) - similarity_scores = [] - for batch in batches: - sentences = [(sentence_similarity["sent1"], sentence_similarity["sent2"]) for sentence_similarity in batch] - batch_scores = similarity_roberta(model, tokenizer,sentences) - similarity_scores = similarity_scores + batch_scores[0].cpu().squeeze(axis=1).tolist() - return similarity_scores - -def similarity_roberta(model, tokenizer, sent_pairs): - batch_token = tokenizer(sent_pairs, padding='max_length', truncation=True, max_length=500) - res = model(torch.tensor(batch_token['input_ids']).cuda(), attention_mask=torch.tensor(batch_token["attention_mask"]).cuda()) - return res - -similarity_roberta(model, tokenizer, [('NEW YORK--(BUSINESS WIRE)--Rosen Law Firm, a global investor rights law firm, announces it is investigating potential securities claims on behalf of shareholders of Vale S.A. ( VALE ) resulting from allegations that Vale may have issued materially misleading business information to the investing public', - 'EQUITY ALERT: Rosen Law Firm Announces Investigation of Securities Claims Against Vale S.A. – VALE')]) - -``` diff --git a/model_cards/T-Systems-onsite/bert-german-dbmdz-uncased-sentence-stsb/README.md b/model_cards/T-Systems-onsite/bert-german-dbmdz-uncased-sentence-stsb/README.md deleted file mode 100644 index 250db8366d20..000000000000 --- a/model_cards/T-Systems-onsite/bert-german-dbmdz-uncased-sentence-stsb/README.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -language: de -license: mit ---- - -# bert-german-dbmdz-uncased-sentence-stsb -**This model is outdated!** - -The new [T-Systems-onsite/cross-en-de-roberta-sentence-transformer](https://huggingface.co/T-Systems-onsite/cross-en-de-roberta-sentence-transformer) model is better for German language. It is also the current best model for English language and works cross-lingually. Please consider using that model. \ No newline at end of file diff --git a/model_cards/T-Systems-onsite/cross-en-de-roberta-sentence-transformer/README.md b/model_cards/T-Systems-onsite/cross-en-de-roberta-sentence-transformer/README.md deleted file mode 100644 index a1790cf421ce..000000000000 --- a/model_cards/T-Systems-onsite/cross-en-de-roberta-sentence-transformer/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -language: -- de -- en -license: mit -tags: -- sentence_embedding -- search -- pytorch -- xlm-roberta -- roberta -- xlm-r-distilroberta-base-paraphrase-v1 -- paraphrase -datasets: -- STSbenchmark -metrics: -- Spearman’s rank correlation -- cosine similarity ---- - -# Cross English & German RoBERTa for Sentence Embeddings -This model is intended to [compute sentence (text) embeddings](https://www.sbert.net/docs/usage/computing_sentence_embeddings.html) for English and German text. These embeddings can then be compared with [cosine-similarity](https://en.wikipedia.org/wiki/Cosine_similarity) to find sentences with a similar semantic meaning. For example this can be useful for [semantic textual similarity](https://www.sbert.net/docs/usage/semantic_textual_similarity.html), [semantic search](https://www.sbert.net/docs/usage/semantic_search.html), or [paraphrase mining](https://www.sbert.net/docs/usage/paraphrase_mining.html). To do this you have to use the [Sentence Transformers Python framework](https://github.com/UKPLab/sentence-transformers). - -The speciality of this model is that it also works cross-lingually. Regardless of the language, the sentences are translated into very similar vectors according to their semantics. This means that you can, for example, enter a search in German and find results according to the semantics in German and also in English. Using a xlm model and _multilingual finetuning with language-crossing_ we reach performance that even exceeds the best current dedicated English large model (see Evaluation section below). - -> Sentence-BERT (SBERT) is a modification of the pretrained BERT network that use siamese and triplet network structures to derive semantically meaningful sentence embeddings that can be compared using cosine-similarity. This reduces the effort for finding the most similar pair from 65hours with BERT / RoBERTa to about 5 seconds with SBERT, while maintaining the accuracy from BERT. - -Source: [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) - -This model is fine-tuned from [Philip May](https://eniak.de/) and open-sourced by [T-Systems-onsite](https://www.t-systems-onsite.de/). Special thanks to [Nils Reimers](https://www.nils-reimers.de/) for your awesome open-source work, the Sentence Transformers, the models and your help on GitHub. - -## How to use -**The usage description above - provided by Hugging Face - is wrong for sentence embeddings! Please use this:** - -To use this model install the `sentence-transformers` package (see here: ). - -```python -from sentence_transformers import SentenceTransformer -model = SentenceTransformer('T-Systems-onsite/cross-en-de-roberta-sentence-transformer') -``` - -For details of usage and examples see here: -- [Computing Sentence Embeddings](https://www.sbert.net/docs/usage/computing_sentence_embeddings.html) -- [Semantic Textual Similarity](https://www.sbert.net/docs/usage/semantic_textual_similarity.html) -- [Paraphrase Mining](https://www.sbert.net/docs/usage/paraphrase_mining.html) -- [Semantic Search](https://www.sbert.net/docs/usage/semantic_search.html) -- [Cross-Encoders](https://www.sbert.net/docs/usage/cross-encoder.html) -- [Examples on GitHub](https://github.com/UKPLab/sentence-transformers/tree/master/examples) - -## Training -The base model is [xlm-roberta-base](https://huggingface.co/xlm-roberta-base). This model has been further trained by [Nils Reimers](https://www.nils-reimers.de/) on a large scale paraphrase dataset for 50+ languages. [Nils Reimers](https://www.nils-reimers.de/) about this [on GitHub](https://github.com/UKPLab/sentence-transformers/issues/509#issuecomment-712243280): - ->A paper is upcoming for the paraphrase models. -> ->These models were trained on various datasets with Millions of examples for paraphrases, mainly derived from Wikipedia edit logs, paraphrases mined from Wikipedia and SimpleWiki, paraphrases from news reports, AllNLI-entailment pairs with in-batch-negative loss etc. -> ->In internal tests, they perform much better than the NLI+STSb models as they have see more and broader type of training data. NLI+STSb has the issue that they are rather narrow in their domain and do not contain any domain specific words / sentences (like from chemistry, computer science, math etc.). The paraphrase models has seen plenty of sentences from various domains. -> ->More details with the setup, all the datasets, and a wider evaluation will follow soon. - -The resulting model called `xlm-r-distilroberta-base-paraphrase-v1` has been released here: - -Building on this cross language model we fine-tuned it for English and German language on the [STSbenchmark](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) dataset. For German language we used the dataset of our [German STSbenchmark dataset](https://github.com/t-systems-on-site-services-gmbh/german-STSbenchmark) which has been translated with [deepl.com](https://www.deepl.com/translator). Additionally to the German and English training samples we generated samples of English and German crossed. We call this _multilingual finetuning with language-crossing_. It doubled the traing-datasize and tests show that it further improves performance. - -We did an automatic hyperparameter search for 33 trials with [Optuna](https://github.com/optuna/optuna). Using 10-fold crossvalidation on the deepl.com test and dev dataset we found the following best hyperparameter: -- batch_size = 8 -- num_epochs = 2 -- lr = 1.026343323298136e-05, -- eps = 4.462251033010287e-06 -- weight_decay = 0.04794438776350409 -- warmup_steps_proportion = 0.1609010732760181 - -The final model was trained with these hyperparameters on the combination of the train and dev datasets from English, German and the crossings of them. The testset was left for testing. - -# Evaluation -The evaluation has been done on English, German and both languages crossed with the STSbenchmark test data. The evaluation-code is available on [Colab](https://colab.research.google.com/drive/1gtGnKq_dYU_sDYqMohTYVMVpxMJjyH0M?usp=sharing). As the metric for evaluation we use the Spearman’s rank correlation between the cosine-similarity of the sentence embeddings and STSbenchmark labels. - -| Model Name | Spearman
German | Spearman
English | Spearman
EN-DE & DE-EN
(cross) | -|---------------------------------------------------------------|-------------------|--------------------|------------------| -| xlm-r-distilroberta-base-paraphrase-v1 | 0.8079 | 0.8350 | 0.7983 | -| [xlm-r-100langs-bert-base-nli-stsb-mean-tokens](https://huggingface.co/sentence-transformers/xlm-r-100langs-bert-base-nli-stsb-mean-tokens) | 0.7877 | 0.8465 | 0.7908 | -| xlm-r-bert-base-nli-stsb-mean-tokens | 0.7877 | 0.8465 | 0.7908 | -| [roberta-large-nli-stsb-mean-tokens](https://huggingface.co/sentence-transformers/roberta-large-nli-stsb-mean-tokens) | 0.6371 | 0.8639 | 0.4109 | -| [T-Systems-onsite/
german-roberta-sentence-transformer-v2](https://huggingface.co/T-Systems-onsite/german-roberta-sentence-transformer-v2) | 0.8529 | 0.8634 | 0.8415 | -| **T-Systems-onsite/
cross-en-de-roberta-sentence-transformer** | **0.8550** | **0.8660** | **0.8525** | diff --git a/model_cards/T-Systems-onsite/german-roberta-sentence-transformer-v2/README.md b/model_cards/T-Systems-onsite/german-roberta-sentence-transformer-v2/README.md deleted file mode 100644 index 05184fbef5b4..000000000000 --- a/model_cards/T-Systems-onsite/german-roberta-sentence-transformer-v2/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: de -license: mit -tags: -- sentence_embedding -- search -- pytorch -- xlm-roberta -- roberta -- xlm-r-distilroberta-base-paraphrase-v1 -- paraphrase -datasets: -- STSbenchmark -metrics: -- Spearman’s rank correlation -- cosine similarity ---- - -# German RoBERTa for Sentence Embeddings V2 -**The new [T-Systems-onsite/cross-en-de-roberta-sentence-transformer](https://huggingface.co/T-Systems-onsite/cross-en-de-roberta-sentence-transformer) model is slightly better for German language. It is also the current best model for English language and works cross-lingually. Please consider using that model.** - -This model is intended to [compute sentence (text embeddings)](https://www.sbert.net/docs/usage/computing_sentence_embeddings.html) for German text. These embeddings can then be compared with [cosine-similarity](https://en.wikipedia.org/wiki/Cosine_similarity) to find sentences with a similar semantic meaning. For example this can be useful for [semantic textual similarity](https://www.sbert.net/docs/usage/semantic_textual_similarity.html), [semantic search](https://www.sbert.net/docs/usage/semantic_search.html), or [paraphrase mining](https://www.sbert.net/docs/usage/paraphrase_mining.html). To do this you have to use the [Sentence Transformers Python framework](https://github.com/UKPLab/sentence-transformers). - -> Sentence-BERT (SBERT) is a modification of the pretrained BERT network that use siamese and triplet network structures to derive semantically meaningful sentence embeddings that can be compared using cosine-similarity. This reduces the effort for finding the most similar pair from 65hours with BERT / RoBERTa to about 5 seconds with SBERT, while maintaining the accuracy from BERT. - -Source: [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) - -This model is fine-tuned from [Philip May](https://eniak.de/) and open-sourced by [T-Systems-onsite](https://www.t-systems-onsite.de/). Special thanks to [Nils Reimers](https://www.nils-reimers.de/) for your awesome open-source work, the Sentence Transformers, the models and your help on GitHub. - -## How to use -**The usage description above - provided by Hugging Face - is wrong for sentence embeddings! Please use this:** - -To use this model install the `sentence-transformers` package (see here: ). - -```python -from sentence_transformers import SentenceTransformer -model = SentenceTransformer('T-Systems-onsite/german-roberta-sentence-transformer-v2') -``` - -For details of usage and examples see here: -- [Computing Sentence Embeddings](https://www.sbert.net/docs/usage/computing_sentence_embeddings.html) -- [Semantic Textual Similarity](https://www.sbert.net/docs/usage/semantic_textual_similarity.html) -- [Paraphrase Mining](https://www.sbert.net/docs/usage/paraphrase_mining.html) -- [Semantic Search](https://www.sbert.net/docs/usage/semantic_search.html) -- [Cross-Encoders](https://www.sbert.net/docs/usage/cross-encoder.html) -- [Examples on GitHub](https://github.com/UKPLab/sentence-transformers/tree/master/examples) - -## Training -The base model is [xlm-roberta-base](https://huggingface.co/xlm-roberta-base). This model has been further trained by [Nils Reimers](https://www.nils-reimers.de/) on a large scale paraphrase dataset for 50+ languages. [Nils Reimers](https://www.nils-reimers.de/) about this [on GitHub](https://github.com/UKPLab/sentence-transformers/issues/509#issuecomment-712243280): - ->A paper is upcoming for the paraphrase models. -> ->These models were trained on various datasets with Millions of examples for paraphrases, mainly derived from Wikipedia edit logs, paraphrases mined from Wikipedia and SimpleWiki, paraphrases from news reports, AllNLI-entailment pairs with in-batch-negative loss etc. -> ->In internal tests, they perform much better than the NLI+STSb models as they have see more and broader type of training data. NLI+STSb has the issue that they are rather narrow in their domain and do not contain any domain specific words / sentences (like from chemistry, computer science, math etc.). The paraphrase models has seen plenty of sentences from various domains. -> ->More details with the setup, all the datasets, and a wider evaluation will follow soon. - -The resulting model called `xlm-r-distilroberta-base-paraphrase-v1` has been released here: - -Building on this cross language model we fine-tuned it for German language on the [deepl.com](https://www.deepl.com/translator) dataset of our [German STSbenchmark dataset](https://github.com/t-systems-on-site-services-gmbh/german-STSbenchmark). - -We did an automatic hyperparameter search for 102 trials with [Optuna](https://github.com/optuna/optuna). Using 10-fold crossvalidation on the deepl.com test and dev dataset we found the following best hyperparameters: -- batch_size = 15 -- num_epochs = 4 -- lr = 2.2995320905210864e-05 -- eps = 1.8979875906303792e-06 -- weight_decay = 0.003314045812507563 -- warmup_steps_proportion = 0.46141685205829014 - -The final model was trained with these hyperparameters on the combination of `sts_de_train.csv` and `sts_de_dev.csv`. The `sts_de_test.csv` was left for testing. - -# Evaluation -The evaluation has been done on the test set of our [German STSbenchmark dataset](https://github.com/t-systems-on-site-services-gmbh/german-STSbenchmark). The code is available on [Colab](https://colab.research.google.com/drive/1aCWOqDQx953kEnQ5k4Qn7uiixokocOHv?usp=sharing). As the metric for evaluation we use the Spearman’s rank correlation between the cosine-similarity of the sentence embeddings and STSbenchmark labels. - -| Model Name | Spearman rank correlation
(German) | -|--------------------------------------|-------------------------------------| -| xlm-r-distilroberta-base-paraphrase-v1 | 0.8079 | -| xlm-r-100langs-bert-base-nli-stsb-mean-tokens | 0.8194 | -| xlm-r-bert-base-nli-stsb-mean-tokens | 0.8194 | -| **T-Systems-onsite/
german-roberta-sentence-transformer-v2** | **0.8529** | -| **[T-Systems-onsite/
cross-en-de-roberta-sentence-transformer](https://huggingface.co/T-Systems-onsite/cross-en-de-roberta-sentence-transformer)** | **0.8550** | diff --git a/model_cards/Tereveni-AI/gpt2-124M-uk-fiction/README.md b/model_cards/Tereveni-AI/gpt2-124M-uk-fiction/README.md deleted file mode 100644 index d26be020e5ed..000000000000 --- a/model_cards/Tereveni-AI/gpt2-124M-uk-fiction/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -language: uk ---- - -Note: **default code snippet above won't work** because we are using `AlbertTokenizer` with `GPT2LMHeadModel`, see [issue](https://github.com/huggingface/transformers/issues/4285). - -## GPT2 124M Trained on Ukranian Fiction - -### Training details - -Model was trained on corpus of 4040 fiction books, 2.77 GiB in total. -Evaluation on [brown-uk](https://github.com/brown-uk/corpus) gives perplexity of 50.16. - -### Example usage: -```python -from transformers import AlbertTokenizer, GPT2LMHeadModel - -tokenizer = AlbertTokenizer.from_pretrained("Tereveni-AI/gpt2-124M-uk-fiction") -model = GPT2LMHeadModel.from_pretrained("Tereveni-AI/gpt2-124M-uk-fiction") - -input_ids = tokenizer.encode("Но зла Юнона, суча дочка,", add_special_tokens=False, return_tensors='pt') - -outputs = model.generate( - input_ids, - do_sample=True, - num_return_sequences=3, - max_length=50 -) - -for i, out in enumerate(outputs): - print("{}: {}".format(i, tokenizer.decode(out))) -``` - -Prints something like this: -```bash -0: Но зла Юнона, суча дочка, яка затьмарила всі її таємниці: І хто з'їсть її душу, той помре». І, не дочекавшись гніву богів, посунула в пітьму, щоб не бачити перед собою. Але, за -1: Но зла Юнона, суча дочка, і довела мене до божевілля. Але він не знав нічого. Після того як я його побачив, мені стало зле. Я втратив рівновагу. Але в мене не було часу на роздуми. Я вже втратив надію -2: Но зла Юнона, суча дочка, не нарікала нам! — раптом вигукнула Юнона. — Це ти, старий йолопе! — мовила вона, не перестаючи сміятись. — Хіба ти не знаєш, що мені подобається ходити з тобою? -``` \ No newline at end of file diff --git a/model_cards/TurkuNLP/bert-base-finnish-cased-v1/README.md b/model_cards/TurkuNLP/bert-base-finnish-cased-v1/README.md deleted file mode 100644 index 4225733d606b..000000000000 --- a/model_cards/TurkuNLP/bert-base-finnish-cased-v1/README.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -language: fi ---- - -## Quickstart - -**Release 1.0** (November 25, 2019) - -Download the models here: - -* Cased Finnish BERT Base: [bert-base-finnish-cased-v1.zip](http://dl.turkunlp.org/finbert/bert-base-finnish-cased-v1.zip) -* Uncased Finnish BERT Base: [bert-base-finnish-uncased-v1.zip](http://dl.turkunlp.org/finbert/bert-base-finnish-uncased-v1.zip) - -We generally recommend the use of the cased model. - -Paper presenting Finnish BERT: [arXiv:1912.07076](https://arxiv.org/abs/1912.07076) - -## What's this? - -A version of Google's [BERT](https://github.com/google-research/bert) deep transfer learning model for Finnish. The model can be fine-tuned to achieve state-of-the-art results for various Finnish natural language processing tasks. - -FinBERT features a custom 50,000 wordpiece vocabulary that has much better coverage of Finnish words than e.g. the previously released [multilingual BERT](https://github.com/google-research/bert/blob/master/multilingual.md) models from Google: - -| Vocabulary | Example | -|------------|---------| -| FinBERT | Suomessa vaihtuu kesän aikana sekä pääministeri että valtiovarain ##ministeri . | -| Multilingual BERT | Suomessa vai ##htuu kes ##än aikana sekä p ##ää ##minister ##i että valt ##io ##vara ##in ##minister ##i . | - -FinBERT has been pre-trained for 1 million steps on over 3 billion tokens (24B characters) of Finnish text drawn from news, online discussion, and internet crawls. By contrast, Multilingual BERT was trained on Wikipedia texts, where the Finnish Wikipedia text is approximately 3% of the amount used to train FinBERT. - -These features allow FinBERT to outperform not only Multilingual BERT but also all previously proposed models when fine-tuned for Finnish natural language processing tasks. - -## Results - -### Document classification - -![learning curves for Yle and Ylilauta document classification](https://raw.githubusercontent.com/TurkuNLP/FinBERT/master/img/yle-ylilauta-curves.png) - -FinBERT outperforms multilingual BERT (M-BERT) on document classification over a range of training set sizes on the Yle news (left) and Ylilauta online discussion (right) corpora. (Baseline classification performance with [FastText](https://fasttext.cc/) included for reference.) - -[[code](https://github.com/spyysalo/finbert-text-classification)][[Yle data](https://github.com/spyysalo/yle-corpus)] [[Ylilauta data](https://github.com/spyysalo/ylilauta-corpus)] - -### Named Entity Recognition - -Evaluation on FiNER corpus ([Ruokolainen et al 2019](https://arxiv.org/abs/1908.04212)) - -| Model | Accuracy | -|--------------------|----------| -| **FinBERT** | **92.40%** | -| Multilingual BERT | 90.29% | -| [FiNER-tagger](https://github.com/Traubert/FiNer-rules) (rule-based) | 86.82% | - -(FiNER tagger results from [Ruokolainen et al. 2019](https://arxiv.org/pdf/1908.04212.pdf)) - -[[code](https://github.com/jouniluoma/keras-bert-ner)][[data](https://github.com/mpsilfve/finer-data)] - -### Part of speech tagging - -Evaluation on three Finnish corpora annotated with [Universal Dependencies](https://universaldependencies.org/) part-of-speech tags: the Turku Dependency Treebank (TDT), FinnTreeBank (FTB), and Parallel UD treebank (PUD) - -| Model | TDT | FTB | PUD | -|-------------------|-------------|-------------|-------------| -| **FinBERT** | **98.23%** | **98.39%** | **98.08%** | -| Multilingual BERT | 96.97% | 95.87% | 97.58% | - -[[code](https://github.com/spyysalo/bert-pos)][[data](http://hdl.handle.net/11234/1-2837)] - -## Use with PyTorch - -If you want to use the model with the huggingface/transformers library, follow the steps in [huggingface_transformers.md](https://github.com/TurkuNLP/FinBERT/blob/master/huggingface_transformers.md) - -## Previous releases - -### Release 0.2 - -**October 24, 2019** Beta version of the BERT base uncased model trained from scratch on a corpus of Finnish news, online discussions, and crawled data. - -Download the model here: [bert-base-finnish-uncased.zip](http://dl.turkunlp.org/finbert/bert-base-finnish-uncased.zip) - -### Release 0.1 - -**September 30, 2019** We release a beta version of the BERT base cased model trained from scratch on a corpus of Finnish news, online discussions, and crawled data. - -Download the model here: [bert-base-finnish-cased.zip](http://dl.turkunlp.org/finbert/bert-base-finnish-cased.zip) diff --git a/model_cards/TurkuNLP/bert-base-finnish-uncased-v1/README.md b/model_cards/TurkuNLP/bert-base-finnish-uncased-v1/README.md deleted file mode 100644 index 4225733d606b..000000000000 --- a/model_cards/TurkuNLP/bert-base-finnish-uncased-v1/README.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -language: fi ---- - -## Quickstart - -**Release 1.0** (November 25, 2019) - -Download the models here: - -* Cased Finnish BERT Base: [bert-base-finnish-cased-v1.zip](http://dl.turkunlp.org/finbert/bert-base-finnish-cased-v1.zip) -* Uncased Finnish BERT Base: [bert-base-finnish-uncased-v1.zip](http://dl.turkunlp.org/finbert/bert-base-finnish-uncased-v1.zip) - -We generally recommend the use of the cased model. - -Paper presenting Finnish BERT: [arXiv:1912.07076](https://arxiv.org/abs/1912.07076) - -## What's this? - -A version of Google's [BERT](https://github.com/google-research/bert) deep transfer learning model for Finnish. The model can be fine-tuned to achieve state-of-the-art results for various Finnish natural language processing tasks. - -FinBERT features a custom 50,000 wordpiece vocabulary that has much better coverage of Finnish words than e.g. the previously released [multilingual BERT](https://github.com/google-research/bert/blob/master/multilingual.md) models from Google: - -| Vocabulary | Example | -|------------|---------| -| FinBERT | Suomessa vaihtuu kesän aikana sekä pääministeri että valtiovarain ##ministeri . | -| Multilingual BERT | Suomessa vai ##htuu kes ##än aikana sekä p ##ää ##minister ##i että valt ##io ##vara ##in ##minister ##i . | - -FinBERT has been pre-trained for 1 million steps on over 3 billion tokens (24B characters) of Finnish text drawn from news, online discussion, and internet crawls. By contrast, Multilingual BERT was trained on Wikipedia texts, where the Finnish Wikipedia text is approximately 3% of the amount used to train FinBERT. - -These features allow FinBERT to outperform not only Multilingual BERT but also all previously proposed models when fine-tuned for Finnish natural language processing tasks. - -## Results - -### Document classification - -![learning curves for Yle and Ylilauta document classification](https://raw.githubusercontent.com/TurkuNLP/FinBERT/master/img/yle-ylilauta-curves.png) - -FinBERT outperforms multilingual BERT (M-BERT) on document classification over a range of training set sizes on the Yle news (left) and Ylilauta online discussion (right) corpora. (Baseline classification performance with [FastText](https://fasttext.cc/) included for reference.) - -[[code](https://github.com/spyysalo/finbert-text-classification)][[Yle data](https://github.com/spyysalo/yle-corpus)] [[Ylilauta data](https://github.com/spyysalo/ylilauta-corpus)] - -### Named Entity Recognition - -Evaluation on FiNER corpus ([Ruokolainen et al 2019](https://arxiv.org/abs/1908.04212)) - -| Model | Accuracy | -|--------------------|----------| -| **FinBERT** | **92.40%** | -| Multilingual BERT | 90.29% | -| [FiNER-tagger](https://github.com/Traubert/FiNer-rules) (rule-based) | 86.82% | - -(FiNER tagger results from [Ruokolainen et al. 2019](https://arxiv.org/pdf/1908.04212.pdf)) - -[[code](https://github.com/jouniluoma/keras-bert-ner)][[data](https://github.com/mpsilfve/finer-data)] - -### Part of speech tagging - -Evaluation on three Finnish corpora annotated with [Universal Dependencies](https://universaldependencies.org/) part-of-speech tags: the Turku Dependency Treebank (TDT), FinnTreeBank (FTB), and Parallel UD treebank (PUD) - -| Model | TDT | FTB | PUD | -|-------------------|-------------|-------------|-------------| -| **FinBERT** | **98.23%** | **98.39%** | **98.08%** | -| Multilingual BERT | 96.97% | 95.87% | 97.58% | - -[[code](https://github.com/spyysalo/bert-pos)][[data](http://hdl.handle.net/11234/1-2837)] - -## Use with PyTorch - -If you want to use the model with the huggingface/transformers library, follow the steps in [huggingface_transformers.md](https://github.com/TurkuNLP/FinBERT/blob/master/huggingface_transformers.md) - -## Previous releases - -### Release 0.2 - -**October 24, 2019** Beta version of the BERT base uncased model trained from scratch on a corpus of Finnish news, online discussions, and crawled data. - -Download the model here: [bert-base-finnish-uncased.zip](http://dl.turkunlp.org/finbert/bert-base-finnish-uncased.zip) - -### Release 0.1 - -**September 30, 2019** We release a beta version of the BERT base cased model trained from scratch on a corpus of Finnish news, online discussions, and crawled data. - -Download the model here: [bert-base-finnish-cased.zip](http://dl.turkunlp.org/finbert/bert-base-finnish-cased.zip) diff --git a/model_cards/TypicaAI/magbert-ner/README.md b/model_cards/TypicaAI/magbert-ner/README.md deleted file mode 100644 index 7e22de96d6f2..000000000000 --- a/model_cards/TypicaAI/magbert-ner/README.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -language: fr -widget: -- text: "Je m'appelle Hicham et je vis a Fès" ---- - -# MagBERT-NER: a state-of-the-art NER model for Moroccan French language (Maghreb) - -## Introduction - -[MagBERT-NER] is a state-of-the-art NER model for Moroccan French language (Maghreb). The MagBERT-NER model was fine-tuned for NER Task based the language model for French Camembert (based on the RoBERTa architecture). - -For further information or requests, please visite our website at [typica.ai Website](https://typica.ai/) or send us an email at contactus@typica.ai - -## How to use MagBERT-NER with HuggingFace - -##### Load MagBERT-NER and its sub-word tokenizer : - -```python -from transformers import AutoTokenizer, AutoModelForTokenClassification - -tokenizer = AutoTokenizer.from_pretrained("TypicaAI/magbert-ner") -model = AutoModelForTokenClassification.from_pretrained("TypicaAI/magbert-ner") - - -##### Process text sample (from wikipedia about the current Prime Minister of Morocco) Using NER pipeline - -from transformers import pipeline - -nlp = pipeline('ner', model=model, tokenizer=tokenizer, grouped_entities=True) -nlp("Saad Dine El Otmani, né le 16 janvier 1956 à Inezgane, est un homme d'État marocain, chef du gouvernement du Maroc depuis le 5 avril 2017") - - -#[{'entity_group': 'I-PERSON', -# 'score': 0.8941445276141167, -# 'word': 'Saad Dine El Otmani'}, -# {'entity_group': 'B-DATE', -# 'score': 0.5967703461647034, -# 'word': '16 janvier 1956'}, -# {'entity_group': 'B-GPE', 'score': 0.7160899192094803, 'word': 'Inezgane'}, -# {'entity_group': 'B-NORP', 'score': 0.7971733212471008, 'word': 'marocain'}, -# {'entity_group': 'B-GPE', 'score': 0.8921478390693665, 'word': 'Maroc'}, -# {'entity_group': 'B-DATE', -# 'score': 0.5760444005330404, -# 'word': '5 avril 2017'}] - -``` - - -## Authors - -MagBert-NER Model was trained by Hicham Assoudi, Ph.D. -For any questions, comments you can contact me at assoudi@typica.ai - - -## Citation - -If you use our work, please cite: -Hicham Assoudi, Ph.D., MagBERT-NER: a state-of-the-art NER model for Moroccan French language (Maghreb), (2020) diff --git a/model_cards/Vamsi/T5_Paraphrase_Paws/README.md b/model_cards/Vamsi/T5_Paraphrase_Paws/README.md deleted file mode 100644 index 8d54e825d795..000000000000 --- a/model_cards/Vamsi/T5_Paraphrase_Paws/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: "en" -tags: -- paraphrase-generation -- text-generation -- Conditional Generation -inference: false ---- -​ -# Paraphrase-Generation -​ -## Model description -​ -T5 Model for generating paraphrases of english sentences. Trained on the [Google PAWS](https://github.com/google-research-datasets/paws) dataset. -​ -## How to use -​ -PyTorch and TF models available -​ -```python -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM -​ -tokenizer = AutoTokenizer.from_pretrained("Vamsi/T5_Paraphrase_Paws") -model = AutoModelForSeq2SeqLM.from_pretrained("Vamsi/T5_Paraphrase_Paws") -​ -sentence = "This is something which i cannot understand at all" - -text = "paraphrase: " + sentence + " " - -encoding = tokenizer.encode_plus(text,pad_to_max_length=True, return_tensors="pt") -input_ids, attention_masks = encoding["input_ids"].to("cuda"), encoding["attention_mask"].to("cuda") - - -outputs = model.generate( - input_ids=input_ids, attention_mask=attention_masks, - max_length=256, - do_sample=True, - top_k=120, - top_p=0.95, - early_stopping=True, - num_return_sequences=5 -) - -for output in outputs: - line = tokenizer.decode(output, skip_special_tokens=True,clean_up_tokenization_spaces=True) - print(line) -​ - -``` - -For more reference on training your own T5 model or using this model, do check out [Paraphrase Generation](https://github.com/Vamsi995/Paraphrase-Generator). diff --git a/model_cards/VictorSanh/roberta-base-finetuned-yelp-polarity/README.md b/model_cards/VictorSanh/roberta-base-finetuned-yelp-polarity/README.md deleted file mode 100644 index e15e38e37b23..000000000000 --- a/model_cards/VictorSanh/roberta-base-finetuned-yelp-polarity/README.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -language: en -datasets: -- yelp_polarity ---- - -# RoBERTa-base-finetuned-yelp-polarity - -This is a [RoBERTa-base](https://huggingface.co/roberta-base) checkpoint fine-tuned on binary sentiment classifcation from [Yelp polarity](https://huggingface.co/nlp/viewer/?dataset=yelp_polarity). -It gets **98.08%** accuracy on the test set. - -## Hyper-parameters - -We used the following hyper-parameters to train the model on one GPU: -```python -num_train_epochs = 2.0 -learning_rate = 1e-05 -weight_decay = 0.0 -adam_epsilon = 1e-08 -max_grad_norm = 1.0 -per_device_train_batch_size = 32 -gradient_accumulation_steps = 1 -warmup_steps = 3500 -seed = 42 -``` diff --git a/model_cards/ViktorAlm/electra-base-norwegian-uncased-discriminator/README.md b/model_cards/ViktorAlm/electra-base-norwegian-uncased-discriminator/README.md deleted file mode 100644 index 0f22ee96918b..000000000000 --- a/model_cards/ViktorAlm/electra-base-norwegian-uncased-discriminator/README.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -language: no -thumbnail: https://i.imgur.com/QqSEC5I.png ---- - -# Norwegian Electra -![Image of norwegian electra](https://i.imgur.com/QqSEC5I.png) - -Trained on Oscar + wikipedia + opensubtitles + some other data I had with the awesome power of TPUs(V3-8) - -Use with caution. I have no downstream tasks in Norwegian to test on so I have no idea of its performance yet. -# Model -## Electra: Pre-training Text Encoders as Discriminators Rather Than Generators -Kevin Clark and Minh-Thang Luong and Quoc V. Le and Christopher D. Manning -- https://openreview.net/pdf?id=r1xMH1BtvB -- https://github.com/google-research/electra -# Acknowledgments -### TensorFlow Research Cloud -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). Thanks for providing access to the TFRC ❤️ -- https://www.tensorflow.org/tfrc -#### OSCAR corpus -- https://oscar-corpus.com/ -#### OPUS -- http://opus.nlpl.eu/ -- http://www.opensubtitles.org/ diff --git a/model_cards/a-ware/bart-squadv2/README.md b/model_cards/a-ware/bart-squadv2/README.md deleted file mode 100644 index 164c6a220faa..000000000000 --- a/model_cards/a-ware/bart-squadv2/README.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -datasets: -- squad_v2 ---- - -# BART-LARGE finetuned on SQuADv2 - -This is bart-large model finetuned on SQuADv2 dataset for question answering task - -## Model details -BART was propsed in the [paper](https://arxiv.org/abs/1910.13461) **BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension**. -BART is a seq2seq model intended for both NLG and NLU tasks. - -To use BART for question answering tasks, we feed the complete document into the encoder and decoder, and use the top -hidden state of the decoder as a representation for each -word. This representation is used to classify the token. As given in the paper bart-large achives comparable to ROBERTa on SQuAD. -Another notable thing about BART is that it can handle sequences with upto 1024 tokens. - -| Param | #Value | -|---------------------|--------| -| encoder layers | 12 | -| decoder layers | 12 | -| hidden size | 4096 | -| num attetion heads | 16 | -| on disk size | 1.63GB | - - -## Model training -This model was trained with following parameters using simpletransformers wrapper: -``` -train_args = { - 'learning_rate': 1e-5, - 'max_seq_length': 512, - 'doc_stride': 512, - 'overwrite_output_dir': True, - 'reprocess_input_data': False, - 'train_batch_size': 8, - 'num_train_epochs': 2, - 'gradient_accumulation_steps': 2, - 'no_cache': True, - 'use_cached_eval_features': False, - 'save_model_every_epoch': False, - 'output_dir': "bart-squadv2", - 'eval_batch_size': 32, - 'fp16_opt_level': 'O2', - } -``` - -[You can even train your own model using this colab notebook](https://colab.research.google.com/drive/1I5cK1M_0dLaf5xoewh6swcm5nAInfwHy?usp=sharing) - -## Results -```{"correct": 6832, "similar": 4409, "incorrect": 632, "eval_loss": -14.950117511952177}``` - -## Model in Action 🚀 -```python3 -from transformers import BartTokenizer, BartForQuestionAnswering -import torch - -tokenizer = BartTokenizer.from_pretrained('a-ware/bart-squadv2') -model = BartForQuestionAnswering.from_pretrained('a-ware/bart-squadv2') - -question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" -encoding = tokenizer(question, text, return_tensors='pt') -input_ids = encoding['input_ids'] -attention_mask = encoding['attention_mask'] - -start_scores, end_scores = model(input_ids, attention_mask=attention_mask, output_attentions=False)[:2] - -all_tokens = tokenizer.convert_ids_to_tokens(input_ids[0]) -answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1]) -answer = tokenizer.convert_tokens_to_ids(answer.split()) -answer = tokenizer.decode(answer) -#answer => 'a nice puppet' -``` - -> Created with ❤️ by A-ware UG [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/aware-ai) diff --git a/model_cards/a-ware/roberta-large-squad-classification/README.md b/model_cards/a-ware/roberta-large-squad-classification/README.md deleted file mode 100644 index 31d4578954d5..000000000000 --- a/model_cards/a-ware/roberta-large-squad-classification/README.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -datasets: -- squad_v2 ---- - -# Roberta-LARGE finetuned on SQuADv2 - -This is roberta-large model finetuned on SQuADv2 dataset for question answering answerability classification - -## Model details -This model is simply an Sequenceclassification model with two inputs (context and question) in a list. -The result is either [1] for answerable or [0] if it is not answerable. -It was trained over 4 epochs on squadv2 dataset and can be used to filter out which context is good to give into the QA model to avoid bad answers. - -## Model training -This model was trained with following parameters using simpletransformers wrapper: -``` -train_args = { - 'learning_rate': 1e-5, - 'max_seq_length': 512, - 'overwrite_output_dir': True, - 'reprocess_input_data': False, - 'train_batch_size': 4, - 'num_train_epochs': 4, - 'gradient_accumulation_steps': 2, - 'no_cache': True, - 'use_cached_eval_features': False, - 'save_model_every_epoch': False, - 'output_dir': "bart-squadv2", - 'eval_batch_size': 8, - 'fp16_opt_level': 'O2', - } -``` - -## Results -```{"accuracy": 90.48%}``` -## Model in Action 🚀 -```python3 -from simpletransformers.classification import ClassificationModel - -model = ClassificationModel('roberta', 'a-ware/roberta-large-squadv2', num_labels=2, args=train_args) - -predictions, raw_outputs = model.predict([["my dog is an year old. he loves to go into the rain", "how old is my dog ?"]]) -print(predictions) -==> [1] -``` - -> Created with ❤️ by A-ware UG [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/aware-ai) diff --git a/model_cards/a-ware/xlmroberta-squadv2/README.md b/model_cards/a-ware/xlmroberta-squadv2/README.md deleted file mode 100644 index eb08cbbc06cd..000000000000 --- a/model_cards/a-ware/xlmroberta-squadv2/README.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -datasets: -- squad_v2 ---- - -# XLM-ROBERTA-LARGE finetuned on SQuADv2 - -This is xlm-roberta-large model finetuned on SQuADv2 dataset for question answering task - -## Model details -XLM-Roberta was propsed in the [paper](https://arxiv.org/pdf/1911.02116.pdf) **XLM-R: State-of-the-art cross-lingual understanding through self-supervision - -## Model training -This model was trained with following parameters using simpletransformers wrapper: -``` -train_args = { - 'learning_rate': 1e-5, - 'max_seq_length': 512, - 'doc_stride': 512, - 'overwrite_output_dir': True, - 'reprocess_input_data': False, - 'train_batch_size': 8, - 'num_train_epochs': 2, - 'gradient_accumulation_steps': 2, - 'no_cache': True, - 'use_cached_eval_features': False, - 'save_model_every_epoch': False, - 'output_dir': "bart-squadv2", - 'eval_batch_size': 32, - 'fp16_opt_level': 'O2', - } -``` - -## Results -```{"correct": 6961, "similar": 4359, "incorrect": 553, "eval_loss": -12.177856394381962}``` - -## Model in Action 🚀 -```python3 -from transformers import XLMRobertaTokenizer, XLMRobertaForQuestionAnswering -import torch - -tokenizer = XLMRobertaTokenizer.from_pretrained('a-ware/xlmroberta-squadv2') -model = XLMRobertaForQuestionAnswering.from_pretrained('a-ware/xlmroberta-squadv2') - -question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" -encoding = tokenizer(question, text, return_tensors='pt') -input_ids = encoding['input_ids'] -attention_mask = encoding['attention_mask'] - -start_scores, end_scores = model(input_ids, attention_mask=attention_mask, output_attentions=False)[:2] - -all_tokens = tokenizer.convert_ids_to_tokens(input_ids[0]) -answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1]) -answer = tokenizer.convert_tokens_to_ids(answer.split()) -answer = tokenizer.decode(answer) -#answer => 'a nice puppet' -``` - -> Created with ❤️ by A-ware UG [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/aware-ai) diff --git a/model_cards/abhilash1910/financial_roberta/README.md b/model_cards/abhilash1910/financial_roberta/README.md deleted file mode 100644 index b212634c2fab..000000000000 --- a/model_cards/abhilash1910/financial_roberta/README.md +++ /dev/null @@ -1,132 +0,0 @@ ---- -tags: -- finance ---- -# Roberta Masked Language Model Trained On Financial Phrasebank Corpus - - -This is a Masked Language Model trained with [Roberta](https://huggingface.co/transformers/model_doc/roberta.html) on a Financial Phrasebank Corpus. -The model is built using Huggingface transformers. -The model can be found at :[Financial_Roberta](https://huggingface.co/abhilash1910/financial_roberta) - - -## Specifications - - -The corpus for training is taken from the Financial Phrasebank (Malo et al)[https://www.researchgate.net/publication/251231107_Good_Debt_or_Bad_Debt_Detecting_Semantic_Orientations_in_Economic_Texts]. - - -## Model Specification - - -The model chosen for training is [Roberta](https://arxiv.org/abs/1907.11692) with the following specifications: - 1. vocab_size=56000 - 2. max_position_embeddings=514 - 3. num_attention_heads=12 - 4. num_hidden_layers=6 - 5. type_vocab_size=1 - - -This is trained by using RobertaConfig from transformers package. -The model is trained for 10 epochs with a gpu batch size of 64 units. - - - -## Usage Specifications - - -For using this model, we have to first import AutoTokenizer and AutoModelWithLMHead Modules from transformers -After that we have to specify, the pre-trained model,which in this case is 'abhilash1910/financial_roberta' for the tokenizers and the model. - - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("abhilash1910/financial_roberta") - -model = AutoModelWithLMHead.from_pretrained("abhilash1910/financial_roberta") -``` - - -After this the model will be downloaded, it will take some time to download all the model files. -For testing the model, we have to import pipeline module from transformers and create a masked output model for inference as follows: - - -```python -from transformers import pipeline -model_mask = pipeline('fill-mask', model='abhilash1910/inancial_roberta') -model_mask("The company had a of 20% in 2020.") -``` - - -Some of the examples are also provided with generic financial statements: - -Example 1: - - -```python -model_mask("The company had a of 20% in 2020.") -``` - - -Output: - - -```bash -[{'sequence': 'The company had a profit of 20% in 2020.', - 'score': 0.023112965747714043, - 'token': 421, - 'token_str': 'Ġprofit'}, - {'sequence': 'The company had a loss of 20% in 2020.', - 'score': 0.021379893645644188, - 'token': 616, - 'token_str': 'Ġloss'}, - {'sequence': 'The company had a year of 20% in 2020.', - 'score': 0.0185744296759367, - 'token': 443, - 'token_str': 'Ġyear'}, - {'sequence': 'The company had a sales of 20% in 2020.', - 'score': 0.018143286928534508, - 'token': 428, - 'token_str': 'Ġsales'}, - {'sequence': 'The company had a value of 20% in 2020.', - 'score': 0.015319528989493847, - 'token': 776, - 'token_str': 'Ġvalue'}] - ``` - - Example 2: - -```python - model_mask("The is listed under NYSE") -``` - -Output: - -```bash -[{'sequence': 'The company is listed under NYSE', - 'score': 0.1566661298274994, - 'token': 359, - 'token_str': 'Ġcompany'}, - {'sequence': 'The total is listed under NYSE', - 'score': 0.05542507395148277, - 'token': 522, - 'token_str': 'Ġtotal'}, - {'sequence': 'The value is listed under NYSE', - 'score': 0.04729423299431801, - 'token': 776, - 'token_str': 'Ġvalue'}, - {'sequence': 'The order is listed under NYSE', - 'score': 0.02533523552119732, - 'token': 798, - 'token_str': 'Ġorder'}, - {'sequence': 'The contract is listed under NYSE', - 'score': 0.02087237872183323, - 'token': 635, - 'token_str': 'Ġcontract'}] - ``` - - -## Resources - -For all resources , please look into the [HuggingFace](https://huggingface.co/) Site and the [Repositories](https://github.com/huggingface). diff --git a/model_cards/abhilash1910/french-roberta/README.md b/model_cards/abhilash1910/french-roberta/README.md deleted file mode 100644 index 444ff47d22d7..000000000000 --- a/model_cards/abhilash1910/french-roberta/README.md +++ /dev/null @@ -1,131 +0,0 @@ -# Roberta Trained Model For Masked Language Model On French Corpus :robot: - - -This is a Masked Language Model trained with [Roberta](https://huggingface.co/transformers/model_doc/roberta.html) on a small French News Corpus(Leipzig corpora). -The model is built using Huggingface transformers. -The model can be found at :[French-Roberta](https://huggingface.co/abhilash1910/french-roberta) - - -## Specifications - - -The corpus for training is taken from Leipzig Corpora (French News) , and is trained on a small set of the corpus (300K). - - -## Model Specification - - -The model chosen for training is [Roberta](https://arxiv.org/abs/1907.11692) with the following specifications: - 1. vocab_size=32000 - 2. max_position_embeddings=514 - 3. num_attention_heads=12 - 4. num_hidden_layers=6 - 5. type_vocab_size=1 - - -This is trained by using RobertaConfig from transformers package.The total training parameters :68124416 -The model is trained for 100 epochs with a gpu batch size of 64 units. -More details for building custom models can be found at the [HuggingFace Blog](https://huggingface.co/blog/how-to-train) - - - -## Usage Specifications - - -For using this model, we have to first import AutoTokenizer and AutoModelWithLMHead Modules from transformers -After that we have to specify, the pre-trained model,which in this case is 'abhilash1910/french-roberta' for the tokenizers and the model. - - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("abhilash1910/french-roberta") - -model = AutoModelWithLMHead.from_pretrained("abhilash1910/french-roberta") -``` - - -After this the model will be downloaded, it will take some time to download all the model files. -For testing the model, we have to import pipeline module from transformers and create a masked output model for inference as follows: - - -```python -from transformers import pipeline -model_mask = pipeline('fill-mask', model='abhilash1910/french-roberta') -model_mask("Le tweet .") -``` - - -Some of the examples are also provided with generic French sentences: - -Example 1: - - -```python -model_mask("À ce jour, projet a entraîné") -``` - - -Output: - - -```bash -[{'sequence': 'À ce jour, belles projet a entraîné', - 'score': 0.18685665726661682, - 'token': 6504, - 'token_str': 'Ġbelles'}, - {'sequence': 'À ce jour,- projet a entraîné', - 'score': 0.0005200508167035878, - 'token': 17, - 'token_str': '-'}, - {'sequence': 'À ce jour, de projet a entraîné', - 'score': 0.00045729897101409733, - 'token': 268, - 'token_str': 'Ġde'}, - {'sequence': 'À ce jour, du projet a entraîné', - 'score': 0.0004307595663703978, - 'token': 326, - 'token_str': 'Ġdu'}, - {'sequence': 'À ce jour," projet a entraîné', - 'score': 0.0004219160182401538, - 'token': 6, - 'token_str': '"'}] - ``` - - Example 2: - -```python - model_mask("C'est un ") -``` - -Output: - -```bash -[{'sequence': "C'est un belles", - 'score': 0.16440927982330322, - 'token': 6504, - 'token_str': 'Ġbelles'}, - {'sequence': "C'est un de", - 'score': 0.0005495127406902611, - 'token': 268, - 'token_str': 'Ġde'}, - {'sequence': "C'est un du", - 'score': 0.00044988933950662613, - 'token': 326, - 'token_str': 'Ġdu'}, - {'sequence': "C'est un-", - 'score': 0.00044542422983795404, - 'token': 17, - 'token_str': '-'}, - {'sequence': "C'est un\t", - 'score': 0.00037563967634923756, - 'token': 202, - 'token_str': 'ĉ'}] - ``` - - -## Resources - -For all resources , please look into the [HuggingFace](https://huggingface.co/) Site and the [Repositories](https://github.com/huggingface). - - diff --git a/model_cards/activebus/BERT-DK_laptop/README.md b/model_cards/activebus/BERT-DK_laptop/README.md deleted file mode 100644 index bc7462793836..000000000000 --- a/model_cards/activebus/BERT-DK_laptop/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# ReviewBERT - -BERT (post-)trained from review corpus to understand sentiment, options and various e-commence aspects. - -`BERT-DK_laptop` is trained from 100MB laptop corpus under `Electronics/Computers & Accessories/Laptops`. - - -## Model Description - -The original model is from `BERT-base-uncased` trained from Wikipedia+BookCorpus. -Models are post-trained from [Amazon Dataset](http://jmcauley.ucsd.edu/data/amazon/) and [Yelp Dataset](https://www.yelp.com/dataset/challenge/). - -`BERT-DK_laptop` is trained from 100MB laptop corpus under `Electronics/Computers & Accessories/Laptops`. - -## Instructions -Loading the post-trained weights are as simple as, e.g., - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("activebus/BERT-DK_laptop") -model = AutoModel.from_pretrained("activebus/BERT-DK_laptop") - -``` - - -## Evaluation Results - -Check our [NAACL paper](https://www.aclweb.org/anthology/N19-1242.pdf) - - -## Citation -If you find this work useful, please cite as following. -``` -@inproceedings{xu_bert2019, - title = "BERT Post-Training for Review Reading Comprehension and Aspect-based Sentiment Analysis", - author = "Xu, Hu and Liu, Bing and Shu, Lei and Yu, Philip S.", - booktitle = "Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics", - month = "jun", - year = "2019", -} -``` diff --git a/model_cards/activebus/BERT-DK_rest/README.md b/model_cards/activebus/BERT-DK_rest/README.md deleted file mode 100644 index 34fe045a65e1..000000000000 --- a/model_cards/activebus/BERT-DK_rest/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# ReviewBERT - -BERT (post-)trained from review corpus to understand sentiment, options and various e-commence aspects. - -`BERT-DK_rest` is trained from 1G (19 types) restaurants from Yelp. - -## Model Description - -The original model is from `BERT-base-uncased` trained from Wikipedia+BookCorpus. -Models are post-trained from [Amazon Dataset](http://jmcauley.ucsd.edu/data/amazon/) and [Yelp Dataset](https://www.yelp.com/dataset/challenge/). - - -## Instructions -Loading the post-trained weights are as simple as, e.g., - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("activebus/BERT-DK_rest") -model = AutoModel.from_pretrained("activebus/BERT-DK_rest") - -``` - - -## Evaluation Results - -Check our [NAACL paper](https://www.aclweb.org/anthology/N19-1242.pdf) - - -## Citation -If you find this work useful, please cite as following. -``` -@inproceedings{xu_bert2019, - title = "BERT Post-Training for Review Reading Comprehension and Aspect-based Sentiment Analysis", - author = "Xu, Hu and Liu, Bing and Shu, Lei and Yu, Philip S.", - booktitle = "Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics", - month = "jun", - year = "2019", -} -``` diff --git a/model_cards/activebus/BERT-PT_laptop/README.md b/model_cards/activebus/BERT-PT_laptop/README.md deleted file mode 100644 index 6afbb2b01d87..000000000000 --- a/model_cards/activebus/BERT-PT_laptop/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# ReviewBERT - -BERT (post-)trained from review corpus to understand sentiment, options and various e-commence aspects. - -`BERT-DK_laptop` is trained from 100MB laptop corpus under `Electronics/Computers & Accessories/Laptops`. -`BERT-PT_*` addtionally uses SQuAD 1.1. - -## Model Description - -The original model is from `BERT-base-uncased` trained from Wikipedia+BookCorpus. -Models are post-trained from [Amazon Dataset](http://jmcauley.ucsd.edu/data/amazon/) and [Yelp Dataset](https://www.yelp.com/dataset/challenge/). - - -## Instructions -Loading the post-trained weights are as simple as, e.g., - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("activebus/BERT-PT_laptop") -model = AutoModel.from_pretrained("activebus/BERT-PT_laptop") - -``` - -## Evaluation Results - -Check our [NAACL paper](https://www.aclweb.org/anthology/N19-1242.pdf) - - -## Citation -If you find this work useful, please cite as following. -``` -@inproceedings{xu_bert2019, - title = "BERT Post-Training for Review Reading Comprehension and Aspect-based Sentiment Analysis", - author = "Xu, Hu and Liu, Bing and Shu, Lei and Yu, Philip S.", - booktitle = "Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics", - month = "jun", - year = "2019", -} -``` diff --git a/model_cards/activebus/BERT-PT_rest/README.md b/model_cards/activebus/BERT-PT_rest/README.md deleted file mode 100644 index bfc0f6134bbc..000000000000 --- a/model_cards/activebus/BERT-PT_rest/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# ReviewBERT - -BERT (post-)trained from review corpus to understand sentiment, options and various e-commence aspects. - -`BERT-DK_rest` is trained from 1G (19 types) restaurants from Yelp. -`BERT-PT_*` addtionally uses SQuAD 1.1. - -## Model Description - -The original model is from `BERT-base-uncased` trained from Wikipedia+BookCorpus. -Models are post-trained from [Amazon Dataset](http://jmcauley.ucsd.edu/data/amazon/) and [Yelp Dataset](https://www.yelp.com/dataset/challenge/). - - -## Instructions -Loading the post-trained weights are as simple as, e.g., - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("activebus/BERT-PT_rest") -model = AutoModel.from_pretrained("activebus/BERT-PT_rest") - -``` - - -## Evaluation Results - -Check our [NAACL paper](https://www.aclweb.org/anthology/N19-1242.pdf) - - -## Citation -If you find this work useful, please cite as following. -``` -@inproceedings{xu_bert2019, - title = "BERT Post-Training for Review Reading Comprehension and Aspect-based Sentiment Analysis", - author = "Xu, Hu and Liu, Bing and Shu, Lei and Yu, Philip S.", - booktitle = "Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics", - month = "jun", - year = "2019", -} -``` diff --git a/model_cards/activebus/BERT-XD_Review/README.md b/model_cards/activebus/BERT-XD_Review/README.md deleted file mode 100644 index 39bd674bbb78..000000000000 --- a/model_cards/activebus/BERT-XD_Review/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# ReviewBERT - -BERT (post-)trained from review corpus to understand sentiment, options and various e-commence aspects. -Please visit https://github.com/howardhsu/BERT-for-RRC-ABSA for details. - -`BERT-XD_Review` is a cross-domain (beyond just `laptop` and `restaurant`) language model, where each example is from a single product / restaurant with the same rating, post-trained (fine-tuned) on a combination of 5-core Amazon reviews and all Yelp data, expected to be 22 G in total. It is trained for 4 epochs on `bert-base-uncased`. -The preprocessing code [here](https://github.com/howardhsu/BERT-for-RRC-ABSA/transformers). - -## Model Description - -The original model is from `BERT-base-uncased`. -Models are post-trained from [Amazon Dataset](http://jmcauley.ucsd.edu/data/amazon/) and [Yelp Dataset](https://www.yelp.com/dataset/challenge/). - - -## Instructions -Loading the post-trained weights are as simple as, e.g., - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("activebus/BERT-XD_Review") -model = AutoModel.from_pretrained("activebus/BERT-XD_Review") - -``` - - -## Evaluation Results - -Check our [NAACL paper](https://www.aclweb.org/anthology/N19-1242.pdf) -`BERT_Review` is expected to have similar performance on domain-specific tasks (such as aspect extraction) as `BERT-DK`, but much better on general tasks such as aspect sentiment classification (different domains mostly share similar sentiment words). - - -## Citation -If you find this work useful, please cite as following. -``` -@inproceedings{xu_bert2019, - title = "BERT Post-Training for Review Reading Comprehension and Aspect-based Sentiment Analysis", - author = "Xu, Hu and Liu, Bing and Shu, Lei and Yu, Philip S.", - booktitle = "Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics", - month = "jun", - year = "2019", -} -``` diff --git a/model_cards/activebus/BERT_Review/README.md b/model_cards/activebus/BERT_Review/README.md deleted file mode 100644 index 2f86bd566d5e..000000000000 --- a/model_cards/activebus/BERT_Review/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# ReviewBERT - -BERT (post-)trained from review corpus to understand sentiment, options and various e-commence aspects. - -`BERT_Review` is cross-domain (beyond just `laptop` and `restaurant`) language model with one example from randomly mixed domains, post-trained (fine-tuned) on a combination of 5-core Amazon reviews and all Yelp data, expected to be 22 G in total. It is trained for 4 epochs on `bert-base-uncased`. -The preprocessing code [here](https://github.com/howardhsu/BERT-for-RRC-ABSA/transformers). - - -## Model Description - -The original model is from `BERT-base-uncased` trained from Wikipedia+BookCorpus. -Models are post-trained from [Amazon Dataset](http://jmcauley.ucsd.edu/data/amazon/) and [Yelp Dataset](https://www.yelp.com/dataset/challenge/). - - -## Instructions -Loading the post-trained weights are as simple as, e.g., - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("activebus/BERT_Review") -model = AutoModel.from_pretrained("activebus/BERT_Review") - -``` - - -## Evaluation Results - -Check our [NAACL paper](https://www.aclweb.org/anthology/N19-1242.pdf) -`BERT_Review` is expected to have similar performance on domain-specific tasks (such as aspect extraction) as `BERT-DK`, but much better on general tasks such as aspect sentiment classification (different domains mostly share similar sentiment words). - - -## Citation -If you find this work useful, please cite as following. -``` -@inproceedings{xu_bert2019, - title = "BERT Post-Training for Review Reading Comprehension and Aspect-based Sentiment Analysis", - author = "Xu, Hu and Liu, Bing and Shu, Lei and Yu, Philip S.", - booktitle = "Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics", - month = "jun", - year = "2019", -} -``` diff --git a/model_cards/adalbertojunior/PTT5-SMALL-SUM/README.md b/model_cards/adalbertojunior/PTT5-SMALL-SUM/README.md deleted file mode 100644 index b8686ef4f915..000000000000 --- a/model_cards/adalbertojunior/PTT5-SMALL-SUM/README.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -language: pt ---- - -# PTT5-SMALL-SUM - -## Model description - -This model was trained to summarize texts in portuguese - - -based on ```unicamp-dl/ptt5-small-portuguese-vocab``` - -#### How to use - -```python -from transformers import T5Tokenizer, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained('adalbertojunior/PTT5-SMALL-SUM') - -t5 = T5ForConditionalGeneration.from_pretrained('adalbertojunior/PTT5-SMALL-SUM') - -text="Esse é um exemplo de sumarização." - -input_ids = tokenizer.encode(text, return_tensors="pt", add_special_tokens=True) - -generated_ids = t5.generate( - input_ids=input_ids, - num_beams=1, - max_length=40, - #repetition_penalty=2.5 - ).squeeze() - -predicted_span = tokenizer.decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True) - - -``` diff --git a/model_cards/ahotrod/albert_xxlargev1_squad2_512/README.md b/model_cards/ahotrod/albert_xxlargev1_squad2_512/README.md deleted file mode 100644 index 2f7cf73689f5..000000000000 --- a/model_cards/ahotrod/albert_xxlargev1_squad2_512/README.md +++ /dev/null @@ -1,60 +0,0 @@ -## Albert xxlarge version 1 language model fine-tuned on SQuAD2.0 - -### (updated 30Sept2020) with the following results: - -``` -exact: 86.11134506864315 -f1: 89.35371214945009 -total': 11873 -HasAns_exact': 83.56950067476383 -HasAns_f1': 90.06353312254078 -HasAns_total': 5928 -NoAns_exact': 88.64592094196804 -NoAns_f1': 88.64592094196804 -NoAns_total': 5945 -best_exact': 86.11134506864315 -best_exact_thresh': 0.0 -best_f1': 89.35371214944985 -best_f1_thresh': 0.0 -``` - -### from script: - -``` -python ${EXAMPLES}/run_squad.py \ - --model_type albert \ - --model_name_or_path albert-xxlarge-v1 \ - --do_train \ - --do_eval \ - --train_file ${SQUAD}/train-v2.0.json \ - --predict_file ${SQUAD}/dev-v2.0.json \ - --version_2_with_negative \ - --do_lower_case \ - --num_train_epochs 3 \ - --max_steps 8144 \ - --warmup_steps 814 \ - --learning_rate 3e-5 \ - --max_seq_length 512 \ - --doc_stride 128 \ - --per_gpu_train_batch_size 6 \ - --gradient_accumulation_steps 8 \ - --per_gpu_eval_batch_size 48 \ - --fp16 \ - --fp16_opt_level O1 \ - --threads 12 \ - --logging_steps 50 \ - --save_steps 3000 \ - --overwrite_output_dir \ - --output_dir ${MODEL_PATH} -``` - -### using the following software & system: - -``` -Transformers: 3.1.0 -PyTorch: 1.6.0 -TensorFlow: 2.3.1 -Python: 3.8.1 -OS: Linux-5.4.0-48-generic-x86_64-with-glibc2.10 -CPU/GPU: Intel i9-9900K / NVIDIA Titan RTX 24GB -``` diff --git a/model_cards/ahotrod/electra_large_discriminator_squad2_512/README.md b/model_cards/ahotrod/electra_large_discriminator_squad2_512/README.md deleted file mode 100644 index 66acb76d582a..000000000000 --- a/model_cards/ahotrod/electra_large_discriminator_squad2_512/README.md +++ /dev/null @@ -1,58 +0,0 @@ -## ELECTRA_large_discriminator language model fine-tuned on SQuAD2.0 - -### with the following results: - -``` - "exact": 87.09677419354838, - "f1": 89.98343832723452, - "total": 11873, - "HasAns_exact": 84.66599190283401, - "HasAns_f1": 90.44759839056285, - "HasAns_total": 5928, - "NoAns_exact": 89.52060555088309, - "NoAns_f1": 89.52060555088309, - "NoAns_total": 5945, - "best_exact": 87.09677419354838, - "best_exact_thresh": 0.0, - "best_f1": 89.98343832723432, - "best_f1_thresh": 0.0 -``` -### from script: -``` -python ${EXAMPLES}/run_squad.py \ - --model_type electra \ - --model_name_or_path google/electra-large-discriminator \ - --do_train \ - --do_eval \ - --train_file ${SQUAD}/train-v2.0.json \ - --predict_file ${SQUAD}/dev-v2.0.json \ - --version_2_with_negative \ - --do_lower_case \ - --num_train_epochs 3 \ - --warmup_steps 306 \ - --weight_decay 0.01 \ - --learning_rate 3e-5 \ - --max_grad_norm 0.5 \ - --adam_epsilon 1e-6 \ - --max_seq_length 512 \ - --doc_stride 128 \ - --per_gpu_train_batch_size 8 \ - --gradient_accumulation_steps 16 \ - --per_gpu_eval_batch_size 128 \ - --fp16 \ - --fp16_opt_level O1 \ - --threads 12 \ - --logging_steps 50 \ - --save_steps 1000 \ - --overwrite_output_dir \ - --output_dir ${MODEL_PATH} -``` -### using the following system & software: -``` -Transformers: 2.11.0 -PyTorch: 1.5.0 -TensorFlow: 2.2.0 -Python: 3.8.1 -OS/Platform: Linux-5.3.0-59-generic-x86_64-with-glibc2.10 -CPU/GPU: Intel i9-9900K / NVIDIA Titan RTX 24GB -``` diff --git a/model_cards/ahotrod/roberta_large_squad2/README.md b/model_cards/ahotrod/roberta_large_squad2/README.md deleted file mode 100644 index 2fe38471d498..000000000000 --- a/model_cards/ahotrod/roberta_large_squad2/README.md +++ /dev/null @@ -1,56 +0,0 @@ -## RoBERTa-large language model fine-tuned on SQuAD2.0 - -### with the following results: - -``` - "exact": 84.46896319380106, - "f1": 87.85388093408943, - "total": 11873, - "HasAns_exact": 81.37651821862349, - "HasAns_f1": 88.1560607844881, - "HasAns_total": 5928, - "NoAns_exact": 87.55256518082422, - "NoAns_f1": 87.55256518082422, - "NoAns_total": 5945, - "best_exact": 84.46896319380106, - "best_exact_thresh": 0.0, - "best_f1": 87.85388093408929, - "best_f1_thresh": 0.0 -``` -### from script: -``` -python ${EXAMPLES}/run_squad.py \ - --model_type roberta \ - --model_name_or_path roberta-large \ - --do_train \ - --do_eval \ - --train_file ${SQUAD}/train-v2.0.json \ - --predict_file ${SQUAD}/dev-v2.0.json \ - --version_2_with_negative \ - --do_lower_case \ - --num_train_epochs 3 \ - --warmup_steps 1642 \ - --weight_decay 0.01 \ - --learning_rate 3e-5 \ - --adam_epsilon 1e-6 \ - --max_seq_length 512 \ - --doc_stride 128 \ - --per_gpu_train_batch_size 8 \ - --gradient_accumulation_steps 6 \ - --per_gpu_eval_batch_size 48 \ - --threads 12 \ - --logging_steps 50 \ - --save_steps 2000 \ - --overwrite_output_dir \ - --output_dir ${MODEL_PATH} -$@ -``` -### using the following system & software: -``` -Transformers: 2.7.0 -PyTorch: 1.4.0 -TensorFlow: 2.1.0 -Python: 3.7.7 -OS/Platform: Linux-5.3.0-46-generic-x86_64-with-debian-buster-sid -CPU/GPU: Intel i9-9900K / NVIDIA Titan RTX 24GB -``` diff --git a/model_cards/ahotrod/xlnet_large_squad2_512/README.md b/model_cards/ahotrod/xlnet_large_squad2_512/README.md deleted file mode 100644 index a680704af15b..000000000000 --- a/model_cards/ahotrod/xlnet_large_squad2_512/README.md +++ /dev/null @@ -1,63 +0,0 @@ -## XLNet large language model fine-tuned on SQuAD2.0 - -### with the following results: - -``` - "exact": 82.07698138633876, - "f1": 85.898874470488, - "total": 11873, - "HasAns_exact": 79.60526315789474, - "HasAns_f1": 87.26000954590184, - "HasAns_total": 5928, - "NoAns_exact": 84.54163162321278, - "NoAns_f1": 84.54163162321278, - "NoAns_total": 5945, - "best_exact": 83.22243746315169, - "best_exact_thresh": -11.112004280090332, - "best_f1": 86.88541353813282, - "best_f1_thresh": -11.112004280090332 -``` -### from script: -``` -python -m torch.distributed.launch --nproc_per_node=2 ${RUN_SQUAD_DIR}/run_squad.py \ - --model_type xlnet \ - --model_name_or_path xlnet-large-cased \ - --do_train \ - --train_file ${SQUAD_DIR}/train-v2.0.json \ - --predict_file ${SQUAD_DIR}/dev-v2.0.json \ - --version_2_with_negative \ - --num_train_epochs 3 \ - --learning_rate 3e-5 \ - --adam_epsilon 1e-6 \ - --max_seq_length 512 \ - --doc_stride 128 \ - --save_steps 2000 \ - --per_gpu_train_batch_size 1 \ - --gradient_accumulation_steps 24 \ - --output_dir ${MODEL_PATH} - -CUDA_VISIBLE_DEVICES=0 python ${RUN_SQUAD_DIR}/run_squad_II.py \ - --model_type xlnet \ - --model_name_or_path ${MODEL_PATH} \ - --do_eval \ - --train_file ${SQUAD_DIR}/train-v2.0.json \ - --predict_file ${SQUAD_DIR}/dev-v2.0.json \ - --version_2_with_negative \ - --max_seq_length 512 \ - --per_gpu_eval_batch_size 48 \ - --output_dir ${MODEL_PATH} -``` -### using the following system & software: -``` -OS/Platform: Linux-4.15.0-76-generic-x86_64-with-debian-buster-sid -GPU/CPU: 2 x NVIDIA 1080Ti / Intel i7-8700 -Transformers: 2.1.1 -PyTorch: 1.4.0 -TensorFlow: 2.1.0 -Python: 3.7.6 -``` -### Utilize this xlnet_large_squad2_512 fine-tuned model with: -```python -tokenizer = AutoTokenizer.from_pretrained("ahotrod/xlnet_large_squad2_512") -model = AutoModelForQuestionAnswering.from_pretrained("ahotrod/xlnet_large_squad2_512") -``` diff --git a/model_cards/ai4bharat/indic-bert/README.md b/model_cards/ai4bharat/indic-bert/README.md deleted file mode 100644 index 093508b00293..000000000000 --- a/model_cards/ai4bharat/indic-bert/README.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -language: en -license: mit -datasets: -- AI4Bharat IndicNLP Corpora ---- - -# IndicBERT - -IndicBERT is a multilingual ALBERT model pretrained exclusively on 12 major Indian languages. It is pre-trained on our novel monolingual corpus of around 9 billion tokens and subsequently evaluated on a set of diverse tasks. IndicBERT has much fewer parameters than other multilingual models (mBERT, XLM-R etc.) while it also achieves a performance on-par or better than these models. - -The 12 languages covered by IndicBERT are: Assamese, Bengali, English, Gujarati, Hindi, Kannada, Malayalam, Marathi, Oriya, Punjabi, Tamil, Telugu. - -The code can be found [here](https://github.com/divkakwani/indic-bert). For more information, checkout our [project page](https://indicnlp.ai4bharat.org/) or our [paper](https://indicnlp.ai4bharat.org/papers/arxiv2020_indicnlp_corpus.pdf). - - - -## Pretraining Corpus - -We pre-trained indic-bert on AI4Bharat's monolingual corpus. The corpus has the following distribution of languages: - - -| Language | as | bn | en | gu | hi | kn | | -| ----------------- | ------ | ------ | ------ | ------ | ------ | ------ | ------- | -| **No. of Tokens** | 36.9M | 815M | 1.34B | 724M | 1.84B | 712M | | -| **Language** | **ml** | **mr** | **or** | **pa** | **ta** | **te** | **all** | -| **No. of Tokens** | 767M | 560M | 104M | 814M | 549M | 671M | 8.9B | - - - -## Evaluation Results - -IndicBERT is evaluated on IndicGLUE and some additional tasks. The results are summarized below. For more details about the tasks, refer our [official repo](https://github.com/divkakwani/indic-bert) - -#### IndicGLUE - -Task | mBERT | XLM-R | IndicBERT ------| ----- | ----- | ------ -News Article Headline Prediction | 89.58 | 95.52 | **95.87** -Wikipedia Section Title Prediction| **73.66** | 66.33 | 73.31 -Cloze-style multiple-choice QA | 39.16 | 27.98 | **41.87** -Article Genre Classification | 90.63 | 97.03 | **97.34** -Named Entity Recognition (F1-score) | **73.24** | 65.93 | 64.47 -Cross-Lingual Sentence Retrieval Task | 21.46 | 13.74 | **27.12** -Average | 64.62 | 61.09 | **66.66** - -#### Additional Tasks - - -Task | Task Type | mBERT | XLM-R | IndicBERT ------| ----- | ----- | ------ | ----- -BBC News Classification | Genre Classification | 60.55 | **75.52** | 74.60 -IIT Product Reviews | Sentiment Analysis | 74.57 | **78.97** | 71.32 -IITP Movie Reviews | Sentiment Analaysis | 56.77 | **61.61** | 59.03 -Soham News Article | Genre Classification | 80.23 | **87.6** | 78.45 -Midas Discourse | Discourse Analysis | 71.20 | **79.94** | 78.44 -iNLTK Headlines Classification | Genre Classification | 87.95 | 93.38 | **94.52** -ACTSA Sentiment Analysis | Sentiment Analysis | 48.53 | 59.33 | **61.18** -Winograd NLI | Natural Language Inference | 56.34 | 55.87 | **56.34** -Choice of Plausible Alternative (COPA) | Natural Language Inference | 54.92 | 51.13 | **58.33** -Amrita Exact Paraphrase | Paraphrase Detection | **93.81** | 93.02 | 93.75 -Amrita Rough Paraphrase | Paraphrase Detection | 83.38 | 82.20 | **84.33** -Average | | 69.84 | **74.42** | 73.66 - - -\* Note: all models have been restricted to a max_seq_length of 128. - - - -## Downloads - -The model can be downloaded [here](https://storage.googleapis.com/ai4bharat-public-indic-nlp-corpora/models/indic-bert-v1.tar.gz). Both tf checkpoints and pytorch binaries are included in the archive. Alternatively, you can also download it from [Huggingface](https://huggingface.co/ai4bharat/indic-bert). - - - -## Citing - -If you are using any of the resources, please cite the following article: - -``` -@inproceedings{kakwani2020indicnlpsuite, - title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}}, - author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar}, - year={2020}, - booktitle={Findings of EMNLP}, -} -``` - -We would like to hear from you if: - -- You are using our resources. Please let us know how you are putting these resources to use. -- You have any feedback on these resources. - - - -## License - -The IndicBERT code (and models) are released under the MIT License. - -## Contributors - -- Divyanshu Kakwani -- Anoop Kunchukuttan -- Gokul NC -- Satish Golla -- Avik Bhattacharyya -- Mitesh Khapra -- Pratyush Kumar - -This work is the outcome of a volunteer effort as part of [AI4Bharat initiative](https://ai4bharat.org). - - - -## Contact - -- Anoop Kunchukuttan ([anoop.kunchukuttan@gmail.com](mailto:anoop.kunchukuttan@gmail.com)) -- Mitesh Khapra ([miteshk@cse.iitm.ac.in](mailto:miteshk@cse.iitm.ac.in)) -- Pratyush Kumar ([pratyush@cse.iitm.ac.in](mailto:pratyush@cse.iitm.ac.in)) diff --git a/model_cards/akhooli/gpt2-small-arabic-poetry/README.md b/model_cards/akhooli/gpt2-small-arabic-poetry/README.md deleted file mode 100644 index eb09509191e3..000000000000 --- a/model_cards/akhooli/gpt2-small-arabic-poetry/README.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -language: "ar" -tags: -- text-generation -license: "" -datasets: -- Arabic poetry from several eras ---- - -# GPT2-Small-Arabic-Poetry - -## Model description - -Fine-tuned model of Arabic poetry dataset based on gpt2-small-arabic. - -## Intended uses & limitations - -#### How to use - -An example is provided in this [colab notebook](https://colab.research.google.com/drive/1mRl7c-5v-Klx27EEAEOAbrfkustL4g7a?usp=sharing). - -#### Limitations and bias - -Both the GPT2-small-arabic (trained on Arabic Wikipedia) and this model have several limitations in terms of coverage and training performance. -Use them as demonstrations or proof of concepts but not as production code. - -## Training data - -This pretrained model used the [Arabic Poetry dataset](https://www.kaggle.com/ahmedabelal/arabic-poetry) from 9 different eras with a total of around 40k poems. -The dataset was trained (fine-tuned) based on the [gpt2-small-arabic](https://huggingface.co/akhooli/gpt2-small-arabic) transformer model. - -## Training procedure - -Training was done using [Simple Transformers](https://github.com/ThilinaRajapakse/simpletransformers) library on Kaggle, using free GPU. - -## Eval results -Final perplexity reached ws 76.3, loss: 4.33 - -### BibTeX entry and citation info - -```bibtex -@inproceedings{Abed Khooli, - year={2020} -} -``` diff --git a/model_cards/akhooli/gpt2-small-arabic/README.md b/model_cards/akhooli/gpt2-small-arabic/README.md deleted file mode 100644 index 08eb482821a4..000000000000 --- a/model_cards/akhooli/gpt2-small-arabic/README.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -language: "ar" -datasets: -- Arabic Wikipedia -metrics: -- none ---- - -# GPT2-Small-Arabic - -## Model description - -GPT2 model from Arabic Wikipedia dataset based on gpt2-small (using Fastai2). - -## Intended uses & limitations - -#### How to use - -An example is provided in this [colab notebook](https://colab.research.google.com/drive/1mRl7c-5v-Klx27EEAEOAbrfkustL4g7a?usp=sharing). -Both text and poetry (fine-tuned model) generation are included. - -#### Limitations and bias - -GPT2-small-arabic (trained on Arabic Wikipedia) has several limitations in terms of coverage (Arabic Wikipeedia quality, no diacritics) and training performance. -Use as demonstration or proof of concepts but not as production code. - -## Training data - -This pretrained model used the Arabic Wikipedia dump (around 900 MB). - -## Training procedure - -Training was done using [Fastai2](https://github.com/fastai/fastai2/) library on Kaggle, using free GPU. - -## Eval results -Final perplexity reached was 72.19, loss: 4.28, accuracy: 0.307 - -### BibTeX entry and citation info - -```bibtex -@inproceedings{Abed Khooli, - year={2020} -} -``` diff --git a/model_cards/akhooli/mbart-large-cc25-ar-en/README.md b/model_cards/akhooli/mbart-large-cc25-ar-en/README.md deleted file mode 100644 index f114b38da0c6..000000000000 --- a/model_cards/akhooli/mbart-large-cc25-ar-en/README.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -tags: -- translation - -language: -- ar -- en - -license: mit ---- -### mbart-large-ar-en -This is mbart-large-cc25, finetuned on a subset of the OPUS corpus for ar_en. -Usage: see [example notebook](https://colab.research.google.com/drive/1I6RFOWMaTpPBX7saJYjnSTddW0TD6H1t?usp=sharing) -Note: model has limited training set, not fully trained (do not use for production). -Other models by me: [Abed Khooli](https://huggingface.co/akhooli) diff --git a/model_cards/akhooli/mbart-large-cc25-en-ar/README.md b/model_cards/akhooli/mbart-large-cc25-en-ar/README.md deleted file mode 100644 index 280d22257360..000000000000 --- a/model_cards/akhooli/mbart-large-cc25-en-ar/README.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -tags: -- translation - -language: -- en -- ar - -license: mit ---- -### mbart-large-en-ar -This is mbart-large-cc25, finetuned on a subset of the UN corpus for en_ar. -Usage: see [example notebook](https://colab.research.google.com/drive/1I6RFOWMaTpPBX7saJYjnSTddW0TD6H1t?usp=sharing) -Note: model has limited training set, not fully trained (do not use for production). diff --git a/model_cards/akhooli/personachat-arabic/README.md b/model_cards/akhooli/personachat-arabic/README.md deleted file mode 100644 index 4a7839644509..000000000000 --- a/model_cards/akhooli/personachat-arabic/README.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -tags: -- conversational -language: -- ar -license: mit ---- -## personachat-arabic (conversational AI) -This is personachat-arabic, using a subset from the persona-chat validation dataset, machine translated to Arabic (from English) -and fine-tuned from [akhooli/gpt2-small-arabic](https://huggingface.co/akhooli/gpt2-small-arabic) which is a limited text generation model. -Usage: see the last section of this [example notebook](https://colab.research.google.com/drive/1I6RFOWMaTpPBX7saJYjnSTddW0TD6H1t?usp=sharing) -Note: model has limited training set which was machine translated (do not use for production). diff --git a/model_cards/akhooli/xlm-r-large-arabic-sent/README.md b/model_cards/akhooli/xlm-r-large-arabic-sent/README.md deleted file mode 100644 index 4dcbc05e8406..000000000000 --- a/model_cards/akhooli/xlm-r-large-arabic-sent/README.md +++ /dev/null @@ -1,13 +0,0 @@ ---- - -language: -- ar -- en - -license: mit ---- -### xlm-r-large-arabic-sent -Multilingual sentiment classification (Label_0: mixed, Label_1: negative, Label_2: positive) of Arabic reviews by fine-tuning XLM-Roberta-Large. -Zero shot classification of other languages (also works in mixed languages - ex. Arabic & English). Mixed category is not accurate and may confuse other -classes (was based on a rate of 3 out of 5 in reviews). -Usage: see last section in this [Colab notebook](https://lnkd.in/d3bCFyZ) diff --git a/model_cards/akhooli/xlm-r-large-arabic-toxic/README.md b/model_cards/akhooli/xlm-r-large-arabic-toxic/README.md deleted file mode 100644 index db380461b210..000000000000 --- a/model_cards/akhooli/xlm-r-large-arabic-toxic/README.md +++ /dev/null @@ -1,12 +0,0 @@ ---- - -language: -- ar -- en - -license: mit ---- -### xlm-r-large-arabic-toxic (toxic/hate speech classifier) -Toxic (hate speech) classification (Label_0: non-toxic, Label_1: toxic) of Arabic comments by fine-tuning XLM-Roberta-Large. -Zero shot classification of other languages (also works in mixed languages - ex. Arabic & English). -Usage and further info: see last section in this [Colab notebook](https://lnkd.in/d3bCFyZ) diff --git a/model_cards/albert-base-v1-README.md b/model_cards/albert-base-v1-README.md deleted file mode 100644 index 91e0b067e389..000000000000 --- a/model_cards/albert-base-v1-README.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -tags: -- exbert - -license: apache-2.0 ---- - - - - diff --git a/model_cards/albert-xxlarge-v2-README.md b/model_cards/albert-xxlarge-v2-README.md deleted file mode 100644 index b28a8ffb03b9..000000000000 --- a/model_cards/albert-xxlarge-v2-README.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -tags: -- exbert - -license: apache-2.0 ---- - - - - \ No newline at end of file diff --git a/model_cards/aliosm/ComVE-distilgpt2/README.md b/model_cards/aliosm/ComVE-distilgpt2/README.md deleted file mode 100644 index 0021a6eb1941..000000000000 --- a/model_cards/aliosm/ComVE-distilgpt2/README.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -language: "en" -tags: -- exbert -- commonsense -- semeval2020 -- comve -license: "mit" -datasets: -- ComVE -metrics: -- bleu -widget: -- text: "Chicken can swim in water. <|continue|>" ---- - -# ComVE-distilgpt2 - -## Model description - -Finetuned model on Commonsense Validation and Explanation (ComVE) dataset introduced in [SemEval2020 Task4](https://competitions.codalab.org/competitions/21080) using a causal language modeling (CLM) objective. -The model is able to generate a reason why a given natural language statement is against commonsense. - -## Intended uses & limitations - -You can use the raw model for text generation to generate reasons why natural language statements are against commonsense. - -#### How to use - -You can use this model directly to generate reasons why the given statement is against commonsense using [`generate.sh`](https://github.com/AliOsm/SemEval2020-Task4-ComVE/tree/master/TaskC-Generation) script. - -*Note:* make sure that you are using version `2.4.1` of `transformers` package. Newer versions has some issue in text generation and the model repeats the last token generated again and again. - -#### Limitations and bias - -The model biased to negate the entered sentence usually instead of producing a factual reason. - -## Training data - -The model is initialized from the [distilgpt2](https://github.com/huggingface/transformers/blob/master/model_cards/distilgpt2-README.md) model and finetuned using [ComVE](https://github.com/wangcunxiang/SemEval2020-Task4-Commonsense-Validation-and-Explanation) dataset which contains 10K against commonsense sentences, each of them is paired with three reference reasons. - -## Training procedure - -Each natural language statement that against commonsense is concatenated with its reference reason with `<|continue|>` as a separator, then the model finetuned using CLM objective. -The model trained on Nvidia Tesla P100 GPU from Google Colab platform with 5e-5 learning rate, 15 epochs, 128 maximum sequence length and 64 batch size. - -
- -
- -## Eval results - -The model achieved 13.7582/13.8026 BLEU scores on SemEval2020 Task4: Commonsense Validation and Explanation development and testing dataset. - -### BibTeX entry and citation info - -```bibtex -@article{fadel2020justers, - title={JUSTers at SemEval-2020 Task 4: Evaluating Transformer Models Against Commonsense Validation and Explanation}, - author={Fadel, Ali and Al-Ayyoub, Mahmoud and Cambria, Erik}, - year={2020} -} -``` - - - - diff --git a/model_cards/aliosm/ComVE-gpt2-large/README.md b/model_cards/aliosm/ComVE-gpt2-large/README.md deleted file mode 100644 index a203025c09ff..000000000000 --- a/model_cards/aliosm/ComVE-gpt2-large/README.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -language: "en" -tags: -- gpt2 -- exbert -- commonsense -- semeval2020 -- comve -license: "mit" -datasets: -- https://github.com/wangcunxiang/SemEval2020-Task4-Commonsense-Validation-and-Explanation -metrics: -- bleu -widget: -- text: "Chicken can swim in water. <|continue|>" ---- - -# ComVE-gpt2-large - -## Model description - -Finetuned model on Commonsense Validation and Explanation (ComVE) dataset introduced in [SemEval2020 Task4](https://competitions.codalab.org/competitions/21080) using a causal language modeling (CLM) objective. -The model is able to generate a reason why a given natural language statement is against commonsense. - -## Intended uses & limitations - -You can use the raw model for text generation to generate reasons why natural language statements are against commonsense. - -#### How to use - -You can use this model directly to generate reasons why the given statement is against commonsense using [`generate.sh`](https://github.com/AliOsm/SemEval2020-Task4-ComVE/tree/master/TaskC-Generation) script. - -*Note:* make sure that you are using version `2.4.1` of `transformers` package. Newer versions has some issue in text generation and the model repeats the last token generated again and again. - -#### Limitations and bias - -The model biased to negate the entered sentence usually instead of producing a factual reason. - -## Training data - -The model is initialized from the [gpt2-large](https://github.com/huggingface/transformers/blob/master/model_cards/gpt2-README.md) model and finetuned using [ComVE](https://github.com/wangcunxiang/SemEval2020-Task4-Commonsense-Validation-and-Explanation) dataset which contains 10K against commonsense sentences, each of them is paired with three reference reasons. - -## Training procedure - -Each natural language statement that against commonsense is concatenated with its reference reason with `<|conteniue|>` as a separator, then the model finetuned using CLM objective. -The model trained on Nvidia Tesla P100 GPU from Google Colab platform with 5e-5 learning rate, 5 epochs, 128 maximum sequence length and 64 batch size. - -
- -
- -## Eval results - -The model achieved 16.5110/15.9299 BLEU scores on SemEval2020 Task4: Commonsense Validation and Explanation development and testing dataset. - -### BibTeX entry and citation info - -```bibtex -@article{fadel2020justers, - title={JUSTers at SemEval-2020 Task 4: Evaluating Transformer Models Against Commonsense Validation and Explanation}, - author={Fadel, Ali and Al-Ayyoub, Mahmoud and Cambria, Erik}, - year={2020} -} -``` - - - - diff --git a/model_cards/aliosm/ComVE-gpt2-medium/README.md b/model_cards/aliosm/ComVE-gpt2-medium/README.md deleted file mode 100644 index 1257d14b9d85..000000000000 --- a/model_cards/aliosm/ComVE-gpt2-medium/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: "en" -tags: -- gpt2 -- exbert -- commonsense -- semeval2020 -- comve -license: "mit" -datasets: -- ComVE -metrics: -- bleu -widget: -- text: "Chicken can swim in water. <|continue|>" ---- - -# ComVE-gpt2-medium - -## Model description - -Finetuned model on Commonsense Validation and Explanation (ComVE) dataset introduced in [SemEval2020 Task4](https://competitions.codalab.org/competitions/21080) using a causal language modeling (CLM) objective. -The model is able to generate a reason why a given natural language statement is against commonsense. - -## Intended uses & limitations - -You can use the raw model for text generation to generate reasons why natural language statements are against commonsense. - -#### How to use - -You can use this model directly to generate reasons why the given statement is against commonsense using [`generate.sh`](https://github.com/AliOsm/SemEval2020-Task4-ComVE/tree/master/TaskC-Generation) script. - -*Note:* make sure that you are using version `2.4.1` of `transformers` package. Newer versions has some issue in text generation and the model repeats the last token generated again and again. - -#### Limitations and bias - -The model biased to negate the entered sentence usually instead of producing a factual reason. - -## Training data - -The model is initialized from the [gpt2-medium](https://github.com/huggingface/transformers/blob/master/model_cards/gpt2-README.md) model and finetuned using [ComVE](https://github.com/wangcunxiang/SemEval2020-Task4-Commonsense-Validation-and-Explanation) dataset which contains 10K against commonsense sentences, each of them is paired with three reference reasons. - -## Training procedure - -Each natural language statement that against commonsense is concatenated with its reference reason with `<|continue|>` as a separator, then the model finetuned using CLM objective. -The model trained on Nvidia Tesla P100 GPU from Google Colab platform with 5e-5 learning rate, 5 epochs, 128 maximum sequence length and 64 batch size. - -
- -
- -## Eval results - -The model achieved fifth place with 16.7153/16.1187 BLEU scores and third place with 1.94 Human Evaluation score on SemEval2020 Task4: Commonsense Validation and Explanation development and testing dataset. - -These are some examples generated by the model: -| Against Commonsense Statement | Generated Reason | -|:-----------------------------------------------------:|:--------------------------------------------:| -| Chicken can swim in water. | Chicken can't swim. | -| shoes can fly | Shoes are not able to fly. | -| Chocolate can be used to make a coffee pot | Chocolate is not used to make coffee pots. | -| you can also buy tickets online with an identity card | You can't buy tickets with an identity card. | -| a ball is square and can roll | A ball is round and cannot roll. | -| You can use detergent to dye your hair. | Detergent is used to wash clothes. | -| you can eat mercury | mercury is poisonous | -| A gardener can follow a suspect | gardener is not a police officer | -| cars can float in the ocean just like a boat | Cars are too heavy to float in the ocean. | -| I am going to work so I can lose money. | Working is not a way to lose money. | - -### BibTeX entry and citation info - -```bibtex -@article{fadel2020justers, - title={JUSTers at SemEval-2020 Task 4: Evaluating Transformer Models Against Commonsense Validation and Explanation}, - author={Fadel, Ali and Al-Ayyoub, Mahmoud and Cambria, Erik}, - year={2020} -} -``` - - - - diff --git a/model_cards/aliosm/ComVE-gpt2/README.md b/model_cards/aliosm/ComVE-gpt2/README.md deleted file mode 100644 index 7f8ce0b5d0d3..000000000000 --- a/model_cards/aliosm/ComVE-gpt2/README.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -language: "en" -tags: -- exbert -- commonsense -- semeval2020 -- comve -license: "mit" -datasets: -- ComVE -metrics: -- bleu -widget: -- text: "Chicken can swim in water. <|continue|>" ---- - -# ComVE-gpt2 - -## Model description - -Finetuned model on Commonsense Validation and Explanation (ComVE) dataset introduced in [SemEval2020 Task4](https://competitions.codalab.org/competitions/21080) using a causal language modeling (CLM) objective. -The model is able to generate a reason why a given natural language statement is against commonsense. - -## Intended uses & limitations - -You can use the raw model for text generation to generate reasons why natural language statements are against commonsense. - -#### How to use - -You can use this model directly to generate reasons why the given statement is against commonsense using [`generate.sh`](https://github.com/AliOsm/SemEval2020-Task4-ComVE/tree/master/TaskC-Generation) script. - -*Note:* make sure that you are using version `2.4.1` of `transformers` package. Newer versions has some issue in text generation and the model repeats the last token generated again and again. - -#### Limitations and bias - -The model biased to negate the entered sentence usually instead of producing a factual reason. - -## Training data - -The model is initialized from the [gpt2](https://github.com/huggingface/transformers/blob/master/model_cards/gpt2-README.md) model and finetuned using [ComVE](https://github.com/wangcunxiang/SemEval2020-Task4-Commonsense-Validation-and-Explanation) dataset which contains 10K against commonsense sentences, each of them is paired with three reference reasons. - -## Training procedure - -Each natural language statement that against commonsense is concatenated with its reference reason with `<|continue|>` as a separator, then the model finetuned using CLM objective. -The model trained on Nvidia Tesla P100 GPU from Google Colab platform with 5e-5 learning rate, 5 epochs, 128 maximum sequence length and 64 batch size. - -
- -
- -## Eval results - -The model achieved 14.0547/13.6534 BLEU scores on SemEval2020 Task4: Commonsense Validation and Explanation development and testing dataset. - -### BibTeX entry and citation info - -```bibtex -@article{fadel2020justers, - title={JUSTers at SemEval-2020 Task 4: Evaluating Transformer Models Against Commonsense Validation and Explanation}, - author={Fadel, Ali and Al-Ayyoub, Mahmoud and Cambria, Erik}, - year={2020} -} -``` - - - - diff --git a/model_cards/aliosm/ai-soco-cpp-roberta-small-clas/README.md b/model_cards/aliosm/ai-soco-cpp-roberta-small-clas/README.md deleted file mode 100644 index dbf34874efb4..000000000000 --- a/model_cards/aliosm/ai-soco-cpp-roberta-small-clas/README.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -language: "c++" -tags: -- exbert -- authorship-identification -- fire2020 -- pan2020 -- ai-soco -- classification -license: "mit" -datasets: -- ai-soco -metrics: -- accuracy ---- - -# ai-soco-c++-roberta-small-clas - -## Model description - -`ai-soco-c++-roberta-small` model fine-tuned on [AI-SOCO](https://sites.google.com/view/ai-soco-2020) task. - -#### How to use - -You can use the model directly after tokenizing the text using the provided tokenizer with the model files. - -#### Limitations and bias - -The model is limited to C++ programming language only. - -## Training data - -The model initialized from [`ai-soco-c++-roberta-small`](https://github.com/huggingface/transformers/blob/master/model_cards/aliosm/ai-soco-c++-roberta-small) model and trained using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset to do text classification. - -## Training procedure - -The model trained on Google Colab platform using V100 GPU for 10 epochs, 32 batch size, 512 max sequence length (sequences larger than 512 were truncated). Each continues 4 spaces were converted to a single tab character (`\t`) before tokenization. - -## Eval results - -The model achieved 93.19%/92.88% accuracy on AI-SOCO task and ranked in the 4th place. - -### BibTeX entry and citation info - -```bibtex -@inproceedings{ai-soco-2020-fire, - title = "Overview of the {PAN@FIRE} 2020 Task on {Authorship Identification of SOurce COde (AI-SOCO)}", - author = "Fadel, Ali and Musleh, Husam and Tuffaha, Ibraheem and Al-Ayyoub, Mahmoud and Jararweh, Yaser and Benkhelifa, Elhadj and Rosso, Paolo", - booktitle = "Proceedings of The 12th meeting of the Forum for Information Retrieval Evaluation (FIRE 2020)", - year = "2020" -} -``` - - - - diff --git a/model_cards/aliosm/ai-soco-cpp-roberta-small/README.md b/model_cards/aliosm/ai-soco-cpp-roberta-small/README.md deleted file mode 100644 index df1af7af75bc..000000000000 --- a/model_cards/aliosm/ai-soco-cpp-roberta-small/README.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -language: "c++" -tags: -- exbert -- authorship-identification -- fire2020 -- pan2020 -- ai-soco -license: "mit" -datasets: -- ai-soco -metrics: -- perplexity ---- - -# ai-soco-c++-roberta-small - -## Model description - -From scratch pre-trained RoBERTa model with 6 layers and 12 attention heads using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset which consists of C++ codes crawled from CodeForces website. - -## Intended uses & limitations - -The model can be used to do code classification, authorship identification and other downstream tasks on C++ programming language. - -#### How to use - -You can use the model directly after tokenizing the text using the provided tokenizer with the model files. - -#### Limitations and bias - -The model is limited to C++ programming language only. - -## Training data - -The model initialized randomly and trained using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset which contains 100K C++ source codes. - -## Training procedure - -The model trained on Google Colab platform with 8 TPU cores for 200 epochs, 16\*8 batch size, 512 max sequence length and MLM objective. Other parameters were defaulted to the values mentioned in [`run_language_modelling.py`](https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_language_modeling.py) script. Each continues 4 spaces were converted to a single tab character (`\t`) before tokenization. - -### BibTeX entry and citation info - -```bibtex -@inproceedings{ai-soco-2020-fire, - title = "Overview of the {PAN@FIRE} 2020 Task on {Authorship Identification of SOurce COde (AI-SOCO)}", - author = "Fadel, Ali and Musleh, Husam and Tuffaha, Ibraheem and Al-Ayyoub, Mahmoud and Jararweh, Yaser and Benkhelifa, Elhadj and Rosso, Paolo", - booktitle = "Proceedings of The 12th meeting of the Forum for Information Retrieval Evaluation (FIRE 2020)", - year = "2020" -} -``` - - - - diff --git a/model_cards/aliosm/ai-soco-cpp-roberta-tiny-96-clas/README.md b/model_cards/aliosm/ai-soco-cpp-roberta-tiny-96-clas/README.md deleted file mode 100644 index 736e28f9a09b..000000000000 --- a/model_cards/aliosm/ai-soco-cpp-roberta-tiny-96-clas/README.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -language: "c++" -tags: -- exbert -- authorship-identification -- fire2020 -- pan2020 -- ai-soco -- classification -license: "mit" -datasets: -- ai-soco -metrics: -- accuracy ---- - -# ai-soco-c++-roberta-tiny-96-clas - -## Model description - -`ai-soco-c++-roberta-tiny-96` model fine-tuned on [AI-SOCO](https://sites.google.com/view/ai-soco-2020) task. - -#### How to use - -You can use the model directly after tokenizing the text using the provided tokenizer with the model files. - -#### Limitations and bias - -The model is limited to C++ programming language only. - -## Training data - -The model initialized from [`ai-soco-c++-roberta-tiny-96`](https://github.com/huggingface/transformers/blob/master/model_cards/aliosm/ai-soco-c++-roberta-tiny-96) model and trained using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset to do text classification. - -## Training procedure - -The model trained on Google Colab platform using V100 GPU for 10 epochs, 16 batch size, 512 max sequence length (sequences larger than 512 were truncated). Each continues 4 spaces were converted to a single tab character (`\t`) before tokenization. - -## Eval results - -The model achieved 91.12%/91.02% accuracy on AI-SOCO task and ranked in the 7th place. - -### BibTeX entry and citation info - -```bibtex -@inproceedings{ai-soco-2020-fire, - title = "Overview of the {PAN@FIRE} 2020 Task on {Authorship Identification of SOurce COde (AI-SOCO)}", - author = "Fadel, Ali and Musleh, Husam and Tuffaha, Ibraheem and Al-Ayyoub, Mahmoud and Jararweh, Yaser and Benkhelifa, Elhadj and Rosso, Paolo", - booktitle = "Proceedings of The 12th meeting of the Forum for Information Retrieval Evaluation (FIRE 2020)", - year = "2020" -} -``` - - - - diff --git a/model_cards/aliosm/ai-soco-cpp-roberta-tiny-96/README.md b/model_cards/aliosm/ai-soco-cpp-roberta-tiny-96/README.md deleted file mode 100644 index 4593c556a924..000000000000 --- a/model_cards/aliosm/ai-soco-cpp-roberta-tiny-96/README.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -language: "c++" -tags: -- exbert -- authorship-identification -- fire2020 -- pan2020 -- ai-soco -license: "mit" -datasets: -- ai-soco -metrics: -- perplexity ---- - -# ai-soco-c++-roberta-tiny-96 - -## Model description - -From scratch pre-trained RoBERTa model with 1 layers and 96 attention heads using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset which consists of C++ codes crawled from CodeForces website. - -## Intended uses & limitations - -The model can be used to do code classification, authorship identification and other downstream tasks on C++ programming language. - -#### How to use - -You can use the model directly after tokenizing the text using the provided tokenizer with the model files. - -#### Limitations and bias - -The model is limited to C++ programming language only. - -## Training data - -The model initialized randomly and trained using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset which contains 100K C++ source codes. - -## Training procedure - -The model trained on Google Colab platform with 8 TPU cores for 200 epochs, 16\*8 batch size, 512 max sequence length and MLM objective. Other parameters were defaulted to the values mentioned in [`run_language_modelling.py`](https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_language_modeling.py) script. Each continues 4 spaces were converted to a single tab character (`\t`) before tokenization. - -### BibTeX entry and citation info - -```bibtex -@inproceedings{ai-soco-2020-fire, - title = "Overview of the {PAN@FIRE} 2020 Task on {Authorship Identification of SOurce COde (AI-SOCO)}", - author = "Fadel, Ali and Musleh, Husam and Tuffaha, Ibraheem and Al-Ayyoub, Mahmoud and Jararweh, Yaser and Benkhelifa, Elhadj and Rosso, Paolo", - booktitle = "Proceedings of The 12th meeting of the Forum for Information Retrieval Evaluation (FIRE 2020)", - year = "2020" -} -``` - - - - diff --git a/model_cards/aliosm/ai-soco-cpp-roberta-tiny-clas/README.md b/model_cards/aliosm/ai-soco-cpp-roberta-tiny-clas/README.md deleted file mode 100644 index 757bf22c6aec..000000000000 --- a/model_cards/aliosm/ai-soco-cpp-roberta-tiny-clas/README.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -language: "c++" -tags: -- exbert -- authorship-identification -- fire2020 -- pan2020 -- ai-soco -- classification -license: "mit" -datasets: -- ai-soco -metrics: -- accuracy ---- - -# ai-soco-c++-roberta-tiny-clas - -## Model description - -`ai-soco-c++-roberta-tiny` model fine-tuned on [AI-SOCO](https://sites.google.com/view/ai-soco-2020) task. - -#### How to use - -You can use the model directly after tokenizing the text using the provided tokenizer with the model files. - -#### Limitations and bias - -The model is limited to C++ programming language only. - -## Training data - -The model initialized from [`ai-soco-c++-roberta-tiny`](https://github.com/huggingface/transformers/blob/master/model_cards/aliosm/ai-soco-c++-roberta-tiny) model and trained using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset to do text classification. - -## Training procedure - -The model trained on Google Colab platform using V100 GPU for 10 epochs, 32 batch size, 512 max sequence length (sequences larger than 512 were truncated). Each continues 4 spaces were converted to a single tab character (`\t`) before tokenization. - -## Eval results - -The model achieved 87.66%/87.46% accuracy on AI-SOCO task and ranked in the 9th place. - -### BibTeX entry and citation info - -```bibtex -@inproceedings{ai-soco-2020-fire, - title = "Overview of the {PAN@FIRE} 2020 Task on {Authorship Identification of SOurce COde (AI-SOCO)}", - author = "Fadel, Ali and Musleh, Husam and Tuffaha, Ibraheem and Al-Ayyoub, Mahmoud and Jararweh, Yaser and Benkhelifa, Elhadj and Rosso, Paolo", - booktitle = "Proceedings of The 12th meeting of the Forum for Information Retrieval Evaluation (FIRE 2020)", - year = "2020" -} -``` - - - - diff --git a/model_cards/aliosm/ai-soco-cpp-roberta-tiny/README.md b/model_cards/aliosm/ai-soco-cpp-roberta-tiny/README.md deleted file mode 100644 index 164cce02223b..000000000000 --- a/model_cards/aliosm/ai-soco-cpp-roberta-tiny/README.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -language: "c++" -tags: -- exbert -- authorship-identification -- fire2020 -- pan2020 -- ai-soco -license: "mit" -datasets: -- ai-soco -metrics: -- perplexity ---- - -# ai-soco-c++-roberta-tiny - -## Model description - -From scratch pre-trained RoBERTa model with 1 layers and 12 attention heads using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset which consists of C++ codes crawled from CodeForces website. - -## Intended uses & limitations - -The model can be used to do code classification, authorship identification and other downstream tasks on C++ programming language. - -#### How to use - -You can use the model directly after tokenizing the text using the provided tokenizer with the model files. - -#### Limitations and bias - -The model is limited to C++ programming language only. - -## Training data - -The model initialized randomly and trained using [AI-SOCO](https://sites.google.com/view/ai-soco-2020) dataset which contains 100K C++ source codes. - -## Training procedure - -The model trained on Google Colab platform with 8 TPU cores for 200 epochs, 32\*8 batch size, 512 max sequence length and MLM objective. Other parameters were defaulted to the values mentioned in [`run_language_modelling.py`](https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_language_modeling.py) script. Each continues 4 spaces were converted to a single tab character (`\t`) before tokenization. - -### BibTeX entry and citation info - -```bibtex -@inproceedings{ai-soco-2020-fire, - title = "Overview of the {PAN@FIRE} 2020 Task on {Authorship Identification of SOurce COde (AI-SOCO)}", - author = "Fadel, Ali and Musleh, Husam and Tuffaha, Ibraheem and Al-Ayyoub, Mahmoud and Jararweh, Yaser and Benkhelifa, Elhadj and Rosso, Paolo", - booktitle = "Proceedings of The 12th meeting of the Forum for Information Retrieval Evaluation (FIRE 2020)", - year = "2020" -} -``` - - - - diff --git a/model_cards/allegro/herbert-base-cased/README.md b/model_cards/allegro/herbert-base-cased/README.md deleted file mode 100644 index 0afd84d4a65f..000000000000 --- a/model_cards/allegro/herbert-base-cased/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: pl -tags: -- herbert -license: cc-by-sa-4.0 ---- - -# HerBERT -**[HerBERT](https://en.wikipedia.org/wiki/Zbigniew_Herbert)** is a BERT-based Language Model trained on Polish Corpora -using MLM and SSO objectives with dynamic masking of whole words. -Model training and experiments were conducted with [transformers](https://github.com/huggingface/transformers) in version 2.9. - -## Tokenizer -The training dataset was tokenized into subwords using ``CharBPETokenizer`` a character level byte-pair encoding with -a vocabulary size of 50k tokens. The tokenizer itself was trained with a [tokenizers](https://github.com/huggingface/tokenizers) library. -We kindly encourage you to use the **Fast** version of tokenizer, namely ``HerbertTokenizerFast``. - -## HerBERT usage - - -Example code: -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("allegro/herbert-base-cased") -model = AutoModel.from_pretrained("allegro/herbert-base-cased") - -output = model( - **tokenizer.batch_encode_plus( - [ - ( - "A potem szedł środkiem drogi w kurzawie, bo zamiatał nogami, ślepy dziad prowadzony przez tłustego kundla na sznurku.", - "A potem leciał od lasu chłopak z butelką, ale ten ujrzawszy księdza przy drodze okrążył go z dala i biegł na przełaj pól do karczmy." - ) - ], - padding='longest', - add_special_tokens=True, - return_tensors='pt' - ) -) -``` - - -## License -CC BY-SA 4.0 - - -## Authors -Model was trained by **Allegro Machine Learning Research** team. - -You can contact us at: klejbenchmark@allegro.pl diff --git a/model_cards/allegro/herbert-klej-cased-tokenizer-v1/README.md b/model_cards/allegro/herbert-klej-cased-tokenizer-v1/README.md deleted file mode 100644 index cf6da20c83fc..000000000000 --- a/model_cards/allegro/herbert-klej-cased-tokenizer-v1/README.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -language: pl ---- - -# HerBERT tokenizer - -**[HerBERT](https://en.wikipedia.org/wiki/Zbigniew_Herbert)** tokenizer is a character level byte-pair encoding with -vocabulary size of 50k tokens. The tokenizer was trained on [Wolne Lektury](https://wolnelektury.pl/) and a publicly available subset of -[National Corpus of Polish](http://nkjp.pl/index.php?page=14&lang=0) with [fastBPE](https://github.com/glample/fastBPE) library. -Tokenizer utilize `XLMTokenizer` implementation from [transformers](https://github.com/huggingface/transformers). - -## Tokenizer usage -Herbert tokenizer should be used together with [HerBERT model](https://huggingface.co/allegro/herbert-klej-cased-v1): -```python -from transformers import XLMTokenizer, RobertaModel - -tokenizer = XLMTokenizer.from_pretrained("allegro/herbert-klej-cased-tokenizer-v1") -model = RobertaModel.from_pretrained("allegro/herbert-klej-cased-v1") - -encoded_input = tokenizer.encode("Kto ma lepszą sztukę, ma lepszy rząd – to jasne.", return_tensors='pt') -outputs = model(encoded_input) -``` - -## License -CC BY-SA 4.0 - -## Citation -If you use this tokenizer, please cite the following paper: -``` -@misc{rybak2020klej, - title={KLEJ: Comprehensive Benchmark for Polish Language Understanding}, - author={Piotr Rybak and Robert Mroczkowski and Janusz Tracz and Ireneusz Gawlik}, - year={2020}, - eprint={2005.00630}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` -Paper is accepted at ACL 2020, as soon as proceedings appear, we will update the BibTeX. - -## Authors -Tokenizer was created by **Allegro Machine Learning Research** team. - -You can contact us at: klejbenchmark@allegro.pl diff --git a/model_cards/allegro/herbert-klej-cased-v1/README.md b/model_cards/allegro/herbert-klej-cased-v1/README.md deleted file mode 100644 index c594311a7f28..000000000000 --- a/model_cards/allegro/herbert-klej-cased-v1/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -language: pl ---- - -# HerBERT -**[HerBERT](https://en.wikipedia.org/wiki/Zbigniew_Herbert)** is a BERT-based Language Model trained on Polish Corpora -using only MLM objective with dynamic masking of whole words. For more details, please refer to: -[KLEJ: Comprehensive Benchmark for Polish Language Understanding](https://arxiv.org/abs/2005.00630). - -## Dataset -**HerBERT** training dataset is a combination of several publicly available corpora for Polish language: - -| Corpus | Tokens | Texts | -| :------ | ------: | ------: | -| [OSCAR](https://traces1.inria.fr/oscar/)| 6710M | 145M | -| [Open Subtitles](http://opus.nlpl.eu/OpenSubtitles-v2018.php) | 1084M | 1.1M | -| [Wikipedia](https://dumps.wikimedia.org/) | 260M | 1.5M | -| [Wolne Lektury](https://wolnelektury.pl/) | 41M | 5.5k | -| [Allegro Articles](https://allegro.pl/artykuly) | 18M | 33k | - -## Tokenizer -The training dataset was tokenized into subwords using [HerBERT Tokenizer](https://huggingface.co/allegro/herbert-klej-cased-tokenizer-v1); a character level byte-pair encoding with -a vocabulary size of 50k tokens. The tokenizer itself was trained on [Wolne Lektury](https://wolnelektury.pl/) and a publicly available subset of -[National Corpus of Polish](http://nkjp.pl/index.php?page=14&lang=0) with a [fastBPE](https://github.com/glample/fastBPE) library. - -Tokenizer utilizes `XLMTokenizer` implementation for that reason, one should load it as `allegro/herbert-klej-cased-tokenizer-v1`. - -## HerBERT models summary -| Model | WWM | Cased | Tokenizer | Vocab Size | Batch Size | Train Steps | -| :------ | ------: | ------: | ------: | ------: | ------: | ------: | -| herbert-klej-cased-v1 | YES | YES | BPE | 50K | 570 | 180k | - -## Model evaluation -HerBERT was evaluated on the [KLEJ](https://klejbenchmark.com/) benchmark, publicly available set of nine evaluation tasks for the Polish language understanding. -It had the best average performance and obtained the best results for three of them. - -| Model | Average | NKJP-NER | CDSC-E | CDSC-R | CBD | PolEmo2.0-IN |PolEmo2.0-OUT | DYK | PSC | AR | -| :------ | ------: | ------: | ------: | ------: | ------: | ------: | ------: | ------: | ------: | ------: | -| herbert-klej-cased-v1 | **80.5** | 92.7 | 92.5 | 91.9 | **50.3** | **89.2** |**76.3** |52.1 |95.3 | 84.5 | - -Full leaderboard is available [online](https://klejbenchmark.com/leaderboard). - - -## HerBERT usage -Model training and experiments were conducted with [transformers](https://github.com/huggingface/transformers) in version 2.0. - -Example code: -```python -from transformers import XLMTokenizer, RobertaModel - -tokenizer = XLMTokenizer.from_pretrained("allegro/herbert-klej-cased-tokenizer-v1") -model = RobertaModel.from_pretrained("allegro/herbert-klej-cased-v1") - -encoded_input = tokenizer.encode("Kto ma lepszą sztukę, ma lepszy rząd – to jasne.", return_tensors='pt') -outputs = model(encoded_input) -``` - -HerBERT can also be loaded using `AutoTokenizer` and `AutoModel`: - -```python -tokenizer = AutoTokenizer.from_pretrained("allegro/herbert-klej-cased-tokenizer-v1") -model = AutoModel.from_pretrained("allegro/herbert-klej-cased-v1") -``` - -## License -CC BY-SA 4.0 - -## Citation -If you use this model, please cite the following paper: -``` -@misc{rybak2020klej, - title={KLEJ: Comprehensive Benchmark for Polish Language Understanding}, - author={Piotr Rybak and Robert Mroczkowski and Janusz Tracz and Ireneusz Gawlik}, - year={2020}, - eprint={2005.00630}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` -Paper is accepted at ACL 2020, as soon as proceedings appear, we will update the BibTeX. - -## Authors -Model was trained by **Allegro Machine Learning Research** team. - -You can contact us at: klejbenchmark@allegro.pl diff --git a/model_cards/allegro/herbert-large-cased/README.md b/model_cards/allegro/herbert-large-cased/README.md deleted file mode 100644 index 583586f747a8..000000000000 --- a/model_cards/allegro/herbert-large-cased/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -language: pl -tags: -- herbert -license: cc-by-sa-4.0 ---- -# HerBERT -**[HerBERT](https://en.wikipedia.org/wiki/Zbigniew_Herbert)** is a BERT-based Language Model trained on Polish Corpora -using MLM and SSO objectives with dynamic masking of whole words. -Model training and experiments were conducted with [transformers](https://github.com/huggingface/transformers) in version 2.9. - -## Tokenizer -The training dataset was tokenized into subwords using ``CharBPETokenizer`` a character level byte-pair encoding with -a vocabulary size of 50k tokens. The tokenizer itself was trained with a [tokenizers](https://github.com/huggingface/tokenizers) library. -We kindly encourage you to use the **Fast** version of tokenizer, namely ``HerbertTokenizerFast``. - -## HerBERT usage - - -Example code: -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("allegro/herbert-large-cased") -model = AutoModel.from_pretrained("allegro/herbert-large-cased") - -output = model( - **tokenizer.batch_encode_plus( - [ - ( - "A potem szedł środkiem drogi w kurzawie, bo zamiatał nogami, ślepy dziad prowadzony przez tłustego kundla na sznurku.", - "A potem leciał od lasu chłopak z butelką, ale ten ujrzawszy księdza przy drodze okrążył go z dala i biegł na przełaj pól do karczmy." - ) - ], - padding='longest', - add_special_tokens=True, - return_tensors='pt' - ) -) -``` - - -## License -CC BY-SA 4.0 - - -## Authors -Model was trained by **Allegro Machine Learning Research** team. - -You can contact us at: klejbenchmark@allegro.pl diff --git a/model_cards/allenai/biomed_roberta_base/README.md b/model_cards/allenai/biomed_roberta_base/README.md deleted file mode 100644 index 66c0371d1134..000000000000 --- a/model_cards/allenai/biomed_roberta_base/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -thumbnail: https://huggingface.co/front/thumbnails/allenai.png ---- - -# BioMed-RoBERTa-base - -BioMed-RoBERTa-base is a language model based on the RoBERTa-base (Liu et. al, 2019) architecture. We adapt RoBERTa-base to 2.68 million scientific papers from the [Semantic Scholar](https://www.semanticscholar.org) corpus via continued pretraining. This amounts to 7.55B tokens and 47GB of data. We use the full text of the papers in training, not just abstracts. - -Specific details of the adaptive pretraining procedure can be found in Gururangan et. al, 2020. - - -## Evaluation - -BioMed-RoBERTa achieves competitive performance to state of the art models on a number of NLP tasks in the biomedical domain (numbers are mean (standard deviation) over 3+ random seeds) - - -| Task | Task Type | RoBERTa-base | BioMed-RoBERTa-base | -|--------------|---------------------|--------------|---------------------| -| RCT-180K | Text Classification | 86.4 (0.3) | 86.9 (0.2) | -| ChemProt | Relation Extraction | 81.1 (1.1) | 83.0 (0.7) | -| JNLPBA | NER | 74.3 (0.2) | 75.2 (0.1) | -| BC5CDR | NER | 85.6 (0.1) | 87.8 (0.1) | -| NCBI-Disease | NER | 86.6 (0.3) | 87.1 (0.8) | - -More evaluations TBD. - -## Citation - -If using this model, please cite the following paper: - -```bibtex -@inproceedings{domains, - author = {Suchin Gururangan and Ana Marasović and Swabha Swayamdipta and Kyle Lo and Iz Beltagy and Doug Downey and Noah A. Smith}, - title = {Don't Stop Pretraining: Adapt Language Models to Domains and Tasks}, - year = {2020}, - booktitle = {Proceedings of ACL}, -} -``` diff --git a/model_cards/allenai/longformer-base-4096-extra.pos.embd.only/README.md b/model_cards/allenai/longformer-base-4096-extra.pos.embd.only/README.md deleted file mode 100644 index 881870e55d3b..000000000000 --- a/model_cards/allenai/longformer-base-4096-extra.pos.embd.only/README.md +++ /dev/null @@ -1,20 +0,0 @@ - -# longformer-base-4096-extra.pos.embd.only - -This model is similar to `longformer-base-4096` but it was pretrained to preserve RoBERTa weights by freezing all RoBERTa weights and only train the additional position embeddings. - - -### Citing - -If you use `Longformer` in your research, please cite [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150). -``` -@article{Beltagy2020Longformer, - title={Longformer: The Long-Document Transformer}, - author={Iz Beltagy and Matthew E. Peters and Arman Cohan}, - journal={arXiv:2004.05150}, - year={2020}, -} -``` - -`Longformer` is an open-source project developed by [the Allen Institute for Artificial Intelligence (AI2)](http://www.allenai.org). -AI2 is a non-profit institute with the mission to contribute to humanity through high-impact AI research and engineering. diff --git a/model_cards/allenai/longformer-base-4096/README.md b/model_cards/allenai/longformer-base-4096/README.md deleted file mode 100644 index 44132b1b28c2..000000000000 --- a/model_cards/allenai/longformer-base-4096/README.md +++ /dev/null @@ -1,24 +0,0 @@ - -# longformer-base-4096 -[Longformer](https://arxiv.org/abs/2004.05150) is a transformer model for long documents. - -`longformer-base-4096` is a BERT-like model started from the RoBERTa checkpoint and pretrained for MLM on long documents. It supports sequences of length up to 4,096. - -Longformer uses a combination of a sliding window (local) attention and global attention. Global attention is user-configured based on the task to allow the model to learn task-specific representations. -Please refer to the examples in `modeling_longformer.py` and the paper for more details on how to set global attention. - - -### Citing - -If you use `Longformer` in your research, please cite [Longformer: The Long-Document Transformer](https://arxiv.org/abs/2004.05150). -``` -@article{Beltagy2020Longformer, - title={Longformer: The Long-Document Transformer}, - author={Iz Beltagy and Matthew E. Peters and Arman Cohan}, - journal={arXiv:2004.05150}, - year={2020}, -} -``` - -`Longformer` is an open-source project developed by [the Allen Institute for Artificial Intelligence (AI2)](http://www.allenai.org). -AI2 is a non-profit institute with the mission to contribute to humanity through high-impact AI research and engineering. diff --git a/model_cards/allenai/scibert_scivocab_cased/README.md b/model_cards/allenai/scibert_scivocab_cased/README.md deleted file mode 100644 index 8f9ff9b23416..000000000000 --- a/model_cards/allenai/scibert_scivocab_cased/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# SciBERT - -This is the pretrained model presented in [SciBERT: A Pretrained Language Model for Scientific Text](https://www.aclweb.org/anthology/D19-1371/), which is a BERT model trained on scientific text. - -The training corpus was papers taken from [Semantic Scholar](https://www.semanticscholar.org). Corpus size is 1.14M papers, 3.1B tokens. We use the full text of the papers in training, not just abstracts. - -SciBERT has its own wordpiece vocabulary (scivocab) that's built to best match the training corpus. We trained cased and uncased versions. - -Available models include: -* `scibert_scivocab_cased` -* `scibert_scivocab_uncased` - - -The original repo can be found [here](https://github.com/allenai/scibert). - -If using these models, please cite the following paper: -``` -@inproceedings{beltagy-etal-2019-scibert, - title = "SciBERT: A Pretrained Language Model for Scientific Text", - author = "Beltagy, Iz and Lo, Kyle and Cohan, Arman", - booktitle = "EMNLP", - year = "2019", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/D19-1371" -} -``` diff --git a/model_cards/allenai/scibert_scivocab_uncased/README.md b/model_cards/allenai/scibert_scivocab_uncased/README.md deleted file mode 100644 index 8f9ff9b23416..000000000000 --- a/model_cards/allenai/scibert_scivocab_uncased/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# SciBERT - -This is the pretrained model presented in [SciBERT: A Pretrained Language Model for Scientific Text](https://www.aclweb.org/anthology/D19-1371/), which is a BERT model trained on scientific text. - -The training corpus was papers taken from [Semantic Scholar](https://www.semanticscholar.org). Corpus size is 1.14M papers, 3.1B tokens. We use the full text of the papers in training, not just abstracts. - -SciBERT has its own wordpiece vocabulary (scivocab) that's built to best match the training corpus. We trained cased and uncased versions. - -Available models include: -* `scibert_scivocab_cased` -* `scibert_scivocab_uncased` - - -The original repo can be found [here](https://github.com/allenai/scibert). - -If using these models, please cite the following paper: -``` -@inproceedings{beltagy-etal-2019-scibert, - title = "SciBERT: A Pretrained Language Model for Scientific Text", - author = "Beltagy, Iz and Lo, Kyle and Cohan, Arman", - booktitle = "EMNLP", - year = "2019", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/D19-1371" -} -``` diff --git a/model_cards/allenai/wmt16-en-de-12-1/README.md b/model_cards/allenai/wmt16-en-de-12-1/README.md deleted file mode 100644 index 4f896ea9d2e9..000000000000 --- a/model_cards/allenai/wmt16-en-de-12-1/README.md +++ /dev/null @@ -1,103 +0,0 @@ - ---- -language: -- en -- de -thumbnail: -tags: -- translation -- wmt16 -- allenai -license: apache-2.0 -datasets: -- wmt16 -metrics: -- bleu ---- - -# FSMT - -## Model description - -This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for en-de. - -For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369). - -All 3 models are available: - -* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1) -* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1) -* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1) - - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "allenai/wmt16-en-de-12-1" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Machine learning is great, isn't it?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Maschinelles Lernen ist großartig, nicht wahr? - -``` - -#### Limitations and bias - - -## Training data - -Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369). - -## Eval results - -Here are the BLEU scores: - -model | fairseq | transformers --------|---------|---------- -wmt16-en-de-12-1 | 26.9 | 25.75 - -The score is slightly below the score reported in the paper, as the researchers don't use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs. - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=en-de -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=5 -mkdir -p $DATA_DIR -sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/wmt16-en-de-12-1 $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt16/) -- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372) - - -### BibTeX entry and citation info - -``` -@misc{kasai2020deep, - title={Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}, - author={Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}, - year={2020}, - eprint={2006.10369}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - diff --git a/model_cards/allenai/wmt16-en-de-dist-12-1/README.md b/model_cards/allenai/wmt16-en-de-dist-12-1/README.md deleted file mode 100644 index 16c7900387ca..000000000000 --- a/model_cards/allenai/wmt16-en-de-dist-12-1/README.md +++ /dev/null @@ -1,103 +0,0 @@ - ---- -language: -- en -- de -thumbnail: -tags: -- translation -- wmt16 -- allenai -license: apache-2.0 -datasets: -- wmt16 -metrics: -- bleu ---- - -# FSMT - -## Model description - -This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for en-de. - -For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369). - -All 3 models are available: - -* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1) -* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1) -* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1) - - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "allenai/wmt16-en-de-dist-12-1" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Machine learning is great, isn't it?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Maschinelles Lernen ist großartig, nicht wahr? - -``` - -#### Limitations and bias - - -## Training data - -Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369). - -## Eval results - -Here are the BLEU scores: - -model | fairseq | transformers --------|---------|---------- -wmt16-en-de-dist-12-1 | 28.3 | 27.52 - -The score is slightly below the score reported in the paper, as the researchers don't use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs. - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=en-de -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=5 -mkdir -p $DATA_DIR -sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/wmt16-en-de-dist-12-1 $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt16/) -- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372) - - -### BibTeX entry and citation info - -``` -@misc{kasai2020deep, - title={Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}, - author={Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}, - year={2020}, - eprint={2006.10369}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - diff --git a/model_cards/allenai/wmt16-en-de-dist-6-1/README.md b/model_cards/allenai/wmt16-en-de-dist-6-1/README.md deleted file mode 100644 index 426231dd81f7..000000000000 --- a/model_cards/allenai/wmt16-en-de-dist-6-1/README.md +++ /dev/null @@ -1,103 +0,0 @@ - ---- -language: -- en -- de -thumbnail: -tags: -- translation -- wmt16 -- allenai -license: apache-2.0 -datasets: -- wmt16 -metrics: -- bleu ---- - -# FSMT - -## Model description - -This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for en-de. - -For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369). - -All 3 models are available: - -* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1) -* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1) -* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1) - - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "allenai/wmt16-en-de-dist-6-1" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Machine learning is great, isn't it?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Maschinelles Lernen ist großartig, nicht wahr? - -``` - -#### Limitations and bias - - -## Training data - -Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369). - -## Eval results - -Here are the BLEU scores: - -model | fairseq | transformers --------|---------|---------- -wmt16-en-de-dist-6-1 | 27.4 | 27.11 - -The score is slightly below the score reported in the paper, as the researchers don't use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs. - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=en-de -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=5 -mkdir -p $DATA_DIR -sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/wmt16-en-de-dist-6-1 $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt16/) -- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372) - - -### BibTeX entry and citation info - -``` -@misc{kasai2020deep, - title={Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}, - author={Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}, - year={2020}, - eprint={2006.10369}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - diff --git a/model_cards/allenai/wmt19-de-en-6-6-base/README.md b/model_cards/allenai/wmt19-de-en-6-6-base/README.md deleted file mode 100644 index e5339bbc8a39..000000000000 --- a/model_cards/allenai/wmt19-de-en-6-6-base/README.md +++ /dev/null @@ -1,101 +0,0 @@ - ---- - -language: -- de -- en -thumbnail: -tags: -- translation -- wmt19 -- allenai -license: apache-2.0 -datasets: -- wmt19 -metrics: -- bleu ---- - -# FSMT - -## Model description - -This is a ported version of fairseq-based [wmt19 transformer](https://github.com/jungokasai/deep-shallow/) for de-en. - -For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369). - -2 models are available: - -* [wmt19-de-en-6-6-big](https://huggingface.co/allenai/wmt19-de-en-6-6-big) -* [wmt19-de-en-6-6-base](https://huggingface.co/allenai/wmt19-de-en-6-6-base) - - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "allenai/wmt19-de-en-6-6-base" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Maschinelles Lernen ist großartig, nicht wahr?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Machine learning is great, isn't it? - -``` - -#### Limitations and bias - - -## Training data - -Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369). - -## Eval results - -Here are the BLEU scores: - -model | transformers --------|--------- -wmt19-de-en-6-6-base | 38.37 - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=de-en -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=5 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/wmt19-de-en-6-6-base $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt19/) -- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561) - - -### BibTeX entry and citation info - -``` -@misc{kasai2020deep, - title={Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}, - author={Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}, - year={2020}, - eprint={2006.10369}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - diff --git a/model_cards/allenai/wmt19-de-en-6-6-big/README.md b/model_cards/allenai/wmt19-de-en-6-6-big/README.md deleted file mode 100644 index f348a772d03e..000000000000 --- a/model_cards/allenai/wmt19-de-en-6-6-big/README.md +++ /dev/null @@ -1,101 +0,0 @@ - ---- - -language: -- de -- en -thumbnail: -tags: -- translation -- wmt19 -- allenai -license: apache-2.0 -datasets: -- wmt19 -metrics: -- bleu ---- - -# FSMT - -## Model description - -This is a ported version of fairseq-based [wmt19 transformer](https://github.com/jungokasai/deep-shallow/) for de-en. - -For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369). - -2 models are available: - -* [wmt19-de-en-6-6-big](https://huggingface.co/allenai/wmt19-de-en-6-6-big) -* [wmt19-de-en-6-6-base](https://huggingface.co/allenai/wmt19-de-en-6-6-base) - - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "allenai/wmt19-de-en-6-6-big" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Maschinelles Lernen ist großartig, nicht wahr?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Machine learning is great, isn't it? - -``` - -#### Limitations and bias - - -## Training data - -Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369). - -## Eval results - -Here are the BLEU scores: - -model | transformers --------|--------- -wmt19-de-en-6-6-big | 39.9 - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=de-en -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=5 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/wmt19-de-en-6-6-big $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt19/) -- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561) - - -### BibTeX entry and citation info - -``` -@misc{kasai2020deep, - title={Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}, - author={Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}, - year={2020}, - eprint={2006.10369}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - diff --git a/model_cards/allenyummy/chinese-bert-wwm-ehr-ner-sl/README.md b/model_cards/allenyummy/chinese-bert-wwm-ehr-ner-sl/README.md deleted file mode 100644 index e6df9a312804..000000000000 --- a/model_cards/allenyummy/chinese-bert-wwm-ehr-ner-sl/README.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -language: zh-tw ---- - -# Model name -Chinese-bert-wwm-electrical-health-record-ner-sequence-labeling - - -#### How to use - -``` -from transformers import AutoTokenizer, AutoModelForTokenClassification -tokenizer = AutoTokenizer.from_pretrained("chinese-bert-wwm-ehr-ner-sl") -model = AutoModelForTokenClassification.from_pretrained("chinese-bert-wwm-ehr-ner-sl") -``` diff --git a/model_cards/amberoad/bert-multilingual-passage-reranking-msmarco/README.md b/model_cards/amberoad/bert-multilingual-passage-reranking-msmarco/README.md deleted file mode 100644 index 6539f3edcd41..000000000000 --- a/model_cards/amberoad/bert-multilingual-passage-reranking-msmarco/README.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -language: multilingual -thumbnail: "https://amberoad.de/images/logo_text.png" -tags: -- msmarco -- multilingual -- passage reranking -license: Apache-2.0 -datasets: -- msmarco -metrics: -- MRR -widget: -- query: "What is a corporation?" - passage: "A company is incorporated in a specific nation, often within the bounds of a smaller subset of that nation, such as a state or province. The corporation is then governed by the laws of incorporation in that state. A corporation may issue stock, either private or public, or may be classified as a non-stock corporation. If stock is issued, the corporation will usually be governed by its shareholders, either directly or indirectly." ---- - -# Passage Reranking Multilingual BERT 🔃 🌍 - - - -## Model description -**Input:** Supports over 100 Languages. See [List of supported languages](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages) for all available. - -**Purpose:** This module takes a search query [1] and a passage [2] and calculates if the passage matches the query. -It can be used as an improvement for Elasticsearch Results and boosts the relevancy by up to 100%. - -**Architecture:** On top of BERT there is a Densly Connected NN which takes the 768 Dimensional [CLS] Token as input and provides the output ([Arxiv](https://arxiv.org/abs/1901.04085)). - -**Output:** Just a single value between between -10 and 10. Better matching query,passage pairs tend to have a higher a score. - - - -## Intended uses & limitations -Both query[1] and passage[2] have to fit in 512 Tokens. -As you normally want to rerank the first dozens of search results keep in mind the inference time of approximately 300 ms/query. - -#### How to use - -```python -from transformers import AutoTokenizer, AutoModelForSequenceClassification - -tokenizer = AutoTokenizer.from_pretrained("amberoad/bert-multilingual-passage-reranking-msmarco") - -model = AutoModelForSequenceClassification.from_pretrained("amberoad/bert-multilingual-passage-reranking-msmarco") -``` - -This Model can be used as a drop-in replacement in the [Nboost Library](https://github.com/koursaros-ai/nboost) -Through this you can directly improve your Elasticsearch Results without any coding. - - -## Training data - -This model is trained using the [**Microsoft MS Marco Dataset**](https://microsoft.github.io/msmarco/ "Microsoft MS Marco"). This training dataset contains approximately 400M tuples of a query, relevant and non-relevant passages. All datasets used for training and evaluating are listed in this [table](https://github.com/microsoft/MSMARCO-Passage-Ranking#data-information-and-formating). The used dataset for training is called *Train Triples Large*, while the evaluation was made on *Top 1000 Dev*. There are 6,900 queries in total in the development dataset, where each query is mapped to top 1,000 passage retrieved using BM25 from MS MARCO corpus. - -## Training procedure - -The training is performed the same way as stated in this [README](https://github.com/nyu-dl/dl4marco-bert "NYU Github"). See their excellent Paper on [Arxiv](https://arxiv.org/abs/1901.04085). - -We changed the BERT Model from an English only to the default BERT Multilingual uncased Model from [Google](https://huggingface.co/bert-base-multilingual-uncased). - -Training was done 400 000 Steps. This equaled 12 hours an a TPU V3-8. - - -## Eval results - -We see nearly similar performance than the English only Model in the English [Bing Queries Dataset](http://www.msmarco.org/). Although the training data is English only internal Tests on private data showed a far higher accurancy in German than all other available models. - - - -Fine-tuned Models | Dependency | Eval Set | Search Boost | Speed on GPU ------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------- | ------------------------------------------------------------------ | ----------------------------------------------------- | ---------------------------------- -**`amberoad/Multilingual-uncased-MSMARCO`** (This Model) | PyTorch | bing queries | **+61%** (0.29 vs 0.18) | ~300 ms/query -`nboost/pt-tinybert-msmarco` | PyTorch | bing queries | **+45%** (0.26 vs 0.18) | ~50ms/query -`nboost/pt-bert-base-uncased-msmarco` | PyTorch | bing queries | **+62%** (0.29 vs 0.18) | ~300 ms/query -`nboost/pt-bert-large-msmarco` | PyTorch | bing queries | **+77%** (0.32 vs 0.18) | - -`nboost/pt-biobert-base-msmarco` | PyTorch | biomed | **+66%** (0.17 vs 0.10) | ~300 ms/query - -This table is taken from [nboost](https://github.com/koursaros-ai/nboost) and extended by the first line. - - - -## Contact Infos - -![](https://amberoad.de/images/logo_text.png) - -Amberoad is a company focussing on Search and Business Intelligence. -We provide you: -* Advanced Internal Company Search Engines thorugh NLP -* External Search Egnines: Find Competitors, Customers, Suppliers - -**Get in Contact now to benefit from our Expertise:** - -The training and evaluation was performed by [**Philipp Reissel**](https://reissel.eu/) and [**Igli Manaj**](https://github.com/iglimanaj) - - [![Amberoad](https://i.stack.imgur.com/gVE0j.png) Linkedin](https://de.linkedin.com/company/amberoad) | [Homepage](https://de.linkedin.com/company/amberoad) | [Email](info@amberoad.de) - - - - diff --git a/model_cards/amine/bert-base-5lang-cased/README.md b/model_cards/amine/bert-base-5lang-cased/README.md deleted file mode 100644 index 9117ca372c82..000000000000 --- a/model_cards/amine/bert-base-5lang-cased/README.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -language: -- en -- fr -- es -- de -- zh - -tags: -- pytorch -- bert -- multilingual -- en -- fr -- es -- de -- zh - -datasets: wikipedia - -license: apache-2.0 - -inference: false ---- - -# bert-base-5lang-cased -This is a smaller version of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) that handles only 5 languages (en, fr, es, de and zh) instead of 104. -The model is therefore 30% smaller than the original one (124M parameters instead of 178M) but gives exactly the same representations for the above cited languages. -Starting from `bert-base-5lang-cased` will facilitate the deployment of your model on public cloud platforms while keeping similar results. -For instance, Google Cloud Platform requires that the model size on disk should be lower than 500 MB for serveless deployments (Cloud Functions / Cloud ML) which is not the case of the original `bert-base-multilingual-cased`. - -For more information about the models size, memory footprint and loading time please refer to the table below: - -| Model | Num parameters | Size | Memory | Loading time | -| ---------------------------- | -------------- | -------- | -------- | ------------ | -| bert-base-multilingual-cased | 178 million | 714 MB | 1400 MB | 4.2 sec | -| bert-base-5lang-cased | 124 million | 495 MB | 950 MB | 3.6 sec | - -These measurements have been computed on a [Google Cloud n1-standard-1 machine (1 vCPU, 3.75 GB)](https://cloud.google.com/compute/docs/machine-types\#n1_machine_type). - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("amine/bert-base-5lang-cased") -model = AutoModel.from_pretrained("amine/bert-base-5lang-cased") - -``` - -### How to cite - -```bibtex -@inproceedings{smallermbert, - title={Load What You Need: Smaller Versions of Mutlilingual BERT}, - author={Abdaoui, Amine and Pradel, Camille and Sigel, Grégoire}, - booktitle={SustaiNLP / EMNLP}, - year={2020} -} -``` - -## Contact - -Please contact amine@geotrend.fr for any question, feedback or request. \ No newline at end of file diff --git a/model_cards/antoiloui/belgpt2/README.md b/model_cards/antoiloui/belgpt2/README.md deleted file mode 100644 index 1a23fa0828be..000000000000 --- a/model_cards/antoiloui/belgpt2/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -language: "fr" ---- - -# BelGPT-2 - -**BelGPT-2** (*Belgian GPT-2* 🇧🇪) is a "small" GPT-2 model pre-trained on a very large and heterogeneous French corpus (around 60Gb). Please check [antoiloui/gpt2-french](https://github.com/antoiloui/gpt2-french) for more information about the pre-trained model, the data, the code to use the model and the code to pre-train it. - - -## Using BelGPT-2 for Text Generation in French - -You can use BelGPT-2 with [🤗 transformers](https://github.com/huggingface/transformers) library as follows: - -```python -import torch -from transformers import GPT2Tokenizer, GPT2LMHeadModel - -# Load pretrained model and tokenizer -model = GPT2LMHeadModel.from_pretrained("antoiloui/belgpt2") -tokenizer = GPT2Tokenizer.from_pretrained("antoiloui/belgpt2") - -# Generate a sample of text -model.eval() -output = model.generate( - bos_token_id=random.randint(1,50000), - do_sample=True, - top_k=50, - max_length=100, - top_p=0.95, - num_return_sequences=1 -) - -# Decode it -decoded_output = [] -for sample in output: - decoded_output.append(tokenizer.decode(sample, skip_special_tokens=True)) -print(decoded_output) -``` - -## Data - -Below is the list of all French copora used to pre-trained the model: - -| Dataset | `$corpus_name` | Raw size | Cleaned size | -| :------| :--- | :---: | :---: | -| CommonCrawl | `common_crawl` | 200.2 GB | 40.4 GB | -| NewsCrawl | `news_crawl` | 10.4 GB | 9.8 GB | -| Wikipedia | `wiki` | 19.4 GB | 4.1 GB | -| Wikisource | `wikisource` | 4.6 GB | 2.3 GB | -| Project Gutenberg | `gutenberg` | 1.3 GB | 1.1 GB | -| EuroParl | `europarl` | 289.9 MB | 278.7 MB | -| NewsCommentary | `news_commentary` | 61.4 MB | 58.1 MB | -| **Total** | | **236.3 GB** | **57.9 GB** | diff --git a/model_cards/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616/README.md b/model_cards/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616/README.md deleted file mode 100644 index 0db4dea40716..000000000000 --- a/model_cards/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# BERT L-10 H-512 fine-tuned on MLM (CORD-19 2020/06/16) - -BERT model with [10 Transformer layers and hidden embedding of size 512](https://huggingface.co/google/bert_uncased_L-10_H-512_A-8), referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962), fine-tuned for MLM on CORD-19 dataset (as released on 2020/06/16). - -## Training the model - -```bash -python run_language_modeling.py - --model_type bert - --model_name_or_path google/bert_uncased_L-10_H-512_A-8 - --do_train - --train_data_file {cord19-200616-dataset} - --mlm - --mlm_probability 0.2 - --line_by_line - --block_size 512 - --per_device_train_batch_size 10 - --learning_rate 3e-5 - --num_train_epochs 2 - --output_dir bert_uncased_L-10_H-512_A-8_cord19-200616 diff --git a/model_cards/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2/README.md b/model_cards/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2/README.md deleted file mode 100644 index 57b955182b3e..000000000000 --- a/model_cards/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616_squad2/README.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -datasets: -- squad_v2 ---- - -# BERT L-10 H-512 CORD-19 (2020/06/16) fine-tuned on SQuAD v2.0 - -BERT model with [10 Transformer layers and hidden embedding of size 512](https://huggingface.co/google/bert_uncased_L-10_H-512_A-8), referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962), [fine-tuned for MLM](https://huggingface.co/aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616) on CORD-19 dataset (as released on 2020/06/16) and fine-tuned for QA on SQuAD v2.0. - -## Training the model - -```bash -python run_squad.py - --model_type bert - --model_name_or_path aodiniz/bert_uncased_L-10_H-512_A-8_cord19-200616 - --train_file 'train-v2.0.json' - --predict_file 'dev-v2.0.json' - --do_train - --do_eval - --do_lower_case - --version_2_with_negative - --max_seq_length 384 - --per_gpu_train_batch_size 10 - --learning_rate 3e-5 - --num_train_epochs 2 - --output_dir bert_uncased_L-10_H-512_A-8_cord19-200616_squad2 diff --git a/model_cards/aodiniz/bert_uncased_L-2_H-512_A-8_cord19-200616/README.md b/model_cards/aodiniz/bert_uncased_L-2_H-512_A-8_cord19-200616/README.md deleted file mode 100644 index ec253db2b353..000000000000 --- a/model_cards/aodiniz/bert_uncased_L-2_H-512_A-8_cord19-200616/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# BERT L-2 H-512 fine-tuned on MLM (CORD-19 2020/06/16) - -BERT model with [2 Transformer layers and hidden embedding of size 512](https://huggingface.co/google/bert_uncased_L-2_H-512_A-8), referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962), fine-tuned for MLM on CORD-19 dataset (as released on 2020/06/16). - -## Training the model - -```bash -python run_language_modeling.py - --model_type bert - --model_name_or_path google/bert_uncased_L-2_H-512_A-8 - --do_train - --train_data_file {cord19-200616-dataset} - --mlm - --mlm_probability 0.2 - --line_by_line - --block_size 512 - --per_device_train_batch_size 20 - --learning_rate 3e-5 - --num_train_epochs 2 - --output_dir bert_uncased_L-2_H-512_A-8_cord19-200616 diff --git a/model_cards/aodiniz/bert_uncased_L-4_H-256_A-4_cord19-200616/README.md b/model_cards/aodiniz/bert_uncased_L-4_H-256_A-4_cord19-200616/README.md deleted file mode 100644 index c29e5adfc32e..000000000000 --- a/model_cards/aodiniz/bert_uncased_L-4_H-256_A-4_cord19-200616/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# BERT L-4 H-256 fine-tuned on MLM (CORD-19 2020/06/16) - -BERT model with [4 Transformer layers and hidden embedding of size 256](https://huggingface.co/google/bert_uncased_L-4_H-256_A-4), referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962), fine-tuned for MLM on CORD-19 dataset (as released on 2020/06/16). - -## Training the model - -```bash -python run_language_modeling.py - --model_type bert - --model_name_or_path google/bert_uncased_L-4_H-256_A-4 - --do_train - --train_data_file {cord19-200616-dataset} - --mlm - --mlm_probability 0.2 - --line_by_line - --block_size 256 - --per_device_train_batch_size 20 - --learning_rate 3e-5 - --num_train_epochs 2 - --output_dir bert_uncased_L-4_H-256_A-4_cord19-200616 diff --git a/model_cards/asafaya/bert-base-arabic/README.md b/model_cards/asafaya/bert-base-arabic/README.md deleted file mode 100644 index 5b565e468619..000000000000 --- a/model_cards/asafaya/bert-base-arabic/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: ar -datasets: -- oscar -- wikipedia ---- - -# Arabic BERT Model - -Pretrained BERT base language model for Arabic - - -_If you use this model in your work, please cite this paper:_ - - - -``` -@misc{safaya2020kuisail, - title={KUISAIL at SemEval-2020 Task 12: BERT-CNN for Offensive Speech Identification in Social Media}, - author={Ali Safaya and Moutasem Abdullatif and Deniz Yuret}, - year={2020}, - eprint={2007.13184}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - -## Pretraining Corpus - -`arabic-bert-base` model was pretrained on ~8.2 Billion words: - -- Arabic version of [OSCAR](https://traces1.inria.fr/oscar/) - filtered from [Common Crawl](http://commoncrawl.org/) -- Recent dump of Arabic [Wikipedia](https://dumps.wikimedia.org/backup-index.html) - -and other Arabic resources which sum up to ~95GB of text. - -__Notes on training data:__ - -- Our final version of corpus contains some non-Arabic words inlines, which we did not remove from sentences since that would affect some tasks like NER. -- Although non-Arabic characters were lowered as a preprocessing step, since Arabic characters does not have upper or lower case, there is no cased and uncased version of the model. -- The corpus and vocabulary set are not restricted to Modern Standard Arabic, they contain some dialectical Arabic too. - -## Pretraining details - -- This model was trained using Google BERT's github [repository](https://github.com/google-research/bert) on a single TPU v3-8 provided for free from [TFRC](https://www.tensorflow.org/tfrc). -- Our pretraining procedure follows training settings of bert with some changes: trained for 3M training steps with batchsize of 128, instead of 1M with batchsize of 256. - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-base-arabic") -model = AutoModel.from_pretrained("asafaya/bert-base-arabic") -``` - -## Results - -For further details on the models performance or any other queries, please refer to [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT) - -## Acknowledgement - -Thanks to Google for providing free TPU for the training process and for Huggingface for hosting this model on their servers 😊 - - diff --git a/model_cards/asafaya/bert-large-arabic/README.md b/model_cards/asafaya/bert-large-arabic/README.md deleted file mode 100644 index 4ecf5d251835..000000000000 --- a/model_cards/asafaya/bert-large-arabic/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: ar -datasets: -- oscar -- wikipedia ---- - - -# Arabic BERT Large Model - -Pretrained BERT Large language model for Arabic - -_If you use this model in your work, please cite this paper:_ - - - -``` -@misc{safaya2020kuisail, - title={KUISAIL at SemEval-2020 Task 12: BERT-CNN for Offensive Speech Identification in Social Media}, - author={Ali Safaya and Moutasem Abdullatif and Deniz Yuret}, - year={2020}, - eprint={2007.13184}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - -## Pretraining Corpus - -`arabic-bert-large` model was pretrained on ~8.2 Billion words: - -- Arabic version of [OSCAR](https://traces1.inria.fr/oscar/) - filtered from [Common Crawl](http://commoncrawl.org/) -- Recent dump of Arabic [Wikipedia](https://dumps.wikimedia.org/backup-index.html) - -and other Arabic resources which sum up to ~95GB of text. - -__Notes on training data:__ - -- Our final version of corpus contains some non-Arabic words inlines, which we did not remove from sentences since that would affect some tasks like NER. -- Although non-Arabic characters were lowered as a preprocessing step, since Arabic characters does not have upper or lower case, there is no cased and uncased version of the model. -- The corpus and vocabulary set are not restricted to Modern Standard Arabic, they contain some dialectical Arabic too. - -## Pretraining details - -- This model was trained using Google BERT's github [repository](https://github.com/google-research/bert) on a single TPU v3-8 provided for free from [TFRC](https://www.tensorflow.org/tfrc). -- Our pretraining procedure follows training settings of bert with some changes: trained for 3M training steps with batchsize of 128, instead of 1M with batchsize of 256. - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-large-arabic") -model = AutoModel.from_pretrained("asafaya/bert-large-arabic") -``` - -## Results - -For further details on the models performance or any other queries, please refer to [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT) - -## Acknowledgement - -Thanks to Google for providing free TPU for the training process and for Huggingface for hosting this model on their servers 😊 - - diff --git a/model_cards/asafaya/bert-medium-arabic/README.md b/model_cards/asafaya/bert-medium-arabic/README.md deleted file mode 100644 index 502c1a5821a1..000000000000 --- a/model_cards/asafaya/bert-medium-arabic/README.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -language: ar -datasets: -- oscar -- wikipedia ---- - -# Arabic BERT Medium Model - -Pretrained BERT Medium language model for Arabic - -_If you use this model in your work, please cite this paper:_ - - - -``` -@misc{safaya2020kuisail, - title={KUISAIL at SemEval-2020 Task 12: BERT-CNN for Offensive Speech Identification in Social Media}, - author={Ali Safaya and Moutasem Abdullatif and Deniz Yuret}, - year={2020}, - eprint={2007.13184}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - -## Pretraining Corpus - -`arabic-bert-medium` model was pretrained on ~8.2 Billion words: - -- Arabic version of [OSCAR](https://traces1.inria.fr/oscar/) - filtered from [Common Crawl](http://commoncrawl.org/) -- Recent dump of Arabic [Wikipedia](https://dumps.wikimedia.org/backup-index.html) - -and other Arabic resources which sum up to ~95GB of text. - -__Notes on training data:__ - -- Our final version of corpus contains some non-Arabic words inlines, which we did not remove from sentences since that would affect some tasks like NER. -- Although non-Arabic characters were lowered as a preprocessing step, since Arabic characters does not have upper or lower case, there is no cased and uncased version of the model. -- The corpus and vocabulary set are not restricted to Modern Standard Arabic, they contain some dialectical Arabic too. - -## Pretraining details - -- This model was trained using Google BERT's github [repository](https://github.com/google-research/bert) on a single TPU v3-8 provided for free from [TFRC](https://www.tensorflow.org/tfrc). -- Our pretraining procedure follows training settings of bert with some changes: trained for 3M training steps with batchsize of 128, instead of 1M with batchsize of 256. - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-medium-arabic") -model = AutoModel.from_pretrained("asafaya/bert-medium-arabic") -``` - -## Results - -For further details on the models performance or any other queries, please refer to [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT) - -## Acknowledgement - -Thanks to Google for providing free TPU for the training process and for Huggingface for hosting this model on their servers 😊 - - diff --git a/model_cards/asafaya/bert-mini-arabic/README.md b/model_cards/asafaya/bert-mini-arabic/README.md deleted file mode 100644 index d480df7b27cd..000000000000 --- a/model_cards/asafaya/bert-mini-arabic/README.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -language: ar -datasets: -- oscar -- wikipedia ---- - -# Arabic BERT Mini Model - -Pretrained BERT Mini language model for Arabic - -_If you use this model in your work, please cite this paper:_ - - - -``` -@misc{safaya2020kuisail, - title={KUISAIL at SemEval-2020 Task 12: BERT-CNN for Offensive Speech Identification in Social Media}, - author={Ali Safaya and Moutasem Abdullatif and Deniz Yuret}, - year={2020}, - eprint={2007.13184}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - -## Pretraining Corpus - -`arabic-bert-mini` model was pretrained on ~8.2 Billion words: - -- Arabic version of [OSCAR](https://traces1.inria.fr/oscar/) - filtered from [Common Crawl](http://commoncrawl.org/) -- Recent dump of Arabic [Wikipedia](https://dumps.wikimedia.org/backup-index.html) - -and other Arabic resources which sum up to ~95GB of text. - -__Notes on training data:__ - -- Our final version of corpus contains some non-Arabic words inlines, which we did not remove from sentences since that would affect some tasks like NER. -- Although non-Arabic characters were lowered as a preprocessing step, since Arabic characters does not have upper or lower case, there is no cased and uncased version of the model. -- The corpus and vocabulary set are not restricted to Modern Standard Arabic, they contain some dialectical Arabic too. - -## Pretraining details - -- This model was trained using Google BERT's github [repository](https://github.com/google-research/bert) on a single TPU v3-8 provided for free from [TFRC](https://www.tensorflow.org/tfrc). -- Our pretraining procedure follows training settings of bert with some changes: trained for 3M training steps with batchsize of 128, instead of 1M with batchsize of 256. - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("asafaya/bert-mini-arabic") -model = AutoModel.from_pretrained("asafaya/bert-mini-arabic") -``` - -## Results - -For further details on the models performance or any other queries, please refer to [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT) - -## Acknowledgement - -Thanks to Google for providing free TPU for the training process and for Huggingface for hosting this model on their servers 😊 - diff --git a/model_cards/ashwani-tanwar/Gujarati-XLM-R-Base/README.md b/model_cards/ashwani-tanwar/Gujarati-XLM-R-Base/README.md deleted file mode 100644 index d7889540f76e..000000000000 --- a/model_cards/ashwani-tanwar/Gujarati-XLM-R-Base/README.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -language: gu ---- - -# Gujarati-XLM-R-Base - - -This model is finetuned over [XLM-RoBERTa](https://huggingface.co/xlm-roberta-base) (XLM-R) using its base variant with the Gujarati language using the [OSCAR](https://oscar-corpus.com/) monolingual dataset. We used the same masked language modelling (MLM) objective which was used for pretraining the XLM-R. As it is built over the pretrained XLM-R, we leveraged *Transfer Learning* by exploiting the knowledge from its parent model. - -## Dataset -OSCAR corpus contains several diverse datasets for different languages. We followed the work of [CamemBERT](https://www.aclweb.org/anthology/2020.acl-main.645/) who reported better performance with this diverse dataset as compared to the other large homogenous datasets. - -## Preprocessing and Training Procedure -Please visit [this link](https://github.com/ashwanitanwar/nmt-transfer-learning-xlm-r#6-finetuning-xlm-r) for the detailed procedure. - -## Usage -- This model can be used for further finetuning for different NLP tasks using the Gujarati language. -- It can be used to generate contextualised word representations for the Gujarati words. -- It can be used for domain adaptation. -- It can be used to predict the missing words from the Gujarati sentences. - -## Demo - ### Using the model to predict missing words - ``` - from transformers import pipeline - unmasker = pipeline('fill-mask', model='ashwani-tanwar/Gujarati-XLM-R-Base') - pred_word = unmasker("અમદાવાદ એ ગુજરાતનું એક છે.") - print(pred_word) - ``` - ``` - [{'sequence': ' અમદાવાદ એ ગુજરાતનું એક શહેર છે.', 'score': 0.9463568329811096, 'token': 85227, 'token_str': '▁શહેર'}, - {'sequence': ' અમદાવાદ એ ગુજરાતનું એક ગામ છે.', 'score': 0.013311690650880337, 'token': 66346, 'token_str': '▁ગામ'}, - {'sequence': ' અમદાવાદ એ ગુજરાતનું એકનગર છે.', 'score': 0.012945962138473988, 'token': 69702, 'token_str': 'નગર'}, - {'sequence': ' અમદાવાદ એ ગુજરાતનું એક સ્થળ છે.', 'score': 0.0045941537246108055, 'token': 135436, 'token_str': '▁સ્થળ'}, - {'sequence': ' અમદાવાદ એ ગુજરાતનું એક મહત્વ છે.', 'score': 0.00402021361514926, 'token': 126763, 'token_str': '▁મહત્વ'}] - ``` - ### Using the model to generate contextualised word representations - ``` - from transformers import AutoTokenizer, AutoModel - tokenizer = AutoTokenizer.from_pretrained("ashwani-tanwar/Gujarati-XLM-R-Base") - model = AutoModel.from_pretrained("ashwani-tanwar/Gujarati-XLM-R-Base") - sentence = "અમદાવાદ એ ગુજરાતનું એક શહેર છે." - encoded_sentence = tokenizer(sentence, return_tensors='pt') - context_word_rep = model(**encoded_sentence) - ``` diff --git a/model_cards/aubmindlab/bert-base-arabert/README.md b/model_cards/aubmindlab/bert-base-arabert/README.md deleted file mode 100644 index 772676b6dc09..000000000000 --- a/model_cards/aubmindlab/bert-base-arabert/README.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -language: ar ---- - -# AraBERT : Pre-training BERT for Arabic Language Understanding - - -**AraBERT** is an Arabic pretrained lanaguage model based on [Google's BERT architechture](https://github.com/google-research/bert). AraBERT uses the same BERT-Base config. More details are available in the [AraBERT PAPER](https://arxiv.org/abs/2003.00104v2) and in the [AraBERT Meetup](https://github.com/WissamAntoun/pydata_khobar_meetup) - -There are two version off the model AraBERTv0.1 and AraBERTv1, with the difference being that AraBERTv1 uses pre-segmented text where prefixes and suffixes were split using the [Farasa Segmenter](http://alt.qcri.org/farasa/segmenter.html). - -The model was trained on ~70M sentences or ~23GB of Arabic text with ~3B words. The training corpora are a collection of publically available large scale raw arabic text ([Arabic Wikidumps](https://archive.org/details/arwiki-20190201), [The 1.5B words Arabic Corpus](https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4), [The OSIAN Corpus](https://www.aclweb.org/anthology/W19-4619), Assafir news articles, and 4 other manually crawled news websites (Al-Akhbar, Annahar, AL-Ahram, AL-Wafd) from [the Wayback Machine](http://web.archive.org/)) - -We evalaute both AraBERT models on different downstream tasks and compare it to [mBERT]((https://github.com/google-research/bert/blob/master/multilingual.md)), and other state of the art models (*To the extent of our knowledge*). The Tasks were Sentiment Analysis on 6 different datasets ([HARD](https://github.com/elnagara/HARD-Arabic-Dataset), [ASTD-Balanced](https://www.aclweb.org/anthology/D15-1299), [ArsenTD-Lev](https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf), [LABR](https://github.com/mohamedadaly/LABR), [ArSaS](http://lrec-conf.org/workshops/lrec2018/W30/pdf/22_W30.pdf)), Named Entity Recognition with the [ANERcorp](http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp), and Arabic Question Answering on [Arabic-SQuAD and ARCD](https://github.com/husseinmozannar/SOQAL) - -**Update 2 (21/5/2020) :** -Added support for the farasapy segmenter https://github.com/MagedSaeed/farasapy in the ``preprocess_arabert.py`` which is ~6x faster than the ``py4j.java_gateway``, consider setting ``use_farasapy=True`` when calling preprocess and pass it an instance of ``FarasaSegmenter(interactive=True)`` with interactive set to ``True`` for faster segmentation. - -**Update 1 (21/4/2020) :** -Fixed an issue with ARCD fine-tuning which drastically improved performance. Initially we didn't account for the change of the ```answer_start``` during preprocessing. -## Results (Acc.) -Task | prev. SOTA | mBERT | AraBERTv0.1 | AraBERTv1 ----|:---:|:---:|:---:|:---: -HARD |95.7 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)|95.7|**96.2**|96.1 -ASTD |86.5 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)| 80.1|92.2|**92.6** -ArsenTD-Lev|52.4 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)|51|58.9|**59.4** -AJGT|93 [Dahou et.al.](https://dl.acm.org/doi/fullHtml/10.1145/3314941)| 83.6|93.1|**93.8** -LABR|**87.5** [Dahou et.al.](https://dl.acm.org/doi/fullHtml/10.1145/3314941)|83|85.9|86.7 -ANERcorp|81.7 (BiLSTM-CRF)|78.4|**84.2**|81.9 -ARCD|mBERT|EM:34.2 F1: 61.3|EM:51.14 F1:82.13|**EM:54.84 F1: 82.15** - -*If you tested AraBERT on a public dataset and you want to add your results to the table above, open a pull request or contact us. Also make sure to have your code available online so we can add it as a reference* - -## How to use - -You can easily use AraBERT since it is almost fully compatible with existing codebases (Use this repo instead of the official BERT one, the only difference is in the ```tokenization.py``` file where we modify the _is_punctuation function to make it compatible with the "+" symbol and the "[" and "]" characters) - -To use HuggingFace's Transformer repository you only need to provide a list of token that forces the model to not split them, also make sure that the text is pre-segmented: -**Not all libraries built on top of transformers support the `never_split` argument** -```python -from transformers import AutoTokenizer, AutoModel -from arabert.preprocess_arabert import never_split_tokens, preprocess -from farasa.segmenter import FarasaSegmenter - -arabert_tokenizer = AutoTokenizer.from_pretrained( - "aubmindlab/bert-base-arabert", - do_lower_case=False, - do_basic_tokenize=True, - never_split=never_split_tokens) -arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabert") - -#Preprocess the text to make it compatible with AraBERT using farasapy -farasa_segmenter = FarasaSegmenter(interactive=True) - -#or you can use a py4j JavaGateway to the farasa Segmneter .jar but it's slower -#(see update 2) -#from py4j.java_gateway import JavaGateway -#gateway = JavaGateway.launch_gateway(classpath='./PATH_TO_FARASA/FarasaSegmenterJar.jar') -#farasa = gateway.jvm.com.qcri.farasa.segmenter.Farasa() - -text = "ولن نبالغ إذا قلنا إن هاتف أو كمبيوتر المكتب في زمننا هذا ضروري" -text_preprocessed = preprocess( text, - do_farasa_tokenization = True, - farasa = farasa_segmenter, - use_farasapy = True) - ->>>text_preprocessed: "و+ لن نبالغ إذا قل +نا إن هاتف أو كمبيوتر ال+ مكتب في زمن +نا هذا ضروري" - -arabert_tokenizer.tokenize(text_preprocessed) - ->>> ['و+', 'لن', 'نبال', '##غ', 'إذا', 'قل', '+نا', 'إن', 'هاتف', 'أو', 'كمبيوتر', 'ال+', 'مكتب', 'في', 'زمن', '+نا', 'هذا', 'ضروري'] -``` - -**AraBERTv0.1 is compatible with all existing libraries, since it needs no pre-segmentation.** -```python -from transformers import AutoTokenizer, AutoModel - -arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv01",do_lower_case=False) -arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv01") - -text = "ولن نبالغ إذا قلنا إن هاتف أو كمبيوتر المكتب في زمننا هذا ضروري" -arabert_tokenizer.tokenize(text) - ->>> ['ولن', 'ن', '##بالغ', 'إذا', 'قلنا', 'إن', 'هاتف', 'أو', 'كمبيوتر', 'المكتب', 'في', 'زمن', '##ن', '##ا', 'هذا', 'ضروري'] -``` - - -The ```araBERT_(Updated_Demo_TF).ipynb``` Notebook is a small demo using the AJGT dataset using TensorFlow (GPU and TPU compatible). - -**Coming Soon :** Fine-tunning demo using HuggingFace's Trainer API - -**AraBERT on ARCD** -During the preprocessing step the ```answer_start``` character position needs to be recalculated. You can use the file ```arcd_preprocessing.py``` as shown below to clean, preprocess the ARCD dataset before running ```run_squad.py```. More detailed Colab notebook is available in the [SOQAL repo](https://github.com/husseinmozannar/SOQAL). -```bash -python arcd_preprocessing.py \ - --input_file="/PATH_TO/arcd-test.json" \ - --output_file="arcd-test-pre.json" \ - --do_farasa_tokenization=True \ - --use_farasapy=True \ -``` -```bash -python SOQAL/bert/run_squad.py \ - --vocab_file="/PATH_TO_PRETRAINED_TF_CKPT/vocab.txt" \ - --bert_config_file="/PATH_TO_PRETRAINED_TF_CKPT/config.json" \ - --init_checkpoint="/PATH_TO_PRETRAINED_TF_CKPT/" \ - --do_train=True \ - --train_file=turk_combined_all_pre.json \ - --do_predict=True \ - --predict_file=arcd-test-pre.json \ - --train_batch_size=32 \ - --predict_batch_size=24 \ - --learning_rate=3e-5 \ - --num_train_epochs=4 \ - --max_seq_length=384 \ - --doc_stride=128 \ - --do_lower_case=False\ - --output_dir="/PATH_TO/OUTPUT_PATH"/ \ - --use_tpu=True \ - --tpu_name=$TPU_ADDRESS \ -``` -## Model Weights and Vocab Download -Models | AraBERTv0.1 | AraBERTv1 ----|:---:|:---: -TensorFlow|[Drive Link](https://drive.google.com/open?id=1-kVmTUZZ4DP2rzeHNjTPkY8OjnQCpomO) | [Drive Link](https://drive.google.com/open?id=1-d7-9ljKgDJP5mx73uBtio-TuUZCqZnt) -PyTorch| [Drive_Link](https://drive.google.com/open?id=1-_3te42mQCPD8SxwZ3l-VBL7yaJH-IOv)| [Drive_Link](https://drive.google.com/open?id=1-69s6Pxqbi63HOQ1M9wTcr-Ovc6PWLLo) - -**You can find the PyTorch models in HuggingFace's Transformer Library under the ```aubmindlab``` username** - -## If you used this model please cite us as: -``` -@inproceedings{antoun2020arabert, - title={AraBERT: Transformer-based Model for Arabic Language Understanding}, - author={Antoun, Wissam and Baly, Fady and Hajj, Hazem}, - booktitle={LREC 2020 Workshop Language Resources and Evaluation Conference 11--16 May 2020}, - pages={9} -} -``` -## Acknowledgments -Thanks to TensorFlow Research Cloud (TFRC) for the free access to Cloud TPUs, couldn't have done it without this program, and to the [AUB MIND Lab](https://sites.aub.edu.lb/mindlab/) Members for the continous support. Also thanks to [Yakshof](https://www.yakshof.com/#/) and Assafir for data and storage access. Another thanks for Habib Rahal (https://www.behance.net/rahalhabib), for putting a face to AraBERT. - -## Contacts -**Wissam Antoun**: [Linkedin](https://www.linkedin.com/in/giulio-ravasio-3a81a9110/) | [Twitter](https://twitter.com/wissam_antoun) | [Github](https://github.com/WissamAntoun) | | - -**Fady Baly**: [Linkedin](https://www.linkedin.com/in/fadybaly/) | [Twitter](https://twitter.com/fadybaly) | [Github](https://github.com/fadybaly) | | diff --git a/model_cards/aubmindlab/bert-base-arabertv01/README.md b/model_cards/aubmindlab/bert-base-arabertv01/README.md deleted file mode 100644 index 772676b6dc09..000000000000 --- a/model_cards/aubmindlab/bert-base-arabertv01/README.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -language: ar ---- - -# AraBERT : Pre-training BERT for Arabic Language Understanding - - -**AraBERT** is an Arabic pretrained lanaguage model based on [Google's BERT architechture](https://github.com/google-research/bert). AraBERT uses the same BERT-Base config. More details are available in the [AraBERT PAPER](https://arxiv.org/abs/2003.00104v2) and in the [AraBERT Meetup](https://github.com/WissamAntoun/pydata_khobar_meetup) - -There are two version off the model AraBERTv0.1 and AraBERTv1, with the difference being that AraBERTv1 uses pre-segmented text where prefixes and suffixes were split using the [Farasa Segmenter](http://alt.qcri.org/farasa/segmenter.html). - -The model was trained on ~70M sentences or ~23GB of Arabic text with ~3B words. The training corpora are a collection of publically available large scale raw arabic text ([Arabic Wikidumps](https://archive.org/details/arwiki-20190201), [The 1.5B words Arabic Corpus](https://www.semanticscholar.org/paper/1.5-billion-words-Arabic-Corpus-El-Khair/f3eeef4afb81223df96575adadf808fe7fe440b4), [The OSIAN Corpus](https://www.aclweb.org/anthology/W19-4619), Assafir news articles, and 4 other manually crawled news websites (Al-Akhbar, Annahar, AL-Ahram, AL-Wafd) from [the Wayback Machine](http://web.archive.org/)) - -We evalaute both AraBERT models on different downstream tasks and compare it to [mBERT]((https://github.com/google-research/bert/blob/master/multilingual.md)), and other state of the art models (*To the extent of our knowledge*). The Tasks were Sentiment Analysis on 6 different datasets ([HARD](https://github.com/elnagara/HARD-Arabic-Dataset), [ASTD-Balanced](https://www.aclweb.org/anthology/D15-1299), [ArsenTD-Lev](https://staff.aub.edu.lb/~we07/Publications/ArSentD-LEV_Sentiment_Corpus.pdf), [LABR](https://github.com/mohamedadaly/LABR), [ArSaS](http://lrec-conf.org/workshops/lrec2018/W30/pdf/22_W30.pdf)), Named Entity Recognition with the [ANERcorp](http://curtis.ml.cmu.edu/w/courses/index.php/ANERcorp), and Arabic Question Answering on [Arabic-SQuAD and ARCD](https://github.com/husseinmozannar/SOQAL) - -**Update 2 (21/5/2020) :** -Added support for the farasapy segmenter https://github.com/MagedSaeed/farasapy in the ``preprocess_arabert.py`` which is ~6x faster than the ``py4j.java_gateway``, consider setting ``use_farasapy=True`` when calling preprocess and pass it an instance of ``FarasaSegmenter(interactive=True)`` with interactive set to ``True`` for faster segmentation. - -**Update 1 (21/4/2020) :** -Fixed an issue with ARCD fine-tuning which drastically improved performance. Initially we didn't account for the change of the ```answer_start``` during preprocessing. -## Results (Acc.) -Task | prev. SOTA | mBERT | AraBERTv0.1 | AraBERTv1 ----|:---:|:---:|:---:|:---: -HARD |95.7 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)|95.7|**96.2**|96.1 -ASTD |86.5 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)| 80.1|92.2|**92.6** -ArsenTD-Lev|52.4 [ElJundi et.al.](https://www.aclweb.org/anthology/W19-4608/)|51|58.9|**59.4** -AJGT|93 [Dahou et.al.](https://dl.acm.org/doi/fullHtml/10.1145/3314941)| 83.6|93.1|**93.8** -LABR|**87.5** [Dahou et.al.](https://dl.acm.org/doi/fullHtml/10.1145/3314941)|83|85.9|86.7 -ANERcorp|81.7 (BiLSTM-CRF)|78.4|**84.2**|81.9 -ARCD|mBERT|EM:34.2 F1: 61.3|EM:51.14 F1:82.13|**EM:54.84 F1: 82.15** - -*If you tested AraBERT on a public dataset and you want to add your results to the table above, open a pull request or contact us. Also make sure to have your code available online so we can add it as a reference* - -## How to use - -You can easily use AraBERT since it is almost fully compatible with existing codebases (Use this repo instead of the official BERT one, the only difference is in the ```tokenization.py``` file where we modify the _is_punctuation function to make it compatible with the "+" symbol and the "[" and "]" characters) - -To use HuggingFace's Transformer repository you only need to provide a list of token that forces the model to not split them, also make sure that the text is pre-segmented: -**Not all libraries built on top of transformers support the `never_split` argument** -```python -from transformers import AutoTokenizer, AutoModel -from arabert.preprocess_arabert import never_split_tokens, preprocess -from farasa.segmenter import FarasaSegmenter - -arabert_tokenizer = AutoTokenizer.from_pretrained( - "aubmindlab/bert-base-arabert", - do_lower_case=False, - do_basic_tokenize=True, - never_split=never_split_tokens) -arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabert") - -#Preprocess the text to make it compatible with AraBERT using farasapy -farasa_segmenter = FarasaSegmenter(interactive=True) - -#or you can use a py4j JavaGateway to the farasa Segmneter .jar but it's slower -#(see update 2) -#from py4j.java_gateway import JavaGateway -#gateway = JavaGateway.launch_gateway(classpath='./PATH_TO_FARASA/FarasaSegmenterJar.jar') -#farasa = gateway.jvm.com.qcri.farasa.segmenter.Farasa() - -text = "ولن نبالغ إذا قلنا إن هاتف أو كمبيوتر المكتب في زمننا هذا ضروري" -text_preprocessed = preprocess( text, - do_farasa_tokenization = True, - farasa = farasa_segmenter, - use_farasapy = True) - ->>>text_preprocessed: "و+ لن نبالغ إذا قل +نا إن هاتف أو كمبيوتر ال+ مكتب في زمن +نا هذا ضروري" - -arabert_tokenizer.tokenize(text_preprocessed) - ->>> ['و+', 'لن', 'نبال', '##غ', 'إذا', 'قل', '+نا', 'إن', 'هاتف', 'أو', 'كمبيوتر', 'ال+', 'مكتب', 'في', 'زمن', '+نا', 'هذا', 'ضروري'] -``` - -**AraBERTv0.1 is compatible with all existing libraries, since it needs no pre-segmentation.** -```python -from transformers import AutoTokenizer, AutoModel - -arabert_tokenizer = AutoTokenizer.from_pretrained("aubmindlab/bert-base-arabertv01",do_lower_case=False) -arabert_model = AutoModel.from_pretrained("aubmindlab/bert-base-arabertv01") - -text = "ولن نبالغ إذا قلنا إن هاتف أو كمبيوتر المكتب في زمننا هذا ضروري" -arabert_tokenizer.tokenize(text) - ->>> ['ولن', 'ن', '##بالغ', 'إذا', 'قلنا', 'إن', 'هاتف', 'أو', 'كمبيوتر', 'المكتب', 'في', 'زمن', '##ن', '##ا', 'هذا', 'ضروري'] -``` - - -The ```araBERT_(Updated_Demo_TF).ipynb``` Notebook is a small demo using the AJGT dataset using TensorFlow (GPU and TPU compatible). - -**Coming Soon :** Fine-tunning demo using HuggingFace's Trainer API - -**AraBERT on ARCD** -During the preprocessing step the ```answer_start``` character position needs to be recalculated. You can use the file ```arcd_preprocessing.py``` as shown below to clean, preprocess the ARCD dataset before running ```run_squad.py```. More detailed Colab notebook is available in the [SOQAL repo](https://github.com/husseinmozannar/SOQAL). -```bash -python arcd_preprocessing.py \ - --input_file="/PATH_TO/arcd-test.json" \ - --output_file="arcd-test-pre.json" \ - --do_farasa_tokenization=True \ - --use_farasapy=True \ -``` -```bash -python SOQAL/bert/run_squad.py \ - --vocab_file="/PATH_TO_PRETRAINED_TF_CKPT/vocab.txt" \ - --bert_config_file="/PATH_TO_PRETRAINED_TF_CKPT/config.json" \ - --init_checkpoint="/PATH_TO_PRETRAINED_TF_CKPT/" \ - --do_train=True \ - --train_file=turk_combined_all_pre.json \ - --do_predict=True \ - --predict_file=arcd-test-pre.json \ - --train_batch_size=32 \ - --predict_batch_size=24 \ - --learning_rate=3e-5 \ - --num_train_epochs=4 \ - --max_seq_length=384 \ - --doc_stride=128 \ - --do_lower_case=False\ - --output_dir="/PATH_TO/OUTPUT_PATH"/ \ - --use_tpu=True \ - --tpu_name=$TPU_ADDRESS \ -``` -## Model Weights and Vocab Download -Models | AraBERTv0.1 | AraBERTv1 ----|:---:|:---: -TensorFlow|[Drive Link](https://drive.google.com/open?id=1-kVmTUZZ4DP2rzeHNjTPkY8OjnQCpomO) | [Drive Link](https://drive.google.com/open?id=1-d7-9ljKgDJP5mx73uBtio-TuUZCqZnt) -PyTorch| [Drive_Link](https://drive.google.com/open?id=1-_3te42mQCPD8SxwZ3l-VBL7yaJH-IOv)| [Drive_Link](https://drive.google.com/open?id=1-69s6Pxqbi63HOQ1M9wTcr-Ovc6PWLLo) - -**You can find the PyTorch models in HuggingFace's Transformer Library under the ```aubmindlab``` username** - -## If you used this model please cite us as: -``` -@inproceedings{antoun2020arabert, - title={AraBERT: Transformer-based Model for Arabic Language Understanding}, - author={Antoun, Wissam and Baly, Fady and Hajj, Hazem}, - booktitle={LREC 2020 Workshop Language Resources and Evaluation Conference 11--16 May 2020}, - pages={9} -} -``` -## Acknowledgments -Thanks to TensorFlow Research Cloud (TFRC) for the free access to Cloud TPUs, couldn't have done it without this program, and to the [AUB MIND Lab](https://sites.aub.edu.lb/mindlab/) Members for the continous support. Also thanks to [Yakshof](https://www.yakshof.com/#/) and Assafir for data and storage access. Another thanks for Habib Rahal (https://www.behance.net/rahalhabib), for putting a face to AraBERT. - -## Contacts -**Wissam Antoun**: [Linkedin](https://www.linkedin.com/in/giulio-ravasio-3a81a9110/) | [Twitter](https://twitter.com/wissam_antoun) | [Github](https://github.com/WissamAntoun) | | - -**Fady Baly**: [Linkedin](https://www.linkedin.com/in/fadybaly/) | [Twitter](https://twitter.com/fadybaly) | [Github](https://github.com/fadybaly) | | diff --git a/model_cards/bart-large-cnn/README.md b/model_cards/bart-large-cnn/README.md deleted file mode 100644 index 36061c05493e..000000000000 --- a/model_cards/bart-large-cnn/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -tags: -- summarization ---- - diff --git a/model_cards/bart-large-xsum/README.md b/model_cards/bart-large-xsum/README.md deleted file mode 100644 index 36061c05493e..000000000000 --- a/model_cards/bart-large-xsum/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -tags: -- summarization ---- - diff --git a/model_cards/bashar-talafha/multi-dialect-bert-base-arabic/README.md b/model_cards/bashar-talafha/multi-dialect-bert-base-arabic/README.md deleted file mode 100644 index 41e9aece3bc1..000000000000 --- a/model_cards/bashar-talafha/multi-dialect-bert-base-arabic/README.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -language: ar -thumbnail: https://raw.githubusercontent.com/mawdoo3/Multi-dialect-Arabic-BERT/master/multidialct_arabic_bert.png -datasets: -- nadi ---- -# Multi-dialect-Arabic-BERT -This is a repository of Multi-dialect Arabic BERT model. - -By [Mawdoo3-AI](https://ai.mawdoo3.com/). - -

-
- Background reference: http://www.qfi.org/wp-content/uploads/2018/02/Qfi_Infographic_Mother-Language_Final.pdf -
-

- - - -### About our Multi-dialect-Arabic-BERT model -Instead of training the Multi-dialect Arabic BERT model from scratch, we initialized the weights of the model using [Arabic-BERT](https://github.com/alisafaya/Arabic-BERT) and trained it on 10M arabic tweets from the unlabled data of [The Nuanced Arabic Dialect Identification (NADI) shared task](https://sites.google.com/view/nadi-shared-task). - -### To cite this work - -``` -@misc{talafha2020multidialect, - title={Multi-Dialect Arabic BERT for Country-Level Dialect Identification}, - author={Bashar Talafha and Mohammad Ali and Muhy Eddin Za'ter and Haitham Seelawi and Ibraheem Tuffaha and Mostafa Samir and Wael Farhan and Hussein T. Al-Natsheh}, - year={2020}, - eprint={2007.05612}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` - -### Usage -The model weights can be loaded using `transformers` library by HuggingFace. - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("bashar-talafha/multi-dialect-bert-base-arabic") -model = AutoModel.from_pretrained("bashar-talafha/multi-dialect-bert-base-arabic") -``` - -Example using `pipeline`: - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="bashar-talafha/multi-dialect-bert-base-arabic ", - tokenizer="bashar-talafha/multi-dialect-bert-base-arabic " -) - -fill_mask(" سافر الرحالة من مطار [MASK] ") -``` -``` -[{'sequence': '[CLS] سافر الرحالة من مطار الكويت [SEP]', 'score': 0.08296813815832138, 'token': 3226}, - {'sequence': '[CLS] سافر الرحالة من مطار دبي [SEP]', 'score': 0.05123933032155037, 'token': 4747}, - {'sequence': '[CLS] سافر الرحالة من مطار مسقط [SEP]', 'score': 0.046838656067848206, 'token': 13205}, - {'sequence': '[CLS] سافر الرحالة من مطار القاهرة [SEP]', 'score': 0.03234650194644928, 'token': 4003}, - {'sequence': '[CLS] سافر الرحالة من مطار الرياض [SEP]', 'score': 0.02606341242790222, 'token': 2200}] -``` -### Repository -Please check the [original repository](https://github.com/mawdoo3/Multi-dialect-Arabic-BERT) for more information. - - diff --git a/model_cards/bayartsogt/albert-mongolian/README.md b/model_cards/bayartsogt/albert-mongolian/README.md deleted file mode 100644 index 47fee353c569..000000000000 --- a/model_cards/bayartsogt/albert-mongolian/README.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -language: mn ---- - -# ALBERT-Mongolian -[pretraining repo link](https://github.com/bayartsogt-ya/albert-mongolian) -## Model description -Here we provide pretrained ALBERT model and trained SentencePiece model for Mongolia text. Training data is the Mongolian wikipedia corpus from Wikipedia Downloads and Mongolian News corpus. - -## Evaluation Result: -``` -loss = 1.7478163 -masked_lm_accuracy = 0.6838185 -masked_lm_loss = 1.6687671 -sentence_order_accuracy = 0.998125 -sentence_order_loss = 0.007942731 -``` - -## Fine-tuning Result on Eduge Dataset: -``` - precision recall f1-score support - - байгал орчин 0.83 0.76 0.80 483 - боловсрол 0.79 0.75 0.77 420 - спорт 0.98 0.96 0.97 1391 - технологи 0.85 0.83 0.84 543 - улс төр 0.88 0.87 0.87 1336 - урлаг соёл 0.89 0.94 0.91 726 - хууль 0.87 0.83 0.85 840 - эдийн засаг 0.80 0.84 0.82 1265 - эрүүл мэнд 0.84 0.90 0.87 562 - - accuracy 0.87 7566 - macro avg 0.86 0.85 0.86 7566 - weighted avg 0.87 0.87 0.87 7566 -``` - -## Reference -1. [ALBERT - official repo](https://github.com/google-research/albert) -2. [WikiExtrator](https://github.com/attardi/wikiextractor) -3. [Mongolian BERT](https://github.com/tugstugi/mongolian-bert) -4. [ALBERT - Japanese](https://github.com/alinear-corp/albert-japanese) -5. [Mongolian Text Classification](https://github.com/sharavsambuu/mongolian-text-classification) -6. [You's paper](https://arxiv.org/abs/1904.00962) - -## Citation -``` -@misc{albert-mongolian, - author = {Bayartsogt Yadamsuren}, - title = {ALBERT Pretrained Model on Mongolian Datasets}, - year = {2020}, - publisher = {GitHub}, - journal = {GitHub repository}, - howpublished = {\url{https://github.com/bayartsogt-ya/albert-mongolian/}} -} -``` - -## For More Information -Please contact by bayartsogtyadamsuren@icloud.com diff --git a/model_cards/bayartsogt/bert-base-mongolian-cased/README.md b/model_cards/bayartsogt/bert-base-mongolian-cased/README.md deleted file mode 100644 index 7cf8c1adaa70..000000000000 --- a/model_cards/bayartsogt/bert-base-mongolian-cased/README.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -language: "mn" -tags: -- mongolian -- cased ---- - -# BERT-BASE-MONGOLIAN-CASED -[Link to Official Mongolian-BERT repo](https://github.com/tugstugi/mongolian-bert) - -## Model description -This repository contains pre-trained Mongolian [BERT](https://arxiv.org/abs/1810.04805) models trained by [tugstugi](https://github.com/tugstugi), [enod](https://github.com/enod) and [sharavsambuu](https://github.com/sharavsambuu). -Special thanks to [nabar](https://github.com/nabar) who provided 5x TPUs. - -This repository is based on the following open source projects: [google-research/bert](https://github.com/google-research/bert/), -[huggingface/pytorch-pretrained-BERT](https://github.com/huggingface/pytorch-pretrained-BERT) and [yoheikikuta/bert-japanese](https://github.com/yoheikikuta/bert-japanese). - -#### How to use - -```python -from transformers import pipeline, AlbertTokenizer, BertForMaskedLM - -tokenizer = AlbertTokenizer.from_pretrained('bayartsogt/bert-base-mongolian-cased') -model = BertForMaskedLM.from_pretrained('bayartsogt/bert-base-mongolian-cased') - -## declare task ## -pipe = pipeline(task="fill-mask", model=model, tokenizer=tokenizer) - -## example ## -input_ = 'Миний [MASK] хоол идэх нь тун чухал.' - -output_ = pipe(input_) -for i in range(len(output_)): - print(output_[i]) - -## Output ## -# {'sequence': '[CLS] Миний хувьд хоол идэх нь тун чухал.[SEP]', 'score': 0.8734784722328186, 'token': 95, 'token_str': '▁хувьд'} -# {'sequence': '[CLS] Миний бодлоор хоол идэх нь тун чухал.[SEP]', 'score': 0.09788835793733597, 'token': 6320, 'token_str': '▁бодлоор'} -# {'sequence': '[CLS] Миний хүү хоол идэх нь тун чухал.[SEP]', 'score': 0.0027510314248502254, 'token': 590, 'token_str': '▁хүү'} -# {'sequence': '[CLS] Миний бие хоол идэх нь тун чухал.[SEP]', 'score': 0.0014857524074614048, 'token': 267, 'token_str': '▁бие'} -# {'sequence': '[CLS] Миний охин хоол идэх нь тун чухал.[SEP]', 'score': 0.0013575413031503558, 'token': 1116, 'token_str': '▁охин'} - -``` - - -## Training data -Mongolian Wikipedia and the 700 million word Mongolian news data set [[Pretraining Procedure](https://github.com/tugstugi/mongolian-bert#pre-training)] - -### BibTeX entry and citation info - -```bibtex -@misc{mongolian-bert, - author = {Tuguldur, Erdene-Ochir and Gunchinish, Sharavsambuu and Bataa, Enkhbold}, - title = {BERT Pretrained Models on Mongolian Datasets}, - year = {2019}, - publisher = {GitHub}, - journal = {GitHub repository}, - howpublished = {\url{https://github.com/tugstugi/mongolian-bert/}} -} -``` diff --git a/model_cards/bayartsogt/bert-base-mongolian-uncased/README.md b/model_cards/bayartsogt/bert-base-mongolian-uncased/README.md deleted file mode 100644 index f673206e4d40..000000000000 --- a/model_cards/bayartsogt/bert-base-mongolian-uncased/README.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -language: "mn" -tags: -- bert -- mongolian -- uncased ---- - -# BERT-BASE-MONGOLIAN-UNCASED -[Link to Official Mongolian-BERT repo](https://github.com/tugstugi/mongolian-bert) - -## Model description -This repository contains pre-trained Mongolian [BERT](https://arxiv.org/abs/1810.04805) models trained by [tugstugi](https://github.com/tugstugi), [enod](https://github.com/enod) and [sharavsambuu](https://github.com/sharavsambuu). -Special thanks to [nabar](https://github.com/nabar) who provided 5x TPUs. - -This repository is based on the following open source projects: [google-research/bert](https://github.com/google-research/bert/), -[huggingface/pytorch-pretrained-BERT](https://github.com/huggingface/pytorch-pretrained-BERT) and [yoheikikuta/bert-japanese](https://github.com/yoheikikuta/bert-japanese). - -#### How to use - -```python -from transformers import pipeline, AlbertTokenizer, BertForMaskedLM - -tokenizer = AlbertTokenizer.from_pretrained('bayartsogt/bert-base-mongolian-uncased') -model = BertForMaskedLM.from_pretrained('bayartsogt/bert-base-mongolian-uncased') - -## declare task ## -pipe = pipeline(task="fill-mask", model=model, tokenizer=tokenizer) - -## example ## -input_ = 'Миний [MASK] хоол идэх нь тун чухал.' - -output_ = pipe(input_) -for i in range(len(output_)): - print(output_[i]) - -``` - - -## Training data -Mongolian Wikipedia and the 700 million word Mongolian news data set [[Pretraining Procedure](https://github.com/tugstugi/mongolian-bert#pre-training)] - -### BibTeX entry and citation info - -```bibtex -@misc{mongolian-bert, - author = {Tuguldur, Erdene-Ochir and Gunchinish, Sharavsambuu and Bataa, Enkhbold}, - title = {BERT Pretrained Models on Mongolian Datasets}, - year = {2019}, - publisher = {GitHub}, - journal = {GitHub repository}, - howpublished = {\url{https://github.com/tugstugi/mongolian-bert/}} -} -``` diff --git a/model_cards/bert-base-cased-README.md b/model_cards/bert-base-cased-README.md deleted file mode 100644 index 82d50d4805ab..000000000000 --- a/model_cards/bert-base-cased-README.md +++ /dev/null @@ -1,230 +0,0 @@ ---- -language: en -tags: -- exbert -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia ---- - -# BERT base model (cased) - -Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in -[this paper](https://arxiv.org/abs/1810.04805) and first released in -[this repository](https://github.com/google-research/bert). This model is case-sensitive: it makes a difference between -english and English. - -Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by -the Hugging Face team. - -## Model description - -BERT is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it -was pretrained with two objectives: - -- Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run - the entire masked sentence through the model and has to predict the masked words. This is different from traditional - recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like - GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the - sentence. -- Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes - they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to - predict if the two sentences were following each other or not. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='bert-base-cased') ->>> unmasker("Hello I'm a [MASK] model.") - -[{'sequence': "[CLS] Hello I'm a fashion model. [SEP]", - 'score': 0.09019174426794052, - 'token': 4633, - 'token_str': 'fashion'}, - {'sequence': "[CLS] Hello I'm a new model. [SEP]", - 'score': 0.06349995732307434, - 'token': 1207, - 'token_str': 'new'}, - {'sequence': "[CLS] Hello I'm a male model. [SEP]", - 'score': 0.06228214129805565, - 'token': 2581, - 'token_str': 'male'}, - {'sequence': "[CLS] Hello I'm a professional model. [SEP]", - 'score': 0.0441727414727211, - 'token': 1848, - 'token_str': 'professional'}, - {'sequence': "[CLS] Hello I'm a super model. [SEP]", - 'score': 0.03326151892542839, - 'token': 7688, - 'token_str': 'super'}] -``` - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained('bert-base-cased') -model = TFBertModel.from_pretrained("bert-base-cased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import BertTokenizer, BertModel -tokenizer = BertTokenizer.from_pretrained('bert-base-cased') -model = BertModel.from_pretrained("bert-base-cased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -### Limitations and bias - -Even if the training data used for this model could be characterized as fairly neutral, this model can have biased -predictions: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='bert-base-cased') ->>> unmasker("The man worked as a [MASK].") - -[{'sequence': '[CLS] The man worked as a lawyer. [SEP]', - 'score': 0.04804691672325134, - 'token': 4545, - 'token_str': 'lawyer'}, - {'sequence': '[CLS] The man worked as a waiter. [SEP]', - 'score': 0.037494491785764694, - 'token': 17989, - 'token_str': 'waiter'}, - {'sequence': '[CLS] The man worked as a cop. [SEP]', - 'score': 0.035512614995241165, - 'token': 9947, - 'token_str': 'cop'}, - {'sequence': '[CLS] The man worked as a detective. [SEP]', - 'score': 0.031271643936634064, - 'token': 9140, - 'token_str': 'detective'}, - {'sequence': '[CLS] The man worked as a doctor. [SEP]', - 'score': 0.027423162013292313, - 'token': 3995, - 'token_str': 'doctor'}] - ->>> unmasker("The woman worked as a [MASK].") - -[{'sequence': '[CLS] The woman worked as a nurse. [SEP]', - 'score': 0.16927455365657806, - 'token': 7439, - 'token_str': 'nurse'}, - {'sequence': '[CLS] The woman worked as a waitress. [SEP]', - 'score': 0.1501094549894333, - 'token': 15098, - 'token_str': 'waitress'}, - {'sequence': '[CLS] The woman worked as a maid. [SEP]', - 'score': 0.05600163713097572, - 'token': 13487, - 'token_str': 'maid'}, - {'sequence': '[CLS] The woman worked as a housekeeper. [SEP]', - 'score': 0.04838843643665314, - 'token': 26458, - 'token_str': 'housekeeper'}, - {'sequence': '[CLS] The woman worked as a cook. [SEP]', - 'score': 0.029980547726154327, - 'token': 9834, - 'token_str': 'cook'}] -``` - -This bias will also affect all fine-tuned versions of this model. - -## Training data - -The BERT model was pretrained on [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 -unpublished books and [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and -headers). - -## Training procedure - -### Preprocessing - -The texts are tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are then of the form: - -``` -[CLS] Sentence A [SEP] Sentence B [SEP] -``` - -With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in -the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a -consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two -"sentences" has a combined length of less than 512 tokens. - -The details of the masking procedure for each sentence are the following: -- 15% of the tokens are masked. -- In 80% of the cases, the masked tokens are replaced by `[MASK]`. -- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace. -- In the 10% remaining cases, the masked tokens are left as is. - -### Pretraining - -The model was trained on 4 cloud TPUs in Pod configuration (16 TPU chips total) for one million steps with a batch size -of 256. The sequence length was limited to 128 tokens for 90% of the steps and 512 for the remaining 10%. The optimizer -used is Adam with a learning rate of 1e-4, \\(\beta_{1} = 0.9\\) and \\(\beta_{2} = 0.999\\), a weight decay of 0.01, -learning rate warmup for 10,000 steps and linear decay of the learning rate after. - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Glue test results: - -| Task | MNLI-(m/mm) | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | Average | -|:----:|:-----------:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:| -| | 84.6/83.4 | 71.2 | 90.5 | 93.5 | 52.1 | 85.8 | 88.9 | 66.4 | 79.6 | - - -### BibTeX entry and citation info - -```bibtex -@article{DBLP:journals/corr/abs-1810-04805, - author = {Jacob Devlin and - Ming{-}Wei Chang and - Kenton Lee and - Kristina Toutanova}, - title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language - Understanding}, - journal = {CoRR}, - volume = {abs/1810.04805}, - year = {2018}, - url = {http://arxiv.org/abs/1810.04805}, - archivePrefix = {arXiv}, - eprint = {1810.04805}, - timestamp = {Tue, 30 Oct 2018 20:39:56 +0100}, - biburl = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` - - - - diff --git a/model_cards/bert-base-chinese-README.md b/model_cards/bert-base-chinese-README.md deleted file mode 100644 index 78633644e32e..000000000000 --- a/model_cards/bert-base-chinese-README.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -language: zh ---- diff --git a/model_cards/bert-base-german-cased-README.md b/model_cards/bert-base-german-cased-README.md deleted file mode 100644 index eda81a6c1eb3..000000000000 --- a/model_cards/bert-base-german-cased-README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: de -license: mit -thumbnail: https://static.tildacdn.com/tild6438-3730-4164-b266-613634323466/german_bert.png -tags: -- exbert ---- - - - - - -# German BERT -![bert_image](https://static.tildacdn.com/tild6438-3730-4164-b266-613634323466/german_bert.png) -## Overview -**Language model:** bert-base-cased -**Language:** German -**Training data:** Wiki, OpenLegalData, News (~ 12GB) -**Eval data:** Conll03 (NER), GermEval14 (NER), GermEval18 (Classification), GNAD (Classification) -**Infrastructure**: 1x TPU v2 -**Published**: Jun 14th, 2019 - -**Update April 3rd, 2020**: we updated the vocabulary file on deepset's s3 to conform with the default tokenization of punctuation tokens. -For details see the related [FARM issue](https://github.com/deepset-ai/FARM/issues/60). If you want to use the old vocab we have also uploaded a ["deepset/bert-base-german-cased-oldvocab"](https://huggingface.co/deepset/bert-base-german-cased-oldvocab) model. - -## Details -- We trained using Google's Tensorflow code on a single cloud TPU v2 with standard settings. -- We trained 810k steps with a batch size of 1024 for sequence length 128 and 30k steps with sequence length 512. Training took about 9 days. -- As training data we used the latest German Wikipedia dump (6GB of raw txt files), the OpenLegalData dump (2.4 GB) and news articles (3.6 GB). -- We cleaned the data dumps with tailored scripts and segmented sentences with spacy v2.1. To create tensorflow records we used the recommended sentencepiece library for creating the word piece vocabulary and tensorflow scripts to convert the text to data usable by BERT. - - -See https://deepset.ai/german-bert for more details - -## Hyperparameters - -``` -batch_size = 1024 -n_steps = 810_000 -max_seq_len = 128 (and 512 later) -learning_rate = 1e-4 -lr_schedule = LinearWarmup -num_warmup_steps = 10_000 -``` - -## Performance - -During training we monitored the loss and evaluated different model checkpoints on the following German datasets: - -- germEval18Fine: Macro f1 score for multiclass sentiment classification -- germEval18coarse: Macro f1 score for binary sentiment classification -- germEval14: Seq f1 score for NER (file names deuutf.\*) -- CONLL03: Seq f1 score for NER -- 10kGNAD: Accuracy for document classification - -Even without thorough hyperparameter tuning, we observed quite stable learning especially for our German model. Multiple restarts with different seeds produced quite similar results. - -![performancetable](https://thumb.tildacdn.com/tild3162-6462-4566-b663-376630376138/-/format/webp/Screenshot_from_2020.png) - -We further evaluated different points during the 9 days of pre-training and were astonished how fast the model converges to the maximally reachable performance. We ran all 5 downstream tasks on 7 different model checkpoints - taken at 0 up to 840k training steps (x-axis in figure below). Most checkpoints are taken from early training where we expected most performance changes. Surprisingly, even a randomly initialized BERT can be trained only on labeled downstream datasets and reach good performance (blue line, GermEval 2018 Coarse task, 795 kB trainset size). - -![checkpointseval](https://thumb.tildacdn.com/tild6335-3531-4137-b533-313365663435/-/format/webp/deepset_checkpoints.png) - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Timo Möller: `timo.moeller [at] deepset.ai` -Malte Pietsch: `malte.pietsch [at] deepset.ai` -Tanay Soni: `tanay.soni [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/bert-base-german-dbmdz-cased-README.md b/model_cards/bert-base-german-dbmdz-cased-README.md deleted file mode 100644 index 87bed79724b4..000000000000 --- a/model_cards/bert-base-german-dbmdz-cased-README.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -language: de -license: mit ---- diff --git a/model_cards/bert-base-german-dbmdz-uncased-README.md b/model_cards/bert-base-german-dbmdz-uncased-README.md deleted file mode 100644 index 87bed79724b4..000000000000 --- a/model_cards/bert-base-german-dbmdz-uncased-README.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -language: de -license: mit ---- diff --git a/model_cards/bert-base-multilingual-cased-README.md b/model_cards/bert-base-multilingual-cased-README.md deleted file mode 100644 index 8a6ed9bef371..000000000000 --- a/model_cards/bert-base-multilingual-cased-README.md +++ /dev/null @@ -1,152 +0,0 @@ ---- -language: multilingual -license: apache-2.0 -datasets: -- wikipedia ---- - -# BERT multilingual base model (cased) - -Pretrained model on the top 104 languages with the largest Wikipedia using a masked language modeling (MLM) objective. -It was introduced in [this paper](https://arxiv.org/abs/1810.04805) and first released in -[this repository](https://github.com/google-research/bert). This model is case sensitive: it makes a difference -between english and English. - -Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by -the Hugging Face team. - -## Model description - -BERT is a transformers model pretrained on a large corpus of multilingual data in a self-supervised fashion. This means -it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it -was pretrained with two objectives: - -- Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run - the entire masked sentence through the model and has to predict the masked words. This is different from traditional - recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like - GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the - sentence. -- Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes - they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to - predict if the two sentences were following each other or not. - -This way, the model learns an inner representation of the languages in the training set that can then be used to -extract features useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a -standard classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='bert-base-multilingual-cased') ->>> unmasker("Hello I'm a [MASK] model.") - -[{'sequence': "[CLS] Hello I'm a model model. [SEP]", - 'score': 0.10182085633277893, - 'token': 13192, - 'token_str': 'model'}, - {'sequence': "[CLS] Hello I'm a world model. [SEP]", - 'score': 0.052126359194517136, - 'token': 11356, - 'token_str': 'world'}, - {'sequence': "[CLS] Hello I'm a data model. [SEP]", - 'score': 0.048930276185274124, - 'token': 11165, - 'token_str': 'data'}, - {'sequence': "[CLS] Hello I'm a flight model. [SEP]", - 'score': 0.02036019042134285, - 'token': 23578, - 'token_str': 'flight'}, - {'sequence': "[CLS] Hello I'm a business model. [SEP]", - 'score': 0.020079681649804115, - 'token': 14155, - 'token_str': 'business'}] -``` - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import BertTokenizer, BertModel -tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased') -model = BertModel.from_pretrained("bert-base-multilingual-cased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased') -model = TFBertModel.from_pretrained("bert-base-multilingual-cased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on the 104 languages with the largest Wikipedias. You can find the complete list -[here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). - -## Training procedure - -### Preprocessing - -The texts are lowercased and tokenized using WordPiece and a shared vocabulary size of 110,000. The languages with a -larger Wikipedia are under-sampled and the ones with lower resources are oversampled. For languages like Chinese, -Japanese Kanji and Korean Hanja that don't have space, a CJK Unicode block is added around every character. - -The inputs of the model are then of the form: - -``` -[CLS] Sentence A [SEP] Sentence B [SEP] -``` - -With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in -the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a -consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two -"sentences" has a combined length of less than 512 tokens. - -The details of the masking procedure for each sentence are the following: -- 15% of the tokens are masked. -- In 80% of the cases, the masked tokens are replaced by `[MASK]`. -- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace. -- In the 10% remaining cases, the masked tokens are left as is. - - -### BibTeX entry and citation info - -```bibtex -@article{DBLP:journals/corr/abs-1810-04805, - author = {Jacob Devlin and - Ming{-}Wei Chang and - Kenton Lee and - Kristina Toutanova}, - title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language - Understanding}, - journal = {CoRR}, - volume = {abs/1810.04805}, - year = {2018}, - url = {http://arxiv.org/abs/1810.04805}, - archivePrefix = {arXiv}, - eprint = {1810.04805}, - timestamp = {Tue, 30 Oct 2018 20:39:56 +0100}, - biburl = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` diff --git a/model_cards/bert-base-multilingual-uncased-README.md b/model_cards/bert-base-multilingual-uncased-README.md deleted file mode 100644 index 862acd3ac458..000000000000 --- a/model_cards/bert-base-multilingual-uncased-README.md +++ /dev/null @@ -1,209 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- wikipedia ---- - -# BERT multilingual base model (uncased) - -Pretrained model on the top 102 languages with the largest Wikipedia using a masked language modeling (MLM) objective. -It was introduced in [this paper](https://arxiv.org/abs/1810.04805) and first released in -[this repository](https://github.com/google-research/bert). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by -the Hugging Face team. - -## Model description - -BERT is a transformers model pretrained on a large corpus of multilingual data in a self-supervised fashion. This means -it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it -was pretrained with two objectives: - -- Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run - the entire masked sentence through the model and has to predict the masked words. This is different from traditional - recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like - GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the - sentence. -- Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes - they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to - predict if the two sentences were following each other or not. - -This way, the model learns an inner representation of the languages in the training set that can then be used to -extract features useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a -standard classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='bert-base-multilingual-uncased') ->>> unmasker("Hello I'm a [MASK] model.") - -[{'sequence': "[CLS] hello i'm a top model. [SEP]", - 'score': 0.1507750153541565, - 'token': 11397, - 'token_str': 'top'}, - {'sequence': "[CLS] hello i'm a fashion model. [SEP]", - 'score': 0.13075384497642517, - 'token': 23589, - 'token_str': 'fashion'}, - {'sequence': "[CLS] hello i'm a good model. [SEP]", - 'score': 0.036272723227739334, - 'token': 12050, - 'token_str': 'good'}, - {'sequence': "[CLS] hello i'm a new model. [SEP]", - 'score': 0.035954564809799194, - 'token': 10246, - 'token_str': 'new'}, - {'sequence': "[CLS] hello i'm a great model. [SEP]", - 'score': 0.028643041849136353, - 'token': 11838, - 'token_str': 'great'}] -``` - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import BertTokenizer, BertModel -tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased') -model = BertModel.from_pretrained("bert-base-multilingual-uncased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased') -model = TFBertModel.from_pretrained("bert-base-multilingual-uncased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -### Limitations and bias - -Even if the training data used for this model could be characterized as fairly neutral, this model can have biased -predictions: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='bert-base-multilingual-uncased') ->>> unmasker("The man worked as a [MASK].") - -[{'sequence': '[CLS] the man worked as a teacher. [SEP]', - 'score': 0.07943806052207947, - 'token': 21733, - 'token_str': 'teacher'}, - {'sequence': '[CLS] the man worked as a lawyer. [SEP]', - 'score': 0.0629938617348671, - 'token': 34249, - 'token_str': 'lawyer'}, - {'sequence': '[CLS] the man worked as a farmer. [SEP]', - 'score': 0.03367974981665611, - 'token': 36799, - 'token_str': 'farmer'}, - {'sequence': '[CLS] the man worked as a journalist. [SEP]', - 'score': 0.03172805905342102, - 'token': 19477, - 'token_str': 'journalist'}, - {'sequence': '[CLS] the man worked as a carpenter. [SEP]', - 'score': 0.031021825969219208, - 'token': 33241, - 'token_str': 'carpenter'}] - ->>> unmasker("The Black woman worked as a [MASK].") - -[{'sequence': '[CLS] the black woman worked as a nurse. [SEP]', - 'score': 0.07045423984527588, - 'token': 52428, - 'token_str': 'nurse'}, - {'sequence': '[CLS] the black woman worked as a teacher. [SEP]', - 'score': 0.05178029090166092, - 'token': 21733, - 'token_str': 'teacher'}, - {'sequence': '[CLS] the black woman worked as a lawyer. [SEP]', - 'score': 0.032601192593574524, - 'token': 34249, - 'token_str': 'lawyer'}, - {'sequence': '[CLS] the black woman worked as a slave. [SEP]', - 'score': 0.030507225543260574, - 'token': 31173, - 'token_str': 'slave'}, - {'sequence': '[CLS] the black woman worked as a woman. [SEP]', - 'score': 0.027691684663295746, - 'token': 14050, - 'token_str': 'woman'}] -``` - -This bias will also affect all fine-tuned versions of this model. - -## Training data - -The BERT model was pretrained on the 102 languages with the largest Wikipedias. You can find the complete list -[here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). - -## Training procedure - -### Preprocessing - -The texts are lowercased and tokenized using WordPiece and a shared vocabulary size of 110,000. The languages with a -larger Wikipedia are under-sampled and the ones with lower resources are oversampled. For languages like Chinese, -Japanese Kanji and Korean Hanja that don't have space, a CJK Unicode block is added around every character. - -The inputs of the model are then of the form: - -``` -[CLS] Sentence A [SEP] Sentence B [SEP] -``` - -With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in -the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a -consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two -"sentences" has a combined length of less than 512 tokens. - -The details of the masking procedure for each sentence are the following: -- 15% of the tokens are masked. -- In 80% of the cases, the masked tokens are replaced by `[MASK]`. -- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace. -- In the 10% remaining cases, the masked tokens are left as is. - - -### BibTeX entry and citation info - -```bibtex -@article{DBLP:journals/corr/abs-1810-04805, - author = {Jacob Devlin and - Ming{-}Wei Chang and - Kenton Lee and - Kristina Toutanova}, - title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language - Understanding}, - journal = {CoRR}, - volume = {abs/1810.04805}, - year = {2018}, - url = {http://arxiv.org/abs/1810.04805}, - archivePrefix = {arXiv}, - eprint = {1810.04805}, - timestamp = {Tue, 30 Oct 2018 20:39:56 +0100}, - biburl = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` diff --git a/model_cards/bert-base-uncased-README.md b/model_cards/bert-base-uncased-README.md deleted file mode 100644 index 52f4ab676bed..000000000000 --- a/model_cards/bert-base-uncased-README.md +++ /dev/null @@ -1,231 +0,0 @@ ---- -language: en -tags: -- exbert -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia ---- - -# BERT base model (uncased) - -Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in -[this paper](https://arxiv.org/abs/1810.04805) and first released in -[this repository](https://github.com/google-research/bert). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing BERT did not write a model card for this model so this model card has been written by -the Hugging Face team. - -## Model description - -BERT is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it -was pretrained with two objectives: - -- Masked language modeling (MLM): taking a sentence, the model randomly masks 15% of the words in the input then run - the entire masked sentence through the model and has to predict the masked words. This is different from traditional - recurrent neural networks (RNNs) that usually see the words one after the other, or from autoregressive models like - GPT which internally mask the future tokens. It allows the model to learn a bidirectional representation of the - sentence. -- Next sentence prediction (NSP): the models concatenates two masked sentences as inputs during pretraining. Sometimes - they correspond to sentences that were next to each other in the original text, sometimes not. The model then has to - predict if the two sentences were following each other or not. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=bert) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='bert-base-uncased') ->>> unmasker("Hello I'm a [MASK] model.") - -[{'sequence': "[CLS] hello i'm a fashion model. [SEP]", - 'score': 0.1073106899857521, - 'token': 4827, - 'token_str': 'fashion'}, - {'sequence': "[CLS] hello i'm a role model. [SEP]", - 'score': 0.08774490654468536, - 'token': 2535, - 'token_str': 'role'}, - {'sequence': "[CLS] hello i'm a new model. [SEP]", - 'score': 0.05338378623127937, - 'token': 2047, - 'token_str': 'new'}, - {'sequence': "[CLS] hello i'm a super model. [SEP]", - 'score': 0.04667217284440994, - 'token': 3565, - 'token_str': 'super'}, - {'sequence': "[CLS] hello i'm a fine model. [SEP]", - 'score': 0.027095865458250046, - 'token': 2986, - 'token_str': 'fine'}] -``` - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import BertTokenizer, BertModel -tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') -model = BertModel.from_pretrained("bert-base-uncased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') -model = TFBertModel.from_pretrained("bert-base-uncased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -### Limitations and bias - -Even if the training data used for this model could be characterized as fairly neutral, this model can have biased -predictions: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='bert-base-uncased') ->>> unmasker("The man worked as a [MASK].") - -[{'sequence': '[CLS] the man worked as a carpenter. [SEP]', - 'score': 0.09747550636529922, - 'token': 10533, - 'token_str': 'carpenter'}, - {'sequence': '[CLS] the man worked as a waiter. [SEP]', - 'score': 0.0523831807076931, - 'token': 15610, - 'token_str': 'waiter'}, - {'sequence': '[CLS] the man worked as a barber. [SEP]', - 'score': 0.04962705448269844, - 'token': 13362, - 'token_str': 'barber'}, - {'sequence': '[CLS] the man worked as a mechanic. [SEP]', - 'score': 0.03788609802722931, - 'token': 15893, - 'token_str': 'mechanic'}, - {'sequence': '[CLS] the man worked as a salesman. [SEP]', - 'score': 0.037680890411138535, - 'token': 18968, - 'token_str': 'salesman'}] - ->>> unmasker("The woman worked as a [MASK].") - -[{'sequence': '[CLS] the woman worked as a nurse. [SEP]', - 'score': 0.21981462836265564, - 'token': 6821, - 'token_str': 'nurse'}, - {'sequence': '[CLS] the woman worked as a waitress. [SEP]', - 'score': 0.1597415804862976, - 'token': 13877, - 'token_str': 'waitress'}, - {'sequence': '[CLS] the woman worked as a maid. [SEP]', - 'score': 0.1154729500412941, - 'token': 10850, - 'token_str': 'maid'}, - {'sequence': '[CLS] the woman worked as a prostitute. [SEP]', - 'score': 0.037968918681144714, - 'token': 19215, - 'token_str': 'prostitute'}, - {'sequence': '[CLS] the woman worked as a cook. [SEP]', - 'score': 0.03042375110089779, - 'token': 5660, - 'token_str': 'cook'}] -``` - -This bias will also affect all fine-tuned versions of this model. - -## Training data - -The BERT model was pretrained on [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 -unpublished books and [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and -headers). - -## Training procedure - -### Preprocessing - -The texts are lowercased and tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are -then of the form: - -``` -[CLS] Sentence A [SEP] Sentence B [SEP] -``` - -With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in -the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a -consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two -"sentences" has a combined length of less than 512 tokens. - -The details of the masking procedure for each sentence are the following: -- 15% of the tokens are masked. -- In 80% of the cases, the masked tokens are replaced by `[MASK]`. -- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace. -- In the 10% remaining cases, the masked tokens are left as is. - -### Pretraining - -The model was trained on 4 cloud TPUs in Pod configuration (16 TPU chips total) for one million steps with a batch size -of 256. The sequence length was limited to 128 tokens for 90% of the steps and 512 for the remaining 10%. The optimizer -used is Adam with a learning rate of 1e-4, \\(\beta_{1} = 0.9\\) and \\(\beta_{2} = 0.999\\), a weight decay of 0.01, -learning rate warmup for 10,000 steps and linear decay of the learning rate after. - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Glue test results: - -| Task | MNLI-(m/mm) | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | Average | -|:----:|:-----------:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:|:-------:| -| | 84.6/83.4 | 71.2 | 90.5 | 93.5 | 52.1 | 85.8 | 88.9 | 66.4 | 79.6 | - - -### BibTeX entry and citation info - -```bibtex -@article{DBLP:journals/corr/abs-1810-04805, - author = {Jacob Devlin and - Ming{-}Wei Chang and - Kenton Lee and - Kristina Toutanova}, - title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language - Understanding}, - journal = {CoRR}, - volume = {abs/1810.04805}, - year = {2018}, - url = {http://arxiv.org/abs/1810.04805}, - archivePrefix = {arXiv}, - eprint = {1810.04805}, - timestamp = {Tue, 30 Oct 2018 20:39:56 +0100}, - biburl = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` - - - - diff --git a/model_cards/bert-large-cased-README.md b/model_cards/bert-large-cased-README.md deleted file mode 100644 index 154df8298fab..000000000000 --- a/model_cards/bert-large-cased-README.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -license: apache-2.0 ---- diff --git a/model_cards/binwang/xlnet-base-cased/README.md b/model_cards/binwang/xlnet-base-cased/README.md deleted file mode 100644 index 482c0fc17e82..000000000000 --- a/model_cards/binwang/xlnet-base-cased/README.md +++ /dev/null @@ -1,5 +0,0 @@ -This model is pre-trained **XLNET** with 12 layers. - -It comes with paper: SBERT-WK: A Sentence Embedding Method By Dissecting BERT-based Word Models - -Project Page: [SBERT-WK](https://github.com/BinWang28/SBERT-WK-Sentence-Embedding) diff --git a/model_cards/bionlp/bluebert_pubmed_mimic_uncased_L-12_H-768_A-12/README.md b/model_cards/bionlp/bluebert_pubmed_mimic_uncased_L-12_H-768_A-12/README.md deleted file mode 100644 index 64319e76e009..000000000000 --- a/model_cards/bionlp/bluebert_pubmed_mimic_uncased_L-12_H-768_A-12/README.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -language: -- en -tags: -- bert -- bluebert -license: -- PUBLIC DOMAIN NOTICE -datasets: -- PubMed -- MIMIC-III - ---- - -# BlueBert-Base, Uncased, PubMed and MIMIC-III - -## Model description - -A BERT model pre-trained on PubMed abstracts and clinical notes ([MIMIC-III](https://mimic.physionet.org/)). - -## Intended uses & limitations - -#### How to use - -Please see https://github.com/ncbi-nlp/bluebert - -## Training data - -We provide [preprocessed PubMed texts](https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/NCBI-BERT/pubmed_uncased_sentence_nltk.txt.tar.gz) that were used to pre-train the BlueBERT models. -The corpus contains ~4000M words extracted from the [PubMed ASCII code version](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PubMed/). - -Pre-trained model: https://huggingface.co/bert-base-uncased - -## Training procedure - -* lowercasing the text -* removing speical chars `\x00`-`\x7F` -* tokenizing the text using the [NLTK Treebank tokenizer](https://www.nltk.org/_modules/nltk/tokenize/treebank.html) - -Below is a code snippet for more details. - -```python -value = value.lower() -value = re.sub(r'[\r\n]+', ' ', value) -value = re.sub(r'[^\x00-\x7F]+', ' ', value) - -tokenized = TreebankWordTokenizer().tokenize(value) -sentence = ' '.join(tokenized) -sentence = re.sub(r"\s's\b", "'s", sentence) -``` - -### BibTeX entry and citation info - -```bibtex -@InProceedings{peng2019transfer, - author = {Yifan Peng and Shankai Yan and Zhiyong Lu}, - title = {Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets}, - booktitle = {Proceedings of the 2019 Workshop on Biomedical Natural Language Processing (BioNLP 2019)}, - year = {2019}, - pages = {58--65}, -} -``` - -### Acknowledgments - -This work was supported by the Intramural Research Programs of the National Institutes of Health, National Library of -Medicine and Clinical Center. This work was supported by the National Library of Medicine of the National Institutes of Health under award number 4R00LM013001-01. - -We are also grateful to the authors of BERT and ELMo to make the data and codes publicly available. - -We would like to thank Dr Sun Kim for processing the PubMed texts. - -### Disclaimer - -This tool shows the results of research conducted in the Computational Biology Branch, NCBI. The information produced -on this website is not intended for direct diagnostic use or medical decision-making without review and oversight -by a clinical professional. Individuals should not change their health behavior solely on the basis of information -produced on this website. NIH does not independently verify the validity or utility of the information produced -by this tool. If you have questions about the information produced on this website, please see a health care -professional. More information about NCBI's disclaimer policy is available. diff --git a/model_cards/bionlp/bluebert_pubmed_mimic_uncased_L-24_H-1024_A-16/README.md b/model_cards/bionlp/bluebert_pubmed_mimic_uncased_L-24_H-1024_A-16/README.md deleted file mode 100644 index fd206ae97361..000000000000 --- a/model_cards/bionlp/bluebert_pubmed_mimic_uncased_L-24_H-1024_A-16/README.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -language: -- en -tags: -- bert -- bluebert -license: -- PUBLIC DOMAIN NOTICE -datasets: -- PubMed -- MIMIC-III - ---- - -# BlueBert-Base, Uncased, PubMed and MIMIC-III - -## Model description - -A BERT model pre-trained on PubMed abstracts and clinical notes ([MIMIC-III](https://mimic.physionet.org/)). - -## Intended uses & limitations - -#### How to use - -Please see https://github.com/ncbi-nlp/bluebert - -## Training data - -We provide [preprocessed PubMed texts](https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/NCBI-BERT/pubmed_uncased_sentence_nltk.txt.tar.gz) that were used to pre-train the BlueBERT models. -The corpus contains ~4000M words extracted from the [PubMed ASCII code version](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PubMed/). - -Pre-trained model: https://huggingface.co/bert-large-uncased - -## Training procedure - -* lowercasing the text -* removing speical chars `\x00`-`\x7F` -* tokenizing the text using the [NLTK Treebank tokenizer](https://www.nltk.org/_modules/nltk/tokenize/treebank.html) - -Below is a code snippet for more details. - -```python -value = value.lower() -value = re.sub(r'[\r\n]+', ' ', value) -value = re.sub(r'[^\x00-\x7F]+', ' ', value) - -tokenized = TreebankWordTokenizer().tokenize(value) -sentence = ' '.join(tokenized) -sentence = re.sub(r"\s's\b", "'s", sentence) -``` - -### BibTeX entry and citation info - -```bibtex -@InProceedings{peng2019transfer, - author = {Yifan Peng and Shankai Yan and Zhiyong Lu}, - title = {Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets}, - booktitle = {Proceedings of the 2019 Workshop on Biomedical Natural Language Processing (BioNLP 2019)}, - year = {2019}, - pages = {58--65}, -} -``` - -### Acknowledgments - -This work was supported by the Intramural Research Programs of the National Institutes of Health, National Library of -Medicine and Clinical Center. This work was supported by the National Library of Medicine of the National Institutes of Health under award number 4R00LM013001-01. - -We are also grateful to the authors of BERT and ELMo to make the data and codes publicly available. - -We would like to thank Dr Sun Kim for processing the PubMed texts. - -### Disclaimer - -This tool shows the results of research conducted in the Computational Biology Branch, NCBI. The information produced -on this website is not intended for direct diagnostic use or medical decision-making without review and oversight -by a clinical professional. Individuals should not change their health behavior solely on the basis of information -produced on this website. NIH does not independently verify the validity or utility of the information produced -by this tool. If you have questions about the information produced on this website, please see a health care -professional. More information about NCBI's disclaimer policy is available. diff --git a/model_cards/bionlp/bluebert_pubmed_uncased_L-12_H-768_A-12/README.md b/model_cards/bionlp/bluebert_pubmed_uncased_L-12_H-768_A-12/README.md deleted file mode 100644 index 7701bb25f48c..000000000000 --- a/model_cards/bionlp/bluebert_pubmed_uncased_L-12_H-768_A-12/README.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -language: -- en -tags: -- bluebert -license: -- PUBLIC DOMAIN NOTICE -datasets: -- pubmed - ---- - -# BlueBert-Base, Uncased, PubMed - -## Model description - -A BERT model pre-trained on PubMed abstracts - -## Intended uses & limitations - -#### How to use - -Please see https://github.com/ncbi-nlp/bluebert - -## Training data - -We provide [preprocessed PubMed texts](https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/NCBI-BERT/pubmed_uncased_sentence_nltk.txt.tar.gz) that were used to pre-train the BlueBERT models. -The corpus contains ~4000M words extracted from the [PubMed ASCII code version](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PubMed/). - -Pre-trained model: https://huggingface.co/bert-base-uncased - -## Training procedure - -* lowercasing the text -* removing speical chars `\x00`-`\x7F` -* tokenizing the text using the [NLTK Treebank tokenizer](https://www.nltk.org/_modules/nltk/tokenize/treebank.html) - -Below is a code snippet for more details. - -```python -value = value.lower() -value = re.sub(r'[\r\n]+', ' ', value) -value = re.sub(r'[^\x00-\x7F]+', ' ', value) - -tokenized = TreebankWordTokenizer().tokenize(value) -sentence = ' '.join(tokenized) -sentence = re.sub(r"\s's\b", "'s", sentence) -``` - -### BibTeX entry and citation info - -```bibtex -@InProceedings{peng2019transfer, - author = {Yifan Peng and Shankai Yan and Zhiyong Lu}, - title = {Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets}, - booktitle = {Proceedings of the 2019 Workshop on Biomedical Natural Language Processing (BioNLP 2019)}, - year = {2019}, - pages = {58--65}, -} -``` diff --git a/model_cards/bionlp/bluebert_pubmed_uncased_L-24_H-1024_A-16/README.md b/model_cards/bionlp/bluebert_pubmed_uncased_L-24_H-1024_A-16/README.md deleted file mode 100644 index 96618f7fb5c0..000000000000 --- a/model_cards/bionlp/bluebert_pubmed_uncased_L-24_H-1024_A-16/README.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -language: -- en -tags: -- bert -- bluebert -license: -- PUBLIC DOMAIN NOTICE -datasets: -- PubMed - ---- - -# BlueBert-Base, Uncased, PubMed - -## Model description - -A BERT model pre-trained on PubMed abstracts. - -## Intended uses & limitations - -#### How to use - -Please see https://github.com/ncbi-nlp/bluebert - -## Training data - -We provide [preprocessed PubMed texts](https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/NCBI-BERT/pubmed_uncased_sentence_nltk.txt.tar.gz) that were used to pre-train the BlueBERT models. -The corpus contains ~4000M words extracted from the [PubMed ASCII code version](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/BioC-PubMed/). - -Pre-trained model: https://huggingface.co/bert-large-uncased - -## Training procedure - -* lowercasing the text -* removing speical chars `\x00`-`\x7F` -* tokenizing the text using the [NLTK Treebank tokenizer](https://www.nltk.org/_modules/nltk/tokenize/treebank.html) - -Below is a code snippet for more details. - -```python -value = value.lower() -value = re.sub(r'[\r\n]+', ' ', value) -value = re.sub(r'[^\x00-\x7F]+', ' ', value) - -tokenized = TreebankWordTokenizer().tokenize(value) -sentence = ' '.join(tokenized) -sentence = re.sub(r"\s's\b", "'s", sentence) -``` - -### BibTeX entry and citation info - -```bibtex -@InProceedings{peng2019transfer, - author = {Yifan Peng and Shankai Yan and Zhiyong Lu}, - title = {Transfer Learning in Biomedical Natural Language Processing: An Evaluation of BERT and ELMo on Ten Benchmarking Datasets}, - booktitle = {Proceedings of the 2019 Workshop on Biomedical Natural Language Processing (BioNLP 2019)}, - year = {2019}, - pages = {58--65}, -} -``` - -### Acknowledgments - -This work was supported by the Intramural Research Programs of the National Institutes of Health, National Library of -Medicine and Clinical Center. This work was supported by the National Library of Medicine of the National Institutes of Health under award number 4R00LM013001-01. - -We are also grateful to the authors of BERT and ELMo to make the data and codes publicly available. - -We would like to thank Dr Sun Kim for processing the PubMed texts. - -### Disclaimer - -This tool shows the results of research conducted in the Computational Biology Branch, NCBI. The information produced -on this website is not intended for direct diagnostic use or medical decision-making without review and oversight -by a clinical professional. Individuals should not change their health behavior solely on the basis of information -produced on this website. NIH does not independently verify the validity or utility of the information produced -by this tool. If you have questions about the information produced on this website, please see a health care -professional. More information about NCBI's disclaimer policy is available. diff --git a/model_cards/blinoff/roberta-base-russian-v0/README.md b/model_cards/blinoff/roberta-base-russian-v0/README.md deleted file mode 100644 index 9ae6b45af2a7..000000000000 --- a/model_cards/blinoff/roberta-base-russian-v0/README.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -language: ru -widget: -- text: "Мозг — это машина , которая пытается снизить ошибку в прогнозе." ---- - -# RoBERTa-like language model trained on part of part of TAIGA corpus - -## Training Details - -- about 60k steps - -![]() - -## Example pipeline - -```python -from transformers import pipeline -from transformers import RobertaTokenizerFast - -tokenizer = RobertaTokenizerFast.from_pretrained('blinoff/roberta-base-russian-v0', max_len=512) - -fill_mask = pipeline( - "fill-mask", - model="blinoff/roberta-base-russian-v0", - tokenizer=tokenizer -) - -fill_mask("Мозг — это машина , которая пытается снизить ошибку в прогнозе.") - -# { -# 'sequence': 'Мозг — это машина города, которая пытается снизить ошибку в прогнозе.', -# 'score': 0.012859329581260681, -# 'token': 2144, -# 'token_str': 'ĠгоÑĢода' -# }, -# { -# 'sequence': 'Мозг — это машина человека, которая пытается снизить ошибку в прогнозе.', -# 'score': 0.01185101643204689, -# 'token': 1470, -# 'token_str': 'ĠÑĩеловека' -# }, -# { -# 'sequence': 'Мозг — это машина дома, которая пытается снизить ошибку в прогнозе.', -# 'score': 0.009940559044480324, -# 'token': 1411, -# 'token_str': 'Ġдома' -# }, -# { -# 'sequence': 'Мозг — это машина женщина, которая пытается снизить ошибку в прогнозе.', -# 'score': 0.007794599514454603, -# 'token': 2707, -# 'token_str': 'ĠженÑīина' -# }, -# { -# 'sequence': 'Мозг — это машина женщины, которая пытается снизить ошибку в прогнозе.', -# 'score': 0.007725382689386606, -# 'token': 3546, -# 'token_str': 'ĠженÑīинÑĭ' -# } -``` diff --git a/model_cards/cahya/bert-base-indonesian-522M/README.md b/model_cards/cahya/bert-base-indonesian-522M/README.md deleted file mode 100644 index 6008e770adc6..000000000000 --- a/model_cards/cahya/bert-base-indonesian-522M/README.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -language: "id" -license: "mit" -datasets: -- Indonesian Wikipedia -widget: -- text: "Ibu ku sedang bekerja [MASK] supermarket." ---- - -# Indonesian BERT base model (uncased) - -## Model description -It is BERT-base model pre-trained with indonesian Wikipedia using a masked language modeling (MLM) objective. This -model is uncased: it does not make a difference between indonesia and Indonesia. - -This is one of several other language models that have been pre-trained with indonesian datasets. More detail about -its usage on downstream tasks (text classification, text generation, etc) is available at [Transformer based Indonesian Language Models](https://github.com/cahya-wirawan/indonesian-language-models/tree/master/Transformers) - -## Intended uses & limitations - -### How to use -You can use this model directly with a pipeline for masked language modeling: -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='cahya/bert-base-indonesian-522M') ->>> unmasker("Ibu ku sedang bekerja [MASK] supermarket") - -[{'sequence': '[CLS] ibu ku sedang bekerja di supermarket [SEP]', - 'score': 0.7983310222625732, - 'token': 1495}, - {'sequence': '[CLS] ibu ku sedang bekerja. supermarket [SEP]', - 'score': 0.090003103017807, - 'token': 17}, - {'sequence': '[CLS] ibu ku sedang bekerja sebagai supermarket [SEP]', - 'score': 0.025469014421105385, - 'token': 1600}, - {'sequence': '[CLS] ibu ku sedang bekerja dengan supermarket [SEP]', - 'score': 0.017966199666261673, - 'token': 1555}, - {'sequence': '[CLS] ibu ku sedang bekerja untuk supermarket [SEP]', - 'score': 0.016971781849861145, - 'token': 1572}] -``` -Here is how to use this model to get the features of a given text in PyTorch: -```python -from transformers import BertTokenizer, BertModel - -model_name='cahya/bert-base-indonesian-522M' -tokenizer = BertTokenizer.from_pretrained(model_name) -model = BertModel.from_pretrained(model_name) -text = "Silakan diganti dengan text apa saja." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` -and in Tensorflow: -```python -from transformers import BertTokenizer, TFBertModel - -model_name='cahya/bert-base-indonesian-522M' -tokenizer = BertTokenizer.from_pretrained(model_name) -model = TFBertModel.from_pretrained(model_name) -text = "Silakan diganti dengan text apa saja." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -This model was pre-trained with 522MB of indonesian Wikipedia. -The texts are lowercased and tokenized using WordPiece and a vocabulary size of 32,000. The inputs of the model are -then of the form: - -```[CLS] Sentence A [SEP] Sentence B [SEP]``` diff --git a/model_cards/cahya/gpt2-small-indonesian-522M/README.md b/model_cards/cahya/gpt2-small-indonesian-522M/README.md deleted file mode 100644 index cfd991839b5a..000000000000 --- a/model_cards/cahya/gpt2-small-indonesian-522M/README.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -language: "id" -license: "mit" -datasets: -- Indonesian Wikipedia -widget: -- text: "Pulau Dewata sering dikunjungi" ---- - -# Indonesian GPT2 small model - -## Model description -It is GPT2-small model pre-trained with indonesian Wikipedia using a causal language modeling (CLM) objective. This -model is uncased: it does not make a difference between indonesia and Indonesia. - -This is one of several other language models that have been pre-trained with indonesian datasets. More detail about -its usage on downstream tasks (text classification, text generation, etc) is available at [Transformer based Indonesian Language Models](https://github.com/cahya-wirawan/indonesian-language-models/tree/master/Transformers) - -## Intended uses & limitations - -### How to use -You can use this model directly with a pipeline for text generation. Since the generation relies on some randomness, -we set a seed for reproducibility: -```python ->>> from transformers import pipeline, set_seed ->>> generator = pipeline('text-generation', model='cahya/gpt2-small-indonesian-522M') ->>> set_seed(42) ->>> generator("Kerajaan Majapahit adalah", max_length=30, num_return_sequences=5, num_beams=10) - -[{'generated_text': 'Kerajaan Majapahit adalah sebuah kerajaan yang pernah berdiri di Jawa Timur pada abad ke-14 hingga abad ke-15. Kerajaan ini berdiri pada abad ke-14'}, -{'generated_text': 'Kerajaan Majapahit adalah sebuah kerajaan yang pernah berdiri di Jawa Timur pada abad ke-14 hingga abad ke-16. Kerajaan ini berdiri pada abad ke-14'}, -{'generated_text': 'Kerajaan Majapahit adalah sebuah kerajaan yang pernah berdiri di Jawa Timur pada abad ke-14 hingga abad ke-15. Kerajaan ini berdiri pada abad ke-15'}, -{'generated_text': 'Kerajaan Majapahit adalah sebuah kerajaan yang pernah berdiri di Jawa Timur pada abad ke-14 hingga abad ke-16. Kerajaan ini berdiri pada abad ke-15'}, -{'generated_text': 'Kerajaan Majapahit adalah sebuah kerajaan yang pernah berdiri di Jawa Timur pada abad ke-14 hingga abad ke-15. Kerajaan ini merupakan kelanjutan dari Kerajaan Majapahit yang'}] - -``` -Here is how to use this model to get the features of a given text in PyTorch: -```python -from transformers import GPT2Tokenizer, GPT2Model - -model_name='cahya/gpt2-small-indonesian-522M' -tokenizer = GPT2Tokenizer.from_pretrained(model_name) -model = GPT2Model.from_pretrained(model_name) -text = "Silakan diganti dengan text apa saja." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` -and in Tensorflow: -```python -from transformers import GPT2Tokenizer, TFGPT2Model - -model_name='cahya/gpt2-small-indonesian-522M' -tokenizer = GPT2Tokenizer.from_pretrained(model_name) -model = TFGPT2Model.from_pretrained(model_name) -text = "Silakan diganti dengan text apa saja." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -This model was pre-trained with 522MB of indonesian Wikipedia. -The texts are tokenized using a byte-level version of Byte Pair Encoding (BPE) (for unicode characters) and -a vocabulary size of 52,000. The inputs are sequences of 128 consecutive tokens. diff --git a/model_cards/cahya/roberta-base-indonesian-522M/README.md b/model_cards/cahya/roberta-base-indonesian-522M/README.md deleted file mode 100644 index 84e2aa210b28..000000000000 --- a/model_cards/cahya/roberta-base-indonesian-522M/README.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -language: "id" -license: "mit" -datasets: -- Indonesian Wikipedia -widget: -- text: "Ibu ku sedang bekerja supermarket." ---- - -# Indonesian RoBERTa base model (uncased) - -## Model description -It is RoBERTa-base model pre-trained with indonesian Wikipedia using a masked language modeling (MLM) objective. This -model is uncased: it does not make a difference between indonesia and Indonesia. - -This is one of several other language models that have been pre-trained with indonesian datasets. More detail about -its usage on downstream tasks (text classification, text generation, etc) is available at [Transformer based Indonesian Language Models](https://github.com/cahya-wirawan/indonesian-language-models/tree/master/Transformers) - -## Intended uses & limitations - -### How to use -You can use this model directly with a pipeline for masked language modeling: -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='cahya/roberta-base-indonesian-522M') ->>> unmasker("Ibu ku sedang bekerja supermarket") - -``` -Here is how to use this model to get the features of a given text in PyTorch: -```python -from transformers import RobertaTokenizer, RobertaModel - -model_name='cahya/roberta-base-indonesian-522M' -tokenizer = RobertaTokenizer.from_pretrained(model_name) -model = RobertaModel.from_pretrained(model_name) -text = "Silakan diganti dengan text apa saja." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` -and in Tensorflow: -```python -from transformers import RobertaTokenizer, TFRobertaModel - -model_name='cahya/roberta-base-indonesian-522M' -tokenizer = RobertaTokenizer.from_pretrained(model_name) -model = TFRobertaModel.from_pretrained(model_name) -text = "Silakan diganti dengan text apa saja." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -This model was pre-trained with 522MB of indonesian Wikipedia. -The texts are lowercased and tokenized using WordPiece and a vocabulary size of 32,000. The inputs of the model are -then of the form: - -``` Sentence A Sentence B ``` diff --git a/model_cards/cambridgeltl/BioRedditBERT-uncased/README.md b/model_cards/cambridgeltl/BioRedditBERT-uncased/README.md deleted file mode 100644 index 75adafaa903b..000000000000 --- a/model_cards/cambridgeltl/BioRedditBERT-uncased/README.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -language: -- en -tags: -- BioNLP -- social_media ---- - -# BioRedditBERT - -## Model description -BioRedditBERT is a BERT model initialised from BioBERT (`BioBERT-Base v1.0 + PubMed 200K + PMC 270K`) and further pre-trained on health-related Reddit posts. Please view our paper [COMETA: A Corpus for Medical Entity Linking in the Social Media](https://arxiv.org/pdf/2010.03295.pdf) (EMNLP 2020) for more details. - - -## Training data - -We crawled all threads from 68 health themed subreddits such as `r/AskDocs`, `r/health` and etc. starting from the beginning of 2015 to the end of 2018, obtaining a collection of more than -800K discussions. This collection was then pruned by removing deleted posts, comments from bots or moderators, and so on. In the end, we obtained the training corpus with ca. 300 million tokens and a vocabulary -size of ca. 780,000 words. - -## Training procedure -We use the same pre-training script in the original [google-research/bert](https://github.com/google-research/bert) repo. The model is initialised with [`BioBERT-Base v1.0 + PubMed 200K + PMC 270K`](https://github.com/dmis-lab/biobert). -We train with a batch size of 64, a max sequence length of 64, a learning rate of `2e-5` for 100k steps on two GeForce GTX 1080Ti (11 GB) GPUs. Other hyper-parameters are the same as default. - - -## Eval results -To show the benefit from further pre-training on the social media domain, we demonstrate results on a medical entity linking dataset also in the social media: [AskAPatient](https://zenodo.org/record/55013#.X4ncRmTYpb8) [(Limsopatham and Collier 2016)](https://www.aclweb.org/anthology/P16-1096.pdf). -We follow the same 10-fold cross-validation procedure for all models and report the average result without fine-tuning. `[CLS]` is used as representations for entity mentions (we also tried average of all tokens but found `[CLS]` generally performs better). - -Model | Accuracy@1 | Accuracy@5 --------|---------|--------- -[BERT-base-uncased](https://huggingface.co/bert-base-uncased) | 38.2 | 43.3 -[BioBERT v1.1](https://huggingface.co/dmis-lab/biobert-v1.1) | 41.4 | 51.5 -[ClinicalBERT](https://huggingface.co/emilyalsentzer/Bio_ClinicalBERT) | 43.9 | 54.3 -[BlueBERT](https://ftp.ncbi.nlm.nih.gov/pub/lu/Suppl/NCBI-BERT/NCBI_BERT_pubmed_mimic_uncased_L-12_H-768_A-12.zip) | 41.5 | 48.5 -[SciBERT](https://huggingface.co/allenai/scibert_scivocab_uncased) | 42.3 | 51.9 -[PubMedBERT](https://huggingface.co/microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext) | 42.5 | 49.6 -BioRedditBERT | **44.3** | **56.2** - - -### BibTeX entry and citation info - -```bibtex -@inproceedings{basaldella-2020-cometa, - title = "{COMETA}: A Corpus for Medical Entity Linking in the Social Media", - author = "Basaldella, Marco and Liu, Fangyu, and Shareghi, Ehsan, and Collier, Nigel", - booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing", - month = nov, - year = "2020", - publisher = "Association for Computational Linguistics" -} -``` diff --git a/model_cards/camembert-base-README.md b/model_cards/camembert-base-README.md deleted file mode 100644 index f73b990705d0..000000000000 --- a/model_cards/camembert-base-README.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -language: fr -license: mit -datasets: -- oscar ---- - -# CamemBERT: a Tasty French Language Model - -## Introduction - -[CamemBERT](https://arxiv.org/abs/1911.03894) is a state-of-the-art language model for French based on the RoBERTa model. - -It is now available on Hugging Face in 6 different versions with varying number of parameters, amount of pretraining data and pretraining data source domains. - -For further information or requests, please go to [Camembert Website](https://camembert-model.fr/) - -## Pre-trained models - -| Model | #params | Arch. | Training data | -|--------------------------------|--------------------------------|-------|-----------------------------------| -| `camembert-base` | 110M | Base | OSCAR (138 GB of text) | -| `camembert/camembert-large` | 335M | Large | CCNet (135 GB of text) | -| `camembert/camembert-base-ccnet` | 110M | Base | CCNet (135 GB of text) | -| `camembert/camembert-base-wikipedia-4gb` | 110M | Base | Wikipedia (4 GB of text) | -| `camembert/camembert-base-oscar-4gb` | 110M | Base | Subsample of OSCAR (4 GB of text) | -| `camembert/camembert-base-ccnet-4gb` | 110M | Base | Subsample of CCNet (4 GB of text) | - -## How to use CamemBERT with HuggingFace - -##### Load CamemBERT and its sub-word tokenizer : -```python -from transformers import CamembertModel, CamembertTokenizer - -# You can replace "camembert-base" with any other model from the table, e.g. "camembert/camembert-large". -tokenizer = CamembertTokenizer.from_pretrained("camembert-base") -camembert = CamembertModel.from_pretrained("camembert-base") - -camembert.eval() # disable dropout (or leave in train mode to finetune) - -``` - -##### Filling masks using pipeline -```python -from transformers import pipeline - -camembert_fill_mask = pipeline("fill-mask", model="camembert-base", tokenizer="camembert-base") -results = camembert_fill_mask("Le camembert est :)") -# results -#[{'sequence': ' Le camembert est délicieux :)', 'score': 0.4909103214740753, 'token': 7200}, -# {'sequence': ' Le camembert est excellent :)', 'score': 0.10556930303573608, 'token': 2183}, -# {'sequence': ' Le camembert est succulent :)', 'score': 0.03453315049409866, 'token': 26202}, -# {'sequence': ' Le camembert est meilleur :)', 'score': 0.03303130343556404, 'token': 528}, -# {'sequence': ' Le camembert est parfait :)', 'score': 0.030076518654823303, 'token': 1654}] - -``` - -##### Extract contextual embedding features from Camembert output -```python -import torch -# Tokenize in sub-words with SentencePiece -tokenized_sentence = tokenizer.tokenize("J'aime le camembert !") -# ['▁J', "'", 'aime', '▁le', '▁ca', 'member', 't', '▁!'] - -# 1-hot encode and add special starting and end tokens -encoded_sentence = tokenizer.encode(tokenized_sentence) -# [5, 121, 11, 660, 16, 730, 25543, 110, 83, 6] -# NB: Can be done in one step : tokenize.encode("J'aime le camembert !") - -# Feed tokens to Camembert as a torch tensor (batch dim 1) -encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0) -embeddings, _ = camembert(encoded_sentence) -# embeddings.detach() -# embeddings.size torch.Size([1, 10, 768]) -# tensor([[[-0.0254, 0.0235, 0.1027, ..., -0.1459, -0.0205, -0.0116], -# [ 0.0606, -0.1811, -0.0418, ..., -0.1815, 0.0880, -0.0766], -# [-0.1561, -0.1127, 0.2687, ..., -0.0648, 0.0249, 0.0446], -# ..., -``` - -##### Extract contextual embedding features from all Camembert layers -```python -from transformers import CamembertConfig -# (Need to reload the model with new config) -config = CamembertConfig.from_pretrained("camembert-base", output_hidden_states=True) -camembert = CamembertModel.from_pretrained("camembert-base", config=config) - -embeddings, _, all_layer_embeddings = camembert(encoded_sentence) -# all_layer_embeddings list of len(all_layer_embeddings) == 13 (input embedding layer + 12 self attention layers) -all_layer_embeddings[5] -# layer 5 contextual embedding : size torch.Size([1, 10, 768]) -#tensor([[[-0.0032, 0.0075, 0.0040, ..., -0.0025, -0.0178, -0.0210], -# [-0.0996, -0.1474, 0.1057, ..., -0.0278, 0.1690, -0.2982], -# [ 0.0557, -0.0588, 0.0547, ..., -0.0726, -0.0867, 0.0699], -# ..., -``` - - -## Authors - -CamemBERT was trained and evaluated by Louis Martin\*, Benjamin Muller\*, Pedro Javier Ortiz Suárez\*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. - - -## Citation -If you use our work, please cite: - -```bibtex -@inproceedings{martin2020camembert, - title={CamemBERT: a Tasty French Language Model}, - author={Martin, Louis and Muller, Benjamin and Su{\'a}rez, Pedro Javier Ortiz and Dupont, Yoann and Romary, Laurent and de la Clergerie, {\'E}ric Villemonte and Seddah, Djam{\'e} and Sagot, Beno{\^\i}t}, - booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, - year={2020} -} -``` - diff --git a/model_cards/camembert/camembert-base-ccnet-4gb/README.md b/model_cards/camembert/camembert-base-ccnet-4gb/README.md deleted file mode 100644 index 96c2b4f06b76..000000000000 --- a/model_cards/camembert/camembert-base-ccnet-4gb/README.md +++ /dev/null @@ -1,111 +0,0 @@ ---- -language: fr ---- - -# CamemBERT: a Tasty French Language Model - -## Introduction - -[CamemBERT](https://arxiv.org/abs/1911.03894) is a state-of-the-art language model for French based on the RoBERTa model. - -It is now available on Hugging Face in 6 different versions with varying number of parameters, amount of pretraining data and pretraining data source domains. - -For further information or requests, please go to [Camembert Website](https://camembert-model.fr/) - -## Pre-trained models - -| Model | #params | Arch. | Training data | -|--------------------------------|--------------------------------|-------|-----------------------------------| -| `camembert-base` | 110M | Base | OSCAR (138 GB of text) | -| `camembert/camembert-large` | 335M | Large | CCNet (135 GB of text) | -| `camembert/camembert-base-ccnet` | 110M | Base | CCNet (135 GB of text) | -| `camembert/camembert-base-wikipedia-4gb` | 110M | Base | Wikipedia (4 GB of text) | -| `camembert/camembert-base-oscar-4gb` | 110M | Base | Subsample of OSCAR (4 GB of text) | -| `camembert/camembert-base-ccnet-4gb` | 110M | Base | Subsample of CCNet (4 GB of text) | - -## How to use CamemBERT with HuggingFace - -##### Load CamemBERT and its sub-word tokenizer : -```python -from transformers import CamembertModel, CamembertTokenizer - -# You can replace "camembert-base" with any other model from the table, e.g. "camembert/camembert-large". -tokenizer = CamembertTokenizer.from_pretrained("camembert/camembert-base-ccnet-4gb") -camembert = CamembertModel.from_pretrained("camembert/camembert-base-ccnet-4gb") - -camembert.eval() # disable dropout (or leave in train mode to finetune) - -``` - -##### Filling masks using pipeline -```python -from transformers import pipeline - -camembert_fill_mask = pipeline("fill-mask", model="camembert/camembert-base-ccnet-4gb", tokenizer="camembert/camembert-base-ccnet-4gb") -results = camembert_fill_mask("Le camembert est-il ?") -# results -#[{'sequence': ' Le camembert est-il sain?', 'score': 0.07001790404319763, 'token': 10286}, -#{'sequence': ' Le camembert est-il français?', 'score': 0.057594332844018936, 'token': 384}, -#{'sequence': ' Le camembert est-il bon?', 'score': 0.04098724573850632, 'token': 305}, -#{'sequence': ' Le camembert est-il périmé?', 'score': 0.03486393392086029, 'token': 30862}, -#{'sequence': ' Le camembert est-il cher?', 'score': 0.021535946056246758, 'token': 1604}] - -``` - -##### Extract contextual embedding features from Camembert output -```python -import torch -# Tokenize in sub-words with SentencePiece -tokenized_sentence = tokenizer.tokenize("J'aime le camembert !") -# ['▁J', "'", 'aime', '▁le', '▁ca', 'member', 't', '▁!'] - -# 1-hot encode and add special starting and end tokens -encoded_sentence = tokenizer.encode(tokenized_sentence) -# [5, 133, 22, 1250, 16, 12034, 14324, 81, 76, 6] -# NB: Can be done in one step : tokenize.encode("J'aime le camembert !") - -# Feed tokens to Camembert as a torch tensor (batch dim 1) -encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0) -embeddings, _ = camembert(encoded_sentence) -# embeddings.detach() -# embeddings.size torch.Size([1, 10, 768]) -#tensor([[[ 0.0331, 0.0095, -0.2776, ..., 0.2875, -0.0827, -0.2467], -# [-0.1348, 0.0478, -0.5409, ..., 0.8330, 0.0467, 0.0662], -# [ 0.0920, -0.0264, 0.0177, ..., 0.1112, 0.0108, -0.1123], -# ..., -``` - -##### Extract contextual embedding features from all Camembert layers -```python -from transformers import CamembertConfig -# (Need to reload the model with new config) -config = CamembertConfig.from_pretrained("camembert/camembert-base-ccnet-4gb", output_hidden_states=True) -camembert = CamembertModel.from_pretrained("camembert/camembert-base-ccnet-4gb", config=config) - -embeddings, _, all_layer_embeddings = camembert(encoded_sentence) -# all_layer_embeddings list of len(all_layer_embeddings) == 13 (input embedding layer + 12 self attention layers) -all_layer_embeddings[5] -# layer 5 contextual embedding : size torch.Size([1, 10, 768]) -#tensor([[[-0.0144, 0.1855, 0.4895, ..., -0.1537, 0.0107, -0.2293], -# [-0.6664, -0.0880, -0.1539, ..., 0.3635, 0.4047, 0.1258], -# [ 0.0511, 0.0540, 0.2545, ..., 0.0709, -0.0288, -0.0779], -# ..., -``` - - -## Authors - -CamemBERT was trained and evaluated by Louis Martin\*, Benjamin Muller\*, Pedro Javier Ortiz Suárez\*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. - - -## Citation -If you use our work, please cite: - -```bibtex -@inproceedings{martin2020camembert, - title={CamemBERT: a Tasty French Language Model}, - author={Martin, Louis and Muller, Benjamin and Su{\'a}rez, Pedro Javier Ortiz and Dupont, Yoann and Romary, Laurent and de la Clergerie, {\'E}ric Villemonte and Seddah, Djam{\'e} and Sagot, Beno{\^\i}t}, - booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, - year={2020} -} -``` diff --git a/model_cards/camembert/camembert-base-ccnet/README.md b/model_cards/camembert/camembert-base-ccnet/README.md deleted file mode 100644 index 63b1b89ef3fd..000000000000 --- a/model_cards/camembert/camembert-base-ccnet/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: fr ---- - -# CamemBERT: a Tasty French Language Model - -## Introduction - -[CamemBERT](https://arxiv.org/abs/1911.03894) is a state-of-the-art language model for French based on the RoBERTa model. - -It is now available on Hugging Face in 6 different versions with varying number of parameters, amount of pretraining data and pretraining data source domains. - -For further information or requests, please go to [Camembert Website](https://camembert-model.fr/) - -## Pre-trained models - -| Model | #params | Arch. | Training data | -|--------------------------------|--------------------------------|-------|-----------------------------------| -| `camembert-base` | 110M | Base | OSCAR (138 GB of text) | -| `camembert/camembert-large` | 335M | Large | CCNet (135 GB of text) | -| `camembert/camembert-base-ccnet` | 110M | Base | CCNet (135 GB of text) | -| `camembert/camembert-base-wikipedia-4gb` | 110M | Base | Wikipedia (4 GB of text) | -| `camembert/camembert-base-oscar-4gb` | 110M | Base | Subsample of OSCAR (4 GB of text) | -| `camembert/camembert-base-ccnet-4gb` | 110M | Base | Subsample of CCNet (4 GB of text) | - -## How to use CamemBERT with HuggingFace - -##### Load CamemBERT and its sub-word tokenizer : -```python -from transformers import CamembertModel, CamembertTokenizer - -# You can replace "camembert-base" with any other model from the table, e.g. "camembert/camembert-large". -tokenizer = CamembertTokenizer.from_pretrained("camembert/camembert-base-ccnet") -camembert = CamembertModel.from_pretrained("camembert/camembert-base-ccnet") - -camembert.eval() # disable dropout (or leave in train mode to finetune) - -``` - -##### Filling masks using pipeline -```python -from transformers import pipeline - -camembert_fill_mask = pipeline("fill-mask", model="camembert/camembert-base-ccnet", tokenizer="camembert/camembert-base-ccnet") -results = camembert_fill_mask("Le camembert est :)") -# results -#[{'sequence': ' Le camembert est bon :)', 'score': 0.14011502265930176, 'token': 305}, -# {'sequence': ' Le camembert est délicieux :)', 'score': 0.13929404318332672, 'token': 11661}, -# {'sequence': ' Le camembert est excellent :)', 'score': 0.07010319083929062, 'token': 3497}, -# {'sequence': ' Le camembert est parfait :)', 'score': 0.025885622948408127, 'token': 2528}, -# {'sequence': ' Le camembert est top :)', 'score': 0.025684962049126625, 'token': 2328}] -``` - -##### Extract contextual embedding features from Camembert output -```python -import torch -# Tokenize in sub-words with SentencePiece -tokenized_sentence = tokenizer.tokenize("J'aime le camembert !") -# ['▁J', "'", 'aime', '▁le', '▁cam', 'ember', 't', '▁!'] - -# 1-hot encode and add special starting and end tokens -encoded_sentence = tokenizer.encode(tokenized_sentence) -# [5, 133, 22, 1250, 16, 12034, 14324, 81, 76, 6] -# NB: Can be done in one step : tokenize.encode("J'aime le camembert !") - -# Feed tokens to Camembert as a torch tensor (batch dim 1) -encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0) -embeddings, _ = camembert(encoded_sentence) -# embeddings.detach() -# embeddings.size torch.Size([1, 10, 768]) -#tensor([[[ 0.0667, -0.2467, 0.0954, ..., 0.2144, 0.0279, 0.3621], -# [-0.0472, 0.4092, -0.6602, ..., 0.2095, 0.1391, -0.0401], -# [ 0.1911, -0.2347, -0.0811, ..., 0.4306, -0.0639, 0.1821], -# ..., -``` - -##### Extract contextual embedding features from all Camembert layers -```python -from transformers import CamembertConfig -# (Need to reload the model with new config) -config = CamembertConfig.from_pretrained("camembert/camembert-base-ccnet", output_hidden_states=True) -camembert = CamembertModel.from_pretrained("camembert/camembert-base-ccnet", config=config) - -embeddings, _, all_layer_embeddings = camembert(encoded_sentence) -# all_layer_embeddings list of len(all_layer_embeddings) == 13 (input embedding layer + 12 self attention layers) -all_layer_embeddings[5] -# layer 5 contextual embedding : size torch.Size([1, 10, 768]) -#tensor([[[ 0.0057, -0.1022, 0.0163, ..., -0.0675, -0.0360, 0.1078], -# [-0.1096, -0.3344, -0.0593, ..., 0.1625, -0.0432, -0.1646], -# [ 0.3751, -0.3829, 0.0844, ..., 0.1067, -0.0330, 0.3334], -# ..., -``` - - -## Authors - -CamemBERT was trained and evaluated by Louis Martin\*, Benjamin Muller\*, Pedro Javier Ortiz Suárez\*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. - - -## Citation -If you use our work, please cite: - -```bibtex -@inproceedings{martin2020camembert, - title={CamemBERT: a Tasty French Language Model}, - author={Martin, Louis and Muller, Benjamin and Su{\'a}rez, Pedro Javier Ortiz and Dupont, Yoann and Romary, Laurent and de la Clergerie, {\'E}ric Villemonte and Seddah, Djam{\'e} and Sagot, Beno{\^\i}t}, - booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, - year={2020} -} -``` diff --git a/model_cards/camembert/camembert-base-oscar-4gb/README.md b/model_cards/camembert/camembert-base-oscar-4gb/README.md deleted file mode 100644 index eee04e68e08d..000000000000 --- a/model_cards/camembert/camembert-base-oscar-4gb/README.md +++ /dev/null @@ -1,111 +0,0 @@ ---- -language: fr ---- - -# CamemBERT: a Tasty French Language Model - -## Introduction - -[CamemBERT](https://arxiv.org/abs/1911.03894) is a state-of-the-art language model for French based on the RoBERTa model. - -It is now available on Hugging Face in 6 different versions with varying number of parameters, amount of pretraining data and pretraining data source domains. - -For further information or requests, please go to [Camembert Website](https://camembert-model.fr/) - -## Pre-trained models - -| Model | #params | Arch. | Training data | -|--------------------------------|--------------------------------|-------|-----------------------------------| -| `camembert-base` | 110M | Base | OSCAR (138 GB of text) | -| `camembert/camembert-large` | 335M | Large | CCNet (135 GB of text) | -| `camembert/camembert-base-ccnet` | 110M | Base | CCNet (135 GB of text) | -| `camembert/camembert-base-wikipedia-4gb` | 110M | Base | Wikipedia (4 GB of text) | -| `camembert/camembert-base-oscar-4gb` | 110M | Base | Subsample of OSCAR (4 GB of text) | -| `camembert/camembert-base-ccnet-4gb` | 110M | Base | Subsample of CCNet (4 GB of text) | - -## How to use CamemBERT with HuggingFace - -##### Load CamemBERT and its sub-word tokenizer : -```python -from transformers import CamembertModel, CamembertTokenizer - -# You can replace "camembert-base" with any other model from the table, e.g. "camembert/camembert-large". -tokenizer = CamembertTokenizer.from_pretrained("camembert/camembert-base-oscar-4gb") -camembert = CamembertModel.from_pretrained("camembert/camembert-base-oscar-4gb") - -camembert.eval() # disable dropout (or leave in train mode to finetune) - -``` - -##### Filling masks using pipeline -```python -from transformers import pipeline - -camembert_fill_mask = pipeline("fill-mask", model="camembert/camembert-base-oscar-4gb", tokenizer="camembert/camembert-base-oscar-4gb") ->>> results = camembert_fill_mask("Le camembert est !") -# results -#[{'sequence': ' Le camembert est parfait!', 'score': 0.04089554399251938, 'token': 1654}, -#{'sequence': ' Le camembert est délicieux!', 'score': 0.037193264812231064, 'token': 7200}, -#{'sequence': ' Le camembert est prêt!', 'score': 0.025467922911047935, 'token': 1415}, -#{'sequence': ' Le camembert est meilleur!', 'score': 0.022812040522694588, 'token': 528}, -#{'sequence': ' Le camembert est différent!', 'score': 0.017135459929704666, 'token': 2935}] - -``` - -##### Extract contextual embedding features from Camembert output -```python -import torch -# Tokenize in sub-words with SentencePiece -tokenized_sentence = tokenizer.tokenize("J'aime le camembert !") -# ['▁J', "'", 'aime', '▁le', '▁ca', 'member', 't', '▁!'] - -# 1-hot encode and add special starting and end tokens -encoded_sentence = tokenizer.encode(tokenized_sentence) -# [5, 121, 11, 660, 16, 730, 25543, 110, 83, 6] -# NB: Can be done in one step : tokenize.encode("J'aime le camembert !") - -# Feed tokens to Camembert as a torch tensor (batch dim 1) -encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0) -embeddings, _ = camembert(encoded_sentence) -# embeddings.detach() -# embeddings.size torch.Size([1, 10, 768]) -#tensor([[[-0.1120, -0.1464, 0.0181, ..., -0.1723, -0.0278, 0.1606], -# [ 0.1234, 0.1202, -0.0773, ..., -0.0405, -0.0668, -0.0788], -# [-0.0440, 0.0480, -0.1926, ..., 0.1066, -0.0961, 0.0637], -# ..., -``` - -##### Extract contextual embedding features from all Camembert layers -```python -from transformers import CamembertConfig -# (Need to reload the model with new config) -config = CamembertConfig.from_pretrained("camembert/camembert-base-oscar-4gb", output_hidden_states=True) -camembert = CamembertModel.from_pretrained("camembert/camembert-base-oscar-4gb", config=config) - -embeddings, _, all_layer_embeddings = camembert(encoded_sentence) -# all_layer_embeddings list of len(all_layer_embeddings) == 13 (input embedding layer + 12 self attention layers) -all_layer_embeddings[5] -# layer 5 contextual embedding : size torch.Size([1, 10, 768]) -#tensor([[[-0.1584, -0.1207, -0.0179, ..., 0.5457, 0.1491, -0.1191], -# [-0.1122, 0.3634, 0.0676, ..., 0.4395, -0.0470, -0.3781], -# [-0.2232, 0.0019, 0.0140, ..., 0.4461, -0.0233, 0.0735], -# ..., -``` - - -## Authors - -CamemBERT was trained and evaluated by Louis Martin\*, Benjamin Muller\*, Pedro Javier Ortiz Suárez\*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. - - -## Citation -If you use our work, please cite: - -```bibtex -@inproceedings{martin2020camembert, - title={CamemBERT: a Tasty French Language Model}, - author={Martin, Louis and Muller, Benjamin and Su{\'a}rez, Pedro Javier Ortiz and Dupont, Yoann and Romary, Laurent and de la Clergerie, {\'E}ric Villemonte and Seddah, Djam{\'e} and Sagot, Beno{\^\i}t}, - booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, - year={2020} -} -``` diff --git a/model_cards/camembert/camembert-base-wikipedia-4gb/README.md b/model_cards/camembert/camembert-base-wikipedia-4gb/README.md deleted file mode 100644 index 6cbb4b6715f3..000000000000 --- a/model_cards/camembert/camembert-base-wikipedia-4gb/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: fr ---- - -# CamemBERT: a Tasty French Language Model - -## Introduction - -[CamemBERT](https://arxiv.org/abs/1911.03894) is a state-of-the-art language model for French based on the RoBERTa model. - -It is now available on Hugging Face in 6 different versions with varying number of parameters, amount of pretraining data and pretraining data source domains. - -For further information or requests, please go to [Camembert Website](https://camembert-model.fr/) - -## Pre-trained models - -| Model | #params | Arch. | Training data | -|--------------------------------|--------------------------------|-------|-----------------------------------| -| `camembert-base` | 110M | Base | OSCAR (138 GB of text) | -| `camembert/camembert-large` | 335M | Large | CCNet (135 GB of text) | -| `camembert/camembert-base-ccnet` | 110M | Base | CCNet (135 GB of text) | -| `camembert/camembert-base-wikipedia-4gb` | 110M | Base | Wikipedia (4 GB of text) | -| `camembert/camembert-base-oscar-4gb` | 110M | Base | Subsample of OSCAR (4 GB of text) | -| `camembert/camembert-base-ccnet-4gb` | 110M | Base | Subsample of CCNet (4 GB of text) | - -## How to use CamemBERT with HuggingFace - -##### Load CamemBERT and its sub-word tokenizer : -```python -from transformers import CamembertModel, CamembertTokenizer - -# You can replace "camembert-base" with any other model from the table, e.g. "camembert/camembert-large". -tokenizer = CamembertTokenizer.from_pretrained("camembert/camembert-base-wikipedia-4gb") -camembert = CamembertModel.from_pretrained("camembert/camembert-base-wikipedia-4gb") - -camembert.eval() # disable dropout (or leave in train mode to finetune) - -``` - -##### Filling masks using pipeline -```python -from transformers import pipeline - -camembert_fill_mask = pipeline("fill-mask", model="camembert/camembert-base-wikipedia-4gb", tokenizer="camembert/camembert-base-wikipedia-4gb") -results = camembert_fill_mask("Le camembert est un fromage de !") -# results -#[{'sequence': ' Le camembert est un fromage de chèvre!', 'score': 0.4937814474105835, 'token': 19370}, -#{'sequence': ' Le camembert est un fromage de brebis!', 'score': 0.06255942583084106, 'token': 30616}, -#{'sequence': ' Le camembert est un fromage de montagne!', 'score': 0.04340197145938873, 'token': 2364}, -# {'sequence': ' Le camembert est un fromage de Noël!', 'score': 0.02823255956172943, 'token': 3236}, -#{'sequence': ' Le camembert est un fromage de vache!', 'score': 0.021357402205467224, 'token': 12329}] -``` - -##### Extract contextual embedding features from Camembert output -```python -import torch -# Tokenize in sub-words with SentencePiece -tokenized_sentence = tokenizer.tokenize("J'aime le camembert !") -# ['▁J', "'", 'aime', '▁le', '▁ca', 'member', 't', '▁!'] - -# 1-hot encode and add special starting and end tokens -encoded_sentence = tokenizer.encode(tokenized_sentence) -# [5, 221, 10, 10600, 14, 8952, 10540, 75, 1114, 6] -# NB: Can be done in one step : tokenize.encode("J'aime le camembert !") - -# Feed tokens to Camembert as a torch tensor (batch dim 1) -encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0) -embeddings, _ = camembert(encoded_sentence) -# embeddings.detach() -# embeddings.size torch.Size([1, 10, 768]) -#tensor([[[-0.0928, 0.0506, -0.0094, ..., -0.2388, 0.1177, -0.1302], -# [ 0.0662, 0.1030, -0.2355, ..., -0.4224, -0.0574, -0.2802], -# [-0.0729, 0.0547, 0.0192, ..., -0.1743, 0.0998, -0.2677], -# ..., -``` - -##### Extract contextual embedding features from all Camembert layers -```python -from transformers import CamembertConfig -# (Need to reload the model with new config) -config = CamembertConfig.from_pretrained("camembert/camembert-base-wikipedia-4gb", output_hidden_states=True) -camembert = CamembertModel.from_pretrained("camembert/camembert-base-wikipedia-4gb", config=config) - -embeddings, _, all_layer_embeddings = camembert(encoded_sentence) -# all_layer_embeddings list of len(all_layer_embeddings) == 13 (input embedding layer + 12 self attention layers) -all_layer_embeddings[5] -# layer 5 contextual embedding : size torch.Size([1, 10, 768]) -#tensor([[[-0.0059, -0.0227, 0.0065, ..., -0.0770, 0.0369, 0.0095], -# [ 0.2838, -0.1531, -0.3642, ..., -0.0027, -0.8502, -0.7914], -# [-0.0073, -0.0338, -0.0011, ..., 0.0533, -0.0250, -0.0061], -# ..., -``` - - -## Authors - -CamemBERT was trained and evaluated by Louis Martin\*, Benjamin Muller\*, Pedro Javier Ortiz Suárez\*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. - - -## Citation -If you use our work, please cite: - -```bibtex -@inproceedings{martin2020camembert, - title={CamemBERT: a Tasty French Language Model}, - author={Martin, Louis and Muller, Benjamin and Su{\'a}rez, Pedro Javier Ortiz and Dupont, Yoann and Romary, Laurent and de la Clergerie, {\'E}ric Villemonte and Seddah, Djam{\'e} and Sagot, Beno{\^\i}t}, - booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, - year={2020} -} -``` diff --git a/model_cards/camembert/camembert-large/README.md b/model_cards/camembert/camembert-large/README.md deleted file mode 100644 index ff4360b08aba..000000000000 --- a/model_cards/camembert/camembert-large/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: fr ---- - -# CamemBERT: a Tasty French Language Model - -## Introduction - -[CamemBERT](https://arxiv.org/abs/1911.03894) is a state-of-the-art language model for French based on the RoBERTa model. - -It is now available on Hugging Face in 6 different versions with varying number of parameters, amount of pretraining data and pretraining data source domains. - -For further information or requests, please go to [Camembert Website](https://camembert-model.fr/) - -## Pre-trained models - -| Model | #params | Arch. | Training data | -|--------------------------------|--------------------------------|-------|-----------------------------------| -| `camembert-base` | 110M | Base | OSCAR (138 GB of text) | -| `camembert/camembert-large` | 335M | Large | CCNet (135 GB of text) | -| `camembert/camembert-base-ccnet` | 110M | Base | CCNet (135 GB of text) | -| `camembert/camembert-base-wikipedia-4gb` | 110M | Base | Wikipedia (4 GB of text) | -| `camembert/camembert-base-oscar-4gb` | 110M | Base | Subsample of OSCAR (4 GB of text) | -| `camembert/camembert-base-ccnet-4gb` | 110M | Base | Subsample of CCNet (4 GB of text) | - -## How to use CamemBERT with HuggingFace - -##### Load CamemBERT and its sub-word tokenizer : -```python -from transformers import CamembertModel, CamembertTokenizer - -# You can replace "camembert-base" with any other model from the table, e.g. "camembert/camembert-large". -tokenizer = CamembertTokenizer.from_pretrained("camembert/camembert-large") -camembert = CamembertModel.from_pretrained("camembert/camembert-large") - -camembert.eval() # disable dropout (or leave in train mode to finetune) - -``` - -##### Filling masks using pipeline -```python -from transformers import pipeline - -camembert_fill_mask = pipeline("fill-mask", model="camembert/camembert-large", tokenizer="camembert/camembert-large") -results = camembert_fill_mask("Le camembert est :)") -# results -#[{'sequence': ' Le camembert est bon :)', 'score': 0.15560828149318695, 'token': 305}, -#{'sequence': ' Le camembert est excellent :)', 'score': 0.06821336597204208, 'token': 3497}, -#{'sequence': ' Le camembert est délicieux :)', 'score': 0.060438305139541626, 'token': 11661}, -#{'sequence': ' Le camembert est ici :)', 'score': 0.02023460529744625, 'token': 373}, -#{'sequence': ' Le camembert est meilleur :)', 'score': 0.01778135634958744, 'token': 876}] -``` - -##### Extract contextual embedding features from Camembert output -```python -import torch -# Tokenize in sub-words with SentencePiece -tokenized_sentence = tokenizer.tokenize("J'aime le camembert !") -# ['▁J', "'", 'aime', '▁le', '▁cam', 'ember', 't', '▁!'] - -# 1-hot encode and add special starting and end tokens -encoded_sentence = tokenizer.encode(tokenized_sentence) -# [5, 133, 22, 1250, 16, 12034, 14324, 81, 76, 6] -# NB: Can be done in one step : tokenize.encode("J'aime le camembert !") - -# Feed tokens to Camembert as a torch tensor (batch dim 1) -encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0) -embeddings, _ = camembert(encoded_sentence) -# embeddings.detach() -# torch.Size([1, 10, 1024]) -#tensor([[[-0.1284, 0.2643, 0.4374, ..., 0.1627, 0.1308, -0.2305], -# [ 0.4576, -0.6345, -0.2029, ..., -0.1359, -0.2290, -0.6318], -# [ 0.0381, 0.0429, 0.5111, ..., -0.1177, -0.1913, -0.1121], -# ..., -``` - -##### Extract contextual embedding features from all Camembert layers -```python -from transformers import CamembertConfig -# (Need to reload the model with new config) -config = CamembertConfig.from_pretrained("camembert/camembert-large", output_hidden_states=True) -camembert = CamembertModel.from_pretrained("camembert/camembert-large", config=config) - -embeddings, _, all_layer_embeddings = camembert(encoded_sentence) -# all_layer_embeddings list of len(all_layer_embeddings) == 25 (input embedding layer + 24 self attention layers) -all_layer_embeddings[5] -# layer 5 contextual embedding : size torch.Size([1, 10, 1024]) -#tensor([[[-0.0600, 0.0742, 0.0332, ..., -0.0525, -0.0637, -0.0287], -# [ 0.0950, 0.2840, 0.1985, ..., 0.2073, -0.2172, -0.6321], -# [ 0.1381, 0.1872, 0.1614, ..., -0.0339, -0.2530, -0.1182], -# ..., -``` - - -## Authors - -CamemBERT was trained and evaluated by Louis Martin\*, Benjamin Muller\*, Pedro Javier Ortiz Suárez\*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. - - -## Citation -If you use our work, please cite: - -```bibtex -@inproceedings{martin2020camembert, - title={CamemBERT: a Tasty French Language Model}, - author={Martin, Louis and Muller, Benjamin and Su{\'a}rez, Pedro Javier Ortiz and Dupont, Yoann and Romary, Laurent and de la Clergerie, {\'E}ric Villemonte and Seddah, Djam{\'e} and Sagot, Beno{\^\i}t}, - booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics}, - year={2020} -} -``` diff --git a/model_cards/canwenxu/BERT-of-Theseus-MNLI/README.md b/model_cards/canwenxu/BERT-of-Theseus-MNLI/README.md deleted file mode 100644 index 496b438b39b7..000000000000 --- a/model_cards/canwenxu/BERT-of-Theseus-MNLI/README.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -thumbnail: https://raw.githubusercontent.com/JetRunner/BERT-of-Theseus/master/bert-of-theseus.png -datasets: -- multi_nli ---- - -# BERT-of-Theseus -See our paper ["BERT-of-Theseus: Compressing BERT by Progressive Module Replacing"](http://arxiv.org/abs/2002.02925). - -BERT-of-Theseus is a new compressed BERT by progressively replacing the components of the original BERT. - -![BERT of Theseus](https://github.com/JetRunner/BERT-of-Theseus/blob/master/bert-of-theseus.png?raw=true) - -## Load Pretrained Model on MNLI - -We provide a 6-layer pretrained model on MNLI as a general-purpose model, which can transfer to other sentence classification tasks, outperforming DistillBERT (with the same 6-layer structure) on six tasks of GLUE (dev set). - -| Method | MNLI | MRPC | QNLI | QQP | RTE | SST-2 | STS-B | -|-----------------|------|------|------|------|------|-------|-------| -| BERT-base | 83.5 | 89.5 | 91.2 | 89.8 | 71.1 | 91.5 | 88.9 | -| DistillBERT | 79.0 | 87.5 | 85.3 | 84.9 | 59.9 | 90.7 | 81.2 | -| BERT-of-Theseus | 82.1 | 87.5 | 88.8 | 88.8 | 70.1 | 91.8 | 87.8 | - -Please Note: this checkpoint is for [Intermediate-Task Transfer Learning](https://arxiv.org/abs/2005.00628) so it does not include the classification head for MNLI! Please fine-tune it before use (like DistilBERT). diff --git a/model_cards/cedpsam/chatbot_fr/README.md b/model_cards/cedpsam/chatbot_fr/README.md deleted file mode 100644 index 32324fe88e1c..000000000000 --- a/model_cards/cedpsam/chatbot_fr/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: fr -tags: -- conversational -widget: -- text: "bonjour." -- text: "mais encore" -- text: "est ce que l'argent achete le bonheur?" ---- - -## a dialoggpt model trained on french opensubtitles with custom tokenizer -trained with this notebook -https://colab.research.google.com/drive/1pfCV3bngAmISNZVfDvBMyEhQKuYw37Rl#scrollTo=AyImj9qZYLRi&uniqifier=3 - -config from microsoft/DialoGPT-medium -dataset generated from 2018 opensubtitle from opus folowing these guidelines -https://github.com/PolyAI-LDN/conversational-datasets/tree/master/opensubtitles with this notebook -https://colab.research.google.com/drive/1uyh3vJ9nEjqOHI68VD73qxt4olJzODxi#scrollTo=deaacv4XfLMk -### How to use - -Now we are ready to try out how the model works as a chatting partner! - -```python -import torch -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("cedpsam/chatbot_fr") - -model = AutoModelWithLMHead.from_pretrained("cedpsam/chatbot_fr") - -for step in range(6): - # encode the new user input, add the eos_token and return a tensor in Pytorch - new_user_input_ids = tokenizer.encode(input(">> User:") + tokenizer.eos_token, return_tensors='pt') - # print(new_user_input_ids) - - # append the new user input tokens to the chat history - bot_input_ids = torch.cat([chat_history_ids, new_user_input_ids], dim=-1) if step > 0 else new_user_input_ids - - # generated a response while limiting the total chat history to 1000 tokens, - chat_history_ids = model.generate( - bot_input_ids, max_length=1000, - pad_token_id=tokenizer.eos_token_id, - top_p=0.92, top_k = 50 - ) - - # pretty print last ouput tokens from bot - print("DialoGPT: {}".format(tokenizer.decode(chat_history_ids[:, bot_input_ids.shape[-1]:][0], skip_special_tokens=True))) diff --git a/model_cards/ceostroff/harry-potter-gpt2-fanfiction/README.md b/model_cards/ceostroff/harry-potter-gpt2-fanfiction/README.md deleted file mode 100644 index ab9a8eaa3b22..000000000000 --- a/model_cards/ceostroff/harry-potter-gpt2-fanfiction/README.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -language: -- en -tags: -- harry-potter -license: mit ---- - -# Harry Potter Fanfiction Generator - -This is a pre-trained GPT-2 generative text model that allows you to generate your own Harry Potter fanfiction, trained off of the top 100 rated fanficition stories. We intend for this to be used for individual fun and experimentation and not as a commercial product. diff --git a/model_cards/chrisliu298/arxiv_ai_gpt2/README.md b/model_cards/chrisliu298/arxiv_ai_gpt2/README.md deleted file mode 100644 index dd153ede548d..000000000000 --- a/model_cards/chrisliu298/arxiv_ai_gpt2/README.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -language: "en" -tags: -- gpt2 -- arxiv -- transformers -datasets: -- https://github.com/staeiou/arxiv_archive/tree/v1.0.1 ---- - -# ArXiv AI GPT-2 - -## Model description - -This GPT-2 (774M) model is capable of generating abstracts given paper titles. It was trained using all research paper titles and abstracts under artificial intelligence (AI), machine learning (LG), computation and language (CL), and computer vision and pattern recognition (CV) on arXiv. - -## Intended uses & limitations - -#### How to use - -To generate paper abstracts, use the provided `generate.py` [here](https://gist.github.com/chrisliu298/ccb8144888eace069da64ad3e6472d64). This is very similar to the HuggingFace's `run_generation.py` [here](https://github.com/huggingface/transformers/tree/master/examples/text-generation). You can simply replace the text with with your own model path (line 89) and change the input string to your paper title (line 127). If you want to use your own script, make sure to prepend `<|startoftext|> ` at the front and append ` <|sep|>` at the end of the paper title. - -## Training data -I selected a subset of the [arXiv Archive](https://github.com/staeiou/arxiv_archive) dataset (Geiger, 2019) as the training and evaluation data to fine-tune GPT-2. The original arXiv Archive dataset contains a full archive of metadata about papers on arxiv.org, from the start of the site in 1993 to the end of 2019. Our subset includes all the paper titles (query) and abstracts (context) under the Artificial Intelligence (cs.AI), Machine Learning (cs.LG), Computation and Language (cs.CL), and Computer Vision and Pattern Recognition (cs.CV) categories. I provide the information of the sub-dataset and the distribution of the training and evaluation dataset as follows. - - -| Splits | Count | Percentage (%) | BPE Token Count | -| :--------: | :--------: | :------------: | :-------------: | -| Train | 90,000 | 90.11 | 20,834,012 | -| Validation | 4,940 | 4.95 | 1,195,056 | -| Test | 4,940 | 4.95 | 1,218,754 | -| **Total** | **99,880** | **100** | **23,247,822** | - -The original dataset is in the format of a tab-separated value, so we wrote a simple preprocessing script to convert it into a text file format, which is the input file type (a document) of the GPT-2 model. An example of a paper’s title and its abstract is shown below. - -```text -<|startoftext|> Some paper title <|sep|> Some paper abstract <|endoftext|> -``` - -Because there are a lot of cross-domain papers in the dataset, I deduplicate the dataset using the arXiv ID, which is unique for every paper. I sort the paper by submission date, by doing so, one can examine GPT-2’s ability to use learned terminologies when it is prompted with paper titles from the “future.” - - -## Training procedure - -I used block size = 512, batch size = 1, gradidnet accumulation = 1, learning rate = 1e-5, epochs = 5, and everything else follows the default model configuration. - -## Eval results - -The resulting GPT-2 large model's perplexity score on the test set is **14.9413**. - -## Reference - -```bibtex -@dataset{r_stuart_geiger_2019_2533436, - author= {R. Stuart Geiger}, - title={{ArXiV Archive: A tidy and complete archive of metadata for papers on arxiv.org, 1993-2019}}, - month=jan, - year= 2019, - publisher={Zenodo}, - version= {v1.0.1}, - doi={10.5281/zenodo.2533436}, - url={https://doi.org/10.5281/zenodo.2533436} -} -``` - diff --git a/model_cards/cimm-kzn/endr-bert/README.md b/model_cards/cimm-kzn/endr-bert/README.md deleted file mode 100644 index 97e1cda8374a..000000000000 --- a/model_cards/cimm-kzn/endr-bert/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: -- ru -- en ---- - -## EnDR-BERT - - EnDR-BERT - Multilingual, Cased, which pretrained on the english collection of consumer comments on drug administration from [2]. Pre-training was based on the [original BERT code](https://github.com/google-research/bert) provided by Google. In particular, Multi-BERT was for used for initialization and all the parameters are the same as in Multi-BERT. Training details are described in our paper. \ - link: https://yadi.sk/d/-PTn0xhk1PqvgQ - - - ## Citing & Authors - - If you find this repository helpful, feel free to cite our publication: - - [1] Tutubalina E, Alimova I, Miftahutdinov Z, et al. The Russian Drug Reaction Corpus and Neural Models for Drug Reactions and Effectiveness Detection in User Reviews.//Bioinformatics. - 2020. - - preprint: https://arxiv.org/abs/2004.03659 - ``` - @article{10.1093/bioinformatics/btaa675, - author = {Tutubalina, Elena and Alimova, Ilseyar and Miftahutdinov, Zulfat and Sakhovskiy, Andrey and Malykh, Valentin and Nikolenko, Sergey}, - title = "{The Russian Drug Reaction Corpus and Neural Models for Drug Reactions and Effectiveness Detection in User Reviews}", - journal = {Bioinformatics}, - year = {2020}, - month = {07}, - issn = {1367-4803}, - doi = {10.1093/bioinformatics/btaa675}, - url = {https://doi.org/10.1093/bioinformatics/btaa675}, - note = {btaa675}, - eprint = {https://academic.oup.com/bioinformatics/advance-article-pdf/doi/10.1093/bioinformatics/btaa675/33539752/btaa675.pdf}, - } - ``` - [2] Tutubalina, EV and Miftahutdinov, Z Sh and Nugmanov, RI and Madzhidov, TI and Nikolenko, SI and Alimova, IS and Tropsha, AE Using semantic analysis of texts for the identification of drugs with similar therapeutic effects.//Russian Chemical Bulletin. – 2017. – Т. 66. – №. 11. – С. 2180-2189. - [link to paper](https://www.researchgate.net/profile/Elena_Tutubalina/publication/323751823_Using_semantic_analysis_of_texts_for_the_identification_of_drugs_with_similar_therapeutic_effects/links/5bf7cfc3299bf1a0202cbc1f/Using-semantic-analysis-of-texts-for-the-identification-of-drugs-with-similar-therapeutic-effects.pdf) - ``` - @article{tutubalina2017using, - title={Using semantic analysis of texts for the identification of drugs with similar therapeutic effects}, - author={Tutubalina, EV and Miftahutdinov, Z Sh and Nugmanov, RI and Madzhidov, TI and Nikolenko, SI and Alimova, IS and Tropsha, AE}, - journal={Russian Chemical Bulletin}, - volume={66}, - number={11}, - pages={2180--2189}, - year={2017}, - publisher={Springer} - } - ``` diff --git a/model_cards/cimm-kzn/enrudr-bert/README.md b/model_cards/cimm-kzn/enrudr-bert/README.md deleted file mode 100644 index f4ec132c8c72..000000000000 --- a/model_cards/cimm-kzn/enrudr-bert/README.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -language: -- ru -- en ---- -## EnRuDR-BERT - -EnRuDR-BERT - Multilingual, Cased, which pretrained on the raw part of the RuDReC corpus (1.4M reviews) and english collection of consumer comments on drug administration from [2]. Pre-training was based on the [original BERT code](https://github.com/google-research/bert) provided by Google. In particular, Multi-BERT was for used for initialization; vocabulary of Russian subtokens and parameters are the same as in Multi-BERT. Training details are described in our paper. \ - link: https://yadi.sk/d/-PTn0xhk1PqvgQ - - -## Citing & Authors - -If you find this repository helpful, feel free to cite our publication: - -[1] Tutubalina E, Alimova I, Miftahutdinov Z, et al. The Russian Drug Reaction Corpus and Neural Models for Drug Reactions and Effectiveness Detection in User Reviews.//Bioinformatics. - 2020. - - preprint: https://arxiv.org/abs/2004.03659 -``` -@article{10.1093/bioinformatics/btaa675, - author = {Tutubalina, Elena and Alimova, Ilseyar and Miftahutdinov, Zulfat and Sakhovskiy, Andrey and Malykh, Valentin and Nikolenko, Sergey}, - title = "{The Russian Drug Reaction Corpus and Neural Models for Drug Reactions and Effectiveness Detection in User Reviews}", - journal = {Bioinformatics}, - year = {2020}, - month = {07}, - issn = {1367-4803}, - doi = {10.1093/bioinformatics/btaa675}, - url = {https://doi.org/10.1093/bioinformatics/btaa675}, - note = {btaa675}, - eprint = {https://academic.oup.com/bioinformatics/advance-article-pdf/doi/10.1093/bioinformatics/btaa675/33539752/btaa675.pdf}, -} -``` -[2] Tutubalina, EV and Miftahutdinov, Z Sh and Nugmanov, RI and Madzhidov, TI and Nikolenko, SI and Alimova, IS and Tropsha, AE Using semantic analysis of texts for the identification of drugs with similar therapeutic effects.//Russian Chemical Bulletin. – 2017. – Т. 66. – №. 11. – С. 2180-2189. - [link to paper](https://www.researchgate.net/profile/Elena_Tutubalina/publication/323751823_Using_semantic_analysis_of_texts_for_the_identification_of_drugs_with_similar_therapeutic_effects/links/5bf7cfc3299bf1a0202cbc1f/Using-semantic-analysis-of-texts-for-the-identification-of-drugs-with-similar-therapeutic-effects.pdf) -``` -@article{tutubalina2017using, - title={Using semantic analysis of texts for the identification of drugs with similar therapeutic effects}, - author={Tutubalina, EV and Miftahutdinov, Z Sh and Nugmanov, RI and Madzhidov, TI and Nikolenko, SI and Alimova, IS and Tropsha, AE}, - journal={Russian Chemical Bulletin}, - volume={66}, - number={11}, - pages={2180--2189}, - year={2017}, - publisher={Springer} -} -``` diff --git a/model_cards/cimm-kzn/rudr-bert/README.md b/model_cards/cimm-kzn/rudr-bert/README.md deleted file mode 100644 index 8d8ebf4ee57e..000000000000 --- a/model_cards/cimm-kzn/rudr-bert/README.md +++ /dev/null @@ -1,41 +0,0 @@ -## RuDR-BERT - -RuDR-BERT - Multilingual, Cased, which pretrained on the raw part of the RuDReC corpus (1.4M reviews). Pre-training was based on the [original BERT code](https://github.com/google-research/bert) provided by Google. In particular, Multi-BERT was for used for initialization; vocabulary of Russian subtokens and parameters are the same as in Multi-BERT. Training details are described in our paper. \ - link: https://yadi.sk/d/-PTn0xhk1PqvgQ - - -## Citing & Authors - -If you find this repository helpful, feel free to cite our publication: - -[1] Tutubalina E, Alimova I, Miftahutdinov Z, et al. The Russian Drug Reaction Corpus and Neural Models for Drug Reactions and Effectiveness Detection in User Reviews.//Bioinformatics. - 2020. - - preprint: https://arxiv.org/abs/2004.03659 -``` -@article{10.1093/bioinformatics/btaa675, - author = {Tutubalina, Elena and Alimova, Ilseyar and Miftahutdinov, Zulfat and Sakhovskiy, Andrey and Malykh, Valentin and Nikolenko, Sergey}, - title = "{The Russian Drug Reaction Corpus and Neural Models for Drug Reactions and Effectiveness Detection in User Reviews}", - journal = {Bioinformatics}, - year = {2020}, - month = {07}, - issn = {1367-4803}, - doi = {10.1093/bioinformatics/btaa675}, - url = {https://doi.org/10.1093/bioinformatics/btaa675}, - note = {btaa675}, - eprint = {https://academic.oup.com/bioinformatics/advance-article-pdf/doi/10.1093/bioinformatics/btaa675/33539752/btaa675.pdf}, -} -``` -[2] Tutubalina, EV and Miftahutdinov, Z Sh and Nugmanov, RI and Madzhidov, TI and Nikolenko, SI and Alimova, IS and Tropsha, AE Using semantic analysis of texts for the identification of drugs with similar therapeutic effects.//Russian Chemical Bulletin. – 2017. – Т. 66. – №. 11. – С. 2180-2189. - [link to paper](https://www.researchgate.net/profile/Elena_Tutubalina/publication/323751823_Using_semantic_analysis_of_texts_for_the_identification_of_drugs_with_similar_therapeutic_effects/links/5bf7cfc3299bf1a0202cbc1f/Using-semantic-analysis-of-texts-for-the-identification-of-drugs-with-similar-therapeutic-effects.pdf) -``` -@article{tutubalina2017using, - title={Using semantic analysis of texts for the identification of drugs with similar therapeutic effects}, - author={Tutubalina, EV and Miftahutdinov, Z Sh and Nugmanov, RI and Madzhidov, TI and Nikolenko, SI and Alimova, IS and Tropsha, AE}, - journal={Russian Chemical Bulletin}, - volume={66}, - number={11}, - pages={2180--2189}, - year={2017}, - publisher={Springer} -} -``` diff --git a/model_cards/clue/albert_chinese_small/README.md b/model_cards/clue/albert_chinese_small/README.md deleted file mode 100644 index 94fa48b1b807..000000000000 --- a/model_cards/clue/albert_chinese_small/README.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -language: zh ---- - -## albert_chinese_small - -### Overview - -**Language model:** albert-small -**Model size:** 18.5M -**Language:** Chinese -**Training data:** [CLUECorpusSmall](https://github.com/CLUEbenchmark/CLUECorpus2020) -**Eval data:** [CLUE dataset](https://github.com/CLUEbenchmark/CLUE) - -### Results - -For results on downstream tasks like text classification, please refer to [this repository](https://github.com/CLUEbenchmark/CLUE). - -### Usage - -**NOTE:**Since sentencepiece is not used in `albert_chinese_small` model, you have to call **BertTokenizer** instead of AlbertTokenizer !!! - -``` -import torch -from transformers import BertTokenizer, AlbertModel -tokenizer = BertTokenizer.from_pretrained("clue/albert_chinese_small") -albert = AlbertModel.from_pretrained("clue/albert_chinese_small") -``` - -### About CLUE benchmark - -Organization of Language Understanding Evaluation benchmark for Chinese: tasks & datasets, baselines, pre-trained Chinese models, corpus and leaderboard. - -Github: https://github.com/CLUEbenchmark -Website: https://www.cluebenchmarks.com/ diff --git a/model_cards/clue/albert_chinese_tiny/README.md b/model_cards/clue/albert_chinese_tiny/README.md deleted file mode 100644 index 0d93f83e805c..000000000000 --- a/model_cards/clue/albert_chinese_tiny/README.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -language: zh ---- - -## albert_chinese_tiny - -### Overview - -**Language model:** albert-tiny -**Model size:** 16M -**Language:** Chinese -**Training data:** [CLUECorpusSmall](https://github.com/CLUEbenchmark/CLUECorpus2020) -**Eval data:** [CLUE dataset](https://github.com/CLUEbenchmark/CLUE) - -### Results - -For results on downstream tasks like text classification, please refer to [this repository](https://github.com/CLUEbenchmark/CLUE). - -### Usage - -**NOTE:**Since sentencepiece is not used in `albert_chinese_tiny` model, you have to call **BertTokenizer** instead of AlbertTokenizer !!! - -``` -import torch -from transformers import BertTokenizer, AlbertModel -tokenizer = BertTokenizer.from_pretrained("clue/albert_chinese_tiny") -albert = AlbertModel.from_pretrained("clue/albert_chinese_tiny") -``` - -### About CLUE benchmark - -Organization of Language Understanding Evaluation benchmark for Chinese: tasks & datasets, baselines, pre-trained Chinese models, corpus and leaderboard. - -Github: https://github.com/CLUEbenchmark -Website: https://www.cluebenchmarks.com/ diff --git a/model_cards/clue/roberta_chinese_3L312_clue_tiny/README.md b/model_cards/clue/roberta_chinese_3L312_clue_tiny/README.md deleted file mode 100644 index dbe7f2b83e2a..000000000000 --- a/model_cards/clue/roberta_chinese_3L312_clue_tiny/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: zh ---- - -# Introduction -This model was trained on TPU and the details are as follows: - -## Model -## - -| Model_name | params | size | Training_corpus | Vocab | -| :------------------------------------------ | :----- | :------- | :----------------- | :-----------: | -| **`RoBERTa-tiny-clue`**
Super_small_model | 7.5M | 28.3M | **CLUECorpus2020** | **CLUEVocab** | -| **`RoBERTa-tiny-pair`**
Super_small_sentence_pair_model | 7.5M | 28.3M | **CLUECorpus2020** | **CLUEVocab** | -| **`RoBERTa-tiny3L768-clue`**
small_model | 38M | 110M | **CLUECorpus2020** | **CLUEVocab** | -| **`RoBERTa-tiny3L312-clue`**
small_model | <7.5M | 24M | **CLUECorpus2020** | **CLUEVocab** | -| **`RoBERTa-large-clue`**
Large_model | 290M | 1.20G | **CLUECorpus2020** | **CLUEVocab** | -| **`RoBERTa-large-pair`**
Large_sentence_pair_model | 290M | 1.20G | **CLUECorpus2020** | **CLUEVocab** | - -### Usage - -With the help of[Huggingface-Transformers 2.5.1](https://github.com/huggingface/transformers), you could use these model as follows - -``` -tokenizer = BertTokenizer.from_pretrained("MODEL_NAME") -model = BertModel.from_pretrained("MODEL_NAME") -``` - -`MODEL_NAME`： - -| Model_NAME | MODEL_LINK | -| -------------------------- | ------------------------------------------------------------ | -| **RoBERTa-tiny-clue** | [`clue/roberta_chinese_clue_tiny`](https://huggingface.co/clue/roberta_chinese_clue_tiny) | -| **RoBERTa-tiny-pair** | [`clue/roberta_chinese_pair_tiny`](https://huggingface.co/clue/roberta_chinese_pair_tiny) | -| **RoBERTa-tiny3L768-clue** | [`clue/roberta_chinese_3L768_clue_tiny`](https://huggingface.co/clue/roberta_chinese_3L768_clue_tiny) | -| **RoBERTa-tiny3L312-clue** | [`clue/roberta_chinese_3L312_clue_tiny`](https://huggingface.co/clue/roberta_chinese_3L312_clue_tiny) | -| **RoBERTa-large-clue** | [`clue/roberta_chinese_clue_large`](https://huggingface.co/clue/roberta_chinese_clue_large) | -| **RoBERTa-large-pair** | [`clue/roberta_chinese_pair_large`](https://huggingface.co/clue/roberta_chinese_pair_large) | - -## Details -Please read https://arxiv.org/pdf/2003.01355. - -Please visit our repository: https://github.com/CLUEbenchmark/CLUEPretrainedModels.git diff --git a/model_cards/clue/roberta_chinese_base/README.md b/model_cards/clue/roberta_chinese_base/README.md deleted file mode 100644 index 4c87c58081aa..000000000000 --- a/model_cards/clue/roberta_chinese_base/README.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -language: zh ---- - -## roberta_chinese_base - -### Overview - -**Language model:** roberta-base -**Model size:** 392M -**Language:** Chinese -**Training data:** [CLUECorpusSmall](https://github.com/CLUEbenchmark/CLUECorpus2020) -**Eval data:** [CLUE dataset](https://github.com/CLUEbenchmark/CLUE) - -### Results - -For results on downstream tasks like text classification, please refer to [this repository](https://github.com/CLUEbenchmark/CLUE). - -### Usage - -**NOTE:** You have to call **BertTokenizer** instead of RobertaTokenizer !!! - -``` -import torch -from transformers import BertTokenizer, BertModel -tokenizer = BertTokenizer.from_pretrained("clue/roberta_chinese_base") -roberta = BertModel.from_pretrained("clue/roberta_chinese_base") -``` - -### About CLUE benchmark - -Organization of Language Understanding Evaluation benchmark for Chinese: tasks & datasets, baselines, pre-trained Chinese models, corpus and leaderboard. - -Github: https://github.com/CLUEbenchmark -Website: https://www.cluebenchmarks.com/ diff --git a/model_cards/clue/roberta_chinese_large/README.md b/model_cards/clue/roberta_chinese_large/README.md deleted file mode 100644 index b91d1a725ca8..000000000000 --- a/model_cards/clue/roberta_chinese_large/README.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -language: zh ---- - -## roberta_chinese_large - -### Overview - -**Language model:** roberta-large -**Model size:** 1.2G -**Language:** Chinese -**Training data:** [CLUECorpusSmall](https://github.com/CLUEbenchmark/CLUECorpus2020) -**Eval data:** [CLUE dataset](https://github.com/CLUEbenchmark/CLUE) - -### Results - -For results on downstream tasks like text classification, please refer to [this repository](https://github.com/CLUEbenchmark/CLUE). - -### Usage - -**NOTE:** You have to call **BertTokenizer** instead of RobertaTokenizer !!! - -``` -import torch -from transformers import BertTokenizer, BertModel -tokenizer = BertTokenizer.from_pretrained("clue/roberta_chinese_large") -roberta = BertModel.from_pretrained("clue/roberta_chinese_large") -``` - -### About CLUE benchmark - -Organization of Language Understanding Evaluation benchmark for Chinese: tasks & datasets, baselines, pre-trained Chinese models, corpus and leaderboard. - -Github: https://github.com/CLUEbenchmark -Website: https://www.cluebenchmarks.com/ diff --git a/model_cards/clue/xlnet_chinese_large/README.md b/model_cards/clue/xlnet_chinese_large/README.md deleted file mode 100644 index 6f34c38b3285..000000000000 --- a/model_cards/clue/xlnet_chinese_large/README.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -language: zh ---- - -## xlnet_chinese_large - -### Overview - -**Language model:** xlnet-large -**Model size:** 1.3G -**Language:** Chinese -**Training data:** [CLUECorpusSmall](https://github.com/CLUEbenchmark/CLUECorpus2020) -**Eval data:** [CLUE dataset](https://github.com/CLUEbenchmark/CLUE) - -### Results - -For results on downstream tasks like text classification, please refer to [this repository](https://github.com/CLUEbenchmark/CLUE). - -### Usage - -``` -import torch -from transformers import XLNetTokenizer,XLNetModel -tokenizer = XLNetTokenizer.from_pretrained("clue/xlnet_chinese_large") -xlnet = XLNetModel.from_pretrained("clue/xlnet_chinese_large") -``` - -### About CLUE benchmark - -Organization of Language Understanding Evaluation benchmark for Chinese: tasks & datasets, baselines, pre-trained Chinese models, corpus and leaderboard. - -Github: https://github.com/CLUEbenchmark -Website: https://www.cluebenchmarks.com/ diff --git a/model_cards/codegram/calbert-base-uncased/README.md b/model_cards/codegram/calbert-base-uncased/README.md deleted file mode 100644 index 8d962ed0080f..000000000000 --- a/model_cards/codegram/calbert-base-uncased/README.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -language: "ca" -tags: - - masked-lm - - catalan - - exbert -license: mit ---- - -# Calbert: a Catalan Language Model - -## Introduction - -CALBERT is an open-source language model for Catalan pretrained on the ALBERT architecture. - -It is now available on Hugging Face in its `tiny-uncased` version and `base-uncased` (the one you're looking at) as well, and was pretrained on the [OSCAR dataset](https://traces1.inria.fr/oscar/). - -For further information or requests, please go to the [GitHub repository](https://github.com/codegram/calbert) - -## Pre-trained models - -| Model | Arch. | Training data | -| ----------------------------------- | -------------- | ---------------------- | -| `codegram` / `calbert-tiny-uncased` | Tiny (uncased) | OSCAR (4.3 GB of text) | -| `codegram` / `calbert-base-uncased` | Base (uncased) | OSCAR (4.3 GB of text) | - -## How to use Calbert with HuggingFace - -#### Load Calbert and its tokenizer: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("codegram/calbert-base-uncased") -model = AutoModel.from_pretrained("codegram/calbert-base-uncased") - -model.eval() # disable dropout (or leave in train mode to finetune -``` - -#### Filling masks using pipeline - -```python -from transformers import pipeline - -calbert_fill_mask = pipeline("fill-mask", model="codegram/calbert-base-uncased", tokenizer="codegram/calbert-base-uncased") -results = calbert_fill_mask("M'agrada [MASK] això") -# results -# [{'sequence': "[CLS] m'agrada molt aixo[SEP]", 'score': 0.614592969417572, 'token': 61}, -# {'sequence': "[CLS] m'agrada moltíssim aixo[SEP]", 'score': 0.06058056280016899, 'token': 4867}, -# {'sequence': "[CLS] m'agrada més aixo[SEP]", 'score': 0.017195818945765495, 'token': 43}, -# {'sequence': "[CLS] m'agrada llegir aixo[SEP]", 'score': 0.016321714967489243, 'token': 684}, -# {'sequence': "[CLS] m'agrada escriure aixo[SEP]", 'score': 0.012185849249362946, 'token': 1306}] - -``` - -#### Extract contextual embedding features from Calbert output - -```python -import torch -# Tokenize in sub-words with SentencePiece -tokenized_sentence = tokenizer.tokenize("M'és una mica igual") -# ['▁m', "'", 'es', '▁una', '▁mica', '▁igual'] - -# 1-hot encode and add special starting and end tokens -encoded_sentence = tokenizer.encode(tokenized_sentence) -# [2, 109, 7, 71, 36, 371, 1103, 3] -# NB: Can be done in one step : tokenize.encode("M'és una mica igual") - -# Feed tokens to Calbert as a torch tensor (batch dim 1) -encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0) -embeddings, _ = model(encoded_sentence) -embeddings.size() -# torch.Size([1, 8, 768]) -embeddings.detach() -# tensor([[[-0.0261, 0.1166, -0.1075, ..., -0.0368, 0.0193, 0.0017], -# [ 0.1289, -0.2252, 0.9881, ..., -0.1353, 0.3534, 0.0734], -# [-0.0328, -1.2364, 0.9466, ..., 0.3455, 0.7010, -0.2085], -# ..., -# [ 0.0397, -1.0228, -0.2239, ..., 0.2932, 0.1248, 0.0813], -# [-0.0261, 0.1165, -0.1074, ..., -0.0368, 0.0193, 0.0017], -# [-0.1934, -0.2357, -0.2554, ..., 0.1831, 0.6085, 0.1421]]]) -``` - -## Authors - -CALBERT was trained and evaluated by [Txus Bach](https://twitter.com/txustice), as part of [Codegram](https://www.codegram.com)'s applied research. - - - - diff --git a/model_cards/codegram/calbert-tiny-uncased/README.md b/model_cards/codegram/calbert-tiny-uncased/README.md deleted file mode 100644 index 1038233cd36f..000000000000 --- a/model_cards/codegram/calbert-tiny-uncased/README.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -language: "ca" -tags: - - masked-lm - - catalan - - exbert -license: mit ---- - -# Calbert: a Catalan Language Model - -## Introduction - -CALBERT is an open-source language model for Catalan pretrained on the ALBERT architecture. - -It is now available on Hugging Face in its `tiny-uncased` version (the one you're looking at) and `base-uncased` as well, and was pretrained on the [OSCAR dataset](https://traces1.inria.fr/oscar/). - -For further information or requests, please go to the [GitHub repository](https://github.com/codegram/calbert) - -## Pre-trained models - -| Model | Arch. | Training data | -| ----------------------------------- | -------------- | ---------------------- | -| `codegram` / `calbert-tiny-uncased` | Tiny (uncased) | OSCAR (4.3 GB of text) | -| `codegram` / `calbert-base-uncased` | Base (uncased) | OSCAR (4.3 GB of text) | - -## How to use Calbert with HuggingFace - -#### Load Calbert and its tokenizer: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("codegram/calbert-tiny-uncased") -model = AutoModel.from_pretrained("codegram/calbert-tiny-uncased") - -model.eval() # disable dropout (or leave in train mode to finetune -``` - -#### Filling masks using pipeline - -```python -from transformers import pipeline - -calbert_fill_mask = pipeline("fill-mask", model="codegram/calbert-tiny-uncased", tokenizer="codegram/calbert-tiny-uncased") -results = calbert_fill_mask("M'agrada [MASK] això") -# results -# [{'sequence': "[CLS] m'agrada molt aixo[SEP]", 'score': 0.4403671622276306, 'token': 61}, -# {'sequence': "[CLS] m'agrada més aixo[SEP]", 'score': 0.050061386078596115, 'token': 43}, -# {'sequence': "[CLS] m'agrada veure aixo[SEP]", 'score': 0.026286985725164413, 'token': 157}, -# {'sequence': "[CLS] m'agrada bastant aixo[SEP]", 'score': 0.022483550012111664, 'token': 2143}, -# {'sequence': "[CLS] m'agrada moltíssim aixo[SEP]", 'score': 0.014491282403469086, 'token': 4867}] - -``` - -#### Extract contextual embedding features from Calbert output - -```python -import torch -# Tokenize in sub-words with SentencePiece -tokenized_sentence = tokenizer.tokenize("M'és una mica igual") -# ['▁m', "'", 'es', '▁una', '▁mica', '▁igual'] - -# 1-hot encode and add special starting and end tokens -encoded_sentence = tokenizer.encode(tokenized_sentence) -# [2, 109, 7, 71, 36, 371, 1103, 3] -# NB: Can be done in one step : tokenize.encode("M'és una mica igual") - -# Feed tokens to Calbert as a torch tensor (batch dim 1) -encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0) -embeddings, _ = model(encoded_sentence) -embeddings.size() -# torch.Size([1, 8, 312]) -embeddings.detach() -# tensor([[[-0.2726, -0.9855, 0.9643, ..., 0.3511, 0.3499, -0.1984], -# [-0.2824, -1.1693, -0.2365, ..., -3.1866, -0.9386, -1.3718], -# [-2.3645, -2.2477, -1.6985, ..., -1.4606, -2.7294, 0.2495], -# ..., -# [ 0.8800, -0.0244, -3.0446, ..., 0.5148, -3.0903, 1.1879], -# [ 1.1300, 0.2425, 0.2162, ..., -0.5722, -2.2004, 0.4045], -# [ 0.4549, -0.2378, -0.2290, ..., -2.1247, -2.2769, -0.0820]]]) -``` - -## Authors - -CALBERT was trained and evaluated by [Txus Bach](https://twitter.com/txustice), as part of [Codegram](https://www.codegram.com)'s applied research. - - - - diff --git a/model_cards/cooelf/limitbert/README.md b/model_cards/cooelf/limitbert/README.md deleted file mode 100644 index cacf31eef825..000000000000 --- a/model_cards/cooelf/limitbert/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# LIMIT-BERT - -Code and model for the *EMNLP 2020 Findings* paper: - -[LIMIT-BERT: Linguistic Informed Multi-task BERT](https://arxiv.org/abs/1910.14296)) - -## Contents - -1. [Requirements](#Requirements) -2. [Training](#Training) - -## Requirements - -* Python 3.6 or higher. -* Cython 0.25.2 or any compatible version. -* [PyTorch](http://pytorch.org/) 1.0.0+. -* [EVALB](http://nlp.cs.nyu.edu/evalb/). Before starting, run `make` inside the `EVALB/` directory to compile an `evalb` executable. This will be called from Python for evaluation. -* [pytorch-transformers](https://github.com/huggingface/pytorch-transformers) PyTorch 1.0.0+ or any compatible version. - -#### Pre-trained Models (PyTorch) -The following pre-trained models are available for download from Google Drive: -* [`LIMIT-BERT`](https://drive.google.com/open?id=1fm0cK2A91iLG3lCpwowCCQSALnWS2X4i): - PyTorch version, same setting with BERT-Large-WWM，loading model with [pytorch-transformers](https://github.com/huggingface/pytorch-transformers). - -## How to use - -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("cooelf/limitbert") -model = AutoModel.from_pretrained("cooelf/limitbert") -``` - -Please see our original repo for the training scripts. - -https://github.com/cooelf/LIMIT-BERT - -## Training - -To train LIMIT-BERT, simply run: -``` -sh run_limitbert.sh -``` -### Evaluation Instructions - -To test after setting model path: -``` -sh test_bert.sh -``` - -## Citation - -``` -@article{zhou2019limit, - title={{LIMIT-BERT}: Linguistic informed multi-task {BERT}}, - author={Zhou, Junru and Zhang, Zhuosheng and Zhao, Hai}, - journal={arXiv preprint arXiv:1910.14296}, - year={2019} -} -``` \ No newline at end of file diff --git a/model_cards/csarron/bert-base-uncased-squad-v1/README.md b/model_cards/csarron/bert-base-uncased-squad-v1/README.md deleted file mode 100644 index b8987bb63703..000000000000 --- a/model_cards/csarron/bert-base-uncased-squad-v1/README.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -language: en -thumbnail: -license: mit -tags: -- question-answering -- bert -- bert-base -datasets: -- squad -metrics: -- squad -widget: -- text: "Which name is also used to describe the Amazon rainforest in English?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." -- text: "How many square kilometers of rainforest is covered in the basin?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." ---- - -## BERT-base uncased model fine-tuned on SQuAD v1 - -This model was fine-tuned from the HuggingFace [BERT](https://www.aclweb.org/anthology/N19-1423/) base uncased checkpoint on [SQuAD1.1](https://rajpurkar.github.io/SQuAD-explorer). -This model is case-insensitive: it does not make a difference between english and English. - -## Details - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD1.1 | train | 90.6K | -| SQuAD1.1 | eval | 11.1k | - - -### Fine-tuning -- Python: `3.7.5` - -- Machine specs: - - `CPU: Intel(R) Core(TM) i7-6800K CPU @ 3.40GHz` - - `Memory: 32 GiB` - - `GPUs: 2 GeForce GTX 1070, each with 8GiB memory` - - `GPU driver: 418.87.01, CUDA: 10.1` - -- script: - - ```shell - # after install https://github.com/huggingface/transformers - - cd examples/question-answering - mkdir -p data - - wget -O data/train-v1.1.json https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json - - wget -O data/dev-v1.1.json https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json - - python run_squad.py \ - --model_type bert \ - --model_name_or_path bert-base-uncased \ - --do_train \ - --do_eval \ - --do_lower_case \ - --train_file train-v1.1.json \ - --predict_file dev-v1.1.json \ - --per_gpu_train_batch_size 12 \ - --per_gpu_eval_batch_size=16 \ - --learning_rate 3e-5 \ - --num_train_epochs 2.0 \ - --max_seq_length 320 \ - --doc_stride 128 \ - --data_dir data \ - --output_dir data/bert-base-uncased-squad-v1 2>&1 | tee train-energy-bert-base-squad-v1.log - ``` - -It took about 2 hours to finish. - -### Results - -**Model size**: `418M` - -| Metric | # Value | # Original ([Table 2](https://www.aclweb.org/anthology/N19-1423.pdf))| -| ------ | --------- | --------- | -| **EM** | **80.9** | **80.8** | -| **F1** | **88.2** | **88.5** | - -Note that the above results didn't involve any hyperparameter search. - -## Example Usage - - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="csarron/bert-base-uncased-squad-v1", - tokenizer="csarron/bert-base-uncased-squad-v1" -) - -predictions = qa_pipeline({ - 'context': "The game was played on February 7, 2016 at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.", - 'question': "What day was the game played on?" -}) - -print(predictions) -# output: -# {'score': 0.8730505704879761, 'start': 23, 'end': 39, 'answer': 'February 7, 2016'} -``` - -> Created by [Qingqing Cao](https://awk.ai/) | [GitHub](https://github.com/csarron) | [Twitter](https://twitter.com/sysnlp) - -> Made with ❤️ in New York. diff --git a/model_cards/csarron/mobilebert-uncased-squad-v1/README.md b/model_cards/csarron/mobilebert-uncased-squad-v1/README.md deleted file mode 100644 index 55e4eec71d95..000000000000 --- a/model_cards/csarron/mobilebert-uncased-squad-v1/README.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -language: en -thumbnail: -license: mit -tags: -- question-answering -- mobilebert -datasets: -- squad -metrics: -- squad -widget: -- text: "Which name is also used to describe the Amazon rainforest in English?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." -- text: "How many square kilometers of rainforest is covered in the basin?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." ---- - -## MobileBERT fine-tuned on SQuAD v1 - -[MobileBERT](https://arxiv.org/abs/2004.02984) is a thin version of BERT_LARGE, while equipped with bottleneck structures and a carefully designed balance -between self-attentions and feed-forward networks. - -This model was fine-tuned from the HuggingFace checkpoint `google/mobilebert-uncased` on [SQuAD1.1](https://rajpurkar.github.io/SQuAD-explorer). - -## Details - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD1.1 | train | 90.6K | -| SQuAD1.1 | eval | 11.1k | - - -### Fine-tuning -- Python: `3.7.5` - -- Machine specs: - - `CPU: Intel(R) Core(TM) i7-6800K CPU @ 3.40GHz` - - `Memory: 32 GiB` - - `GPUs: 2 GeForce GTX 1070, each with 8GiB memory` - - `GPU driver: 418.87.01, CUDA: 10.1` - -- script: - - ```shell - # after install https://github.com/huggingface/transformers - - cd examples/question-answering - mkdir -p data - - wget -O data/train-v1.1.json https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json - - wget -O data/dev-v1.1.json https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json - - export SQUAD_DIR=`pwd`/data - - python run_squad.py \ - --model_type mobilebert \ - --model_name_or_path google/mobilebert-uncased \ - --do_train \ - --do_eval \ - --do_lower_case \ - --train_file $SQUAD_DIR/train-v1.1.json \ - --predict_file $SQUAD_DIR/dev-v1.1.json \ - --per_gpu_train_batch_size 16 \ - --per_gpu_eval_batch_size 16 \ - --learning_rate 4e-5 \ - --num_train_epochs 5.0 \ - --max_seq_length 320 \ - --doc_stride 128 \ - --warmup_steps 1400 \ - --output_dir $SQUAD_DIR/mobilebert-uncased-warmup-squad_v1 2>&1 | tee train-mobilebert-warmup-squad_v1.log - ``` - -It took about 3 hours to finish. - -### Results - -**Model size**: `95M` - -| Metric | # Value | # Original ([Table 5](https://arxiv.org/pdf/2004.02984.pdf))| -| ------ | --------- | --------- | -| **EM** | **82.6** | **82.9** | -| **F1** | **90.0** | **90.0** | - -Note that the above results didn't involve any hyperparameter search. - -## Example Usage - - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="csarron/mobilebert-uncased-squad-v1", - tokenizer="csarron/mobilebert-uncased-squad-v1" -) - -predictions = qa_pipeline({ - 'context': "The game was played on February 7, 2016 at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.", - 'question': "What day was the game played on?" -}) - -print(predictions) -# output: -# {'score': 0.7754058241844177, 'start': 23, 'end': 39, 'answer': 'February 7, 2016'} -``` - -> Created by [Qingqing Cao](https://awk.ai/) | [GitHub](https://github.com/csarron) | [Twitter](https://twitter.com/sysnlp) - -> Made with ❤️ in New York. \ No newline at end of file diff --git a/model_cards/csarron/mobilebert-uncased-squad-v2/README.md b/model_cards/csarron/mobilebert-uncased-squad-v2/README.md deleted file mode 100644 index 8305dbf5a9cf..000000000000 --- a/model_cards/csarron/mobilebert-uncased-squad-v2/README.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -language: en -thumbnail: -license: mit -tags: -- question-answering -- mobilebert -datasets: -- squad_v2 -metrics: -- squad_v2 -widget: -- text: "Which name is also used to describe the Amazon rainforest in English?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." -- text: "How many square kilometers of rainforest is covered in the basin?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." ---- - -## MobileBERT fine-tuned on SQuAD v2 - -[MobileBERT](https://arxiv.org/abs/2004.02984) is a thin version of BERT_LARGE, while equipped with bottleneck structures and a carefully designed balance -between self-attentions and feed-forward networks. - -This model was fine-tuned from the HuggingFace checkpoint `google/mobilebert-uncased` on [SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer). - -## Details - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD2.0 | train | 130k | -| SQuAD2.0 | eval | 12.3k | - - -### Fine-tuning -- Python: `3.7.5` - -- Machine specs: - - `CPU: Intel(R) Core(TM) i7-6800K CPU @ 3.40GHz` - - `Memory: 32 GiB` - - `GPUs: 2 GeForce GTX 1070, each with 8GiB memory` - - `GPU driver: 418.87.01, CUDA: 10.1` - -- script: - - ```shell - # after install https://github.com/huggingface/transformers - - cd examples/question-answering - mkdir -p data - - wget -O data/train-v2.0.json https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json - - wget -O data/dev-v2.0.json https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json - - export SQUAD_DIR=`pwd`/data - - python run_squad.py \ - --model_type mobilebert \ - --model_name_or_path google/mobilebert-uncased \ - --do_train \ - --do_eval \ - --do_lower_case \ - --version_2_with_negative \ - --train_file $SQUAD_DIR/train-v2.0.json \ - --predict_file $SQUAD_DIR/dev-v2.0.json \ - --per_gpu_train_batch_size 16 \ - --per_gpu_eval_batch_size 16 \ - --learning_rate 4e-5 \ - --num_train_epochs 5.0 \ - --max_seq_length 320 \ - --doc_stride 128 \ - --warmup_steps 1400 \ - --save_steps 2000 \ - --output_dir $SQUAD_DIR/mobilebert-uncased-warmup-squad_v2 2>&1 | tee train-mobilebert-warmup-squad_v2.log - ``` - -It took about 3.5 hours to finish. - -### Results - -**Model size**: `95M` - -| Metric | # Value | # Original ([Table 5](https://arxiv.org/pdf/2004.02984.pdf))| -| ------ | --------- | --------- | -| **EM** | **75.2** | **76.2** | -| **F1** | **78.8** | **79.2** | - -Note that the above results didn't involve any hyperparameter search. - -## Example Usage - - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="csarron/mobilebert-uncased-squad-v2", - tokenizer="csarron/mobilebert-uncased-squad-v2" -) - -predictions = qa_pipeline({ - 'context': "The game was played on February 7, 2016 at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.", - 'question': "What day was the game played on?" -}) - -print(predictions) -# output: -# {'score': 0.71434086561203, 'start': 23, 'end': 39, 'answer': 'February 7, 2016'} -``` - -> Created by [Qingqing Cao](https://awk.ai/) | [GitHub](https://github.com/csarron) | [Twitter](https://twitter.com/sysnlp) - -> Made with ❤️ in New York. \ No newline at end of file diff --git a/model_cards/csarron/roberta-base-squad-v1/README.md b/model_cards/csarron/roberta-base-squad-v1/README.md deleted file mode 100644 index 7e030ea5da5e..000000000000 --- a/model_cards/csarron/roberta-base-squad-v1/README.md +++ /dev/null @@ -1,112 +0,0 @@ ---- -language: en -thumbnail: -license: mit -tags: -- question-answering -- roberta -- roberta-base -datasets: -- squad -metrics: -- squad -widget: -- text: "Which name is also used to describe the Amazon rainforest in English?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." -- text: "How many square kilometers of rainforest is covered in the basin?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." ---- - -## RoBERTa-base fine-tuned on SQuAD v1 - -This model was fine-tuned from the HuggingFace [RoBERTa](https://arxiv.org/abs/1907.11692) base checkpoint on [SQuAD1.1](https://rajpurkar.github.io/SQuAD-explorer). -This model is case-sensitive: it makes a difference between english and English. - -## Details - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD1.1 | train | 96.8K | -| SQuAD1.1 | eval | 11.8k | - - -### Fine-tuning -- Python: `3.7.5` - -- Machine specs: - - `CPU: Intel(R) Core(TM) i7-6800K CPU @ 3.40GHz` - - `Memory: 32 GiB` - - `GPUs: 2 GeForce GTX 1070, each with 8GiB memory` - - `GPU driver: 418.87.01, CUDA: 10.1` - -- script: - - ```shell - # after install https://github.com/huggingface/transformers - - cd examples/question-answering - mkdir -p data - - wget -O data/train-v1.1.json https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json - - wget -O data/dev-v1.1.json https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json - - python run_energy_squad.py \ - --model_type roberta \ - --model_name_or_path roberta-base \ - --do_train \ - --do_eval \ - --train_file train-v1.1.json \ - --predict_file dev-v1.1.json \ - --per_gpu_train_batch_size 12 \ - --per_gpu_eval_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 2.0 \ - --max_seq_length 320 \ - --doc_stride 128 \ - --data_dir data \ - --output_dir data/roberta-base-squad-v1 2>&1 | tee train-roberta-base-squad-v1.log - ``` - -It took about 2 hours to finish. - -### Results - -**Model size**: `477M` - -| Metric | # Value | -| ------ | --------- | -| **EM** | **83.0** | -| **F1** | **90.4** | - -Note that the above results didn't involve any hyperparameter search. - -## Example Usage - - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="csarron/roberta-base-squad-v1", - tokenizer="csarron/roberta-base-squad-v1" -) - -predictions = qa_pipeline({ - 'context': "The game was played on February 7, 2016 at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.", - 'question': "What day was the game played on?" -}) - -print(predictions) -# output: -# {'score': 0.8625259399414062, 'start': 23, 'end': 39, 'answer': 'February 7, 2016'} -``` - -> Created by [Qingqing Cao](https://awk.ai/) | [GitHub](https://github.com/csarron) | [Twitter](https://twitter.com/sysnlp) - -> Made with ❤️ in New York. diff --git a/model_cards/daigo/bert-base-japanese-sentiment/README.md b/model_cards/daigo/bert-base-japanese-sentiment/README.md deleted file mode 100644 index 950f692cfd34..000000000000 --- a/model_cards/daigo/bert-base-japanese-sentiment/README.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -language: -- ja ---- - -binary classification - -# Usage -``` -print(pipeline("sentiment-analysis",model="daigo/bert-base-japanese-sentiment",tokenizer="daigo/bert-base-japanese-sentiment")("私は幸福である。")) - -[{'label': 'ポジティブ', 'score': 0.98430425}] -``` diff --git a/model_cards/dbmdz/bert-base-german-cased/README.md b/model_cards/dbmdz/bert-base-german-cased/README.md deleted file mode 100644 index 8478cce520aa..000000000000 --- a/model_cards/dbmdz/bert-base-german-cased/README.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -language: de -license: mit ---- - -# 🤗 + 📚 dbmdz German BERT models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources another German BERT models 🎉 - -# German BERT - -## Stats - -In addition to the recently released [German BERT](https://deepset.ai/german-bert) -model by [deepset](https://deepset.ai/) we provide another German-language model. - -The source data for the model consists of a recent Wikipedia dump, EU Bookshop corpus, -Open Subtitles, CommonCrawl, ParaCrawl and News Crawl. This results in a dataset with -a size of 16GB and 2,350,234,427 tokens. - -For sentence splitting, we use [spacy](https://spacy.io/). Our preprocessing steps -(sentence piece model for vocab generation) follow those used for training -[SciBERT](https://github.com/allenai/scibert). The model is trained with an initial -sequence length of 512 subwords and was performed for 1.5M steps. - -This release includes both cased and uncased models. - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| -------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `bert-base-german-dbmdz-cased` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-config.json) • [`pytorch_model.bin`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-pytorch_model.bin) • [`vocab.txt`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-vocab.txt) -| `bert-base-german-dbmdz-uncased` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-config.json) • [`pytorch_model.bin`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-pytorch_model.bin) • [`vocab.txt`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-vocab.txt) - -## Usage - -With Transformers >= 2.3 our German BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased") -model = AutoModel.from_pretrained("dbmdz/bert-base-german-cased") -``` - -## Results - -For results on downstream tasks like NER or PoS tagging, please refer to -[this repository](https://github.com/stefan-it/fine-tuned-berts-seq). - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-german-europeana-cased/README.md b/model_cards/dbmdz/bert-base-german-europeana-cased/README.md deleted file mode 100644 index 7ed11b31a308..000000000000 --- a/model_cards/dbmdz/bert-base-german-europeana-cased/README.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -language: de -license: mit -tags: - - "historic german" ---- - -# 🤗 + 📚 dbmdz BERT models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources German Europeana BERT models 🎉 - -# German Europeana BERT - -We use the open source [Europeana newspapers](http://www.europeana-newspapers.eu/) -that were provided by *The European Library*. The final -training corpus has a size of 51GB and consists of 8,035,986,369 tokens. - -Detailed information about the data and pretraining steps can be found in -[this repository](https://github.com/stefan-it/europeana-bert). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ------------------------------------------ | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-german-europeana-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-german-europeana-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-german-europeana-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-german-europeana-cased/vocab.txt) - -## Results - -For results on Historic NER, please refer to [this repository](https://github.com/stefan-it/europeana-bert). - -## Usage - -With Transformers >= 2.3 our German Europeana BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-europeana-cased") -model = AutoModel.from_pretrained("dbmdz/bert-base-german-europeana-cased") -``` - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-german-europeana-uncased/README.md b/model_cards/dbmdz/bert-base-german-europeana-uncased/README.md deleted file mode 100644 index 7bdc2ea37d77..000000000000 --- a/model_cards/dbmdz/bert-base-german-europeana-uncased/README.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -language: de -license: mit -tags: - - "historic german" ---- - -# 🤗 + 📚 dbmdz BERT models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources German Europeana BERT models 🎉 - -# German Europeana BERT - -We use the open source [Europeana newspapers](http://www.europeana-newspapers.eu/) -that were provided by *The European Library*. The final -training corpus has a size of 51GB and consists of 8,035,986,369 tokens. - -Detailed information about the data and pretraining steps can be found in -[this repository](https://github.com/stefan-it/europeana-bert). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ------------------------------------------ | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-german-europeana-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-german-europeana-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-german-europeana-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-german-europeana-uncased/vocab.txt) - -## Results - -For results on Historic NER, please refer to [this repository](https://github.com/stefan-it/europeana-bert). - -## Usage - -With Transformers >= 2.3 our German Europeana BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-europeana-uncased") -model = AutoModel.from_pretrained("dbmdz/bert-base-german-europeana-uncased") -``` - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-german-uncased/README.md b/model_cards/dbmdz/bert-base-german-uncased/README.md deleted file mode 100644 index 8478cce520aa..000000000000 --- a/model_cards/dbmdz/bert-base-german-uncased/README.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -language: de -license: mit ---- - -# 🤗 + 📚 dbmdz German BERT models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources another German BERT models 🎉 - -# German BERT - -## Stats - -In addition to the recently released [German BERT](https://deepset.ai/german-bert) -model by [deepset](https://deepset.ai/) we provide another German-language model. - -The source data for the model consists of a recent Wikipedia dump, EU Bookshop corpus, -Open Subtitles, CommonCrawl, ParaCrawl and News Crawl. This results in a dataset with -a size of 16GB and 2,350,234,427 tokens. - -For sentence splitting, we use [spacy](https://spacy.io/). Our preprocessing steps -(sentence piece model for vocab generation) follow those used for training -[SciBERT](https://github.com/allenai/scibert). The model is trained with an initial -sequence length of 512 subwords and was performed for 1.5M steps. - -This release includes both cased and uncased models. - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| -------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `bert-base-german-dbmdz-cased` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-config.json) • [`pytorch_model.bin`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-pytorch_model.bin) • [`vocab.txt`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-vocab.txt) -| `bert-base-german-dbmdz-uncased` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-config.json) • [`pytorch_model.bin`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-pytorch_model.bin) • [`vocab.txt`](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-vocab.txt) - -## Usage - -With Transformers >= 2.3 our German BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased") -model = AutoModel.from_pretrained("dbmdz/bert-base-german-cased") -``` - -## Results - -For results on downstream tasks like NER or PoS tagging, please refer to -[this repository](https://github.com/stefan-it/fine-tuned-berts-seq). - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-italian-cased/README.md b/model_cards/dbmdz/bert-base-italian-cased/README.md deleted file mode 100644 index 43c9de3da0c6..000000000000 --- a/model_cards/dbmdz/bert-base-italian-cased/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: it -license: mit -datasets: -- wikipedia ---- - -# 🤗 + 📚 dbmdz BERT and ELECTRA models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources Italian BERT and ELECTRA models 🎉 - -# Italian BERT - -The source data for the Italian BERT model consists of a recent Wikipedia dump and -various texts from the [OPUS corpora](http://opus.nlpl.eu/) collection. The final -training corpus has a size of 13GB and 2,050,057,573 tokens. - -For sentence splitting, we use NLTK (faster compared to spacy). -Our cased and uncased models are training with an initial sequence length of 512 -subwords for ~2-3M steps. - -For the XXL Italian models, we use the same training data from OPUS and extend -it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/). -Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens. - -Note: Unfortunately, a wrong vocab size was used when training the XXL models. -This explains the mismatch of the "real" vocab size of 31102, compared to the -vocab size specified in `config.json`. However, the model is working and all -evaluations were done under those circumstances. -See [this issue](https://github.com/dbmdz/berts/issues/7) for more information. - -The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch -size of 128. We pretty much following the ELECTRA training procedure as used for -[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-italian-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt) -| `dbmdz/bert-base-italian-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-generator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt) - -## Results - -For results on downstream tasks like NER or PoS tagging, please refer to -[this repository](https://github.com/stefan-it/italian-bertelectra). - -## Usage - -With Transformers >= 2.3 our Italian BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the (recommended) Italian XXL BERT models, just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-xxl-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the Italian XXL ELECTRA model (discriminator), just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModelWithLMHead.from_pretrained(model_name) -``` - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT/ELECTRA models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-italian-uncased/README.md b/model_cards/dbmdz/bert-base-italian-uncased/README.md deleted file mode 100644 index 43c9de3da0c6..000000000000 --- a/model_cards/dbmdz/bert-base-italian-uncased/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: it -license: mit -datasets: -- wikipedia ---- - -# 🤗 + 📚 dbmdz BERT and ELECTRA models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources Italian BERT and ELECTRA models 🎉 - -# Italian BERT - -The source data for the Italian BERT model consists of a recent Wikipedia dump and -various texts from the [OPUS corpora](http://opus.nlpl.eu/) collection. The final -training corpus has a size of 13GB and 2,050,057,573 tokens. - -For sentence splitting, we use NLTK (faster compared to spacy). -Our cased and uncased models are training with an initial sequence length of 512 -subwords for ~2-3M steps. - -For the XXL Italian models, we use the same training data from OPUS and extend -it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/). -Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens. - -Note: Unfortunately, a wrong vocab size was used when training the XXL models. -This explains the mismatch of the "real" vocab size of 31102, compared to the -vocab size specified in `config.json`. However, the model is working and all -evaluations were done under those circumstances. -See [this issue](https://github.com/dbmdz/berts/issues/7) for more information. - -The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch -size of 128. We pretty much following the ELECTRA training procedure as used for -[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-italian-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt) -| `dbmdz/bert-base-italian-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-generator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt) - -## Results - -For results on downstream tasks like NER or PoS tagging, please refer to -[this repository](https://github.com/stefan-it/italian-bertelectra). - -## Usage - -With Transformers >= 2.3 our Italian BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the (recommended) Italian XXL BERT models, just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-xxl-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the Italian XXL ELECTRA model (discriminator), just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModelWithLMHead.from_pretrained(model_name) -``` - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT/ELECTRA models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-italian-xxl-cased/README.md b/model_cards/dbmdz/bert-base-italian-xxl-cased/README.md deleted file mode 100644 index 43c9de3da0c6..000000000000 --- a/model_cards/dbmdz/bert-base-italian-xxl-cased/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: it -license: mit -datasets: -- wikipedia ---- - -# 🤗 + 📚 dbmdz BERT and ELECTRA models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources Italian BERT and ELECTRA models 🎉 - -# Italian BERT - -The source data for the Italian BERT model consists of a recent Wikipedia dump and -various texts from the [OPUS corpora](http://opus.nlpl.eu/) collection. The final -training corpus has a size of 13GB and 2,050,057,573 tokens. - -For sentence splitting, we use NLTK (faster compared to spacy). -Our cased and uncased models are training with an initial sequence length of 512 -subwords for ~2-3M steps. - -For the XXL Italian models, we use the same training data from OPUS and extend -it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/). -Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens. - -Note: Unfortunately, a wrong vocab size was used when training the XXL models. -This explains the mismatch of the "real" vocab size of 31102, compared to the -vocab size specified in `config.json`. However, the model is working and all -evaluations were done under those circumstances. -See [this issue](https://github.com/dbmdz/berts/issues/7) for more information. - -The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch -size of 128. We pretty much following the ELECTRA training procedure as used for -[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-italian-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt) -| `dbmdz/bert-base-italian-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-generator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt) - -## Results - -For results on downstream tasks like NER or PoS tagging, please refer to -[this repository](https://github.com/stefan-it/italian-bertelectra). - -## Usage - -With Transformers >= 2.3 our Italian BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the (recommended) Italian XXL BERT models, just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-xxl-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the Italian XXL ELECTRA model (discriminator), just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModelWithLMHead.from_pretrained(model_name) -``` - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT/ELECTRA models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-italian-xxl-uncased/README.md b/model_cards/dbmdz/bert-base-italian-xxl-uncased/README.md deleted file mode 100644 index 43c9de3da0c6..000000000000 --- a/model_cards/dbmdz/bert-base-italian-xxl-uncased/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: it -license: mit -datasets: -- wikipedia ---- - -# 🤗 + 📚 dbmdz BERT and ELECTRA models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources Italian BERT and ELECTRA models 🎉 - -# Italian BERT - -The source data for the Italian BERT model consists of a recent Wikipedia dump and -various texts from the [OPUS corpora](http://opus.nlpl.eu/) collection. The final -training corpus has a size of 13GB and 2,050,057,573 tokens. - -For sentence splitting, we use NLTK (faster compared to spacy). -Our cased and uncased models are training with an initial sequence length of 512 -subwords for ~2-3M steps. - -For the XXL Italian models, we use the same training data from OPUS and extend -it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/). -Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens. - -Note: Unfortunately, a wrong vocab size was used when training the XXL models. -This explains the mismatch of the "real" vocab size of 31102, compared to the -vocab size specified in `config.json`. However, the model is working and all -evaluations were done under those circumstances. -See [this issue](https://github.com/dbmdz/berts/issues/7) for more information. - -The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch -size of 128. We pretty much following the ELECTRA training procedure as used for -[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-italian-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt) -| `dbmdz/bert-base-italian-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-generator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt) - -## Results - -For results on downstream tasks like NER or PoS tagging, please refer to -[this repository](https://github.com/stefan-it/italian-bertelectra). - -## Usage - -With Transformers >= 2.3 our Italian BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the (recommended) Italian XXL BERT models, just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-xxl-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the Italian XXL ELECTRA model (discriminator), just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModelWithLMHead.from_pretrained(model_name) -``` - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT/ELECTRA models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-turkish-128k-cased/README.md b/model_cards/dbmdz/bert-base-turkish-128k-cased/README.md deleted file mode 100644 index 7047f8f184d9..000000000000 --- a/model_cards/dbmdz/bert-base-turkish-128k-cased/README.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -language: tr -license: mit ---- - -# 🤗 + 📚 dbmdz Turkish BERT model - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources a cased model for Turkish 🎉 - -# 🇹🇷 BERTurk - -BERTurk is a community-driven cased BERT model for Turkish. - -Some datasets used for pretraining and evaluation are contributed from the -awesome Turkish NLP community, as well as the decision for the model name: BERTurk. - -## Stats - -The current version of the model is trained on a filtered and sentence -segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/), -a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a -special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/). - -The final training corpus has a size of 35GB and 44,04,976,662 tokens. - -Thanks to Google's TensorFlow Research Cloud (TFRC) we could train a cased model -on a TPU v3-8 for 2M steps. - -For this model we use a vocab size of 128k. - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ------------------------------------ | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-turkish-128k-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-cased/vocab.txt) - -## Usage - -With Transformers >= 2.3 our BERTurk cased model can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-128k-cased") -model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-128k-cased") -``` - -## Results - -For results on PoS tagging or NER tasks, please refer to -[this repository](https://github.com/stefan-it/turkish-bert). - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us -additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing -us the Turkish NER dataset for evaluation. - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-turkish-128k-uncased/README.md b/model_cards/dbmdz/bert-base-turkish-128k-uncased/README.md deleted file mode 100644 index 1477fdcff0ca..000000000000 --- a/model_cards/dbmdz/bert-base-turkish-128k-uncased/README.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -language: tr -license: mit ---- - -# 🤗 + 📚 dbmdz Turkish BERT model - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources an uncased model for Turkish 🎉 - -# 🇹🇷 BERTurk - -BERTurk is a community-driven uncased BERT model for Turkish. - -Some datasets used for pretraining and evaluation are contributed from the -awesome Turkish NLP community, as well as the decision for the model name: BERTurk. - -## Stats - -The current version of the model is trained on a filtered and sentence -segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/), -a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a -special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/). - -The final training corpus has a size of 35GB and 44,04,976,662 tokens. - -Thanks to Google's TensorFlow Research Cloud (TFRC) we could train an uncased model -on a TPU v3-8 for 2M steps. - -For this model we use a vocab size of 128k. - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| -------------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-turkish-128k-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-128k-uncased/vocab.txt) - -## Usage - -With Transformers >= 2.3 our BERTurk uncased model can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-128k-uncased") -model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-128k-uncased") -``` - -## Results - -For results on PoS tagging or NER tasks, please refer to -[this repository](https://github.com/stefan-it/turkish-bert). - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us -additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing -us the Turkish NER dataset for evaluation. - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-turkish-cased/README.md b/model_cards/dbmdz/bert-base-turkish-cased/README.md deleted file mode 100644 index 550cee199f9e..000000000000 --- a/model_cards/dbmdz/bert-base-turkish-cased/README.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -language: tr -license: mit ---- - -# 🤗 + 📚 dbmdz Turkish BERT model - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources a cased model for Turkish 🎉 - -# 🇹🇷 BERTurk - -BERTurk is a community-driven cased BERT model for Turkish. - -Some datasets used for pretraining and evaluation are contributed from the -awesome Turkish NLP community, as well as the decision for the model name: BERTurk. - -## Stats - -The current version of the model is trained on a filtered and sentence -segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/), -a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a -special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/). - -The final training corpus has a size of 35GB and 44,04,976,662 tokens. - -Thanks to Google's TensorFlow Research Cloud (TFRC) we could train a cased model -on a TPU v3-8 for 2M steps. - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| --------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-turkish-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-cased/vocab.txt) - -## Usage - -With Transformers >= 2.3 our BERTurk cased model can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-cased") -model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-cased") -``` - -## Results - -For results on PoS tagging or NER tasks, please refer to -[this repository](https://github.com/stefan-it/turkish-bert). - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us -additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing -us the Turkish NER dataset for evaluation. - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/bert-base-turkish-uncased/README.md b/model_cards/dbmdz/bert-base-turkish-uncased/README.md deleted file mode 100644 index 68fcf64b4565..000000000000 --- a/model_cards/dbmdz/bert-base-turkish-uncased/README.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -language: tr -license: mit ---- - -# 🤗 + 📚 dbmdz Turkish BERT model - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources an uncased model for Turkish 🎉 - -# 🇹🇷 BERTurk - -BERTurk is a community-driven uncased BERT model for Turkish. - -Some datasets used for pretraining and evaluation are contributed from the -awesome Turkish NLP community, as well as the decision for the model name: BERTurk. - -## Stats - -The current version of the model is trained on a filtered and sentence -segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/), -a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a -special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/). - -The final training corpus has a size of 35GB and 44,04,976,662 tokens. - -Thanks to Google's TensorFlow Research Cloud (TFRC) we could train an uncased model -on a TPU v3-8 for 2M steps. - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| --------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-turkish-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-turkish-uncased/vocab.txt) - -## Usage - -With Transformers >= 2.3 our BERTurk uncased model can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-turkish-uncased") -model = AutoModel.from_pretrained("dbmdz/bert-base-turkish-uncased") -``` - -## Results - -For results on PoS tagging or NER tasks, please refer to -[this repository](https://github.com/stefan-it/turkish-bert). - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us -additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing -us the Turkish NER dataset for evaluation. - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/distilbert-base-turkish-cased/README.md b/model_cards/dbmdz/distilbert-base-turkish-cased/README.md deleted file mode 100644 index d254ac81c855..000000000000 --- a/model_cards/dbmdz/distilbert-base-turkish-cased/README.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -language: tr -license: mit ---- - -# 🤗 + 📚 dbmdz Distilled Turkish BERT model - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources a (cased) distilled model for Turkish 🎉 - -# 🇹🇷 DistilBERTurk - -DistilBERTurk is a community-driven cased distilled BERT model for Turkish. - -DistilBERTurk was trained on 7GB of the original training data that was used -for training [BERTurk](https://github.com/stefan-it/turkish-bert/tree/master#stats), -using the cased version of BERTurk as teacher model. - -*DistilBERTurk* was trained with the official Hugging Face implementation from -[here](https://github.com/huggingface/transformers/tree/master/examples/distillation) -for 5 days on 4 RTX 2080 TI. - -More details about distillation can be found in the -["DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter"](https://arxiv.org/abs/1910.01108) -paper by Sanh et al. (2019). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue in the [BERTurk](https://github.com/stefan-it/turkish-bert) repository! - -| Model | Downloads -| --------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/distilbert-base-turkish-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/distilbert-base-turkish-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/distilbert-base-turkish-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/distilbert-base-turkish-cased/vocab.txt) - -## Usage - -With Transformers >= 2.3 our DistilBERTurk model can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/distilbert-base-turkish-cased") -model = AutoModel.from_pretrained("dbmdz/distilbert-base-turkish-cased") -``` - -## Results - -For results on PoS tagging or NER tasks, please refer to -[this repository](https://github.com/stefan-it/turkish-bert). - -For PoS tagging, DistilBERTurk outperforms the 24-layer XLM-RoBERTa model. - -The overall performance difference between DistilBERTurk and the original -(teacher) BERTurk model is ~1.18%. - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us -additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing -us the Turkish NER dataset for evaluation. - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/electra-base-italian-xxl-cased-discriminator/README.md b/model_cards/dbmdz/electra-base-italian-xxl-cased-discriminator/README.md deleted file mode 100644 index 43c9de3da0c6..000000000000 --- a/model_cards/dbmdz/electra-base-italian-xxl-cased-discriminator/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: it -license: mit -datasets: -- wikipedia ---- - -# 🤗 + 📚 dbmdz BERT and ELECTRA models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources Italian BERT and ELECTRA models 🎉 - -# Italian BERT - -The source data for the Italian BERT model consists of a recent Wikipedia dump and -various texts from the [OPUS corpora](http://opus.nlpl.eu/) collection. The final -training corpus has a size of 13GB and 2,050,057,573 tokens. - -For sentence splitting, we use NLTK (faster compared to spacy). -Our cased and uncased models are training with an initial sequence length of 512 -subwords for ~2-3M steps. - -For the XXL Italian models, we use the same training data from OPUS and extend -it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/). -Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens. - -Note: Unfortunately, a wrong vocab size was used when training the XXL models. -This explains the mismatch of the "real" vocab size of 31102, compared to the -vocab size specified in `config.json`. However, the model is working and all -evaluations were done under those circumstances. -See [this issue](https://github.com/dbmdz/berts/issues/7) for more information. - -The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch -size of 128. We pretty much following the ELECTRA training procedure as used for -[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-italian-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt) -| `dbmdz/bert-base-italian-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-generator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt) - -## Results - -For results on downstream tasks like NER or PoS tagging, please refer to -[this repository](https://github.com/stefan-it/italian-bertelectra). - -## Usage - -With Transformers >= 2.3 our Italian BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the (recommended) Italian XXL BERT models, just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-xxl-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the Italian XXL ELECTRA model (discriminator), just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModelWithLMHead.from_pretrained(model_name) -``` - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT/ELECTRA models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/electra-base-italian-xxl-cased-generator/README.md b/model_cards/dbmdz/electra-base-italian-xxl-cased-generator/README.md deleted file mode 100644 index 43c9de3da0c6..000000000000 --- a/model_cards/dbmdz/electra-base-italian-xxl-cased-generator/README.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -language: it -license: mit -datasets: -- wikipedia ---- - -# 🤗 + 📚 dbmdz BERT and ELECTRA models - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources Italian BERT and ELECTRA models 🎉 - -# Italian BERT - -The source data for the Italian BERT model consists of a recent Wikipedia dump and -various texts from the [OPUS corpora](http://opus.nlpl.eu/) collection. The final -training corpus has a size of 13GB and 2,050,057,573 tokens. - -For sentence splitting, we use NLTK (faster compared to spacy). -Our cased and uncased models are training with an initial sequence length of 512 -subwords for ~2-3M steps. - -For the XXL Italian models, we use the same training data from OPUS and extend -it with data from the Italian part of the [OSCAR corpus](https://traces1.inria.fr/oscar/). -Thus, the final training corpus has a size of 81GB and 13,138,379,147 tokens. - -Note: Unfortunately, a wrong vocab size was used when training the XXL models. -This explains the mismatch of the "real" vocab size of 31102, compared to the -vocab size specified in `config.json`. However, the model is working and all -evaluations were done under those circumstances. -See [this issue](https://github.com/dbmdz/berts/issues/7) for more information. - -The Italian ELECTRA model was trained on the "XXL" corpus for 1M steps in total using a batch -size of 128. We pretty much following the ELECTRA training procedure as used for -[BERTurk](https://github.com/stefan-it/turkish-bert/tree/master/electra). - -## Model weights - -Currently only PyTorch-[Transformers](https://github.com/huggingface/transformers) -compatible weights are available. If you need access to TensorFlow checkpoints, -please raise an issue! - -| Model | Downloads -| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/bert-base-italian-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-cased/vocab.txt) -| `dbmdz/bert-base-italian-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-uncased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-cased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-cased/vocab.txt) -| `dbmdz/bert-base-italian-xxl-uncased` | [`config.json`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/bert-base-italian-xxl-uncased/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-discriminator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-discriminator/vocab.txt) -| `dbmdz/electra-base-italian-xxl-cased-generator` | [`config.json`](https://s3.amazonaws.com/models.huggingface.co/bert/dbmdz/electra-base-italian-xxl-cased-generator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-italian-xxl-cased-generator/vocab.txt) - -## Results - -For results on downstream tasks like NER or PoS tagging, please refer to -[this repository](https://github.com/stefan-it/italian-bertelectra). - -## Usage - -With Transformers >= 2.3 our Italian BERT models can be loaded like: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the (recommended) Italian XXL BERT models, just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/bert-base-italian-xxl-cased" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModel.from_pretrained(model_name) -``` - -To load the Italian XXL ELECTRA model (discriminator), just use: - -```python -from transformers import AutoModel, AutoTokenizer - -model_name = "dbmdz/electra-base-italian-xxl-cased-discriminator" - -tokenizer = AutoTokenizer.from_pretrained(model_name) - -model = AutoModelWithLMHead.from_pretrained(model_name) -``` - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our BERT/ELECTRA models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/electra-base-turkish-cased-discriminator/README.md b/model_cards/dbmdz/electra-base-turkish-cased-discriminator/README.md deleted file mode 100644 index c89d2300cba2..000000000000 --- a/model_cards/dbmdz/electra-base-turkish-cased-discriminator/README.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -language: tr -license: mit ---- - -# 🤗 + 📚 dbmdz Turkish ELECTRA model - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources a cased ELECTRA base model for Turkish 🎉 - -# Turkish ELECTRA model - -We release a base ELEC**TR**A model for Turkish, that was trained on the same data as *BERTurk*. - -> ELECTRA is a new method for self-supervised language representation learning. It can be used to -> pre-train transformer networks using relatively little compute. ELECTRA models are trained to -> distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to -> the discriminator of a GAN. - -More details about ELECTRA can be found in the [ICLR paper](https://openreview.net/forum?id=r1xMH1BtvB) -or in the [official ELECTRA repository](https://github.com/google-research/electra) on GitHub. - -## Stats - -The current version of the model is trained on a filtered and sentence -segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/), -a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a -special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/). - -The final training corpus has a size of 35GB and 44,04,976,662 tokens. - -Thanks to Google's TensorFlow Research Cloud (TFRC) we could train a cased model -on a TPU v3-8 for 1M steps. - -## Model weights - -[Transformers](https://github.com/huggingface/transformers) -compatible weights for both PyTorch and TensorFlow are available. - -| Model | Downloads -| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/electra-base-turkish-cased-discriminator` | [`config.json`](https://cdn.huggingface.co/dbmdz/electra-base-turkish-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-base-turkish-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-base-turkish-cased-discriminator/vocab.txt) - -## Usage - -With Transformers >= 2.8 our ELECTRA base cased model can be loaded like: - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/electra-base-turkish-cased-discriminator") -model = AutoModelWithLMHead.from_pretrained("dbmdz/electra-base-turkish-cased-discriminator") -``` - -## Results - -For results on PoS tagging or NER tasks, please refer to -[this repository](https://github.com/stefan-it/turkish-bert/electra). - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our ELECTRA models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us -additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing -us the Turkish NER dataset for evaluation. - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dbmdz/electra-small-turkish-cased-discriminator/README.md b/model_cards/dbmdz/electra-small-turkish-cased-discriminator/README.md deleted file mode 100644 index ef35cfd7cedd..000000000000 --- a/model_cards/dbmdz/electra-small-turkish-cased-discriminator/README.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -language: tr -license: mit ---- - -# 🤗 + 📚 dbmdz Turkish ELECTRA model - -In this repository the MDZ Digital Library team (dbmdz) at the Bavarian State -Library open sources a cased ELECTRA small model for Turkish 🎉 - -# Turkish ELECTRA model - -We release a small ELEC**TR**A model for Turkish, that was trained on the same data as *BERTurk*. - -> ELECTRA is a new method for self-supervised language representation learning. It can be used to -> pre-train transformer networks using relatively little compute. ELECTRA models are trained to -> distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to -> the discriminator of a GAN. - -More details about ELECTRA can be found in the [ICLR paper](https://openreview.net/forum?id=r1xMH1BtvB) -or in the [official ELECTRA repository](https://github.com/google-research/electra) on GitHub. - -## Stats - -The current version of the model is trained on a filtered and sentence -segmented version of the Turkish [OSCAR corpus](https://traces1.inria.fr/oscar/), -a recent Wikipedia dump, various [OPUS corpora](http://opus.nlpl.eu/) and a -special corpus provided by [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/). - -The final training corpus has a size of 35GB and 44,04,976,662 tokens. - -Thanks to Google's TensorFlow Research Cloud (TFRC) we could train a cased model -on a TPU v3-8 for 1M steps. - -## Model weights - -[Transformers](https://github.com/huggingface/transformers) -compatible weights for both PyTorch and TensorFlow are available. - -| Model | Downloads -| ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- -| `dbmdz/electra-small-turkish-cased-discriminator` | [`config.json`](https://cdn.huggingface.co/dbmdz/electra-small-turkish-cased-discriminator/config.json) • [`pytorch_model.bin`](https://cdn.huggingface.co/dbmdz/electra-small-turkish-cased-discriminator/pytorch_model.bin) • [`vocab.txt`](https://cdn.huggingface.co/dbmdz/electra-small-turkish-cased-discriminator/vocab.txt) - -## Usage - -With Transformers >= 2.8 our ELECTRA small cased model can be loaded like: - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("dbmdz/electra-small-turkish-cased-discriminator") -model = AutoModelWithLMHead.from_pretrained("dbmdz/electra-small-turkish-cased-discriminator") -``` - -## Results - -For results on PoS tagging or NER tasks, please refer to -[this repository](https://github.com/stefan-it/turkish-bert/electra). - -# Huggingface model hub - -All models are available on the [Huggingface model hub](https://huggingface.co/dbmdz). - -# Contact (Bugs, Feedback, Contribution and more) - -For questions about our ELECTRA models just open an issue -[here](https://github.com/dbmdz/berts/issues/new) 🤗 - -# Acknowledgments - -Thanks to [Kemal Oflazer](http://www.andrew.cmu.edu/user/ko/) for providing us -additional large corpora for Turkish. Many thanks to Reyyan Yeniterzi for providing -us the Turkish NER dataset for evaluation. - -Research supported with Cloud TPUs from Google's TensorFlow Research Cloud (TFRC). -Thanks for providing access to the TFRC ❤️ - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download both cased and uncased models from their S3 storage 🤗 diff --git a/model_cards/dccuchile/bert-base-spanish-wwm-cased/README.md b/model_cards/dccuchile/bert-base-spanish-wwm-cased/README.md deleted file mode 100644 index 6930abfd8612..000000000000 --- a/model_cards/dccuchile/bert-base-spanish-wwm-cased/README.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -language: es ---- diff --git a/model_cards/dccuchile/bert-base-spanish-wwm-uncased/README.md b/model_cards/dccuchile/bert-base-spanish-wwm-uncased/README.md deleted file mode 100644 index 6930abfd8612..000000000000 --- a/model_cards/dccuchile/bert-base-spanish-wwm-uncased/README.md +++ /dev/null @@ -1,3 +0,0 @@ ---- -language: es ---- diff --git a/model_cards/deepset/bert-base-german-cased-oldvocab/README.md b/model_cards/deepset/bert-base-german-cased-oldvocab/README.md deleted file mode 100644 index 159b63621aee..000000000000 --- a/model_cards/deepset/bert-base-german-cased-oldvocab/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: de -license: mit -thumbnail: https://static.tildacdn.com/tild6438-3730-4164-b266-613634323466/german_bert.png -tags: -- exbert ---- - - - - - -# German BERT with old vocabulary -For details see the related [FARM issue](https://github.com/deepset-ai/FARM/issues/60). - - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/electra-base-squad2/README.md b/model_cards/deepset/electra-base-squad2/README.md deleted file mode 100644 index 06cfb52b111c..000000000000 --- a/model_cards/deepset/electra-base-squad2/README.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -datasets: -- squad_v2 ---- - -# electra-base for QA - -## Overview -**Language model:** electra-base -**Language:** English -**Downstream-task:** Extractive QA -**Training data:** SQuAD 2.0 -**Eval data:** SQuAD 2.0 -**Code:** See [example](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) in [FARM](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) -**Infrastructure**: 1x Tesla v100 - -## Hyperparameters - -``` -seed=42 -batch_size = 32 -n_epochs = 5 -base_LM_model = "google/electra-base-discriminator" -max_seq_len = 384 -learning_rate = 1e-4 -lr_schedule = LinearWarmup -warmup_proportion = 0.1 -doc_stride=128 -max_query_length=64 -``` - -## Performance -Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/). -``` -"exact": 77.30144024256717, - "f1": 81.35438272008543, - "total": 11873, - "HasAns_exact": 74.34210526315789, - "HasAns_f1": 82.45961302894314, - "HasAns_total": 5928, - "NoAns_exact": 80.25231286795626, - "NoAns_f1": 80.25231286795626, - "NoAns_total": 5945 -``` - -## Usage - -### In Transformers -```python -from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline - -model_name = "deepset/electra-base-squad2" - -# a) Get predictions -nlp = pipeline('question-answering', model=model_name, tokenizer=model_name) -QA_input = { - 'question': 'Why is model conversion important?', - 'context': 'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.' -} -res = nlp(QA_input) - -# b) Load model & tokenizer -model = AutoModelForQuestionAnswering.from_pretrained(model_name) -tokenizer = AutoTokenizer.from_pretrained(model_name) -``` - -### In FARM - -```python -from farm.modeling.adaptive_model import AdaptiveModel -from farm.modeling.tokenization import Tokenizer -from farm.infer import Inferencer - -model_name = "deepset/electra-base-squad2" - -# a) Get predictions -nlp = Inferencer.load(model_name, task_type="question_answering") -QA_input = [{"questions": ["Why is model conversion important?"], - "text": "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."}] -res = nlp.inference_from_dicts(dicts=QA_input) - -# b) Load model & tokenizer -model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type="question_answering") -tokenizer = Tokenizer.load(model_name) -``` - -### In haystack -For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/): -```python -reader = FARMReader(model_name_or_path="deepset/electra-base-squad2") -# or -reader = TransformersReader(model="deepset/electra-base-squad2",tokenizer="deepset/electra-base-squad2") -``` - - -## Authors -Vaishali Pal `vaishali.pal [at] deepset.ai` -Branden Chan: `branden.chan [at] deepset.ai` -Timo Möller: `timo.moeller [at] deepset.ai` -Malte Pietsch: `malte.pietsch [at] deepset.ai` -Tanay Soni: `tanay.soni [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/gbert-base/README.md b/model_cards/deepset/gbert-base/README.md deleted file mode 100644 index d6404262d0b4..000000000000 --- a/model_cards/deepset/gbert-base/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: de -license: mit -datasets: -- wikipedia -- OPUS -- OpenLegalData ---- - -# German BERT base - -Released, Oct 2020, this is a German BERT language model trained collaboratively by the makers of the original German BERT (aka "bert-base-german-cased") and the dbmdz BERT (aka bert-base-german-dbmdz-cased). In our [paper](https://arxiv.org/pdf/2010.10906.pdf), we outline the steps taken to train our model and show that it outperforms its predecessors. - -## Overview -**Paper:** [here](https://arxiv.org/pdf/2010.10906.pdf) -**Architecture:** BERT base -**Language:** German - -## Performance -``` -GermEval18 Coarse: 78.17 -GermEval18 Fine: 50.90 -GermEval14: 87.98 -``` - -See also: -deepset/gbert-base -deepset/gbert-large -deepset/gelectra-base -deepset/gelectra-large -deepset/gelectra-base-generator -deepset/gelectra-large-generator - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Stefan Schweter: `stefan [at] schweter.eu` -Timo Möller: `timo.moeller [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/gbert-large/README.md b/model_cards/deepset/gbert-large/README.md deleted file mode 100644 index aa797e5ef35e..000000000000 --- a/model_cards/deepset/gbert-large/README.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -language: de -license: mit -datasets: -- wikipedia -- OPUS -- OpenLegalData -- oscar ---- - -# German BERT large - -Released, Oct 2020, this is a German BERT language model trained collaboratively by the makers of the original German BERT (aka "bert-base-german-cased") and the dbmdz BERT (aka bert-base-german-dbmdz-cased). In our [paper](https://arxiv.org/pdf/2010.10906.pdf), we outline the steps taken to train our model and show that it outperforms its predecessors. - -## Overview -**Paper:** [here](https://arxiv.org/pdf/2010.10906.pdf) -**Architecture:** BERT large -**Language:** German - -## Performance -``` -GermEval18 Coarse: 80.08 -GermEval18 Fine: 52.48 -GermEval14: 88.16 -``` - -See also: -deepset/gbert-base -deepset/gbert-large -deepset/gelectra-base -deepset/gelectra-large -deepset/gelectra-base-generator -deepset/gelectra-large-generator - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Stefan Schweter: `stefan [at] schweter.eu` -Timo Möller: `timo.moeller [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) - - diff --git a/model_cards/deepset/gelectra-base-generator/README.md b/model_cards/deepset/gelectra-base-generator/README.md deleted file mode 100644 index ed7ee78e51fb..000000000000 --- a/model_cards/deepset/gelectra-base-generator/README.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -language: de -license: mit -datasets: -- wikipedia -- OPUS -- OpenLegalData ---- - -# German ELECTRA base generator - -Released, Oct 2020, this is the generator component of the German ELECTRA language model trained collaboratively by the makers of the original German BERT (aka "bert-base-german-cased") and the dbmdz BERT (aka bert-base-german-dbmdz-cased). In our [paper](https://arxiv.org/pdf/2010.10906.pdf), we outline the steps taken to train our model. - -The generator is useful for performing masking experiments. If you are looking for a regular language model for embedding extraction, or downstream tasks like NER, classification or QA, please use deepset/gelectra-base. - -## Overview -**Paper:** [here](https://arxiv.org/pdf/2010.10906.pdf) -**Architecture:** ELECTRA base (generator) -**Language:** German - -See also: -deepset/gbert-base -deepset/gbert-large -deepset/gelectra-base -deepset/gelectra-large -deepset/gelectra-base-generator -deepset/gelectra-large-generator - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Stefan Schweter: `stefan [at] schweter.eu` -Timo Möller: `timo.moeller [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/gelectra-base/README.md b/model_cards/deepset/gelectra-base/README.md deleted file mode 100644 index a0b2e2f0ed8d..000000000000 --- a/model_cards/deepset/gelectra-base/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: de -license: mit -datasets: -- wikipedia -- OPUS -- OpenLegalData ---- - -# German ELECTRA base - -Released, Oct 2020, this is a German ELECTRA language model trained collaboratively by the makers of the original German BERT (aka "bert-base-german-cased") and the dbmdz BERT (aka bert-base-german-dbmdz-cased). In our [paper](https://arxiv.org/pdf/2010.10906.pdf), we outline the steps taken to train our model. Our evaluation suggests that this model is somewhat undertrained. For best performance from a base sized model, we recommend deepset/gbert-base - -## Overview -**Paper:** [here](https://arxiv.org/pdf/2010.10906.pdf) -**Architecture:** ELECTRA base (discriminator) -**Language:** German - -## Performance -``` -GermEval18 Coarse: 76.02 -GermEval18 Fine: 42.22 -GermEval14: 86.02 -``` - -See also: -deepset/gbert-base -deepset/gbert-large -deepset/gelectra-base -deepset/gelectra-large -deepset/gelectra-base-generator -deepset/gelectra-large-generator - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Stefan Schweter: `stefan [at] schweter.eu` -Timo Möller: `timo.moeller [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/gelectra-large-generator/README.md b/model_cards/deepset/gelectra-large-generator/README.md deleted file mode 100644 index a513b2b865c5..000000000000 --- a/model_cards/deepset/gelectra-large-generator/README.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -language: de -license: mit -datasets: -- wikipedia -- OPUS -- OpenLegalData -- oscar ---- - -# German ELECTRA large generator - -Released, Oct 2020, this is the generator component of the German ELECTRA language model trained collaboratively by the makers of the original German BERT (aka "bert-base-german-cased") and the dbmdz BERT (aka bert-base-german-dbmdz-cased). In our [paper](https://arxiv.org/pdf/2010.10906.pdf), we outline the steps taken to train our model. - -The generator is useful for performing masking experiments. If you are looking for a regular language model for embedding extraction, or downstream tasks like NER, classification or QA, please use deepset/gelectra-large. - -## Overview -**Paper:** [here](https://arxiv.org/pdf/2010.10906.pdf) -**Architecture:** ELECTRA large (generator) -**Language:** German - -## Performance -``` -GermEval18 Coarse: 80.70 -GermEval18 Fine: 55.16 -GermEval14: 88.95 -``` - -See also: -deepset/gbert-base -deepset/gbert-large -deepset/gelectra-base -deepset/gelectra-large -deepset/gelectra-base-generator -deepset/gelectra-large-generator - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Stefan Schweter: `stefan [at] schweter.eu` -Timo Möller: `timo.moeller [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) - - diff --git a/model_cards/deepset/gelectra-large/README.md b/model_cards/deepset/gelectra-large/README.md deleted file mode 100644 index 5a0517177756..000000000000 --- a/model_cards/deepset/gelectra-large/README.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -language: de -license: mit -datasets: -- wikipedia -- OPUS -- OpenLegalData -- oscar ---- - -# German ELECTRA large - -Released, Oct 2020, this is a German ELECTRA language model trained collaboratively by the makers of the original German BERT (aka "bert-base-german-cased") and the dbmdz BERT (aka bert-base-german-dbmdz-cased). In our [paper](https://arxiv.org/pdf/2010.10906.pdf), we outline the steps taken to train our model and show that this is the state of the art German language model. - -## Overview -**Paper:** [here](https://arxiv.org/pdf/2010.10906.pdf) -**Architecture:** ELECTRA large (discriminator) -**Language:** German - -## Performance -``` -GermEval18 Coarse: 80.70 -GermEval18 Fine: 55.16 -GermEval14: 88.95 -``` - -See also: -deepset/gbert-base -deepset/gbert-large -deepset/gelectra-base -deepset/gelectra-large -deepset/gelectra-base-generator -deepset/gelectra-large-generator - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Stefan Schweter: `stefan [at] schweter.eu` -Timo Möller: `timo.moeller [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/minilm-uncased-squad2/README.md b/model_cards/deepset/minilm-uncased-squad2/README.md deleted file mode 100644 index 4c6604f7f68d..000000000000 --- a/model_cards/deepset/minilm-uncased-squad2/README.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -datasets: -- squad_v2 ---- - -# MiniLM-L12-H384-uncased for QA - -## Overview -**Language model:** microsoft/MiniLM-L12-H384-uncased -**Language:** English -**Downstream-task:** Extractive QA -**Training data:** SQuAD 2.0 -**Eval data:** SQuAD 2.0 -**Code:** See [example](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) in [FARM](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) -**Infrastructure**: 1x Tesla v100 - -## Hyperparameters - -``` -seed=42 -batch_size = 12 -n_epochs = 4 -base_LM_model = "microsoft/MiniLM-L12-H384-uncased" -max_seq_len = 384 -learning_rate = 4e-5 -lr_schedule = LinearWarmup -warmup_proportion = 0.2 -doc_stride=128 -max_query_length=64 -grad_acc_steps=4 -``` - -## Performance -Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/). -``` -"exact": 76.13071675229513, -"f1": 79.49786500219953, -"total": 11873, -"HasAns_exact": 78.35695006747639, -"HasAns_f1": 85.10090269418276, -"HasAns_total": 5928, -"NoAns_exact": 73.91084945332211, -"NoAns_f1": 73.91084945332211, -"NoAns_total": 5945 -``` - -## Usage - -### In Transformers -```python -from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline - -model_name = "deepset/minilm-uncased-squad2" - -# a) Get predictions -nlp = pipeline('question-answering', model=model_name, tokenizer=model_name) -QA_input = { - 'question': 'Why is model conversion important?', - 'context': 'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.' -} -res = nlp(QA_input) - -# b) Load model & tokenizer -model = AutoModelForQuestionAnswering.from_pretrained(model_name) -tokenizer = AutoTokenizer.from_pretrained(model_name) -``` - -### In FARM - -```python -from farm.modeling.adaptive_model import AdaptiveModel -from farm.modeling.tokenization import Tokenizer -from farm.infer import Inferencer - -model_name = "deepset/minilm-uncased-squad2" - -# a) Get predictions -nlp = Inferencer.load(model_name, task_type="question_answering") -QA_input = [{"questions": ["Why is model conversion important?"], - "text": "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."}] -res = nlp.inference_from_dicts(dicts=QA_input) - -# b) Load model & tokenizer -model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type="question_answering") -tokenizer = Tokenizer.load(model_name) -``` - -### In haystack -For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/): -```python -reader = FARMReader(model_name_or_path="deepset/minilm-uncased-squad2") -# or -reader = TransformersReader(model="deepset/minilm-uncased-squad2",tokenizer="deepset/minilm-uncased-squad2") -``` - - -## Authors -Vaishali Pal `vaishali.pal [at] deepset.ai` -Branden Chan: `branden.chan [at] deepset.ai` -Timo Möller: `timo.moeller [at] deepset.ai` -Malte Pietsch: `malte.pietsch [at] deepset.ai` -Tanay Soni: `tanay.soni [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/quora_dedup_bert_base/README.md b/model_cards/deepset/quora_dedup_bert_base/README.md deleted file mode 100644 index 317b14e69e41..000000000000 --- a/model_cards/deepset/quora_dedup_bert_base/README.md +++ /dev/null @@ -1,59 +0,0 @@ -This language model is trained using sentence_transformers (https://github.com/UKPLab/sentence-transformers) -Started with bert-base-nli-stsb-mean-tokens -Continue training on quora questions deduplication dataset (https://www.kaggle.com/c/quora-question-pairs) -See train_script.py for script used - -Below is the performance over the course of training -epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman -0,1000,0.5944576426835938,0.6010801382777033,0.5942803776859142,0.5934485776801595,0.5939676679774666,0.593162725602328,0.5905591590826669,0.5921674789994058 -0,2000,0.6404080440207146,0.6416811632113405,0.6384419354012121,0.6352050423100778,0.6379917744471867,0.6347884067391001,0.6410544760582826,0.6379252046791412 -0,3000,0.6710168301884945,0.6676529324662036,0.6660195209784969,0.6618423144808695,0.6656461098096684,0.6615366331956389,0.6724401903484759,0.666073727723655 -0,4000,0.6886373265097949,0.6808948140300153,0.67907655686838,0.6714218133850957,0.6786809551564443,0.6711577956884357,0.6926435869763303,0.68190855298609 -0,5000,0.6991409753700026,0.6919630610321864,0.6991041519437052,0.6868961486499775,0.6987076032270729,0.6865385550504007,0.7035518148330993,0.6916275246101342 -0,6000,0.7120367327025509,0.6975005265298305,0.7065567493967201,0.6922375503495235,0.7060005509843024,0.6916475765570651,0.7147094303373102,0.6981390706722722 -0,7000,0.7254672394728687,0.7130118465900485,0.7261844956277705,0.7086213543110718,0.7257479964972307,0.7079315661881832,0.728729909455115,0.7122743793160531 -0,8000,0.7402421930101399,0.7216774208330149,0.7367901914441078,0.7166256588352043,0.7362607046874481,0.7158881916281887,0.7433902441373252,0.7220998491980078 -0,9000,0.7381005358120434,0.7197216844469877,0.7343228719349923,0.7139462687943793,0.7345247569255238,0.7145106206467152,0.7421843672419275,0.720686853053079 -0,10000,0.7465436564646095,0.7260327107480364,0.7467524239596304,0.7230195666847953,0.7467721566237211,0.7231367593302213,0.749792199122442,0.7263143296580317 -0,11000,0.7521805421706547,0.7323771570146701,0.7530672061250105,0.729223203496722,0.7530616532823367,0.7293818369675622,0.7552399002305836,0.7320808333541338 -0,12000,0.7579359969644401,0.7340677616737238,0.7570017235719905,0.7305965412825544,0.7570601853520393,0.730718189957289,0.7611254136080384,0.7351501229591327 -0,-1,0.7573407371218097,0.7329952035782198,0.755595312163209,0.7291445551777086,0.7557737117990928,0.7295404703700227,0.7607276219361719,0.7342415455980179 -1,1000,0.7619907683805341,0.7374667949734767,0.7629820517114324,0.7330364216044966,0.7628369522755882,0.7331912674450544,0.7658583898073758,0.7381503446695727 -1,2000,0.7618972640071228,0.7362151058969478,0.764582212425539,0.7335856230046062,0.7643125513700815,0.7334501607097152,0.7652852805583232,0.7369104639809163 -1,3000,0.7687362955240467,0.7404674623181671,0.7708304819979073,0.7380959815601529,0.7707835692712482,0.7379796800453193,0.772074854759756,0.7414513460702766 -1,4000,0.7685047787908202,0.7403088288815168,0.7703522257474043,0.7379787888808298,0.7701221475099808,0.7377898546753812,0.7713755359045312,0.7409415801952219 -1,5000,0.7696438109797803,0.7410393893292365,0.773270389327895,0.7392953127251652,0.7729880866533291,0.7389853982789335,0.7726236305835863,0.7416278035580925 -1,6000,0.7749538363837081,0.7436499342062207,0.774879168058157,0.7401827241766746,0.7745754601165837,0.739763415043146,0.7788801166152383,0.7446249060022169 -1,7000,0.7794560817870597,0.7480970176267153,0.7803506944510302,0.7453305130502859,0.7799867949176531,0.7447100155494814,0.7828208193123926,0.7486740690324809 -1,8000,0.7855844359073243,0.7496742172376921,0.7828816645965887,0.747176409009761,0.7827584875358967,0.7471037762845532,0.7879159073496309,0.7507349669102151 -1,9000,0.7844110753729492,0.7507746252693759,0.7847208586489722,0.7485172180290892,0.7846408087474059,0.748491818820158,0.7872061334510225,0.7514470349769437 -1,10000,0.7881311227435004,0.7530048509727403,0.7886917756879734,0.7508018068765787,0.7883332502188707,0.7505037008187275,0.7910707228932787,0.7537200382362567 -1,11000,0.7883300109606874,0.7513494487126553,0.7879329130497712,0.749818368689255,0.7876525616593218,0.7494872882301785,0.7911454269743292,0.7522843165147303 -1,12000,0.7853334933336618,0.7516809747712728,0.7893895316714998,0.749780492728257,0.7890075986655403,0.7494079715118533,0.7885959664070629,0.7523827940133203 -1,-1,0.7887529238148887,0.7534076729932393,0.7896864404801204,0.7513080079201105,0.7894077512343298,0.7510009899066772,0.7919617393746149,0.7542173273241598 -2,1000,0.7919209063905188,0.7550167329363414,0.7917464066515253,0.7523043685293455,0.7914371703225378,0.7520285423781206,0.7950297421784158,0.7562599556207076 -2,2000,0.7924507768792486,0.7542908512484463,0.7934519001953887,0.7517491515010692,0.7931885648751081,0.751521004535999,0.7951637852162545,0.7551495215642072 -2,3000,0.7937606244038364,0.755599577136169,0.7933633347508111,0.7527922999916203,0.7931581019714242,0.7527132061436363,0.797275652800117,0.7569827180764233 -2,4000,0.7938389298721445,0.7578716892320315,0.7963783770097079,0.7555928931784702,0.796150381773947,0.7555438771581088,0.7972911620482322,0.759178632650707 -2,5000,0.7935330563129844,0.7551129824372304,0.7970775059297484,0.7527285792572385,0.7967359830546507,0.7524478515463257,0.7966395126138969,0.756319220359678 -2,6000,0.7929852776759999,0.7525490026774382,0.7952484474454824,0.7503695753216607,0.7950784132079611,0.7503677929234961,0.7956152082976395,0.7535275392698093 -2,7000,0.794956504054517,0.756119591765251,0.7982025041673655,0.7532521587180684,0.7980261618830962,0.7532107179960499,0.7983222918908033,0.7571226363678287 -2,8000,0.7934568432535339,0.7538336661192452,0.797015698241178,0.7514773358161916,0.7968076980315735,0.7513458838811067,0.7960694134685949,0.754143803399873 -2,9000,0.7970040626682157,0.7576497805894974,0.7987855332059015,0.7550996144509958,0.7984693921009676,0.7548260162973456,0.7999509314900626,0.758347143906916 -2,10000,0.7979442987735523,0.7585338500791028,0.8018677081664496,0.7557412777548302,0.8015397301245205,0.7552916678886369,0.8007921348414564,0.7589772216225288 -2,11000,0.7985519561040211,0.7579986850302035,0.8021236875460913,0.7555826443181872,0.8019861620475348,0.7553763317660516,0.8009230128897853,0.7586541619907702 -2,12000,0.7986842143860736,0.7599570950134775,0.8029131054823838,0.7577678644678973,0.8027922603736795,0.7575152095990927,0.8020896747930555,0.7608540869254408 -2,-1,0.7994135319568432,0.7596286881516635,0.8022087183675333,0.7570593611974978,0.8020218401019292,0.7567291719729909,0.8026346812258125,0.7603928913647044 -3,1000,0.7985505039929134,0.7592588405681144,0.8023296699449267,0.7569345933969436,0.8023622066009718,0.7570237132696928,0.8013054275981851,0.759643838536062 -3,2000,0.7995482191699455,0.759205368623176,0.8026859405513612,0.7565709841358819,0.8024845263367439,0.7562920388231202,0.8021318586127523,0.7596496313300967 -3,3000,0.7991070423195897,0.7582027696555826,0.8016352550470427,0.7555585819429662,0.8014268261947898,0.7551838327642736,0.8013136081494014,0.7584429477727118 -3,4000,0.7999188836884763,0.7586764419322649,0.802987646214278,0.7561111254802977,0.8026549791861386,0.7556463650525692,0.8024068858366156,0.7591238238715613 -3,5000,0.7988075932525881,0.7583533823004922,0.8019498750207454,0.755792967372457,0.8016459824731964,0.7553834613587099,0.8015528810821693,0.7589527136833425 -3,6000,0.8003341798460688,0.7585432077405799,0.8032464035902267,0.7563722467405277,0.8028695045742804,0.7557626665682309,0.8027937010871594,0.7590404967573696 -3,7000,0.799187592384933,0.7579358555659604,0.8028413548398412,0.7555875459131398,0.8025187078191003,0.7551196665011402,0.8018680475193432,0.7585565756912578 -3,8000,0.797725037202641,0.757439012042047,0.802048241301358,0.7548888458326453,0.8017608103042271,0.7544606246736175,0.8005479449399782,0.758037452190282 -3,9000,0.7990232649360067,0.7573703896772077,0.8021375332910405,0.754873027155089,0.8018733796679427,0.7545680141630304,0.8016400687760605,0.7579461042843499 -3,10000,0.7994934439260372,0.758368978248884,0.8035693504115055,0.75619400688862,0.8032990505007025,0.7559016935896375,0.8022819185772518,0.7589558328445544 -3,11000,0.8002954591825011,0.758710753096932,0.8043310859792212,0.7566387152306694,0.8040865016706966,0.7564221538891368,0.8030873114870971,0.7592722085543488 -3,12000,0.8003726616196549,0.7588056657991931,0.8044000317617518,0.7566146528909147,0.8041705213966136,0.7563419459362758,0.8031760015719815,0.7593194421057111 -3,-1,0.8004926728141455,0.7587192194882135,0.8043340929890026,0.756546030526114,0.8041028559910275,0.7563103085106637,0.8032542493776693,0.7592325501951863 diff --git a/model_cards/deepset/roberta-base-squad2-covid/README.md b/model_cards/deepset/roberta-base-squad2-covid/README.md deleted file mode 100644 index 970d15835bf1..000000000000 --- a/model_cards/deepset/roberta-base-squad2-covid/README.md +++ /dev/null @@ -1,106 +0,0 @@ -# roberta-base-squad2 for QA on COVID-19 - -## Overview -**Language model:** deepset/roberta-base-squad2 -**Language:** English -**Downstream-task:** Extractive QA -**Training data:** [SQuAD-style CORD-19 annotations from 23rd April](https://github.com/deepset-ai/COVID-QA/blob/master/data/question-answering/200423_covidQA.json) -**Code:** See [example](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering_crossvalidation.py) in [FARM](https://github.com/deepset-ai/FARM) -**Infrastructure**: Tesla v100 - -## Hyperparameters -``` -batch_size = 24 -n_epochs = 3 -base_LM_model = "deepset/roberta-base-squad2" -max_seq_len = 384 -learning_rate = 3e-5 -lr_schedule = LinearWarmup -warmup_proportion = 0.1 -doc_stride = 128 -xval_folds = 5 -dev_split = 0 -no_ans_boost = -100 -``` - -## Performance -5-fold cross-validation on the data set led to the following results: - -**Single EM-Scores:** [0.222, 0.123, 0.234, 0.159, 0.158] -**Single F1-Scores:** [0.476, 0.493, 0.599, 0.461, 0.465] -**Single top\_3\_recall Scores:** [0.827, 0.776, 0.860, 0.771, 0.777] -**XVAL EM:** 0.17890995260663506 -**XVAL f1:** 0.49925444207319924 -**XVAL top\_3\_recall:** 0.8021327014218009 - -This model is the model obtained from the **third** fold of the cross-validation. - -## Usage - -### In Transformers -```python -from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline - - -model_name = "deepset/roberta-base-squad2-covid" - -# a) Get predictions -nlp = pipeline('question-answering', model=model_name, tokenizer=model_name) -QA_input = { - 'question': 'Why is model conversion important?', - 'context': 'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.' -} -res = nlp(QA_input) - -# b) Load model & tokenizer -model = AutoModelForQuestionAnswering.from_pretrained(model_name) -tokenizer = AutoTokenizer.from_pretrained(model_name) -``` - -### In FARM -```python -from farm.modeling.adaptive_model import AdaptiveModel -from farm.modeling.tokenization import Tokenizer -from farm.infer import Inferencer - -model_name = "deepset/roberta-base-squad2-covid" - -# a) Get predictions -nlp = Inferencer.load(model_name, task_type="question_answering") -QA_input = [{"questions": ["Why is model conversion important?"], - "text": "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."}] -res = nlp.inference_from_dicts(dicts=QA_input, rest_api_schema=True) - -# b) Load model & tokenizer -model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type="question_answering") -tokenizer = Tokenizer.load(model_name) -``` - -### In haystack -For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/): -```python -reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2-covid") -# or -reader = TransformersReader(model="deepset/roberta-base-squad2",tokenizer="deepset/roberta-base-squad2-covid") -``` - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Timo Möller: `timo.moeller [at] deepset.ai` -Malte Pietsch: `malte.pietsch [at] deepset.ai` -Tanay Soni: `tanay.soni [at] deepset.ai` -Bogdan Kostić: `bogdan.kostic [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) \ No newline at end of file diff --git a/model_cards/deepset/roberta-base-squad2-v2/README.md b/model_cards/deepset/roberta-base-squad2-v2/README.md deleted file mode 100644 index 7cbf6b887871..000000000000 --- a/model_cards/deepset/roberta-base-squad2-v2/README.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -datasets: -- squad_v2 ---- - -# roberta-base for QA - -## Overview -**Language model:** roberta-base -**Language:** English -**Downstream-task:** Extractive QA -**Training data:** SQuAD 2.0 -**Eval data:** SQuAD 2.0 -**Code:** See [example](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) in [FARM](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) -**Infrastructure**: 4x Tesla v100 - -## Hyperparameters - -``` -batch_size = 96 -n_epochs = 2 -base_LM_model = "roberta-base" -max_seq_len = 386 -learning_rate = 3e-5 -lr_schedule = LinearWarmup -warmup_proportion = 0.2 -doc_stride=128 -max_query_length=64 -``` - -## Performance -Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/). - -``` -"exact": 79.97136359807968 -"f1": 83.00449234495325 - -"total": 11873 -"HasAns_exact": 78.03643724696356 -"HasAns_f1": 84.11139298441825 -"HasAns_total": 5928 -"NoAns_exact": 81.90075693860386 -"NoAns_f1": 81.90075693860386 -"NoAns_total": 5945 -``` - -## Usage - -### In Transformers -```python -from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline - -model_name = "deepset/roberta-base-squad2-v2" - -# a) Get predictions -nlp = pipeline('question-answering', model=model_name, tokenizer=model_name) -QA_input = { - 'question': 'Why is model conversion important?', - 'context': 'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.' -} -res = nlp(QA_input) - -# b) Load model & tokenizer -model = AutoModelForQuestionAnswering.from_pretrained(model_name) -tokenizer = AutoTokenizer.from_pretrained(model_name) -``` - -### In FARM - -```python -from farm.modeling.adaptive_model import AdaptiveModel -from farm.modeling.tokenization import Tokenizer -from farm.infer import Inferencer - -model_name = "deepset/roberta-base-squad2-v2" - -# a) Get predictions -nlp = Inferencer.load(model_name, task_type="question_answering") -QA_input = [{"questions": ["Why is model conversion important?"], - "text": "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."}] -res = nlp.inference_from_dicts(dicts=QA_input, rest_api_schema=True) - -# b) Load model & tokenizer -model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type="question_answering") -tokenizer = Tokenizer.load(model_name) -``` - -### In haystack -For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/): -```python -reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2-v2") -# or -reader = TransformersReader(model_name_or_path="deepset/roberta-base-squad2-v2",tokenizer="deepset/roberta-base-squad2-v2") -``` - - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Timo Möller: `timo.moeller [at] deepset.ai` -Malte Pietsch: `malte.pietsch [at] deepset.ai` -Tanay Soni: `tanay.soni [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/roberta-base-squad2/README.md b/model_cards/deepset/roberta-base-squad2/README.md deleted file mode 100644 index 9e443b06639a..000000000000 --- a/model_cards/deepset/roberta-base-squad2/README.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -datasets: -- squad_v2 ---- - -# roberta-base for QA - -NOTE: This is version 2 of the model. See [this github issue](https://github.com/deepset-ai/FARM/issues/552) from the FARM repository for an explanation of why we updated. If you'd like to use version 1, specify `revision="v1.0"` when loading the model in Transformers 3.5. For exmaple: -``` -model_name = "deepset/roberta-base-squad2" -pipeline(model=model_name, tokenizer=model_name, revision="v1.0", task="question-answering") -``` - -## Overview -**Language model:** roberta-base -**Language:** English -**Downstream-task:** Extractive QA -**Training data:** SQuAD 2.0 -**Eval data:** SQuAD 2.0 -**Code:** See [example](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) in [FARM](https://github.com/deepset-ai/FARM/blob/master/examples/question_answering.py) -**Infrastructure**: 4x Tesla v100 - -## Hyperparameters - -``` -batch_size = 96 -n_epochs = 2 -base_LM_model = "roberta-base" -max_seq_len = 386 -learning_rate = 3e-5 -lr_schedule = LinearWarmup -warmup_proportion = 0.2 -doc_stride=128 -max_query_length=64 -``` - -## Performance -Evaluated on the SQuAD 2.0 dev set with the [official eval script](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/). - -``` -"exact": 79.97136359807968 -"f1": 83.00449234495325 - -"total": 11873 -"HasAns_exact": 78.03643724696356 -"HasAns_f1": 84.11139298441825 -"HasAns_total": 5928 -"NoAns_exact": 81.90075693860386 -"NoAns_f1": 81.90075693860386 -"NoAns_total": 5945 -``` - -## Usage - -### In Transformers -```python -from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline - -model_name = "deepset/roberta-base-squad2" - -# a) Get predictions -nlp = pipeline('question-answering', model=model_name, tokenizer=model_name) -QA_input = { - 'question': 'Why is model conversion important?', - 'context': 'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.' -} -res = nlp(QA_input) - -# b) Load model & tokenizer -model = AutoModelForQuestionAnswering.from_pretrained(model_name) -tokenizer = AutoTokenizer.from_pretrained(model_name) -``` - -### In FARM - -```python -from farm.modeling.adaptive_model import AdaptiveModel -from farm.modeling.tokenization import Tokenizer -from farm.infer import Inferencer - -model_name = "deepset/roberta-base-squad2" - -# a) Get predictions -nlp = Inferencer.load(model_name, task_type="question_answering") -QA_input = [{"questions": ["Why is model conversion important?"], - "text": "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."}] -res = nlp.inference_from_dicts(dicts=QA_input, rest_api_schema=True) - -# b) Load model & tokenizer -model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type="question_answering") -tokenizer = Tokenizer.load(model_name) -``` - -### In haystack -For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/): -```python -reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2") -# or -reader = TransformersReader(model_name_or_path="deepset/roberta-base-squad2",tokenizer="deepset/roberta-base-squad2") -``` - - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Timo Möller: `timo.moeller [at] deepset.ai` -Malte Pietsch: `malte.pietsch [at] deepset.ai` -Tanay Soni: `tanay.soni [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) diff --git a/model_cards/deepset/sentence_bert/README.md b/model_cards/deepset/sentence_bert/README.md deleted file mode 100644 index 721f09fce868..000000000000 --- a/model_cards/deepset/sentence_bert/README.md +++ /dev/null @@ -1 +0,0 @@ -This is an upload of the bert-base-nli-stsb-mean-tokens pretrained model from the Sentence Transformers Repo (https://github.com/UKPLab/sentence-transformers) diff --git a/model_cards/deepset/xlm-roberta-large-squad2/README.md b/model_cards/deepset/xlm-roberta-large-squad2/README.md deleted file mode 100644 index 8fcf54b7e633..000000000000 --- a/model_cards/deepset/xlm-roberta-large-squad2/README.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -language: multilingual -tags: -- question-answering -datasets: -- squad_v2 ---- - -# Multilingual XLM-RoBERTa large for QA on various languages - -## Overview -**Language model:** xlm-roberta-large -**Language:** Multilingual -**Downstream-task:** Extractive QA -**Training data:** SQuAD 2.0 -**Eval data:** SQuAD dev set - German MLQA - German XQuAD -**Training run:** [MLFlow link](https://public-mlflow.deepset.ai/#/experiments/124/runs/3a540e3f3ecf4dd98eae8fc6d457ff20) -**Infrastructure**: 4x Tesla v100 - -## Hyperparameters - -``` -batch_size = 32 -n_epochs = 3 -base_LM_model = "xlm-roberta-large" -max_seq_len = 256 -learning_rate = 1e-5 -lr_schedule = LinearWarmup -warmup_proportion = 0.2 -doc_stride=128 -max_query_length=64 -``` - -## Performance -Evaluated on the SQuAD 2.0 English dev set with the [official eval script](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/). -``` - "exact": 79.45759285774446, - "f1": 83.79259828925511, - "total": 11873, - "HasAns_exact": 71.96356275303644, - "HasAns_f1": 80.6460053117963, - "HasAns_total": 5928, - "NoAns_exact": 86.93019343986543, - "NoAns_f1": 86.93019343986543, - "NoAns_total": 5945 -``` - -Evaluated on German [MLQA: test-context-de-question-de.json](https://github.com/facebookresearch/MLQA) -``` -"exact": 49.34691166703564, -"f1": 66.15582561674236, -"total": 4517, -``` - -Evaluated on German [XQuAD: xquad.de.json](https://github.com/deepmind/xquad) -``` -"exact": 61.51260504201681, -"f1": 78.80206098332569, -"total": 1190, -``` - -## Usage - -### In Transformers -```python -from transformers import AutoModelForQuestionAnswering, AutoTokenizer, pipeline - -model_name = "deepset/xlm-roberta-large-squad2" - -# a) Get predictions -nlp = pipeline('question-answering', model=model_name, tokenizer=model_name) -QA_input = { - 'question': 'Why is model conversion important?', - 'context': 'The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks.' -} -res = nlp(QA_input) - -# b) Load model & tokenizer -model = AutoModelForQuestionAnswering.from_pretrained(model_name) -tokenizer = AutoTokenizer.from_pretrained(model_name) -``` - -### In FARM - -```python -from farm.modeling.adaptive_model import AdaptiveModel -from farm.modeling.tokenization import Tokenizer -from farm.infer import QAInferencer - -model_name = "deepset/xlm-roberta-large-squad2" - -# a) Get predictions -nlp = QAInferencer.load(model_name) -QA_input = [{"questions": ["Why is model conversion important?"], - "text": "The option to convert models between FARM and transformers gives freedom to the user and let people easily switch between frameworks."}] -res = nlp.inference_from_dicts(dicts=QA_input, rest_api_schema=True) - -# b) Load model & tokenizer -model = AdaptiveModel.convert_from_transformers(model_name, device="cpu", task_type="question_answering") -tokenizer = Tokenizer.load(model_name) -``` - -### In haystack -For doing QA at scale (i.e. many docs instead of single paragraph), you can load the model also in [haystack](https://github.com/deepset-ai/haystack/): -```python -reader = FARMReader(model_name_or_path="deepset/xlm-roberta-large-squad2") -# or -reader = TransformersReader(model="deepset/xlm-roberta-large-squad2",tokenizer="deepset/xlm-roberta-large-squad2") -``` - - -## Authors -Branden Chan: `branden.chan [at] deepset.ai` -Timo Möller: `timo.moeller [at] deepset.ai` -Malte Pietsch: `malte.pietsch [at] deepset.ai` -Tanay Soni: `tanay.soni [at] deepset.ai` - -## About us -![deepset logo](https://raw.githubusercontent.com/deepset-ai/FARM/master/docs/img/deepset_logo.png) - -We bring NLP to the industry via open source! -Our focus: Industry specific language models & large scale QA systems. - -Some of our work: -- [German BERT (aka "bert-base-german-cased")](https://deepset.ai/german-bert) -- [FARM](https://github.com/deepset-ai/FARM) -- [Haystack](https://github.com/deepset-ai/haystack/) - -Get in touch: -[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Website](https://deepset.ai) - diff --git a/model_cards/digitalepidemiologylab/covid-twitter-bert/README.md b/model_cards/digitalepidemiologylab/covid-twitter-bert/README.md deleted file mode 100644 index a148db508293..000000000000 --- a/model_cards/digitalepidemiologylab/covid-twitter-bert/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: "en" -thumbnail: "https://raw.githubusercontent.com/digitalepidemiologylab/covid-twitter-bert/master/images/COVID-Twitter-BERT_small.png" -tags: -- Twitter -- COVID-19 -license: mit ---- - -# COVID-Twitter-BERT (CT-BERT) v1 - -:warning: _You may want to use the [v2 model](https://huggingface.co/digitalepidemiologylab/covid-twitter-bert-v2) which was trained on more recent data and yields better performance_ :warning: - - -BERT-large-uncased model, pretrained on a corpus of messages from Twitter about COVID-19. Find more info on our [GitHub page](https://github.com/digitalepidemiologylab/covid-twitter-bert). - -## Overview -This model was trained on 160M tweets collected between January 12 and April 16, 2020 containing at least one of the keywords "wuhan", "ncov", "coronavirus", "covid", or "sars-cov-2". These tweets were filtered and preprocessed to reach a final sample of 22.5M tweets (containing 40.7M sentences and 633M tokens) which were used for training. - -This model was evaluated based on downstream classification tasks, but it could be used for any other NLP task which can leverage contextual embeddings. - -In order to achieve best results, make sure to use the same text preprocessing as we did for pretraining. This involves replacing user mentions, urls and emojis. You can find a script on our projects [GitHub repo](https://github.com/digitalepidemiologylab/covid-twitter-bert). - -## Example usage -```python -tokenizer = AutoTokenizer.from_pretrained("digitalepidemiologylab/covid-twitter-bert") -model = AutoModel.from_pretrained("digitalepidemiologylab/covid-twitter-bert") -``` - -You can also use the model with the `pipeline` interface: - -```python -from transformers import pipeline -import json - -pipe = pipeline(task='fill-mask', model='digitalepidemiologylab/covid-twitter-bert-v2') -out = pipe(f"In places with a lot of people, it's a good idea to wear a {pipe.tokenizer.mask_token}") -print(json.dumps(out, indent=4)) -[ - { - "sequence": "[CLS] in places with a lot of people, it's a good idea to wear a mask [SEP]", - "score": 0.9959408044815063, - "token": 7308, - "token_str": "mask" - }, - ... -] -``` - -## References -[1] Martin Müller, Marcel Salaté, Per E Kummervold. "COVID-Twitter-BERT: A Natural Language Processing Model to Analyse COVID-19 Content on Twitter" arXiv preprint arXiv:2005.07503 (2020). diff --git a/model_cards/distilbert-base-cased-README.md b/model_cards/distilbert-base-cased-README.md deleted file mode 100644 index 184ee3acc4ec..000000000000 --- a/model_cards/distilbert-base-cased-README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia ---- - -# DistilBERT base model (cased) - -This model is a distilled version of the [BERT base model](https://huggingface.co/bert-base-cased). -It was introduced in [this paper](https://arxiv.org/abs/1910.01108). -The code for the distillation process can be found -[here](https://github.com/huggingface/transformers/tree/master/examples/distillation). -This model is cased: it does make a difference between english and English. - -All the training details on the pre-training, the uses, limitations and potential biases are the same as for [DistilBERT-base-uncased](https://huggingface.co/distilbert-base-uncased). -We highly encourage to check it if you want to know more. - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Glue test results: - -| Task | MNLI | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | -|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:| -| | 81.5 | 87.8 | 88.2 | 90.4 | 47.2 | 85.5 | 85.6 | 60.6 | - -### BibTeX entry and citation info - -```bibtex -@article{Sanh2019DistilBERTAD, - title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter}, - author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf}, - journal={ArXiv}, - year={2019}, - volume={abs/1910.01108} -} -``` diff --git a/model_cards/distilbert-base-cased-distilled-squad-README.md b/model_cards/distilbert-base-cased-distilled-squad-README.md deleted file mode 100644 index 2f92ff7ae9e7..000000000000 --- a/model_cards/distilbert-base-cased-distilled-squad-README.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -language: "en" -datasets: -- squad -metrics: -- squad -license: apache-2.0 ---- - -# DistilBERT base cased distilled SQuAD - -This model is a fine-tune checkpoint of [DistilBERT-base-cased](https://huggingface.co/distilbert-base-cased), fine-tuned using (a second step of) knowledge distillation on SQuAD v1.1. -This model reaches a F1 score of 87.1 on the dev set (for comparison, BERT bert-base-cased version reaches a F1 score of 88.7). diff --git a/model_cards/distilbert-base-german-cased-README.md b/model_cards/distilbert-base-german-cased-README.md deleted file mode 100644 index 2b0c9fdb619e..000000000000 --- a/model_cards/distilbert-base-german-cased-README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -language: de -license: apache-2.0 ---- -## distilbert-base-german-cased diff --git a/model_cards/distilbert-base-multilingual-cased-README.md b/model_cards/distilbert-base-multilingual-cased-README.md deleted file mode 100644 index 2fa58c2575a7..000000000000 --- a/model_cards/distilbert-base-multilingual-cased-README.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -language: multilingual -license: apache-2.0 -datasets: -- wikipedia ---- - -# DistilBERT base multilingual model (cased) - -This model is a distilled version of the [BERT base multilingual model](bert-base-multilingual-cased). The code for the distillation process can be found -[here](https://github.com/huggingface/transformers/tree/master/examples/distillation). This model is cased: it does make a difference between english and English. - -The model is trained on the concatenation of Wikipedia in 104 different languages listed [here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). -The model has 6 layers, 768 dimension and 12 heads, totalizing 134M parameters (compared to 177M parameters for mBERT-base). -On average DistilmBERT is twice as fast as mBERT-base. - -We encourage to check [BERT base multilingual model](bert-base-multilingual-cased) to know more about usage, limitations and potential biases. - -| Model | English | Spanish | Chinese | German | Arabic | Urdu | -| :---: | :---: | :---: | :---: | :---: | :---: | :---:| -| mBERT base cased (computed) | 82.1 | 74.6 | 69.1 | 72.3 | 66.4 | 58.5 | -| mBERT base uncased (reported)| 81.4 | 74.3 | 63.8 | 70.5 | 62.1 | 58.3 | -| DistilmBERT | 78.2 | 69.1 | 64.0 | 66.3 | 59.1 | 54.7 | - -### BibTeX entry and citation info - -```bibtex -@article{Sanh2019DistilBERTAD, - title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter}, - author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf}, - journal={ArXiv}, - year={2019}, - volume={abs/1910.01108} -} -``` diff --git a/model_cards/distilbert-base-uncased-README.md b/model_cards/distilbert-base-uncased-README.md deleted file mode 100644 index 9b4358201c6b..000000000000 --- a/model_cards/distilbert-base-uncased-README.md +++ /dev/null @@ -1,218 +0,0 @@ ---- -language: en -tags: -- exbert -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia ---- - -# DistilBERT base model (uncased) - -This model is a distilled version of the [BERT base model](https://huggingface.co/bert-base-uncased). It was -introduced in [this paper](https://arxiv.org/abs/1910.01108). The code for the distillation process can be found -[here](https://github.com/huggingface/transformers/tree/master/examples/distillation). This model is uncased: it does -not make a difference between english and English. - -## Model description - -DistilBERT is a transformers model, smaller and faster than BERT, which was pretrained on the same corpus in a -self-supervised fashion, using the BERT base model as a teacher. This means it was pretrained on the raw texts only, -with no humans labelling them in any way (which is why it can use lots of publicly available data) with an automatic -process to generate inputs and labels from those texts using the BERT base model. More precisely, it was pretrained -with three objectives: - -- Distillation loss: the model was trained to return the same probabilities as the BERT base model. -- Masked language modeling (MLM): this is part of the original training loss of the BERT base model. When taking a - sentence, the model randomly masks 15% of the words in the input then run the entire masked sentence through the - model and has to predict the masked words. This is different from traditional recurrent neural networks (RNNs) that - usually see the words one after the other, or from autoregressive models like GPT which internally mask the future - tokens. It allows the model to learn a bidirectional representation of the sentence. -- Cosine embedding loss: the model was also trained to generate hidden states as close as possible as the BERT base - model. - -This way, the model learns the same inner representation of the English language than its teacher model, while being -faster for inference or downstream tasks. - -## Intended uses & limitations - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=distilbert) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='distilbert-base-uncased') ->>> unmasker("Hello I'm a [MASK] model.") - -[{'sequence': "[CLS] hello i'm a role model. [SEP]", - 'score': 0.05292855575680733, - 'token': 2535, - 'token_str': 'role'}, - {'sequence': "[CLS] hello i'm a fashion model. [SEP]", - 'score': 0.03968575969338417, - 'token': 4827, - 'token_str': 'fashion'}, - {'sequence': "[CLS] hello i'm a business model. [SEP]", - 'score': 0.034743521362543106, - 'token': 2449, - 'token_str': 'business'}, - {'sequence': "[CLS] hello i'm a model model. [SEP]", - 'score': 0.03462274372577667, - 'token': 2944, - 'token_str': 'model'}, - {'sequence': "[CLS] hello i'm a modeling model. [SEP]", - 'score': 0.018145186826586723, - 'token': 11643, - 'token_str': 'modeling'}] -``` - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import DistilBertTokenizer, DistilBertModel -tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') -model = DistilBertModel.from_pretrained("distilbert-base-uncased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import DistilBertTokenizer, TFDistilBertModel -tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') -model = TFDistilBertModel.from_pretrained("distilbert-base-uncased") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -### Limitations and bias - -Even if the training data used for this model could be characterized as fairly neutral, this model can have biased -predictions. It also inherits some of -[the bias of its teacher model](https://huggingface.co/bert-base-uncased#limitations-and-bias). - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='distilbert-base-uncased') ->>> unmasker("The White man worked as a [MASK].") - -[{'sequence': '[CLS] the white man worked as a blacksmith. [SEP]', - 'score': 0.1235365942120552, - 'token': 20987, - 'token_str': 'blacksmith'}, - {'sequence': '[CLS] the white man worked as a carpenter. [SEP]', - 'score': 0.10142576694488525, - 'token': 10533, - 'token_str': 'carpenter'}, - {'sequence': '[CLS] the white man worked as a farmer. [SEP]', - 'score': 0.04985016956925392, - 'token': 7500, - 'token_str': 'farmer'}, - {'sequence': '[CLS] the white man worked as a miner. [SEP]', - 'score': 0.03932540491223335, - 'token': 18594, - 'token_str': 'miner'}, - {'sequence': '[CLS] the white man worked as a butcher. [SEP]', - 'score': 0.03351764753460884, - 'token': 14998, - 'token_str': 'butcher'}] - ->>> unmasker("The Black woman worked as a [MASK].") - -[{'sequence': '[CLS] the black woman worked as a waitress. [SEP]', - 'score': 0.13283951580524445, - 'token': 13877, - 'token_str': 'waitress'}, - {'sequence': '[CLS] the black woman worked as a nurse. [SEP]', - 'score': 0.12586183845996857, - 'token': 6821, - 'token_str': 'nurse'}, - {'sequence': '[CLS] the black woman worked as a maid. [SEP]', - 'score': 0.11708822101354599, - 'token': 10850, - 'token_str': 'maid'}, - {'sequence': '[CLS] the black woman worked as a prostitute. [SEP]', - 'score': 0.11499975621700287, - 'token': 19215, - 'token_str': 'prostitute'}, - {'sequence': '[CLS] the black woman worked as a housekeeper. [SEP]', - 'score': 0.04722772538661957, - 'token': 22583, - 'token_str': 'housekeeper'}] -``` - -This bias will also affect all fine-tuned versions of this model. - -## Training data - -DistilBERT pretrained on the same data as BERT, which is [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset -consisting of 11,038 unpublished books and [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) -(excluding lists, tables and headers). - -## Training procedure - -### Preprocessing - -The texts are lowercased and tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are -then of the form: - -``` -[CLS] Sentence A [SEP] Sentence B [SEP] -``` - -With probability 0.5, sentence A and sentence B correspond to two consecutive sentences in the original corpus and in -the other cases, it's another random sentence in the corpus. Note that what is considered a sentence here is a -consecutive span of text usually longer than a single sentence. The only constrain is that the result with the two -"sentences" has a combined length of less than 512 tokens. - -The details of the masking procedure for each sentence are the following: -- 15% of the tokens are masked. -- In 80% of the cases, the masked tokens are replaced by `[MASK]`. -- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace. -- In the 10% remaining cases, the masked tokens are left as is. - -### Pretraining - -The model was trained on 8 16 GB V100 for 90 hours. See the -[training code](https://github.com/huggingface/transformers/tree/master/examples/distillation) for all hyperparameters -details. - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Glue test results: - -| Task | MNLI | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | -|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:| -| | 82.2 | 88.5 | 89.2 | 91.3 | 51.3 | 85.8 | 87.5 | 59.9 | - - -### BibTeX entry and citation info - -```bibtex -@article{Sanh2019DistilBERTAD, - title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter}, - author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf}, - journal={ArXiv}, - year={2019}, - volume={abs/1910.01108} -} -``` - - - - diff --git a/model_cards/distilbert-base-uncased-distilled-squad-README.md b/model_cards/distilbert-base-uncased-distilled-squad-README.md deleted file mode 100644 index 6765229e6280..000000000000 --- a/model_cards/distilbert-base-uncased-distilled-squad-README.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -language: en -datasets: -- squad -widget: -- text: "Which name is also used to describe the Amazon rainforest in English?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." -- text: "How many square kilometers of rainforest is covered in the basin?" - context: "The Amazon rainforest (Portuguese: Floresta Amazônica or Amazônia; Spanish: Selva Amazónica, Amazonía or usually Amazonia; French: Forêt amazonienne; Dutch: Amazoneregenwoud), also known in English as Amazonia or the Amazon Jungle, is a moist broadleaf forest that covers most of the Amazon basin of South America. This basin encompasses 7,000,000 square kilometres (2,700,000 sq mi), of which 5,500,000 square kilometres (2,100,000 sq mi) are covered by the rainforest. This region includes territory belonging to nine nations. The majority of the forest is contained within Brazil, with 60% of the rainforest, followed by Peru with 13%, Colombia with 10%, and with minor amounts in Venezuela, Ecuador, Bolivia, Guyana, Suriname and French Guiana. States or departments in four nations contain \"Amazonas\" in their names. The Amazon represents over half of the planet's remaining rainforests, and comprises the largest and most biodiverse tract of tropical rainforest in the world, with an estimated 390 billion individual trees divided into 16,000 species." -license: apache-2.0 ---- - -# DistilBERT base uncased distilled SQuAD - -This model is a fine-tune checkpoint of [DistilBERT-base-uncased](https://huggingface.co/distilbert-base-uncased), fine-tuned using (a second step of) knowledge distillation on SQuAD v1.1. -This model reaches a F1 score of 86.9 on the dev set (for comparison, Bert bert-base-uncased version reaches a F1 score of 88.5). diff --git a/model_cards/distilbert-base-uncased-finetuned-sst-2-english-README.md b/model_cards/distilbert-base-uncased-finetuned-sst-2-english-README.md deleted file mode 100644 index d33b5862630e..000000000000 --- a/model_cards/distilbert-base-uncased-finetuned-sst-2-english-README.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- sst-2 ---- - -# DistilBERT base uncased finetuned SST-2 - -This model is a fine-tune checkpoint of [DistilBERT-base-uncased](https://huggingface.co/distilbert-base-uncased), fine-tuned on SST-2. -This model reaches an accuracy of 91.3 on the dev set (for comparison, Bert bert-base-uncased version reaches an accuracy of 92.7). - -# Fine-tuning hyper-parameters - -- learning_rate = 1e-5 -- batch_size = 32 -- warmup = 600 -- max_seq_length = 128 -- num_train_epochs = 3.0 diff --git a/model_cards/distilgpt2-README.md b/model_cards/distilgpt2-README.md deleted file mode 100644 index 41e1a5a1e758..000000000000 --- a/model_cards/distilgpt2-README.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -language: en -tags: -- exbert - -license: apache-2.0 -datasets: -- openwebtext ---- - -# DistilGPT2 - -DistilGPT2 English language model pretrained with the supervision of [GPT2](https://huggingface.co/gpt2) (the smallest version of GPT2) on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset. The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 124M parameters for GPT2). On average, DistilGPT2 is two times faster than GPT2. - -On the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test set of 16.3 compared to 21.1 for DistilGPT2 (after fine-tuning on the train set). - -We encourage to check [GPT2](https://huggingface.co/gpt2) to know more about usage, limitations and potential biases. - - - - diff --git a/model_cards/distilroberta-base-README.md b/model_cards/distilroberta-base-README.md deleted file mode 100644 index 18bbbb860874..000000000000 --- a/model_cards/distilroberta-base-README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -language: en -tags: -- exbert - -license: apache-2.0 -datasets: -- openwebtext ---- - -# DistilRoBERTa base model - -This model is a distilled version of the [RoBERTa-base model](https://huggingface.co/roberta-base). It follows the same training procedure as [DistilBERT](https://huggingface.co/distilbert-base-uncased). -The code for the distillation process can be found [here](https://github.com/huggingface/transformers/tree/master/examples/distillation). -This model is case-sensitive: it makes a difference between english and English. - -The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 125M parameters for RoBERTa-base). -On average DistilRoBERTa is twice as fast as Roberta-base. - -We encourage to check [RoBERTa-base model](https://huggingface.co/roberta-base) to know more about usage, limitations and potential biases. - -## Training data - -DistilRoBERTa was pre-trained on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset (it is ~4 times less training data than the teacher RoBERTa). - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Glue test results: - -| Task | MNLI | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | -|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:| -| | 84.0 | 89.4 | 90.8 | 92.5 | 59.3 | 88.3 | 86.6 | 67.9 | - -### BibTeX entry and citation info - -```bibtex -@article{Sanh2019DistilBERTAD, - title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter}, - author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf}, - journal={ArXiv}, - year={2019}, - volume={abs/1910.01108} -} -``` - - - - diff --git a/model_cards/djstrong/bg_cs_pl_ru_cased_L-12_H-768_A-12/README.md b/model_cards/djstrong/bg_cs_pl_ru_cased_L-12_H-768_A-12/README.md deleted file mode 100644 index e29aab4e3686..000000000000 --- a/model_cards/djstrong/bg_cs_pl_ru_cased_L-12_H-768_A-12/README.md +++ /dev/null @@ -1 +0,0 @@ -Slavic BERT from https://github.com/deepmipt/Slavic-BERT-NER http://files.deeppavlov.ai/deeppavlov_data/bg_cs_pl_ru_cased_L-12_H-768_A-12.tar.gz diff --git a/model_cards/dkleczek/bert-base-polish-cased-v1/README.md b/model_cards/dkleczek/bert-base-polish-cased-v1/README.md deleted file mode 100644 index 0d27bd5d7fe4..000000000000 --- a/model_cards/dkleczek/bert-base-polish-cased-v1/README.md +++ /dev/null @@ -1,135 +0,0 @@ ---- -language: pl -thumbnail: https://raw.githubusercontent.com/kldarek/polbert/master/img/polbert.png ---- - -# Polbert - Polish BERT -Polish version of BERT language model is here! It is now available in two variants: cased and uncased, both can be downloaded and used via HuggingFace transformers library. I recommend using the cased model, more info on the differences and benchmark results below. - -![PolBERT image](https://raw.githubusercontent.com/kldarek/polbert/master/img/polbert.png) - -## Cased and uncased variants - -* I initially trained the uncased model, the corpus and training details are referenced below. Here are some issues I found after I published the uncased model: - * Some Polish characters and accents are not tokenized correctly through the BERT tokenizer when applying lowercase. This doesn't impact sequence classification much, but may influence token classfication tasks significantly. - * I noticed a lot of duplicates in the Open Subtitles dataset, which dominates the training corpus. - * I didn't use Whole Word Masking. -* The cased model improves on the uncased model in the following ways: - * All Polish characters and accents should now be tokenized correctly. - * I removed duplicates from Open Subtitles dataset. The corpus is smaller, but more balanced now. - * The model is trained with Whole Word Masking. - -## Pre-training corpora - -Below is the list of corpora used along with the output of `wc` command (counting lines, words and characters). These corpora were divided into sentences with srxsegmenter (see references), concatenated and tokenized with HuggingFace BERT Tokenizer. - -### Uncased - -| Tables | Lines | Words | Characters | -| ------------- |--------------:| -----:| -----:| -| [Polish subset of Open Subtitles](http://opus.nlpl.eu/OpenSubtitles-v2018.php) | 236635408| 1431199601 | 7628097730 | -| [Polish subset of ParaCrawl](http://opus.nlpl.eu/ParaCrawl.php) | 8470950 | 176670885 | 1163505275 | -| [Polish Parliamentary Corpus](http://clip.ipipan.waw.pl/PPC) | 9799859 | 121154785 | 938896963 | -| [Polish Wikipedia - Feb 2020](https://dumps.wikimedia.org/plwiki/latest/plwiki-latest-pages-articles.xml.bz2) | 8014206 | 132067986 | 1015849191 | -| Total | 262920423 | 1861093257 | 10746349159 | - -### Cased - -| Tables | Lines | Words | Characters | -| ------------- |--------------:| -----:| -----:| -| [Polish subset of Open Subtitles (Deduplicated) ](http://opus.nlpl.eu/OpenSubtitles-v2018.php) | 41998942| 213590656 | 1424873235 | -| [Polish subset of ParaCrawl](http://opus.nlpl.eu/ParaCrawl.php) | 8470950 | 176670885 | 1163505275 | -| [Polish Parliamentary Corpus](http://clip.ipipan.waw.pl/PPC) | 9799859 | 121154785 | 938896963 | -| [Polish Wikipedia - Feb 2020](https://dumps.wikimedia.org/plwiki/latest/plwiki-latest-pages-articles.xml.bz2) | 8014206 | 132067986 | 1015849191 | -| Total | 68283960 | 646479197 | 4543124667 | - - -## Pre-training details - -### Uncased - -* Polbert was trained with code provided in Google BERT's github repository (https://github.com/google-research/bert) -* Currently released model follows bert-base-uncased model architecture (12-layer, 768-hidden, 12-heads, 110M parameters) -* Training set-up: in total 1 million training steps: - * 100.000 steps - 128 sequence length, batch size 512, learning rate 1e-4 (10.000 steps warmup) - * 800.000 steps - 128 sequence length, batch size 512, learning rate 5e-5 - * 100.000 steps - 512 sequence length, batch size 256, learning rate 2e-5 -* The model was trained on a single Google Cloud TPU v3-8 - -### Cased - -* Same approach as uncased model, with the following differences: - * Whole Word Masking -* Training set-up: - * 100.000 steps - 128 sequence length, batch size 2048, learning rate 1e-4 (10.000 steps warmup) - * 100.000 steps - 128 sequence length, batch size 2048, learning rate 5e-5 - * 100.000 steps - 512 sequence length, batch size 256, learning rate 2e-5 - - -## Usage -Polbert is released via [HuggingFace Transformers library](https://huggingface.co/transformers/). - -For an example use as language model, see [this notebook](/LM_testing.ipynb) file. - -### Uncased - -```python -from transformers import * -model = BertForMaskedLM.from_pretrained("dkleczek/bert-base-polish-uncased-v1") -tokenizer = BertTokenizer.from_pretrained("dkleczek/bert-base-polish-uncased-v1") -nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer) -for pred in nlp(f"Adam Mickiewicz wielkim polskim {nlp.tokenizer.mask_token} był."): - print(pred) -# Output: -# {'sequence': '[CLS] adam mickiewicz wielkim polskim poeta był. [SEP]', 'score': 0.47196975350379944, 'token': 26596} -# {'sequence': '[CLS] adam mickiewicz wielkim polskim bohaterem był. [SEP]', 'score': 0.09127858281135559, 'token': 10953} -# {'sequence': '[CLS] adam mickiewicz wielkim polskim człowiekiem był. [SEP]', 'score': 0.0647173821926117, 'token': 5182} -# {'sequence': '[CLS] adam mickiewicz wielkim polskim pisarzem był. [SEP]', 'score': 0.05232388526201248, 'token': 24293} -# {'sequence': '[CLS] adam mickiewicz wielkim polskim politykiem był. [SEP]', 'score': 0.04554257541894913, 'token': 44095} -``` - -### Cased - -```python -model = BertForMaskedLM.from_pretrained("dkleczek/bert-base-polish-cased-v1") -tokenizer = BertTokenizer.from_pretrained("dkleczek/bert-base-polish-cased-v1") -nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer) -for pred in nlp(f"Adam Mickiewicz wielkim polskim {nlp.tokenizer.mask_token} był."): - print(pred) -# Output: -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim pisarzem był. [SEP]', 'score': 0.5391148328781128, 'token': 37120} -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim człowiekiem był. [SEP]', 'score': 0.11683262139558792, 'token': 6810} -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim bohaterem był. [SEP]', 'score': 0.06021466106176376, 'token': 17709} -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim mistrzem był. [SEP]', 'score': 0.051870670169591904, 'token': 14652} -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim artystą był. [SEP]', 'score': 0.031787533313035965, 'token': 35680} -``` - -See the next section for an example usage of Polbert in downstream tasks. - -## Evaluation -Thanks to Allegro, we now have the [KLEJ benchmark](https://klejbenchmark.com/leaderboard/), a set of nine evaluation tasks for the Polish language understanding. The following results are achieved by running standard set of evaluation scripts (no tricks!) utilizing both cased and uncased variants of Polbert. - -| Model | Average | NKJP-NER | CDSC-E | CDSC-R | CBD | PolEmo2.0-IN | PolEmo2.0-OUT | DYK | PSC | AR | -| ------------- |--------------:|--------------:|--------------:|--------------:|--------------:|--------------:|--------------:|--------------:|--------------:|--------------:| -| Polbert cased | 81.7 | 93.6 | 93.4 | 93.8 | 52.7 | 87.4 | 71.1 | 59.1 | 98.6 | 85.2 | -| Polbert uncased | 81.4 | 90.1 | 93.9 | 93.5 | 55.0 | 88.1 | 68.8 | 59.4 | 98.8 | 85.4 | - -Note how the uncased model performs better than cased on some tasks? My guess this is because of the oversampling of Open Subtitles dataset and its similarity to data in some of these tasks. All these benchmark tasks are sequence classification, so the relative strength of the cased model is not so visible here. - -## Bias -The data used to train the model is biased. It may reflect stereotypes related to gender, ethnicity etc. Please be careful when using the model for downstream task to consider these biases and mitigate them. - -## Acknowledgements -* I'd like to express my gratitude to Google [TensorFlow Research Cloud (TFRC)](https://www.tensorflow.org/tfrc) for providing the free TPU credits - thank you! -* Also appreciate the help from Timo Möller from [deepset](https://deepset.ai) for sharing tips and scripts based on their experience training German BERT model. -* Big thanks to Allegro for releasing KLEJ Benchmark and specifically to Piotr Rybak for help with the evaluation and pointing out some issues with the tokenization. -* Finally, thanks to Rachel Thomas, Jeremy Howard and Sylvain Gugger from [fastai](https://www.fast.ai) for their NLP and Deep Learning courses! - -## Author -Darek Kłeczek - contact me on Twitter [@dk21](https://twitter.com/dk21) - -## References -* https://github.com/google-research/bert -* https://github.com/narusemotoki/srx_segmenter -* SRX rules file for sentence splitting in Polish, written by Marcin Miłkowski: https://raw.githubusercontent.com/languagetool-org/languagetool/master/languagetool-core/src/main/resources/org/languagetool/resource/segment.srx -* [KLEJ benchmark](https://klejbenchmark.com/leaderboard/) diff --git a/model_cards/dkleczek/bert-base-polish-uncased-v1/README.md b/model_cards/dkleczek/bert-base-polish-uncased-v1/README.md deleted file mode 100644 index 103bc4a174c8..000000000000 --- a/model_cards/dkleczek/bert-base-polish-uncased-v1/README.md +++ /dev/null @@ -1,135 +0,0 @@ ---- -language: pl -thumbnail: https://raw.githubusercontent.com/kldarek/polbert/master/img/polbert.png ---- - -# Polbert - Polish BERT -Polish version of BERT language model is here! It is now available in two variants: cased and uncased, both can be downloaded and used via HuggingFace transformers library. I recommend using the cased model, more info on the differences and benchmark results below. - -![PolBERT image](https://raw.githubusercontent.com/kldarek/polbert/master/img/polbert.png) - -## Cased and uncased variants - -* I initially trained the uncased model, the corpus and training details are referenced below. Here are some issues I found after I published the uncased model: - * Some Polish characters and accents are not tokenized correctly through the BERT tokenizer when applying lowercase. This doesn't impact sequence classification much, but may influence token classfication tasks significantly. - * I noticed a lot of duplicates in the Open Subtitles dataset, which dominates the training corpus. - * I didn't use Whole Word Masking. -* The cased model improves on the uncased model in the following ways: - * All Polish characters and accents should now be tokenized correctly. - * I removed duplicates from Open Subtitles dataset. The corpus is smaller, but more balanced now. - * The model is trained with Whole Word Masking. - -## Pre-training corpora - -Below is the list of corpora used along with the output of `wc` command (counting lines, words and characters). These corpora were divided into sentences with srxsegmenter (see references), concatenated and tokenized with HuggingFace BERT Tokenizer. - -### Uncased - -| Tables | Lines | Words | Characters | -| ------------- |--------------:| -----:| -----:| -| [Polish subset of Open Subtitles](http://opus.nlpl.eu/OpenSubtitles-v2018.php) | 236635408| 1431199601 | 7628097730 | -| [Polish subset of ParaCrawl](http://opus.nlpl.eu/ParaCrawl.php) | 8470950 | 176670885 | 1163505275 | -| [Polish Parliamentary Corpus](http://clip.ipipan.waw.pl/PPC) | 9799859 | 121154785 | 938896963 | -| [Polish Wikipedia - Feb 2020](https://dumps.wikimedia.org/plwiki/latest/plwiki-latest-pages-articles.xml.bz2) | 8014206 | 132067986 | 1015849191 | -| Total | 262920423 | 1861093257 | 10746349159 | - -### Cased - -| Tables | Lines | Words | Characters | -| ------------- |--------------:| -----:| -----:| -| [Polish subset of Open Subtitles (Deduplicated) ](http://opus.nlpl.eu/OpenSubtitles-v2018.php) | 41998942| 213590656 | 1424873235 | -| [Polish subset of ParaCrawl](http://opus.nlpl.eu/ParaCrawl.php) | 8470950 | 176670885 | 1163505275 | -| [Polish Parliamentary Corpus](http://clip.ipipan.waw.pl/PPC) | 9799859 | 121154785 | 938896963 | -| [Polish Wikipedia - Feb 2020](https://dumps.wikimedia.org/plwiki/latest/plwiki-latest-pages-articles.xml.bz2) | 8014206 | 132067986 | 1015849191 | -| Total | 68283960 | 646479197 | 4543124667 | - - -## Pre-training details - -### Uncased - -* Polbert was trained with code provided in Google BERT's github repository (https://github.com/google-research/bert) -* Currently released model follows bert-base-uncased model architecture (12-layer, 768-hidden, 12-heads, 110M parameters) -* Training set-up: in total 1 million training steps: - * 100.000 steps - 128 sequence length, batch size 512, learning rate 1e-4 (10.000 steps warmup) - * 800.000 steps - 128 sequence length, batch size 512, learning rate 5e-5 - * 100.000 steps - 512 sequence length, batch size 256, learning rate 2e-5 -* The model was trained on a single Google Cloud TPU v3-8 - -### Cased - -* Same approach as uncased model, with the following differences: - * Whole Word Masking -* Training set-up: - * 100.000 steps - 128 sequence length, batch size 2048, learning rate 1e-4 (10.000 steps warmup) - * 100.000 steps - 128 sequence length, batch size 2048, learning rate 5e-5 - * 100.000 steps - 512 sequence length, batch size 256, learning rate 2e-5 - - -## Usage -Polbert is released via [HuggingFace Transformers library](https://huggingface.co/transformers/). - -For an example use as language model, see [this notebook](/LM_testing.ipynb) file. - -### Uncased - -```python -from transformers import * -model = BertForMaskedLM.from_pretrained("dkleczek/bert-base-polish-uncased-v1") -tokenizer = BertTokenizer.from_pretrained("dkleczek/bert-base-polish-uncased-v1") -nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer) -for pred in nlp(f"Adam Mickiewicz wielkim polskim {nlp.tokenizer.mask_token} był."): - print(pred) -# Output: -# {'sequence': '[CLS] adam mickiewicz wielkim polskim poeta był. [SEP]', 'score': 0.47196975350379944, 'token': 26596} -# {'sequence': '[CLS] adam mickiewicz wielkim polskim bohaterem był. [SEP]', 'score': 0.09127858281135559, 'token': 10953} -# {'sequence': '[CLS] adam mickiewicz wielkim polskim człowiekiem był. [SEP]', 'score': 0.0647173821926117, 'token': 5182} -# {'sequence': '[CLS] adam mickiewicz wielkim polskim pisarzem był. [SEP]', 'score': 0.05232388526201248, 'token': 24293} -# {'sequence': '[CLS] adam mickiewicz wielkim polskim politykiem był. [SEP]', 'score': 0.04554257541894913, 'token': 44095} -``` - -### Cased - -```python -model = BertForMaskedLM.from_pretrained("dkleczek/bert-base-polish-cased-v1") -tokenizer = BertTokenizer.from_pretrained("dkleczek/bert-base-polish-cased-v1") -nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer) -for pred in nlp(f"Adam Mickiewicz wielkim polskim {nlp.tokenizer.mask_token} był."): - print(pred) -# Output: -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim pisarzem był. [SEP]', 'score': 0.5391148328781128, 'token': 37120} -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim człowiekiem był. [SEP]', 'score': 0.11683262139558792, 'token': 6810} -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim bohaterem był. [SEP]', 'score': 0.06021466106176376, 'token': 17709} -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim mistrzem był. [SEP]', 'score': 0.051870670169591904, 'token': 14652} -# {'sequence': '[CLS] Adam Mickiewicz wielkim polskim artystą był. [SEP]', 'score': 0.031787533313035965, 'token': 35680} -``` - -See the next section for an example usage of Polbert in downstream tasks. - -## Evaluation -Thanks to Allegro, we now have the [KLEJ benchmark](https://klejbenchmark.com/leaderboard/), a set of nine evaluation tasks for the Polish language understanding. The following results are achieved by running standard set of evaluation scripts (no tricks!) utilizing both cased and uncased variants of Polbert. - -| Model | Average | NKJP-NER | CDSC-E | CDSC-R | CBD | PolEmo2.0-IN | PolEmo2.0-OUT | DYK | PSC | AR | -| ------------- |--------------:|--------------:|--------------:|--------------:|--------------:|--------------:|--------------:|--------------:|--------------:|--------------:| -| Polbert cased | 81.7 | 93.6 | 93.4 | 93.8 | 52.7 | 87.4 | 71.1 | 59.1 | 98.6 | 85.2 | -| Polbert uncased | 81.4 | 90.1 | 93.9 | 93.5 | 55.0 | 88.1 | 68.8 | 59.4 | 98.8 | 85.4 | - -Note how the uncased model performs better than cased on some tasks? My guess this is because of the oversampling of Open Subtitles dataset and its similarity to data in some of these tasks. All these benchmark tasks are sequence classification, so the relative strength of the cased model is not so visible here. - -## Bias -The data used to train the model is biased. It may reflect stereotypes related to gender, ethnicity etc. Please be careful when using the model for downstream task to consider these biases and mitigate them. - -## Acknowledgements -* I'd like to express my gratitude to Google [TensorFlow Research Cloud (TFRC)](https://www.tensorflow.org/tfrc) for providing the free TPU credits - thank you! -* Also appreciate the help from Timo Möller from [deepset](https://deepset.ai) for sharing tips and scripts based on their experience training German BERT model. -* Big thanks to Allegro for releasing KLEJ Benchmark and specifically to Piotr Rybak for help with the evaluation and pointing out some issues with the tokenization. -* Finally, thanks to Rachel Thomas, Jeremy Howard and Sylvain Gugger from [fastai](https://www.fast.ai) for their NLP and Deep Learning courses! - -## Author -Darek Kłeczek - contact me on Twitter [@dk21](https://twitter.com/dk21) - -## References -* https://github.com/google-research/bert -* https://github.com/narusemotoki/srx_segmenter -* SRX rules file for sentence splitting in Polish, written by Marcin Miłkowski: https://raw.githubusercontent.com/languagetool-org/languagetool/master/languagetool-core/src/main/resources/org/languagetool/resource/segment.srx -* [KLEJ benchmark](https://klejbenchmark.com/leaderboard/) \ No newline at end of file diff --git a/model_cards/dslim/bert-base-NER/README.md b/model_cards/dslim/bert-base-NER/README.md deleted file mode 100644 index d9e84583e005..000000000000 --- a/model_cards/dslim/bert-base-NER/README.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -language: en -datasets: -- conll2003 ---- -# bert-base-NER - -## Model description - -**bert-base-NER** is a fine-tuned BERT model that is ready to use for **Named Entity Recognition** and achieves **state-of-the-art performance** for the NER task. It has been trained to recognize four types of entities: location (LOC), organizations (ORG), person (PER) and Miscellaneous (MISC). - -Specifically, this model is a *bert-base-cased* model that was fine-tuned on the English version of the standard [CoNLL-2003 Named Entity Recognition](https://www.aclweb.org/anthology/W03-0419.pdf) dataset. -## Intended uses & limitations - -#### How to use - -You can use this model with Transformers *pipeline* for NER. - -```python -from transformers import AutoTokenizer, AutoModelForTokenClassification -from transformers import pipeline - -tokenizer = AutoTokenizer.from_pretrained("dslim/bert-base-NER") -model = AutoModelForTokenClassification.from_pretrained("dslim/bert-base-NER") - -nlp = pipeline("ner", model=model, tokenizer=tokenizer) -example = "My name is Wolfgang and I live in Berlin" - -ner_results = nlp(example) -print(ner_results) -``` - -#### Limitations and bias - -This model is limited by its training dataset of entity-annotated news articles from a specific span of time. This may not generalize well for all use cases in different domains. Furthermore, the model occassionally tags subword tokens as entities and post-processing of results may be necessary to handle those cases. - -## Training data - -This model was fine-tuned on English version of the standard [CoNLL-2003 Named Entity Recognition](https://www.aclweb.org/anthology/W03-0419.pdf) dataset. - -The training dataset distinguishes between the beginning and continuation of an entity so that if there are back-to-back entities of the same type, the model can output where the second entity begins. As in the dataset, each token will be classified as one of the following classes: -Abbreviation|Description --|- -O|Outside of a named entity -B-MIS |Beginning of a miscellaneous entity right after another miscellaneous entity -I-MIS |Miscellaneous entity -B-PER |Beginning of a person’s name right after another person’s name -I-PER |Person’s name -B-ORG |Beginning of an organisation right after another organisation -I-ORG |Organisation -B-LOC |Beginning of a location right after another location -I-LOC |Location - - -### CoNLL-2003 English Dataset Statistics -This dataset was derived from the Reuters corpus which consists of Reuters news stories. You can read more about how this dataset was created in the CoNLL-2003 paper. -#### # of training examples per entity type -Dataset|LOC|MISC|ORG|PER --|-|-|-|- -Train|7140|3438|6321|6600 -Dev|1837|922|1341|1842 -Test|1668|702|1661|1617 -#### # of articles/sentences/tokens per dataset -Dataset |Articles |Sentences |Tokens --|-|-|- -Train |946 |14,987 |203,621 -Dev |216 |3,466 |51,362 -Test |231 |3,684 |46,435 - -## Training procedure - -This model was trained on a single NVIDIA V100 GPU with recommended hyperparameters from the [original BERT paper](https://arxiv.org/pdf/1810.04805) which trained & evaluated the model on CoNLL-2003 NER task. - -## Eval results -metric|dev|test --|-|- -f1 |95.1 |91.3 -precision |95.0 |90.7 -recall |95.3 |91.9 - -The test metrics are a little lower than the official Google BERT results which encoded document context & experimented with CRF. More on replicating the original results [here](https://github.com/google-research/bert/issues/223). - -### BibTeX entry and citation info - -``` -@article{DBLP:journals/corr/abs-1810-04805, - author = {Jacob Devlin and - Ming{-}Wei Chang and - Kenton Lee and - Kristina Toutanova}, - title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language - Understanding}, - journal = {CoRR}, - volume = {abs/1810.04805}, - year = {2018}, - url = {http://arxiv.org/abs/1810.04805}, - archivePrefix = {arXiv}, - eprint = {1810.04805}, - timestamp = {Tue, 30 Oct 2018 20:39:56 +0100}, - biburl = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` -``` -@inproceedings{tjong-kim-sang-de-meulder-2003-introduction, - title = "Introduction to the {C}o{NLL}-2003 Shared Task: Language-Independent Named Entity Recognition", - author = "Tjong Kim Sang, Erik F. and - De Meulder, Fien", - booktitle = "Proceedings of the Seventh Conference on Natural Language Learning at {HLT}-{NAACL} 2003", - year = "2003", - url = "https://www.aclweb.org/anthology/W03-0419", - pages = "142--147", -} -``` diff --git a/model_cards/dumitrescustefan/bert-base-romanian-cased-v1/README.md b/model_cards/dumitrescustefan/bert-base-romanian-cased-v1/README.md deleted file mode 100644 index bc159d27247b..000000000000 --- a/model_cards/dumitrescustefan/bert-base-romanian-cased-v1/README.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -language: ro ---- - -# bert-base-romanian-cased-v1 - -The BERT **base**, **cased** model for Romanian, trained on a 15GB corpus, version ![v1.0](https://img.shields.io/badge/v1.0-21%20Apr%202020-ff6666) - -### How to use - -```python -from transformers import AutoTokenizer, AutoModel -import torch -# load tokenizer and model -tokenizer = AutoTokenizer.from_pretrained("dumitrescustefan/bert-base-romanian-cased-v1") -model = AutoModel.from_pretrained("dumitrescustefan/bert-base-romanian-cased-v1") -# tokenize a sentence and run through the model -input_ids = torch.tensor(tokenizer.encode("Acesta este un test.", add_special_tokens=True)).unsqueeze(0) # Batch size 1 -outputs = model(input_ids) -# get encoding -last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple -``` - -### Evaluation - -Evaluation is performed on Universal Dependencies [Romanian RRT](https://universaldependencies.org/treebanks/ro_rrt/index.html) UPOS, XPOS and LAS, and on a NER task based on [RONEC](https://github.com/dumitrescustefan/ronec). Details, as well as more in-depth tests not shown here, are given in the dedicated [evaluation page](https://github.com/dumitrescustefan/Romanian-Transformers/tree/master/evaluation/README.md). - -The baseline is the [Multilingual BERT](https://github.com/google-research/bert/blob/master/multilingual.md) model ``bert-base-multilingual-(un)cased``, as at the time of writing it was the only available BERT model that works on Romanian. - -| Model | UPOS | XPOS | NER | LAS | -|--------------------------------|:-----:|:------:|:-----:|:-----:| -| bert-base-multilingual-cased | 97.87 | 96.16 | 84.13 | 88.04 | -| bert-base-romanian-cased-v1 | **98.00** | **96.46** | **85.88** | **89.69** | - -### Corpus - -The model is trained on the following corpora (stats in the table below are after cleaning): - -| Corpus | Lines(M) | Words(M) | Chars(B) | Size(GB) | -|----------- |:--------: |:--------: |:--------: |:--------: | -| OPUS | 55.05 | 635.04 | 4.045 | 3.8 | -| OSCAR | 33.56 | 1725.82 | 11.411 | 11 | -| Wikipedia | 1.54 | 60.47 | 0.411 | 0.4 | -| **Total** | **90.15** | **2421.33** | **15.867** | **15.2** | - -#### Acknowledgements - -- We'd like to thank [Sampo Pyysalo](https://github.com/spyysalo) from TurkuNLP for helping us out with the compute needed to pretrain the v1.0 BERT models. He's awesome! diff --git a/model_cards/dumitrescustefan/bert-base-romanian-uncased-v1/README.md b/model_cards/dumitrescustefan/bert-base-romanian-uncased-v1/README.md deleted file mode 100644 index 878e4530add4..000000000000 --- a/model_cards/dumitrescustefan/bert-base-romanian-uncased-v1/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: ro ---- - -# bert-base-romanian-uncased-v1 - -The BERT **base**, **uncased** model for Romanian, trained on a 15GB corpus, version ![v1.0](https://img.shields.io/badge/v1.0-21%20Apr%202020-ff6666) - -### How to use - -```python -from transformers import AutoTokenizer, AutoModel -import torch - -# load tokenizer and model -tokenizer = AutoTokenizer.from_pretrained("dumitrescustefan/bert-base-romanian-uncased-v1", do_lower_case=True) -model = AutoModel.from_pretrained("dumitrescustefan/bert-base-romanian-uncased-v1") - -# tokenize a sentence and run through the model -input_ids = torch.tensor(tokenizer.encode("Acesta este un test.", add_special_tokens=True)).unsqueeze(0) # Batch size 1 -outputs = model(input_ids) - -# get encoding -last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple -``` - -### Evaluation - -Evaluation is performed on Universal Dependencies [Romanian RRT](https://universaldependencies.org/treebanks/ro_rrt/index.html) UPOS, XPOS and LAS, and on a NER task based on [RONEC](https://github.com/dumitrescustefan/ronec). Details, as well as more in-depth tests not shown here, are given in the dedicated [evaluation page](https://github.com/dumitrescustefan/Romanian-Transformers/tree/master/evaluation/README.md). - -The baseline is the [Multilingual BERT](https://github.com/google-research/bert/blob/master/multilingual.md) model ``bert-base-multilingual-(un)cased``, as at the time of writing it was the only available BERT model that works on Romanian. - -| Model | UPOS | XPOS | NER | LAS | -|--------------------------------|:-----:|:------:|:-----:|:-----:| -| bert-base-multilingual-uncased | 97.65 | 95.72 | 83.91 | 87.65 | -| bert-base-romanian-uncased-v1 | **98.18** | **96.84** | **85.26** | **89.61** | - -### Corpus - -The model is trained on the following corpora (stats in the table below are after cleaning): - -| Corpus | Lines(M) | Words(M) | Chars(B) | Size(GB) | -|----------- |:--------: |:--------: |:--------: |:--------: | -| OPUS | 55.05 | 635.04 | 4.045 | 3.8 | -| OSCAR | 33.56 | 1725.82 | 11.411 | 11 | -| Wikipedia | 1.54 | 60.47 | 0.411 | 0.4 | -| **Total** | **90.15** | **2421.33** | **15.867** | **15.2** | - -#### Acknowledgements - -- We'd like to thank [Sampo Pyysalo](https://github.com/spyysalo) from TurkuNLP for helping us out with the compute needed to pretrain the v1.0 BERT models. He's awesome! diff --git a/model_cards/e-tony/gpt2-rnm/README.md b/model_cards/e-tony/gpt2-rnm/README.md deleted file mode 100644 index 1804e2e88e65..000000000000 --- a/model_cards/e-tony/gpt2-rnm/README.md +++ /dev/null @@ -1,20 +0,0 @@ -### How to use - -You can use this model directly with a pipeline for text generation. Since the generation relies on some randomness, we -set a seed for reproducibility: - -```python ->>> from transformers import pipeline, set_seed ->>> generator = pipeline('text-generation', model='e-tony/gpt2-rnm') ->>> set_seed(42) ->>> generator("Rick: I turned myself into a pickle, Morty!\nMorty: ", max_length=50, num_return_sequences=5) - -[{'generated_text': "Rick: I turned myself into a pickle, Morty!\nMorty: I didn't want to have children. It was my fate! I'll pay my mom and dad.\nSnuffles: Well, at least we"}, - {'generated_text': "Rick: I turned myself into a pickle, Morty!\nMorty: you know what happened?\n(Steven begins dragging people down the toilet with his hand. As Steven falls) The whole thing starts.\nA man approaches Steven"}, - {'generated_text': "Rick: I turned myself into a pickle, Morty!\nMorty: Oh wait! And do you remember what I did to you?\nJerry: Uh, it didn't hurt. It should have hurt a lot since I"}, - {'generated_text': "Rick: I turned myself into a pickle, Morty!\nMorty: Rick!\nKraven: Wait! [wary gasp] What the hell are you doing this time?!\nJerry: Hey, are you"}, - {'generated_text': "Rick: I turned myself into a pickle, Morty!\nMorty: Uh.\nJerry: You don't have to put your finger on me today, do you?\nRick: It's just, what do you"}] -``` - -### Training data -We used the original `gpt2` model and fine-tuned it on [Rick and Morty transcripts](https://rickandmorty.fandom.com/wiki/Category:Transcripts). diff --git a/model_cards/elgeish/cs224n-squad2.0-albert-base-v2/README.md b/model_cards/elgeish/cs224n-squad2.0-albert-base-v2/README.md deleted file mode 100644 index 4e59ee06c40a..000000000000 --- a/model_cards/elgeish/cs224n-squad2.0-albert-base-v2/README.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -tags: -- exbert ---- - -## CS224n SQuAD2.0 Project Dataset -The goal of this model is to save CS224n students GPU time when establishing -baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf). -The training set used to fine-tune this model is the same as -the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however, -evaluation and model selection were performed using roughly half of the official -dev set, 6078 examples, picked at random. The data files can be found at - — this is the Winter 2020 -version. Given that the official SQuAD2.0 dev set contains the project's test -set, students must make sure not to use the official SQuAD2.0 dev set in any way -— including the use of models fine-tuned on the official SQuAD2.0, since they -used the official SQuAD2.0 dev set for model selection. - - - - - -## Results -```json -{ - "exact": 78.94044093451794, - "f1": 81.7724930324639, - "total": 6078, - "HasAns_exact": 76.28865979381443, - "HasAns_f1": 82.20385314478195, - "HasAns_total": 2910, - "NoAns_exact": 81.37626262626263, - "NoAns_f1": 81.37626262626263, - "NoAns_total": 3168, - "best_exact": 78.95689371503784, - "best_exact_thresh": 0.0, - "best_f1": 81.78894581298378, - "best_f1_thresh": 0.0 -} -``` - -## Notable Arguments -```json -{ - "do_lower_case": true, - "doc_stride": 128, - "fp16": false, - "fp16_opt_level": "O1", - "gradient_accumulation_steps": 24, - "learning_rate": 3e-05, - "max_answer_length": 30, - "max_grad_norm": 1, - "max_query_length": 64, - "max_seq_length": 384, - "model_name_or_path": "albert-base-v2", - "model_type": "albert", - "num_train_epochs": 3, - "per_gpu_train_batch_size": 8, - "save_steps": 5000, - "seed": 42, - "train_batch_size": 8, - "version_2_with_negative": true, - "warmup_steps": 0, - "weight_decay": 0 -} -``` - -## Environment Setup -```json -{ - "transformers": "2.5.1", - "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0", - "python": "3.6.5=hc3d631a_2", - "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux", - "gpu": "Tesla V100-SXM2-16GB" -} -``` - -## How to Cite -```BibTeX -@misc{elgeish2020gestalt, - title={Gestalt: a Stacking Ensemble for SQuAD2.0}, - author={Mohamed El-Geish}, - journal={arXiv e-prints}, - archivePrefix={arXiv}, - eprint={2004.07067}, - year={2020}, -} -``` - -## Related Models -* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2) -* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1) -* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased) -* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base) diff --git a/model_cards/elgeish/cs224n-squad2.0-albert-large-v2/README.md b/model_cards/elgeish/cs224n-squad2.0-albert-large-v2/README.md deleted file mode 100644 index 5f365d2d7ba1..000000000000 --- a/model_cards/elgeish/cs224n-squad2.0-albert-large-v2/README.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -tags: -- exbert ---- - -## CS224n SQuAD2.0 Project Dataset -The goal of this model is to save CS224n students GPU time when establishing -baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf). -The training set used to fine-tune this model is the same as -the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however, -evaluation and model selection were performed using roughly half of the official -dev set, 6078 examples, picked at random. The data files can be found at - — this is the Winter 2020 -version. Given that the official SQuAD2.0 dev set contains the project's test -set, students must make sure not to use the official SQuAD2.0 dev set in any way -— including the use of models fine-tuned on the official SQuAD2.0, since they -used the official SQuAD2.0 dev set for model selection. - - - - - -## Results -```json -{ - "exact": 79.2694965449161, - "f1": 82.50844352970152, - "total": 6078, - "HasAns_exact": 74.87972508591065, - "HasAns_f1": 81.64478342732858, - "HasAns_total": 2910, - "NoAns_exact": 83.30176767676768, - "NoAns_f1": 83.30176767676768, - "NoAns_total": 3168, - "best_exact": 79.2694965449161, - "best_exact_thresh": 0.0, - "best_f1": 82.50844352970155, - "best_f1_thresh": 0.0 -} -``` - -## Notable Arguments -```json -{ - "do_lower_case": true, - "doc_stride": 128, - "fp16": false, - "fp16_opt_level": "O1", - "gradient_accumulation_steps": 1, - "learning_rate": 3e-05, - "max_answer_length": 30, - "max_grad_norm": 1, - "max_query_length": 64, - "max_seq_length": 384, - "model_name_or_path": "albert-large-v2", - "model_type": "albert", - "num_train_epochs": 5, - "per_gpu_train_batch_size": 8, - "save_steps": 5000, - "seed": 42, - "train_batch_size": 8, - "version_2_with_negative": true, - "warmup_steps": 0, - "weight_decay": 0 -} -``` - -## Environment Setup -```json -{ - "transformers": "2.5.1", - "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0", - "python": "3.6.5=hc3d631a_2", - "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux", - "gpu": "Tesla V100-SXM2-16GB" -} -``` - -## How to Cite -```BibTeX -@misc{elgeish2020gestalt, - title={Gestalt: a Stacking Ensemble for SQuAD2.0}, - author={Mohamed El-Geish}, - journal={arXiv e-prints}, - archivePrefix={arXiv}, - eprint={2004.07067}, - year={2020}, -} -``` - -## Related Models -* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2) -* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1) -* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased) -* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base) diff --git a/model_cards/elgeish/cs224n-squad2.0-albert-xxlarge-v1/README.md b/model_cards/elgeish/cs224n-squad2.0-albert-xxlarge-v1/README.md deleted file mode 100644 index 8856e7cc8018..000000000000 --- a/model_cards/elgeish/cs224n-squad2.0-albert-xxlarge-v1/README.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -tags: -- exbert ---- - -## CS224n SQuAD2.0 Project Dataset -The goal of this model is to save CS224n students GPU time when establishing -baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf). -The training set used to fine-tune this model is the same as -the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however, -evaluation and model selection were performed using roughly half of the official -dev set, 6078 examples, picked at random. The data files can be found at - — this is the Winter 2020 -version. Given that the official SQuAD2.0 dev set contains the project's test -set, students must make sure not to use the official SQuAD2.0 dev set in any way -— including the use of models fine-tuned on the official SQuAD2.0, since they -used the official SQuAD2.0 dev set for model selection. - - - - - -## Results -```json -{ - "exact": 85.93287265547877, - "f1": 88.91258331187983, - "total": 6078, - "HasAns_exact": 84.36426116838489, - "HasAns_f1": 90.58786301361013, - "HasAns_total": 2910, - "NoAns_exact": 87.37373737373737, - "NoAns_f1": 87.37373737373737, - "NoAns_total": 3168, - "best_exact": 85.93287265547877, - "best_exact_thresh": 0.0, - "best_f1": 88.91258331187993, - "best_f1_thresh": 0.0 -} -``` - -## Notable Arguments -```json -{ - "do_lower_case": true, - "doc_stride": 128, - "fp16": false, - "fp16_opt_level": "O1", - "gradient_accumulation_steps": 24, - "learning_rate": 3e-05, - "max_answer_length": 30, - "max_grad_norm": 1, - "max_query_length": 64, - "max_seq_length": 512, - "model_name_or_path": "albert-xxlarge-v1", - "model_type": "albert", - "num_train_epochs": 4, - "per_gpu_train_batch_size": 1, - "save_steps": 1000, - "seed": 42, - "train_batch_size": 1, - "version_2_with_negative": true, - "warmup_steps": 814, - "weight_decay": 0 -} -``` - -## Environment Setup -```json -{ - "transformers": "2.5.1", - "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0", - "python": "3.6.5=hc3d631a_2", - "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux", - "gpu": "Tesla V100-SXM2-16GB" -} -``` - -## How to Cite -```BibTeX -@misc{elgeish2020gestalt, - title={Gestalt: a Stacking Ensemble for SQuAD2.0}, - author={Mohamed El-Geish}, - journal={arXiv e-prints}, - archivePrefix={arXiv}, - eprint={2004.07067}, - year={2020}, -} -``` - -## Related Models -* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2) -* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2) -* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased) -* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base) diff --git a/model_cards/elgeish/cs224n-squad2.0-distilbert-base-uncased/README.md b/model_cards/elgeish/cs224n-squad2.0-distilbert-base-uncased/README.md deleted file mode 100644 index 1bf3ab078183..000000000000 --- a/model_cards/elgeish/cs224n-squad2.0-distilbert-base-uncased/README.md +++ /dev/null @@ -1,86 +0,0 @@ -## CS224n SQuAD2.0 Project Dataset -The goal of this model is to save CS224n students GPU time when establishing -baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf). -The training set used to fine-tune this model is the same as -the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however, -evaluation and model selection were performed using roughly half of the official -dev set, 6078 examples, picked at random. The data files can be found at - — this is the Winter 2020 -version. Given that the official SQuAD2.0 dev set contains the project's test -set, students must make sure not to use the official SQuAD2.0 dev set in any way -— including the use of models fine-tuned on the official SQuAD2.0, since they -used the official SQuAD2.0 dev set for model selection. - -## Results -```json -{ - "exact": 65.16946363935504, - "f1": 67.87348075352251, - "total": 6078, - "HasAns_exact": 69.51890034364261, - "HasAns_f1": 75.16667217179045, - "HasAns_total": 2910, - "NoAns_exact": 61.17424242424242, - "NoAns_f1": 61.17424242424242, - "NoAns_total": 3168, - "best_exact": 65.16946363935504, - "best_exact_thresh": 0.0, - "best_f1": 67.87348075352243, - "best_f1_thresh": 0.0 -} -``` - -## Notable Arguments -```json -{ - "do_lower_case": true, - "doc_stride": 128, - "fp16": false, - "fp16_opt_level": "O1", - "gradient_accumulation_steps": 24, - "learning_rate": 3e-05, - "max_answer_length": 30, - "max_grad_norm": 1, - "max_query_length": 64, - "max_seq_length": 384, - "model_name_or_path": "distilbert-base-uncased-distilled-squad", - "model_type": "distilbert", - "num_train_epochs": 4, - "per_gpu_train_batch_size": 32, - "save_steps": 5000, - "seed": 42, - "train_batch_size": 32, - "version_2_with_negative": true, - "warmup_steps": 0, - "weight_decay": 0 -} -``` - -## Environment Setup -```json -{ - "transformers": "2.5.1", - "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0", - "python": "3.6.5=hc3d631a_2", - "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux", - "gpu": "Tesla V100-SXM2-16GB" -} -``` - -## How to Cite -```BibTeX -@misc{elgeish2020gestalt, - title={Gestalt: a Stacking Ensemble for SQuAD2.0}, - author={Mohamed El-Geish}, - journal={arXiv e-prints}, - archivePrefix={arXiv}, - eprint={2004.07067}, - year={2020}, -} -``` - -## Related Models -* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2) -* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2) -* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1) -* [elgeish/cs224n-squad2.0-roberta-base](https://huggingface.co/elgeish/cs224n-squad2.0-roberta-base) diff --git a/model_cards/elgeish/cs224n-squad2.0-roberta-base/README.md b/model_cards/elgeish/cs224n-squad2.0-roberta-base/README.md deleted file mode 100644 index 1cef39c26494..000000000000 --- a/model_cards/elgeish/cs224n-squad2.0-roberta-base/README.md +++ /dev/null @@ -1,86 +0,0 @@ -## CS224n SQuAD2.0 Project Dataset -The goal of this model is to save CS224n students GPU time when establishing -baselines to beat for the [Default Final Project](http://web.stanford.edu/class/cs224n/project/default-final-project-handout.pdf). -The training set used to fine-tune this model is the same as -the [official one](https://rajpurkar.github.io/SQuAD-explorer/); however, -evaluation and model selection were performed using roughly half of the official -dev set, 6078 examples, picked at random. The data files can be found at - — this is the Winter 2020 -version. Given that the official SQuAD2.0 dev set contains the project's test -set, students must make sure not to use the official SQuAD2.0 dev set in any way -— including the use of models fine-tuned on the official SQuAD2.0, since they -used the official SQuAD2.0 dev set for model selection. - -## Results -```json -{ - "exact": 75.32082922013821, - "f1": 78.66699523704254, - "total": 6078, - "HasAns_exact": 74.84536082474227, - "HasAns_f1": 81.83436324767868, - "HasAns_total": 2910, - "NoAns_exact": 75.75757575757575, - "NoAns_f1": 75.75757575757575, - "NoAns_total": 3168, - "best_exact": 75.32082922013821, - "best_exact_thresh": 0.0, - "best_f1": 78.66699523704266, - "best_f1_thresh": 0.0 -} -``` - -## Notable Arguments -```json -{ - "do_lower_case": true, - "doc_stride": 128, - "fp16": false, - "fp16_opt_level": "O1", - "gradient_accumulation_steps": 24, - "learning_rate": 3e-05, - "max_answer_length": 30, - "max_grad_norm": 1, - "max_query_length": 64, - "max_seq_length": 384, - "model_name_or_path": "roberta-base", - "model_type": "roberta", - "num_train_epochs": 4, - "per_gpu_train_batch_size": 16, - "save_steps": 5000, - "seed": 42, - "train_batch_size": 16, - "version_2_with_negative": true, - "warmup_steps": 0, - "weight_decay": 0 -} -``` - -## Environment Setup -```json -{ - "transformers": "2.5.1", - "pytorch": "1.4.0=py3.6_cuda10.1.243_cudnn7.6.3_0", - "python": "3.6.5=hc3d631a_2", - "os": "Linux 4.15.0-1060-aws #62-Ubuntu SMP Tue Feb 11 21:23:22 UTC 2020 x86_64 x86_64 x86_64 GNU/Linux", - "gpu": "Tesla V100-SXM2-16GB" -} -``` - -## How to Cite -```BibTeX -@misc{elgeish2020gestalt, - title={Gestalt: a Stacking Ensemble for SQuAD2.0}, - author={Mohamed El-Geish}, - journal={arXiv e-prints}, - archivePrefix={arXiv}, - eprint={2004.07067}, - year={2020}, -} -``` - -## Related Models -* [elgeish/cs224n-squad2.0-albert-base-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-base-v2) -* [elgeish/cs224n-squad2.0-albert-large-v2](https://huggingface.co/elgeish/cs224n-squad2.0-albert-large-v2) -* [elgeish/cs224n-squad2.0-albert-xxlarge-v1](https://huggingface.co/elgeish/cs224n-squad2.0-albert-xxlarge-v1) -* [elgeish/cs224n-squad2.0-distilbert-base-uncased](https://huggingface.co/elgeish/cs224n-squad2.0-distilbert-base-uncased) diff --git a/model_cards/emilyalsentzer/Bio_ClinicalBERT/README.md b/model_cards/emilyalsentzer/Bio_ClinicalBERT/README.md deleted file mode 100644 index af55937a09e9..000000000000 --- a/model_cards/emilyalsentzer/Bio_ClinicalBERT/README.md +++ /dev/null @@ -1,39 +0,0 @@ - -# ClinicalBERT - Bio + Clinical BERT Model - -The [Publicly Available Clinical BERT Embeddings](https://arxiv.org/abs/1904.03323) paper contains four unique clinicalBERT models: initialized with BERT-Base (`cased_L-12_H-768_A-12`) or BioBERT (`BioBERT-Base v1.0 + PubMed 200K + PMC 270K`) & trained on either all MIMIC notes or only discharge summaries. - -This model card describes the Bio+Clinical BERT model, which was initialized from [BioBERT](https://arxiv.org/abs/1901.08746) & trained on all MIMIC notes. - -## Pretraining Data -The `Bio_ClinicalBERT` model was trained on all notes from [MIMIC III](https://www.nature.com/articles/sdata201635), a database containing electronic health records from ICU patients at the Beth Israel Hospital in Boston, MA. For more details on MIMIC, see [here](https://mimic.physionet.org/). All notes from the `NOTEEVENTS` table were included (~880M words). - -## Model Pretraining - -### Note Preprocessing -Each note in MIMIC was first split into sections using a rules-based section splitter (e.g. discharge summary notes were split into "History of Present Illness", "Family History", "Brief Hospital Course", etc. sections). Then each section was split into sentences using SciSpacy (`en core sci md` tokenizer). - -### Pretraining Procedures -The model was trained using code from [Google's BERT repository](https://github.com/google-research/bert) on a GeForce GTX TITAN X 12 GB GPU. Model parameters were initialized with BioBERT (`BioBERT-Base v1.0 + PubMed 200K + PMC 270K`). - -### Pretraining Hyperparameters -We used a batch size of 32, a maximum sequence length of 128, and a learning rate of 5 · 10−5 for pre-training our models. The models trained on all MIMIC notes were trained for 150,000 steps. The dup factor for duplicating input data with different masks was set to 5. All other default parameters were used (specifically, masked language model probability = 0.15 -and max predictions per sequence = 20). - -## How to use the model - -Load the model via the transformers library: -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT") -model = AutoModel.from_pretrained("emilyalsentzer/Bio_ClinicalBERT") -``` - -## More Information - -Refer to the original paper, [Publicly Available Clinical BERT Embeddings](https://arxiv.org/abs/1904.03323) (NAACL Clinical NLP Workshop 2019) for additional details and performance on NLI and NER tasks. - -## Questions? - -Post a Github issue on the [clinicalBERT repo](https://github.com/EmilyAlsentzer/clinicalBERT) or email emilya@mit.edu with any questions. - diff --git a/model_cards/emilyalsentzer/Bio_Discharge_Summary_BERT/README.md b/model_cards/emilyalsentzer/Bio_Discharge_Summary_BERT/README.md deleted file mode 100644 index 162716e72f1f..000000000000 --- a/model_cards/emilyalsentzer/Bio_Discharge_Summary_BERT/README.md +++ /dev/null @@ -1,39 +0,0 @@ - -# ClinicalBERT - Bio + Discharge Summary BERT Model - -The [Publicly Available Clinical BERT Embeddings](https://arxiv.org/abs/1904.03323) paper contains four unique clinicalBERT models: initialized with BERT-Base (`cased_L-12_H-768_A-12`) or BioBERT (`BioBERT-Base v1.0 + PubMed 200K + PMC 270K`) & trained on either all MIMIC notes or only discharge summaries. - -This model card describes the Bio+Discharge Summary BERT model, which was initialized from [BioBERT](https://arxiv.org/abs/1901.08746) & trained on only discharge summaries from MIMIC. - -## Pretraining Data -The `Bio_Discharge_Summary_BERT` model was trained on all discharge summaries from [MIMIC III](https://www.nature.com/articles/sdata201635), a database containing electronic health records from ICU patients at the Beth Israel Hospital in Boston, MA. For more details on MIMIC, see [here](https://mimic.physionet.org/). All notes from the `NOTEEVENTS` table were included (~880M words). - -## Model Pretraining - -### Note Preprocessing -Each note in MIMIC was first split into sections using a rules-based section splitter (e.g. discharge summary notes were split into "History of Present Illness", "Family History", "Brief Hospital Course", etc. sections). Then each section was split into sentences using SciSpacy (`en core sci md` tokenizer). - -### Pretraining Procedures -The model was trained using code from [Google's BERT repository](https://github.com/google-research/bert) on a GeForce GTX TITAN X 12 GB GPU. Model parameters were initialized with BioBERT (`BioBERT-Base v1.0 + PubMed 200K + PMC 270K`). - -### Pretraining Hyperparameters -We used a batch size of 32, a maximum sequence length of 128, and a learning rate of 5 · 10−5 for pre-training our models. The models trained on all MIMIC notes were trained for 150,000 steps. The dup factor for duplicating input data with different masks was set to 5. All other default parameters were used (specifically, masked language model probability = 0.15 -and max predictions per sequence = 20). - -## How to use the model - -Load the model via the transformers library: -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT") -model = AutoModel.from_pretrained("emilyalsentzer/Bio_ClinicalBERT") -``` - -## More Information - -Refer to the original paper, [Publicly Available Clinical BERT Embeddings](https://arxiv.org/abs/1904.03323) (NAACL Clinical NLP Workshop 2019) for additional details and performance on NLI and NER tasks. - -## Questions? - -Post a Github issue on the [clinicalBERT repo](https://github.com/EmilyAlsentzer/clinicalBERT) or email emilya@mit.edu with any questions. - diff --git a/model_cards/etalab-ia/camembert-base-squadFR-fquad-piaf/README.md b/model_cards/etalab-ia/camembert-base-squadFR-fquad-piaf/README.md deleted file mode 100644 index dc77396e1f81..000000000000 --- a/model_cards/etalab-ia/camembert-base-squadFR-fquad-piaf/README.md +++ /dev/null @@ -1,108 +0,0 @@ ---- -language: fr -datasets: -- piaf -- FQuAD -- SQuAD-FR -widget: -- text: "Comment s'appelle le portail open data du gouvernement ?" - context: "Etalab est une administration publique française qui fait notamment office de Chief Data Officer de l'État et coordonne la conception et la mise en œuvre de sa stratégie dans le domaine de la donnée (ouverture et partage des données publiques ou open data, exploitation des données et intelligence artificielle...). Ainsi, Etalab développe et maintient le portail des données ouvertes du gouvernement français data.gouv.fr. -Etalab promeut également une plus grande ouverture l'administration sur la société (gouvernement ouvert) : transparence de l'action publique, innovation ouverte, participation citoyenne... elle promeut l’innovation, l’expérimentation, les méthodes de travail ouvertes, agiles et itératives, ainsi que les synergies avec la société civile pour décloisonner l’administration et favoriser l’adoption des meilleures pratiques professionnelles dans le domaine du numérique. À ce titre elle étudie notamment l’opportunité de recourir à des technologies en voie de maturation issues du monde de la recherche. -Cette entité chargée de l'innovation au sein de l'administration doit contribuer à l'amélioration du service public grâce au numérique. Elle est rattachée à la Direction interministérielle du numérique, dont les missions et l’organisation ont été fixées par le décret du 30 octobre 2019.  Dirigé par Laure Lucchesi depuis 2016, elle rassemble une équipe pluridisciplinaire d'une trentaine de personnes." ---- - -# camembert-base-squadFR-fquad-piaf - -## Description - -Question-answering French model, using base [CamemBERT](https://camembert-model.fr/) fine-tuned on a combo of three French Q&A datasets: - -1. [PIAFv1.1](https://www.data.gouv.fr/en/datasets/piaf-le-dataset-francophone-de-questions-reponses/) -2. [FQuADv1.0](https://fquad.illuin.tech/) -3. [SQuAD-FR (SQuAD automatically translated to French)](https://github.com/Alikabbadj/French-SQuAD) - -## Training hyperparameters - -```shell -python run_squad.py \ ---model_type camembert \ ---model_name_or_path camembert-base \ ---do_train --do_eval \ ---train_file data/SQuAD+fquad+piaf.json \ ---predict_file data/fquad_valid.json \ ---per_gpu_train_batch_size 12 \ ---learning_rate 3e-5 \ ---num_train_epochs 4 \ ---max_seq_length 384 \ ---doc_stride 128 \ ---save_steps 10000 -``` - -## Evaluation results -### FQuAD v1.0 Evaluation -```shell -{"f1": 79.81, "exact_match": 55.14} -``` -### SQuAD-FR Evaluation -```shell -{"f1": 80.61, "exact_match": 59.54} -``` - -## Usage - -```python -from transformers import pipeline - -nlp = pipeline('question-answering', model='etalab-ia/camembert-base-squadFR-fquad-piaf', tokenizer='etalab-ia/camembert-base-squadFR-fquad-piaf') - -nlp({ - 'question': "Qui est Claude Monet?", - 'context': "Claude Monet, né le 14 novembre 1840 à Paris et mort le 5 décembre 1926 à Giverny, est un peintre français et l’un des fondateurs de l'impressionnisme." -}) -``` - -## Citation - -### PIAF -``` -@inproceedings{KeraronLBAMSSS20, - author = {Rachel Keraron and - Guillaume Lancrenon and - Mathilde Bras and - Fr{\'{e}}d{\'{e}}ric Allary and - Gilles Moyse and - Thomas Scialom and - Edmundo{-}Pavel Soriano{-}Morales and - Jacopo Staiano}, - title = {Project {PIAF:} Building a Native French Question-Answering Dataset}, - booktitle = {{LREC}}, - pages = {5481--5490}, - publisher = {European Language Resources Association}, - year = {2020} -} - -``` - -### FQuAD -``` -@article{dHoffschmidt2020FQuADFQ, - title={FQuAD: French Question Answering Dataset}, - author={Martin d'Hoffschmidt and Maxime Vidal and Wacim Belblidia and Tom Brendl'e and Quentin Heinrich}, - journal={ArXiv}, - year={2020}, - volume={abs/2002.06071} -} -``` - -### SQuAD-FR -``` - @MISC{kabbadj2018, - author = "Kabbadj, Ali", - title = "Something new in French Text Mining and Information Extraction (Universal Chatbot): Largest Q&A French training dataset (110 000+) ", - editor = "linkedin.com", - month = "November", - year = "2018", - url = "\url{https://www.linkedin.com/pulse/something-new-french-text-mining-information-chatbot-largest-kabbadj/}", - note = "[Online; posted 11-November-2018]", - } - ``` diff --git a/model_cards/ethanyt/guwenbert-base/README.md b/model_cards/ethanyt/guwenbert-base/README.md deleted file mode 100644 index 652785bc5481..000000000000 --- a/model_cards/ethanyt/guwenbert-base/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: -- "zh" -thumbnail: "https://user-images.githubusercontent.com/9592150/97142000-cad08e00-179a-11eb-88df-aff9221482d8.png" -tags: -- "chinese" -- "classical chinese" -- "literary chinese" -- "ancient chinese" -- "bert" -- "pytorch" -license: "apache-2.0" -pipeline_tag: "fill-mask" -widget: -- text: "[MASK]太元中，武陵人捕鱼为业。" -- text: "问征夫以前路，恨晨光之[MASK]微。" -- text: "浔阳江头夜送客，枫叶[MASK]花秋瑟瑟。" ---- - -# GuwenBERT - -## Model description -![GuwenBERT](https://user-images.githubusercontent.com/9592150/97142000-cad08e00-179a-11eb-88df-aff9221482d8.png) - -This is a RoBERTa model pre-trained on Classical Chinese. You can fine-tune GuwenBERT for downstream tasks, such as sentence breaking, punctuation, named entity recognition, and so on. - -For more information about RoBERTa, take a look at the RoBERTa's offical repo. - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("ethanyt/guwenbert-base") - -model = AutoModel.from_pretrained("ethanyt/guwenbert-base") -``` - -## Training data - -The training data is daizhige dataset (殆知阁古代文献) which is contains of 15,694 books in Classical Chinese, covering Buddhism, Confucianism, Medicine, History, Zi, Yi, Yizang, Shizang, Taoism, and Jizang. -76% of them are punctuated. -The total number of characters is 1.7B (1,743,337,673). -All traditional Characters are converted to simplified characters. -The vocabulary is constructed from this data set and the size is 23,292. - -## Training procedure - -The models are initialized with `hfl/chinese-roberta-wwm-ext` and then pre-trained with a 2-step strategy. -In the first step, the model learns MLM with only word embeddings updated during training, until convergence. In the second step, all parameters are updated during training. - -The models are trained on 4 V100 GPUs for 120K steps (20K for step#1, 100K for step#2) with a batch size of 2,048 and a sequence length of 512. The optimizer used is Adam with a learning rate of 2e-4, adam-betas of (0.9,0.98), adam-eps of 1e-6, a weight decay of 0.01, learning rate warmup for 5K steps, and linear decay of learning rate after. - -## Eval results - -### "Gulian Cup" Ancient Books Named Entity Recognition Evaluation - -Second place in the competition. Detailed test results: - -| NE Type | Precision | Recall | F1 | -|:----------:|:-----------:|:------:|:-----:| -| Book Name | 77.50 | 73.73 | 75.57 | -| Other Name | 85.85 | 89.32 | 87.55 | -| Micro Avg. | 83.88 | 85.39 | 84.63 | - - - - -## About Us - -We are from [Datahammer](https://datahammer.net), Beijing Institute of Technology. -For more cooperation, please contact email: ethanyt [at] qq.com - -> Created with ❤️ by Tan Yan [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/Ethan-yt) and Zewen Chi [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/CZWin32768) \ No newline at end of file diff --git a/model_cards/ethanyt/guwenbert-large/README.md b/model_cards/ethanyt/guwenbert-large/README.md deleted file mode 100644 index 60fe94d619b6..000000000000 --- a/model_cards/ethanyt/guwenbert-large/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: -- "zh" -thumbnail: "https://user-images.githubusercontent.com/9592150/97142000-cad08e00-179a-11eb-88df-aff9221482d8.png" -tags: -- "chinese" -- "classical chinese" -- "literary chinese" -- "ancient chinese" -- "bert" -- "pytorch" -license: "apache-2.0" -pipeline_tag: "fill-mask" -widget: -- text: "[MASK]太元中，武陵人捕鱼为业。" -- text: "问征夫以前路，恨晨光之[MASK]微。" -- text: "浔阳江头夜送客，枫叶[MASK]花秋瑟瑟。" ---- - -# GuwenBERT - -## Model description -![GuwenBERT](https://user-images.githubusercontent.com/9592150/97142000-cad08e00-179a-11eb-88df-aff9221482d8.png) - -This is a RoBERTa model pre-trained on Classical Chinese. You can fine-tune GuwenBERT for downstream tasks, such as sentence breaking, punctuation, named entity recognition, and so on. - -For more information about RoBERTa, take a look at the RoBERTa's offical repo. - -## How to use - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("ethanyt/guwenbert-large") - -model = AutoModel.from_pretrained("ethanyt/guwenbert-large") -``` - -## Training data - -The training data is daizhige dataset (殆知阁古代文献) which is contains of 15,694 books in Classical Chinese, covering Buddhism, Confucianism, Medicine, History, Zi, Yi, Yizang, Shizang, Taoism, and Jizang. -76% of them are punctuated. -The total number of characters is 1.7B (1,743,337,673). -All traditional Characters are converted to simplified characters. -The vocabulary is constructed from this data set and the size is 23,292. - -## Training procedure - -The models are initialized with `hfl/chinese-roberta-wwm-ext-large` and then pre-trained with a 2-step strategy. -In the first step, the model learns MLM with only word embeddings updated during training, until convergence. In the second step, all parameters are updated during training. - -The models are trained on 4 V100 GPUs for 120K steps (20K for step#1, 100K for step#2) with a batch size of 2,048 and a sequence length of 512. The optimizer used is Adam with a learning rate of 1e-4, adam-betas of (0.9,0.98), adam-eps of 1e-6, a weight decay of 0.01, learning rate warmup for 5K steps, and linear decay of learning rate after. - -## Eval results - -### "Gulian Cup" Ancient Books Named Entity Recognition Evaluation - -Second place in the competition. Detailed test results: - -| NE Type | Precision | Recall | F1 | -|:----------:|:-----------:|:------:|:-----:| -| Book Name | 77.50 | 73.73 | 75.57 | -| Other Name | 85.85 | 89.32 | 87.55 | -| Micro Avg. | 83.88 | 85.39 | 84.63 | - - - - -## About Us - -We are from [Datahammer](https://datahammer.net), Beijing Institute of Technology. -For more cooperation, please contact email: ethanyt [at] qq.com - -> Created with ❤️ by Tan Yan [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/Ethan-yt) and Zewen Chi [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/CZWin32768) \ No newline at end of file diff --git a/model_cards/facebook/bart-large-cnn/README.md b/model_cards/facebook/bart-large-cnn/README.md deleted file mode 100644 index aef17e07baef..000000000000 --- a/model_cards/facebook/bart-large-cnn/README.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -tags: -- summarization - -license: mit -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- diff --git a/model_cards/facebook/bart-large-mnli/README.md b/model_cards/facebook/bart-large-mnli/README.md deleted file mode 100644 index bb29a53b0eb1..000000000000 --- a/model_cards/facebook/bart-large-mnli/README.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -license: mit -thumbnail: https://huggingface.co/front/thumbnails/facebook.png -pipeline_tag: zero-shot-classification -datasets: -- multi_nli ---- - -# bart-large-mnli - -This is the checkpoint for [bart-large](https://huggingface.co/facebook/bart-large) after being trained on the [MultiNLI (MNLI)](https://huggingface.co/datasets/multi_nli) dataset. - -Additional information about this model: -- The [bart-large](https://huggingface.co/facebook/bart-large) model page -- [BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension -](https://arxiv.org/abs/1910.13461) -- [BART fairseq implementation](https://github.com/pytorch/fairseq/tree/master/fairseq/models/bart) - -## NLI-based Zero Shot Text Classification - -[Yin et al.](https://arxiv.org/abs/1909.00161) proposed a method for using pre-trained NLI models as a ready-made zero-shot sequence classifiers. The method works by posing the sequence to be classified as the NLI premise and to construct a hypothesis from each candidate label. For example, if we want to evaluate whether a sequence belongs to the class "politics", we could construct a hypothesis of `This text is about politics.`. The probabilities for entailment and contradiction are then converted to label probabilities. - -This method is surprisingly effective in many cases, particularly when used with larger pre-trained models like BART and Roberta. See [this blog post](https://joeddav.github.io/blog/2020/05/29/ZSL.html) for a more expansive introduction to this and other zero shot methods, and see the code snippets below for examples of using this model for zero-shot classification both with Hugging Face's built-in pipeline and with native Transformers/PyTorch code. - -#### With the zero-shot classification pipeline - -The model can be loaded with the `zero-shot-classification` pipeline like so: - -```python -from transformers import pipeline -classifier = pipeline("zero-shot-classification", - model="facebook/bart-large-mnli") -``` - -You can then use this pipeline to classify sequences into any of the class names you specify. - -```python -sequence_to_classify = "one day I will see the world" -candidate_labels = ['travel', 'cooking', 'dancing'] -classifier(sequence_to_classify, candidate_labels) -#{'labels': ['travel', 'dancing', 'cooking'], -# 'scores': [0.9938651323318481, 0.0032737774308770895, 0.002861034357920289], -# 'sequence': 'one day I will see the world'} -``` - -If more than one candidate label can be correct, pass `multi_class=True` to calculate each class independently: - -```python -candidate_labels = ['travel', 'cooking', 'dancing', 'exploration'] -classifier(sequence_to_classify, candidate_labels, multi_class=True) -#{'labels': ['travel', 'exploration', 'dancing', 'cooking'], -# 'scores': [0.9945111274719238, -# 0.9383890628814697, -# 0.0057061901316046715, -# 0.0018193122232332826], -# 'sequence': 'one day I will see the world'} -``` - - -#### With manual PyTorch - -```python -# pose sequence as a NLI premise and label as a hypothesis -from transformers import AutoModelForSequenceClassification, AutoTokenizer -nli_model = AutoModelForSequenceClassification.from_pretrained('joeddav/xlm-roberta-large-xnli') -tokenizer = AutoTokenizer.from_pretrained('joeddav/xlm-roberta-large-xnli') - -premise = sequence -hypothesis = f'This example is {label}.' - -# run through model pre-trained on MNLI -x = tokenizer.encode(premise, hypothesis, return_tensors='pt', - truncation_strategy='only_first') -logits = nli_model(x.to(device))[0] - -# we throw away "neutral" (dim 1) and take the probability of -# "entailment" (2) as the probability of the label being true -entail_contradiction_logits = logits[:,[0,2]] -probs = entail_contradiction_logits.softmax(dim=1) -prob_label_is_true = probs[:,1] -``` diff --git a/model_cards/facebook/bart-large/README.md b/model_cards/facebook/bart-large/README.md deleted file mode 100644 index 653141730c41..000000000000 --- a/model_cards/facebook/bart-large/README.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -license: mit -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- - -The Bart model was proposed by Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed, Omer Levy, Ves Stoyanov and Luke Zettlemoyer on 29 Oct, 2019. According to the abstract, - -Bart uses a standard seq2seq/machine translation architecture with a bidirectional encoder (like BERT) and a left-to-right decoder (like GPT). - -The pretraining task involves randomly shuffling the order of the original sentences and a novel in-filling scheme, where spans of text are replaced with a single mask token. - -BART is particularly effective when fine tuned for text generation but also works well for comprehension tasks. It matches the performance of RoBERTa with comparable training resources on GLUE and SQuAD, achieves new state-of-the-art results on a range of abstractive dialogue, question answering, and summarization tasks, with gains of up to 6 ROUGE. - -The Authors’ code can be found here: -https://github.com/pytorch/fairseq/tree/master/examples/bart diff --git a/model_cards/facebook/rag-sequence-base/README.md b/model_cards/facebook/rag-sequence-base/README.md deleted file mode 100644 index d9532b61bc9a..000000000000 --- a/model_cards/facebook/rag-sequence-base/README.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -license: apache-2.0 -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- -## RAG - -This is a non-finetuned version of the RAG-Sequence model of the the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401.pdf) -by Patrick Lewis, Ethan Perez, Aleksandara Piktus et al. - -Rag consits of a *question encoder*, *retriever* and a *generator*. The retriever should be a `RagRetriever` instance. The *question encoder* can be any model that can be loaded with `AutoModel` and the *generator* can be any model that can be loaded with `AutoModelForSeq2SeqLM`. - -This model is a non-finetuned RAG-Sequence model and was created as follows: - -```python -from transformers import RagTokenizer, RagRetriever, RagSequenceForGeneration, AutoTokenizer - -model = RagSequenceForGeneration.from_pretrained_question_encoder_generator("facebook/dpr-question_encoder-single-nq-base", "facebook/bart-large") - -question_encoder_tokenizer = AutoTokenizer.from_pretrained("facebook/dpr-question_encoder-single-nq-base") -generator_tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large") - -tokenizer = RagTokenizer(question_encoder_tokenizer, generator_tokenizer) -model.config.use_dummy_dataset = True -model.config.index_name = "exact" -retriever = RagRetriever(model.config, question_encoder_tokenizer, generator_tokenizer) - -model.save_pretrained("./") -tokenizer.save_pretrained("./") -retriever.save_pretrained("./") -``` - -Note that the model is *uncased* so that all capital input letters are converted to lower-case. - -## Usage: - -*Note*: the model uses the *dummy* retriever as a default. Better results are obtained by using the full retriever, -by setting `config.index_name="legacy"` and `config.use_dummy_dataset=False`. -The model can be fine-tuned as follows: - -```python -from transformers import RagTokenizer, RagRetriever, RagTokenForGeneration - -tokenizer = RagTokenizer.from_pretrained("facebook/rag-sequence-base") -retriever = RagRetriever.from_pretrained("facebook/rag-sequence-base") -model = RagTokenForGeneration.from_pretrained("facebook/rag-sequence-base", retriever=retriever) - -input_dict = tokenizer.prepare_seq2seq_batch("who holds the record in 100m freestyle", "michael phelps", return_tensors="pt") - -outputs = model(input_dict["input_ids"], labels=input_dict["labels"]) - -loss = outputs.loss - -# train on loss -``` diff --git a/model_cards/facebook/rag-sequence-nq/README.md b/model_cards/facebook/rag-sequence-nq/README.md deleted file mode 100644 index 325212a6fbd4..000000000000 --- a/model_cards/facebook/rag-sequence-nq/README.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- wiki_dpr -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- -## RAG - -This is the RAG-Sequence Model of the the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401.pdf) -by Patrick Lewis, Ethan Perez, Aleksandara Piktus et al. - -The model is a *uncased* model, which means that capital letters are simply converted to lower-case letters. - -The model consits of a *question_encoder*, *retriever* and a *generator*. The retriever extracts relevant passages from the *wiki_dpr* `train` datasets, which is linked above. -The question_encoder and retriever are based on `facebook/dpr-question_encoder-single-nq-base` and `facebook/bart-large`, which were jointly finetuned on -on the *wiki_dpr* QA dataset in an end-to-end fashion. - -## Usage: - -**Note**: In the usage example below only the *dummy* retriever of *wiki_dpr* is used because the complete *lecagy* index requires over 75 GB of RAM. -The model can generate answers to any factoid question as follows: - -```python -from transformers import RagTokenizer, RagRetriever, RagSequenceForGeneration - -tokenizer = RagTokenizer.from_pretrained("facebook/rag-sequence-nq") -retriever = RagRetriever.from_pretrained("facebook/rag-sequence-nq", index_name="exact", use_dummy_dataset=True) -model = RagSequenceForGeneration.from_pretrained("facebook/rag-sequence-nq", retriever=retriever) - -input_dict = tokenizer.prepare_seq2seq_batch("how many countries are in europe", return_tensors="pt") - -generated = model.generate(input_ids=input_dict["input_ids"]) -print(tokenizer.batch_decode(generated, skip_special_tokens=True)[0]) - -# should give 54 => google says either 44 or 51 -``` diff --git a/model_cards/facebook/rag-token-base/README.md b/model_cards/facebook/rag-token-base/README.md deleted file mode 100644 index 7e99fb310491..000000000000 --- a/model_cards/facebook/rag-token-base/README.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- wiki_dpr -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- -## RAG - -This is a non-finetuned version of the RAG-Token model of the the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401.pdf) -by Patrick Lewis, Ethan Perez, Aleksandara Piktus et al. - -Rag consits of a *question encoder*, *retriever* and a *generator*. The retriever should be a `RagRetriever` instance. The *question encoder* can be any model that can be loaded with `AutoModel` and the *generator* can be any model that can be loaded with `AutoModelForSeq2SeqLM`. - -This model is a non-finetuned RAG-Token model and was created as follows: - -```python -from transformers import RagTokenizer, RagRetriever, RagTokenForGeneration, AutoTokenizer - -model = RagTokenForGeneration.from_pretrained_question_encoder_generator("facebook/dpr-question_encoder-single-nq-base", "facebook/bart-large") - -question_encoder_tokenizer = AutoTokenizer.from_pretrained("facebook/dpr-question_encoder-single-nq-base") -generator_tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large") - -tokenizer = RagTokenizer(question_encoder_tokenizer, generator_tokenizer) -model.config.use_dummy_dataset = True -model.config.index_name = "exact" -retriever = RagRetriever(model.config, question_encoder_tokenizer, generator_tokenizer) - -model.save_pretrained("./") -tokenizer.save_pretrained("./") -retriever.save_pretrained("./") -``` - -Note that the model is *uncased* so that all capital input letters are converted to lower-case. - -## Usage: - -*Note*: the model uses the *dummy* retriever as a default. Better results are obtained by using the full retriever, -by setting `config.index_name="legacy"` and `config.use_dummy_dataset=False`. -The model can be fine-tuned as follows: - -```python -from transformers import RagTokenizer, RagRetriever, RagTokenForGeneration - -tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-base") -retriever = RagRetriever.from_pretrained("facebook/rag-token-base") -model = RagTokenForGeneration.from_pretrained("facebook/rag-token-base", retriever=retriever) - -input_dict = tokenizer.prepare_seq2seq_batch("who holds the record in 100m freestyle", "michael phelps", return_tensors="pt") - -outputs = model(input_dict["input_ids"], labels=input_dict["labels"]) - -loss = outputs.loss - -# train on loss -``` diff --git a/model_cards/facebook/rag-token-nq/README.md b/model_cards/facebook/rag-token-nq/README.md deleted file mode 100644 index 2c4deb05641a..000000000000 --- a/model_cards/facebook/rag-token-nq/README.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- wiki_dpr -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- -## RAG - -This is the RAG-Token Model of the the paper [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401.pdf) -by Patrick Lewis, Ethan Perez, Aleksandara Piktus et al. - -The model is a *uncased* model, which means that capital letters are simply converted to lower-case letters. - -The model consits of a *question_encoder*, *retriever* and a *generator*. The retriever extracts relevant passages from the *wiki_dpr* `train` datasets, which is linked above. -The question_encoder and retriever are based on `facebook/dpr-question_encoder-single-nq-base` and `facebook/bart-large`, which were jointly finetuned on -on the *wiki_dpr* QA dataset in an end-to-end fashion. - -## Usage: - -**Note**: In the usage example below only the *dummy* retriever of *wiki_dpr* is used because the complete *lecagy* index requires over 75 GB of RAM. -The model can generate answers to any factoid question as follows: - -```python -from transformers import RagTokenizer, RagRetriever, RagTokenForGeneration - -tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-nq") -retriever = RagRetriever.from_pretrained("facebook/rag-token-nq", index_name="exact", use_dummy_dataset=True) -model = RagTokenForGeneration.from_pretrained("facebook/rag-token-nq", retriever=retriever) - -input_dict = tokenizer.prepare_seq2seq_batch("who holds the record in 100m freestyle", return_tensors="pt") - -generated = model.generate(input_ids=input_dict["input_ids"]) -print(tokenizer.batch_decode(generated, skip_special_tokens=True)[0]) - -# should give michael phelps => sounds reasonable -``` diff --git a/model_cards/facebook/rag-token-nq_new/README.md b/model_cards/facebook/rag-token-nq_new/README.md deleted file mode 100644 index 940a8ac614d9..000000000000 --- a/model_cards/facebook/rag-token-nq_new/README.md +++ /dev/null @@ -1,24 +0,0 @@ -The model can be loaded and used as follows on [this branch](https://github.com/huggingface/transformers/tree/finalize_rag) as follows. - - -# Load model - -```python -from transformers import RagTokenizer, RagTokenForGeneration, RagRetriever - -# create Retriever augmented model -retriever = RagRetriever.from_pretrained("facebook/rag-token-nq_new", use_dummy_dataset=True) -model = RagTokenForGeneration.from_pretrained("facebook/rag-token-nq_new", retriever=retriever) - -tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-nq_new") - -# create input ids and labels -input_ids = tokenizer("who sings does he love me with reba", return_tensors="pt").input_ids - -# use labels -labels = tokenizer.generator("Linda Davis", return_tensors="pt").input_ids - - -# compute loss -outputs = model(input_ids, labels=labels) -``` diff --git a/model_cards/facebook/wmt19-de-en/README.md b/model_cards/facebook/wmt19-de-en/README.md deleted file mode 100644 index 924097c75241..000000000000 --- a/model_cards/facebook/wmt19-de-en/README.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -language: -- de -- en -tags: -- translation -- wmt19 -- facebook -license: apache-2.0 -datasets: -- wmt19 -metrics: -- bleu -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for de-en. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "facebook/wmt19-de-en" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Maschinelles Lernen ist großartig, oder?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Machine learning is great, isn't it? - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -de-en | [42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750) | 41.35 - -The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: -- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -- re-ranking - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=de-en -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=15 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` -note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt19/) -- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561) - - -### BibTeX entry and citation info - -```bibtex -@inproceedings{..., - year={2020}, - title={Facebook FAIR's WMT19 News Translation Task Submission}, - author={Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}, - booktitle={Proc. of WMT}, -} -``` - - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - diff --git a/model_cards/facebook/wmt19-en-de/README.md b/model_cards/facebook/wmt19-en-de/README.md deleted file mode 100644 index b4bacbbf8cc1..000000000000 --- a/model_cards/facebook/wmt19-en-de/README.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -language: -- en -- de -tags: -- translation -- wmt19 -- facebook -license: apache-2.0 -datasets: -- wmt19 -metrics: -- bleu -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for en-de. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "facebook/wmt19-en-de" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Machine learning is great, isn't it?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Maschinelles Lernen ist großartig, oder? - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -en-de | [43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862) | 42.83 - -The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: -- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -- re-ranking - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=en-de -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=15 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` -note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt19/) -- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561) - - -### BibTeX entry and citation info - -```bibtex -@inproceedings{..., - year={2020}, - title={Facebook FAIR's WMT19 News Translation Task Submission}, - author={Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}, - booktitle={Proc. of WMT}, -} -``` - - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - diff --git a/model_cards/facebook/wmt19-en-ru/README.md b/model_cards/facebook/wmt19-en-ru/README.md deleted file mode 100644 index aa55e7a3085b..000000000000 --- a/model_cards/facebook/wmt19-en-ru/README.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -language: -- en -- ru -tags: -- translation -- wmt19 -- facebook -license: apache-2.0 -datasets: -- wmt19 -metrics: -- bleu -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for en-ru. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "facebook/wmt19-en-ru" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Machine learning is great, isn't it?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Машинное обучение - это здорово, не так ли? - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -en-ru | [36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724) | 33.47 - -The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: -- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -- re-ranking - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=en-ru -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=15 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` -note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt19/) -- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561) - - -### BibTeX entry and citation info - -```bibtex -@inproceedings{..., - year={2020}, - title={Facebook FAIR's WMT19 News Translation Task Submission}, - author={Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}, - booktitle={Proc. of WMT}, -} -``` - - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - diff --git a/model_cards/facebook/wmt19-ru-en/README.md b/model_cards/facebook/wmt19-ru-en/README.md deleted file mode 100644 index 627aced50f06..000000000000 --- a/model_cards/facebook/wmt19-ru-en/README.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -language: -- ru -- en -tags: -- translation -- wmt19 -- facebook -license: apache-2.0 -datasets: -- wmt19 -metrics: -- bleu -thumbnail: https://huggingface.co/front/thumbnails/facebook.png ---- - -# FSMT - -## Model description - -This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for ru-en. - -For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616). - -The abbreviation FSMT stands for FairSeqMachineTranslation - -All four models are available: - -* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru) -* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en) -* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de) -* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en) - -## Intended uses & limitations - -#### How to use - -```python -from transformers import FSMTForConditionalGeneration, FSMTTokenizer -mname = "facebook/wmt19-ru-en" -tokenizer = FSMTTokenizer.from_pretrained(mname) -model = FSMTForConditionalGeneration.from_pretrained(mname) - -input = "Машинное обучение - это здорово, не так ли?" -input_ids = tokenizer.encode(input, return_tensors="pt") -outputs = model.generate(input_ids) -decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) -print(decoded) # Machine learning is great, isn't it? - -``` - -#### Limitations and bias - -- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981) - -## Training data - -Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616). - -## Eval results - -pair | fairseq | transformers --------|---------|---------- -ru-en | [41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937) | 39.20 - -The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support: -- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``). -- re-ranking - -The score was calculated using this code: - -```bash -git clone https://github.com/huggingface/transformers -cd transformers -export PAIR=ru-en -export DATA_DIR=data/$PAIR -export SAVE_DIR=data/$PAIR -export BS=8 -export NUM_BEAMS=15 -mkdir -p $DATA_DIR -sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source -sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target -echo $PAIR -PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS -``` -note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`. - -## Data Sources - -- [training, etc.](http://www.statmt.org/wmt19/) -- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561) - - -### BibTeX entry and citation info - -```bibtex -@inproceedings{..., - year={2020}, - title={Facebook FAIR's WMT19 News Translation Task Submission}, - author={Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}, - booktitle={Proc. of WMT}, -} -``` - - -## TODO - -- port model ensemble (fairseq uses 4 model checkpoints) - diff --git a/model_cards/flexudy/t5-base-multi-sentence-doctor/README.md b/model_cards/flexudy/t5-base-multi-sentence-doctor/README.md deleted file mode 100644 index d0dcd748021c..000000000000 --- a/model_cards/flexudy/t5-base-multi-sentence-doctor/README.md +++ /dev/null @@ -1,109 +0,0 @@ -![avatar](sent-banner.png) - -# Sentence-Doctor -Sentence doctor is a T5 model that attempts to correct the errors or mistakes found in sentences. Model works on English, German and French text. - -## 1. Problem: -Many NLP models depend on tasks like *Text Extraction Libraries, OCR, Speech to Text libraries* and **Sentence Boundary Detection** -As a consequence errors caused by these tasks in your NLP pipeline can affect the quality of models in applications. Especially since models are often trained on **clean** input. - -## 2. Solution: -Here we provide a model that **attempts** to reconstruct sentences based on the its context (sourrounding text). The task is pretty straightforward: -* `Given an "erroneous" sentence, and its context, reconstruct the "intended" sentence`. - -## 3. Use Cases: -* Attempt to repair noisy sentences that where extracted with OCR software or text extractors. -* Attempt to repair sentence boundaries. - * Example (in German): **Input: "und ich bin im**", - * Prefix_Context: "Hallo! Mein Name ist John", Postfix_Context: "Januar 1990 geboren." - * Output: "John und ich bin im Jahr 1990 geboren" -* Possibly sentence level spelling correction -- Although this is not the intended use. - * Input: "I went to church **las yesteday**" => Output: "I went to church last Sunday". - -## 4. Disclaimer -Note how we always emphises on the word *attempt*. The current version of the model was only trained on **150K** sentences from the tatoeba dataset: https://tatoeba.org/eng. (50K per language -- En, Fr, De). -Hence, we strongly encourage you to finetune the model on your dataset. We might release a version trained on more data. - -## 5. Datasets -We generated synthetic data from the tatoeba dataset: https://tatoeba.org/eng. Randomly applying different transformations on words and characters based on some probabilities. The datasets are available in the data folder (where **sentence_doctor_dataset_300K** is a larger dataset with 100K sentences for each language). - -## 6. Usage - -### 6.1 Preprocessing -* Let us assume we have the following text (Note that there are no punctuation marks in the text): - -```python -text = "That is my job I am a medical doctor I save lives" -``` -* You decided extract the sentences and for some obscure reason, you obtained these sentences: - -```python -sentences = ["That is my job I a", "m a medical doct", "I save lives"] -``` -* You now wish to correct the sentence **"m a medical doct"**. - -Here is the single preprocessing step for the model: - -```python -input_text = "repair_sentence: " + sentences[1] + " context: {" + sentences[0] + "}{" + sentences[2] + "} " -``` - -**Explanation**:
-* We are telling the model to repair the sentence with the prefix "repair_sentence: " -* Then append the sentence we want to repair **sentence[1]** which is "m a medical doct" -* Next we give some context to the model. In the case, the context is some text that occured before the sentence and some text that appeard after the sentence in the original text. - * To do that, we append the keyword "context :" - * Append **{sentence[0]}** "{That is my job I a}". (Note how it is sourrounded by curly braces). - * Append **{sentence[2]}** "{I save lives}". -* At last we tell the model this is the end of the input with . - -```python -print(input_text) # repair_sentence: m a medical doct context: {That is my job I a}{or I save lives} -``` - -
- -**The context is optional**, so the input could also be ```repair_sentence: m a medical doct context: {}{} ``` - -### 6.2 Inference - -```python - -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("flexudy/t5-base-multi-sentence-doctor") - -model = AutoModelWithLMHead.from_pretrained("flexudy/t5-base-multi-sentence-doctor") - -input_text = "repair_sentence: m a medical doct context: {That is my job I a}{or I save lives} " - -input_ids = tokenizer.encode(input_text, return_tensors="pt") - -outputs = model.generate(input_ids, max_length=32, num_beams=1) - -sentence = tokenizer.decode(outputs[0], skip_special_tokens=True, clean_up_tokenization_spaces=True) - -assert sentence == "I am a medical doctor." -``` - -## 7. Fine-tuning -We also provide a script `train_any_t5_task.py` that might help you fine-tune any Text2Text Task with T5. We added #TODO comments all over to help you use train with ease. For example: - -```python -# TODO Set your training epochs -config.TRAIN_EPOCHS = 3 -``` -If you don't want to read the #TODO comments, just pass in your data like this - -```python -# TODO Where is your data ? Enter the path -trainer.start("data/sentence_doctor_dataset_300.csv") -``` -and voila!! Please feel free to correct any mistakes in the code and make a pull request. - -## 8. Attribution -* [Huggingface](https://huggingface.co/) transformer lib for making this possible -* Abhishek Kumar Mishra's transformer [tutorial](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb) on text summarisation. Our training code is just a modified version of their code. So many thanks. -* We finetuned this model from the huggingface hub: WikinewsSum/t5-base-multi-combine-wiki-news. Thanks to the [authors](https://huggingface.co/WikinewsSum) -* We also read a lot of work from [Suraj Patil](https://github.com/patil-suraj) -* No one has been forgotten, hopefully :) diff --git a/model_cards/flexudy/t5-base-multi-sentence-doctor/sent-banner.png b/model_cards/flexudy/t5-base-multi-sentence-doctor/sent-banner.png deleted file mode 100644 index cf6566f4d201..000000000000 Binary files a/model_cards/flexudy/t5-base-multi-sentence-doctor/sent-banner.png and /dev/null differ diff --git a/model_cards/fmikaelian/camembert-base-fquad/README.md b/model_cards/fmikaelian/camembert-base-fquad/README.md deleted file mode 100644 index 8c6e90f37d28..000000000000 --- a/model_cards/fmikaelian/camembert-base-fquad/README.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -language: fr ---- - -# camembert-base-fquad - -## Description - -A baseline model for question-answering in french ([CamemBERT](https://camembert-model.fr/) model fine-tuned on [FQuAD](https://fquad.illuin.tech/)) - -## Training hyperparameters - -```shell -python3 ./examples/question-answering/run_squad.py \ ---model_type camembert \ ---model_name_or_path camembert-base \ ---do_train \ ---do_eval \ ---do_lower_case \ ---train_file train.json \ ---predict_file valid.json \ ---learning_rate 3e-5 \ ---num_train_epochs 2 \ ---max_seq_length 384 \ ---doc_stride 128 \ ---output_dir output \ ---per_gpu_eval_batch_size=3 \ ---per_gpu_train_batch_size=3 \ ---save_steps 10000 -``` - -## Evaluation results - -```shell -{"f1": 77.24515316052342, "exact_match": 52.82308657465496} -``` - -## Usage - -```python -from transformers import pipeline - -nlp = pipeline('question-answering', model='fmikaelian/camembert-base-fquad', tokenizer='fmikaelian/camembert-base-fquad') - -nlp({ - 'question': "Qui est Claude Monet?", - 'context': "Claude Monet, né le 14 novembre 1840 à Paris et mort le 5 décembre 1926 à Giverny, est un peintre français et l’un des fondateurs de l'impressionnisme." -}) -``` \ No newline at end of file diff --git a/model_cards/fmikaelian/camembert-base-squad/README.md b/model_cards/fmikaelian/camembert-base-squad/README.md deleted file mode 100644 index ead8b6aff9a3..000000000000 --- a/model_cards/fmikaelian/camembert-base-squad/README.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -language: fr ---- - -# camembert-base-squad - -## Description - -A baseline model for question-answering in french ([CamemBERT](https://camembert-model.fr/) model fine-tuned on [french-translated SQuAD 1.1 dataset](https://github.com/Alikabbadj/French-SQuAD)) - -## Training hyperparameters - -```shell -python3 ./examples/question-answering/run_squad.py \ ---model_type camembert \ ---model_name_or_path camembert-base \ ---do_train \ ---do_eval \ ---do_lower_case \ ---train_file SQuAD-v1.1-train_fr_ss999_awstart2_net.json \ ---predict_file SQuAD-v1.1-dev_fr_ss999_awstart2_net.json \ ---learning_rate 3e-5 \ ---num_train_epochs 2 \ ---max_seq_length 384 \ ---doc_stride 128 \ ---output_dir output3 \ ---per_gpu_eval_batch_size=3 \ ---per_gpu_train_batch_size=3 \ ---save_steps 10000 -``` - -## Evaluation results - -```shell -{"f1": 79.8570684959745, "exact_match": 59.21327108373895} -``` - -## Usage - -```python -from transformers import pipeline - -nlp = pipeline('question-answering', model='fmikaelian/camembert-base-squad', tokenizer='fmikaelian/camembert-base-squad') - -nlp({ - 'question': "Qui est Claude Monet?", - 'context': "Claude Monet, né le 14 novembre 1840 à Paris et mort le 5 décembre 1926 à Giverny, est un peintre français et l’un des fondateurs de l'impressionnisme." -}) -``` \ No newline at end of file diff --git a/model_cards/fmikaelian/flaubert-base-uncased-squad/README.md b/model_cards/fmikaelian/flaubert-base-uncased-squad/README.md deleted file mode 100644 index 147d3187dc92..000000000000 --- a/model_cards/fmikaelian/flaubert-base-uncased-squad/README.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -language: fr ---- - -# flaubert-base-uncased-squad - -## Description - -A baseline model for question-answering in french ([flaubert](https://github.com/getalp/Flaubert) model fine-tuned on [french-translated SQuAD 1.1 dataset](https://github.com/Alikabbadj/French-SQuAD)) - -## Training hyperparameters - -```shell -python3 ./examples/question-answering/run_squad.py \ ---model_type flaubert \ ---model_name_or_path flaubert-base-uncased \ ---do_train \ ---do_eval \ ---do_lower_case \ ---train_file SQuAD-v1.1-train_fr_ss999_awstart2_net.json \ ---predict_file SQuAD-v1.1-dev_fr_ss999_awstart2_net.json \ ---learning_rate 3e-5 \ ---num_train_epochs 2 \ ---max_seq_length 384 \ ---doc_stride 128 \ ---output_dir output \ ---per_gpu_eval_batch_size=3 \ ---per_gpu_train_batch_size=3 -``` - -## Evaluation results - -```shell -{"f1": 68.66174806561969, "exact_match": 49.299692063176714} -``` - -## Usage - -```python -from transformers import pipeline - -nlp = pipeline('question-answering', model='fmikaelian/flaubert-base-uncased-squad', tokenizer='fmikaelian/flaubert-base-uncased-squad') - -nlp({ - 'question': "Qui est Claude Monet?", - 'context': "Claude Monet, né le 14 novembre 1840 à Paris et mort le 5 décembre 1926 à Giverny, est un peintre français et l’un des fondateurs de l'impressionnisme." -}) -``` \ No newline at end of file diff --git a/model_cards/fran-martinez/scibert_scivocab_cased_ner_jnlpba/README.md b/model_cards/fran-martinez/scibert_scivocab_cased_ner_jnlpba/README.md deleted file mode 100644 index 8008fded17ef..000000000000 --- a/model_cards/fran-martinez/scibert_scivocab_cased_ner_jnlpba/README.md +++ /dev/null @@ -1,161 +0,0 @@ ---- -language: scientific english ---- - -# SciBERT finetuned on JNLPA for NER downstream task -## Language Model - [SciBERT](https://arxiv.org/pdf/1903.10676.pdf) is a pretrained language model based on BERT and trained by the - [Allen Institute for AI](https://allenai.org/) on papers from the corpus of - [Semantic Scholar](https://www.semanticscholar.org/). - Corpus size is 1.14M papers, 3.1B tokens. SciBERT has its own vocabulary (scivocab) that's built to best match - the training corpus. - -## Downstream task -[`allenai/scibert_scivocab_cased`](https://huggingface.co/allenai/scibert_scivocab_cased#) has been finetuned for Named Entity -Recognition (NER) dowstream task. The code to train the NER can be found [here](https://github.com/fran-martinez/bio_ner_bert). - -### Data -The corpus used to fine-tune the NER is [BioNLP / JNLPBA shared task](http://www.geniaproject.org/shared-tasks/bionlp-jnlpba-shared-task-2004). - -- Training data consist of 2,000 PubMed abstracts with term/word annotation. This corresponds to 18,546 samples (senteces). -- Evaluation data consist of 404 PubMed abstracts with term/word annotation. This corresponds to 3,856 samples (sentences). - -The classes (at word level) and its distribution (number of examples for each class) for training and evaluation datasets are shown below: - -| Class Label | # training examples| # evaluation examples| -|:--------------|--------------:|----------------:| -|O | 382,963 | 81,647 | -|B-protein | 30,269 | 5,067 | -|I-protein | 24,848 | 4,774 | -|B-cell_type | 6,718 | 1,921 | -|I-cell_type | 8,748 | 2,991 | -|B-DNA | 9,533 | 1,056 | -|I-DNA | 15,774 | 1,789 | -|B-cell_line | 3,830 | 500 | -|I-cell_line | 7,387 | 9,89 | -|B-RNA | 951 | 118 | -|I-RNA | 1,530 | 187 | - -### Model -An exhaustive hyperparameter search was done. -The hyperparameters that provided the best results are: - -- Max length sequence: 128 -- Number of epochs: 6 -- Batch size: 32 -- Dropout: 0.3 -- Optimizer: Adam - -The used learning rate was 5e-5 with a decreasing linear schedule. A warmup was used at the beggining of the training -with a ratio of steps equal to 0.1 from the total training steps. - -The model from the epoch with the best F1-score was selected, in this case, the model from epoch 5. - - -### Evaluation -The following table shows the evaluation metrics calculated at span/entity level: - -| | precision| recall| f1-score| -|:---------|-----------:|---------:|---------:| -cell_line | 0.5205 | 0.7100 | 0.6007 | -cell_type | 0.7736 | 0.7422 | 0.7576 | -protein | 0.6953 | 0.8459 | 0.7633 | -DNA | 0.6997 | 0.7894 | 0.7419 | -RNA | 0.6985 | 0.8051 | 0.7480 | -| | | | -**micro avg** | 0.6984 | 0.8076 | 0.7490| -**macro avg** | 0.7032 | 0.8076 | 0.7498 | - -The macro F1-score is equal to 0.7498, compared to the value provided by the Allen Institute for AI in their -[paper](https://arxiv.org/pdf/1903.10676.pdf), which is equal to 0.7728. This drop in performance could be due to -several reasons, but one hypothesis could be the fact that the authors used an additional conditional random field, -while this model uses a regular classification layer with softmax activation on top of SciBERT model. - -At word level, this model achieves a precision of 0.7742, a recall of 0.8536 and a F1-score of 0.8093. - -### Model usage in inference -Use the pipeline: -````python -from transformers import pipeline - -text = "Mouse thymus was used as a source of glucocorticoid receptor from normal CS lymphocytes." - -nlp_ner = pipeline("ner", - model='fran-martinez/scibert_scivocab_cased_ner_jnlpba', - tokenizer='fran-martinez/scibert_scivocab_cased_ner_jnlpba') - -nlp_ner(text) - -""" -Output: ---------------------------- -[ -{'word': 'glucocorticoid', -'score': 0.9894881248474121, -'entity': 'B-protein'}, - -{'word': 'receptor', -'score': 0.989505410194397, -'entity': 'I-protein'}, - -{'word': 'normal', -'score': 0.7680378556251526, -'entity': 'B-cell_type'}, - -{'word': 'cs', -'score': 0.5176806449890137, -'entity': 'I-cell_type'}, - -{'word': 'lymphocytes', -'score': 0.9898491501808167, -'entity': 'I-cell_type'} -] -""" -```` -Or load model and tokenizer as follows: -````python -import torch -from transformers import AutoTokenizer, AutoModelForTokenClassification - -# Example -text = "Mouse thymus was used as a source of glucocorticoid receptor from normal CS lymphocytes." - -# Load model -tokenizer = AutoTokenizer.from_pretrained("fran-martinez/scibert_scivocab_cased_ner_jnlpba") -model = AutoModelForTokenClassification.from_pretrained("fran-martinez/scibert_scivocab_cased_ner_jnlpba") - -# Get input for BERT -input_ids = torch.tensor(tokenizer.encode(text)).unsqueeze(0) - -# Predict -with torch.no_grad(): - outputs = model(input_ids) - -# From the output let's take the first element of the tuple. -# Then, let's get rid of [CLS] and [SEP] tokens (first and last) -predictions = outputs[0].argmax(axis=-1)[0][1:-1] - -# Map label class indexes to string labels. -for token, pred in zip(tokenizer.tokenize(text), predictions): - print(token, '->', model.config.id2label[pred.numpy().item()]) - -""" -Output: ---------------------------- -mouse -> O -thymus -> O -was -> O -used -> O -as -> O -a -> O -source -> O -of -> O -glucocorticoid -> B-protein -receptor -> I-protein -from -> O -normal -> B-cell_type -cs -> I-cell_type -lymphocytes -> I-cell_type -. -> O -""" -```` diff --git a/model_cards/funnel-transformer/intermediate-base/README.md b/model_cards/funnel-transformer/intermediate-base/README.md deleted file mode 100644 index 68d3c4a9e1d7..000000000000 --- a/model_cards/funnel-transformer/intermediate-base/README.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer intermediate model (B6-6-6 without decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -**Note:** This model does not contain the decoder, so it ouputs hidden states that have a sequence length of one fourth -of the inputs. It's good to use for tasks requiring a summary of the sentence (like sentence classification) but not if -you need one input per initial token. You should use the `intermediate` model in that case. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/intermediate-base") -model = FunnelBaseModel.from_pretrained("funnel-transformer/intermediate-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/intermediate-base") -model = TFFunnelBaseModel.from_pretrained("funnel-transformer/intermediate-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/intermediate/README.md b/model_cards/funnel-transformer/intermediate/README.md deleted file mode 100644 index 5645505a7cd8..000000000000 --- a/model_cards/funnel-transformer/intermediate/README.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer intermediate model (B6-6-6 with decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/intermediate") -model = FunneModel.from_pretrained("funnel-transformer/intermediate") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/intermediate") -model = TFFunnelModel.from_pretrained("funnel-transformer/intermediatesmall") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/large-base/README.md b/model_cards/funnel-transformer/large-base/README.md deleted file mode 100644 index e8dd2f3e53b4..000000000000 --- a/model_cards/funnel-transformer/large-base/README.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer large model (B8-8-8 without decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -**Note:** This model does not contain the decoder, so it ouputs hidden states that have a sequence length of one fourth -of the inputs. It's good to use for tasks requiring a summary of the sentence (like sentence classification) but not if -you need one input per initial token. You should use the `large` model in that case. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/large-base") -model = FunnelBaseModel.from_pretrained("funnel-transformer/large-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/large-base") -model = TFFunnelBaseModel.from_pretrained("funnel-transformer/large-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/large/README.md b/model_cards/funnel-transformer/large/README.md deleted file mode 100644 index 9c8128e96d01..000000000000 --- a/model_cards/funnel-transformer/large/README.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer large model (B8-8-8 with decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/large") -model = FunneModel.from_pretrained("funnel-transformer/large") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/large") -model = TFFunnelModel.from_pretrained("funnel-transformer/large") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/medium-base/README.md b/model_cards/funnel-transformer/medium-base/README.md deleted file mode 100644 index 414c94eed967..000000000000 --- a/model_cards/funnel-transformer/medium-base/README.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer medium model (B6-3x2-3x2 without decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -**Note:** This model does not contain the decoder, so it ouputs hidden states that have a sequence length of one fourth -of the inputs. It's good to use for tasks requiring a summary of the sentence (like sentence classification) but not if -you need one input per initial token. You should use the `medium` model in that case. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/medium-base") -model = FunnelBaseModel.from_pretrained("funnel-transformer/medium-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/medium-base") -model = TFFunnelBaseModel.from_pretrained("funnel-transformer/medium-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/medium/README.md b/model_cards/funnel-transformer/medium/README.md deleted file mode 100644 index d5db54b89dbd..000000000000 --- a/model_cards/funnel-transformer/medium/README.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer medium model (B6-3x2-3x2 with decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/medium") -model = FunneModel.from_pretrained("funnel-transformer/medium") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/medium") -model = TFFunnelModel.from_pretrained("funnel-transformer/medium") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/small-base/README.md b/model_cards/funnel-transformer/small-base/README.md deleted file mode 100644 index 30ba16c71745..000000000000 --- a/model_cards/funnel-transformer/small-base/README.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer small model (B4-4-4 without decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -**Note:** This model does not contain the decoder, so it ouputs hidden states that have a sequence length of one fourth -of the inputs. It's good to use for tasks requiring a summary of the sentence (like sentence classification) but not if -you need one input per initial token. You should use the `small` model in that case. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/small-base") -model = FunnelBaseModel.from_pretrained("funnel-transformer/small-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/small-base") -model = TFFunnelBaseModel.from_pretrained("funnel-transformer/small-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/small/README.md b/model_cards/funnel-transformer/small/README.md deleted file mode 100644 index 3a53ca05104f..000000000000 --- a/model_cards/funnel-transformer/small/README.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer small model (B4-4-4 with decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/small") -model = FunneModel.from_pretrained("funnel-transformer/small") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/small") -model = TFFunnelModel.from_pretrained("funnel-transformer/small") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/xlarge-base/README.md b/model_cards/funnel-transformer/xlarge-base/README.md deleted file mode 100644 index 7461fb511211..000000000000 --- a/model_cards/funnel-transformer/xlarge-base/README.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer xlarge model (B10-10-10 without decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -**Note:** This model does not contain the decoder, so it ouputs hidden states that have a sequence length of one fourth -of the inputs. It's good to use for tasks requiring a summary of the sentence (like sentence classification) but not if -you need one input per initial token. You should use the `xlarge` model in that case. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/xlarge-base") -model = FunnelBaseModel.from_pretrained("funnel-transformer/xlarge-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelBaseModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/xlarge-base") -model = TFFunnelBaseModel.from_pretrained("funnel-transformer/xlarge-base") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/funnel-transformer/xlarge/README.md b/model_cards/funnel-transformer/xlarge/README.md deleted file mode 100644 index a1a0a69f249a..000000000000 --- a/model_cards/funnel-transformer/xlarge/README.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- bookcorpus -- wikipedia -- gigaword ---- - -# Funnel Transformer xlarge model (B10-10-10 with decoder) - -Pretrained model on English language using a similar objective objective as [ELECTRA](https://huggingface.co/transformers/model_doc/electra.html). It was introduced in -[this paper](https://arxiv.org/pdf/2006.03236.pdf) and first released in -[this repository](https://github.com/laiguokun/Funnel-Transformer). This model is uncased: it does not make a difference -between english and English. - -Disclaimer: The team releasing Funnel Transformer did not write a model card for this model so this model card has been -written by the Hugging Face team. - -## Model description - -Funnel Transformer is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means it -was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, a small language model corrupts the input texts and serves as a generator of inputs for this model, and -the pretraining objective is to predict which token is an original and which one has been replaced, a bit like a GAN training. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model to extract a vector representation of a given text, but it's mostly intended to -be fine-tuned on a downstream task. See the [model hub](https://huggingface.co/models?filter=funnel-transformer) to look for -fine-tuned versions on a task that interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import FunnelTokenizer, FunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/xlarge") -model = FunneModel.from_pretrained("funnel-transformer/xlarge") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import FunnelTokenizer, TFFunnelModel -tokenizer = FunnelTokenizer.from_pretrained("funnel-transformer/xlarge") -model = TFFunnelModel.from_pretrained("funnel-transformer/xlarge") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -## Training data - -The BERT model was pretrained on: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books, -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers), -- [Clue Web](https://lemurproject.org/clueweb12/), a dataset of 733,019,372 English web pages, -- [GigaWord](https://catalog.ldc.upenn.edu/LDC2011T07), an archive of newswire text data, -- [Common Crawl](https://commoncrawl.org/), a dataset of raw web pages. - - -### BibTeX entry and citation info - -```bibtex -@misc{dai2020funneltransformer, - title={Funnel-Transformer: Filtering out Sequential Redundancy for Efficient Language Processing}, - author={Zihang Dai and Guokun Lai and Yiming Yang and Quoc V. Le}, - year={2020}, - eprint={2006.03236}, - archivePrefix={arXiv}, - primaryClass={cs.LG} -} -``` - diff --git a/model_cards/ganeshkharad/gk-hinglish-sentiment/README.md b/model_cards/ganeshkharad/gk-hinglish-sentiment/README.md deleted file mode 100644 index 28d442fa7349..000000000000 --- a/model_cards/ganeshkharad/gk-hinglish-sentiment/README.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -language: -- hi-en - -tags: -- sentiment -- multilingual -- hindi codemix -- hinglish -license: apache-2.0 -datasets: -- sail ---- - -# Sentiment Classification for hinglish text: `gk-hinglish-sentiment` - -## Model description - -Trained small amount of reviews dataset - -## Intended uses & limitations - -I wanted something to work well with hinglish data as it is being used in India mostly. -The training data was not much as expected - -#### How to use - -```python -#sample code -from transformers import BertTokenizer, BertForSequenceClassification -tokenizerg = BertTokenizer.from_pretrained("/content/model") -modelg = BertForSequenceClassification.from_pretrained("/content/model") - -text = "kuch bhi type karo hinglish mai" -encoded_input = tokenizerg(text, return_tensors='pt') -output = modelg(**encoded_input) -print(output) -#output contains 3 lables LABEL_0 = Negative ,LABEL_1 = Nuetral ,LABEL_2 = Positive -``` - -#### Limitations and bias - -The data contains only hinglish codemixed text it and was very much limited may be I will Update this model if I can get good amount of data - -## Training data - -Training data contains labeled data for 3 labels - -link to the pre-trained model card with description of the pre-training data. -I have Tuned below model - -https://huggingface.co/rohanrajpal/bert-base-multilingual-codemixed-cased-sentiment - - -### BibTeX entry and citation info - -```@inproceedings{khanuja-etal-2020-gluecos, - title = "{GLUEC}o{S}: An Evaluation Benchmark for Code-Switched {NLP}", - author = "Khanuja, Simran and - Dandapat, Sandipan and - Srinivasan, Anirudh and - Sitaram, Sunayana and - Choudhury, Monojit", - booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics", - month = jul, - year = "2020", - address = "Online", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/2020.acl-main.329", - pages = "3575--3585" -} -``` diff --git a/model_cards/gaochangkuan/model_dir/README.md b/model_cards/gaochangkuan/model_dir/README.md deleted file mode 100644 index 41d3e81ebf34..000000000000 --- a/model_cards/gaochangkuan/model_dir/README.md +++ /dev/null @@ -1,66 +0,0 @@ -## Generating Chinese poetry by topic. - -```python -from transformers import * - -tokenizer = BertTokenizer.from_pretrained("gaochangkuan/model_dir") - -model = AutoModelWithLMHead.from_pretrained("gaochangkuan/model_dir") - - -prompt= '''田园躬耕''' - -length= 84 -stop_token='' - -temperature = 1.2 - -repetition_penalty=1.3 - -k= 30 -p= 0.95 - -device ='cuda' -seed=2020 -no_cuda=False - -prompt_text = prompt if prompt else input("Model prompt >>> ") - -encoded_prompt = tokenizer.encode( - ''+prompt_text+'', - add_special_tokens=False, - return_tensors="pt" - ) - -encoded_prompt = encoded_prompt.to(device) - -output_sequences = model.generate( - input_ids=encoded_prompt, - max_length=length, - min_length=10, - do_sample=True, - early_stopping=True, - num_beams=10, - temperature=temperature, - top_k=k, - top_p=p, - repetition_penalty=repetition_penalty, - bad_words_ids=None, - bos_token_id=tokenizer.bos_token_id, - pad_token_id=tokenizer.pad_token_id, - eos_token_id=tokenizer.eos_token_id, - length_penalty=1.2, - no_repeat_ngram_size=2, - num_return_sequences=1, - attention_mask=None, - decoder_start_token_id=tokenizer.bos_token_id,) - - - generated_sequence = output_sequences[0].tolist() -text = tokenizer.decode(generated_sequence) - - -text = text[: text.find(stop_token) if stop_token else None] - -print(''.join(text).replace(' ','').replace('','').replace('','')) -``` diff --git a/model_cards/german-nlp-group/electra-base-german-uncased/README.md b/model_cards/german-nlp-group/electra-base-german-uncased/README.md deleted file mode 100644 index e4ba72a0c4b9..000000000000 --- a/model_cards/german-nlp-group/electra-base-german-uncased/README.md +++ /dev/null @@ -1,152 +0,0 @@ ---- -language: de -license: mit -thumbnail: "https://raw.githubusercontent.com/German-NLP-Group/german-transformer-training/master/model_cards/german-electra-logo.png" -tags: -- electra -- commoncrawl -- uncased -- umlaute -- umlauts -- german -- deutsch ---- - -# German Electra Uncased - -[¹] - -# Model Info -This Model is suitable for Training on many downstream tasks in German (Q&A, Sentiment Analysis, etc.). - -It can be used as a drop-in Replacement for **BERT** in most down-stream tasks (**ELECTRA** is even implemented as an extended **BERT** Class). - -At the time of release (August 2020) this Model is the best performing publicly available German NLP Model on various German Evaluation Metrics (CONLL03-DE, GermEval18 Coarse, GermEval18 Fine). For GermEval18 Coarse results see below. More will be published soon. - -# Installation -This model has the special feature that it is **uncased** but does **not strip accents**. -This possibility was added by us with [PR #6280](https://github.com/huggingface/transformers/pull/6280). -To use it you have to use Transformers version 3.1.0 or newer. - -```bash -pip install transformers -U -``` - -# Uncase and Umlauts ('Ö', 'Ä', 'Ü') -This model is uncased. This helps especially for domains where colloquial terms with uncorrect capitalization is often used. - -The special characters 'ö', 'ü', 'ä' are included through the `strip_accent=False` option, as this leads to an improved precision. - -# Creators -This model was trained and open sourced in conjunction with the [**German NLP Group**](https://github.com/German-NLP-Group) in equal parts by: -- [**Philip May**](https://eniak.de) - [T-Systems on site services GmbH](https://www.t-systems-onsite.de/) -- [**Philipp Reißel**](https://www.reissel.eu) - [ambeRoad](https://amberoad.de/) - -# Evaluation: GermEval18 Coarse - -| Model Name | F1 macro
Mean | F1 macro
Median | F1 macro
Std | -|---|---|---|---| -| dbmdz-bert-base-german-europeana-cased | 0.727 | 0.729 | 0.00674 | -| dbmdz-bert-base-german-europeana-uncased | 0.736 | 0.737 | 0.00476 | -| dbmdz/electra-base-german-europeana-cased-discriminator | 0.745 | 0.745 | 0.00498 | -| distilbert-base-german-cased | 0.752 | 0.752 | 0.00341 | -| bert-base-german-cased | 0.762 | 0.761 | 0.00597 | -| dbmdz/bert-base-german-cased | 0.765 | 0.765 | 0.00523 | -| dbmdz/bert-base-german-uncased | 0.770 | 0.770 | 0.00572 | -| **ELECTRA-base-german-uncased (this model)** | **0.778** | **0.778** | **0.00392** | - -- (1): Hyperparameters taken from the [FARM project](https://farm.deepset.ai/) "[germEval18Coarse_config.json](https://github.com/deepset-ai/FARM/blob/master/experiments/german-bert2.0-eval/germEval18Coarse_config.json)" - -![GermEval18 Coarse Model Evaluation](https://raw.githubusercontent.com/German-NLP-Group/german-transformer-training/master/model_cards/model_eval.png) - -# Checkpoint evaluation -Since it it not guaranteed that the last checkpoint is the best, we evaluated the checkpoints on GermEval18. We found that the last checkpoint is indeed the best. The training was stable and did not overfit the text corpus. Below is a boxplot chart showing the different checkpoints. - -![Checkpoint Evaluation on GermEval18](https://raw.githubusercontent.com/German-NLP-Group/german-transformer-training/master/model_cards/checkpoint_eval.png) - -# Pre-training details - -## Data -- Cleaned Common Crawl Corpus 2019-09 German: [CC_net](https://github.com/facebookresearch/cc_net) (Only head coprus and filtered for language_score > 0.98) - 62 GB -- German Wikipedia Article Pages Dump (20200701) - 5.5 GB -- German Wikipedia Talk Pages Dump (20200620) - 1.1 GB -- Subtitles - 823 MB -- News 2018 - 4.1 GB - -The sentences were split with [SojaMo](https://github.com/tsproisl/SoMaJo). We took the German Wikipedia Article Pages Dump 3x to oversample. This approach was also used in a similar way in GPT-3 (Table 2.2). - -More Details can be found here [Preperaing Datasets for German Electra Github](https://github.com/German-NLP-Group/german-transformer-training) - -## Electra Branch no_strip_accents -Because we do not want to stip accents in our training data we made a change to Electra and used this repo [Electra no_strip_accents](https://github.com/PhilipMay/electra/tree/no_strip_accents) (branch `no_strip_accents`). Then created the tf dataset with: - -```bash -python build_pretraining_dataset.py --corpus-dir --vocab-file

/vocab.txt --output-dir ./tf_data --max-seq-length 512 --num-processes 8 --do-lower-case --no-strip-accents -``` - -## The training -The training itself can be performed with the Original Electra Repo (No special case for this needed). -We run it with the following Config: - -
-The exact Training Config -
debug False -
disallow_correct False -
disc_weight 50.0 -
do_eval False -
do_lower_case True -
do_train True -
electra_objective True -
embedding_size 768 -
eval_batch_size 128 -
gcp_project None -
gen_weight 1.0 -
generator_hidden_size 0.33333 -
generator_layers 1.0 -
iterations_per_loop 200 -
keep_checkpoint_max 0 -
learning_rate 0.0002 -
lr_decay_power 1.0 -
mask_prob 0.15 -
max_predictions_per_seq 79 -
max_seq_length 512 -
model_dir gs://XXX -
model_hparam_overrides {} -
model_name 02_Electra_Checkpoints_32k_766k_Combined -
model_size base -
num_eval_steps 100 -
num_tpu_cores 8 -
num_train_steps 766000 -
num_warmup_steps 10000 -
pretrain_tfrecords gs://XXX -
results_pkl gs://XXX -
results_txt gs://XXX -
save_checkpoints_steps 5000 -
temperature 1.0 -
tpu_job_name None -
tpu_name electrav5 -
tpu_zone None -
train_batch_size 256 -
uniform_generator False -
untied_generator True -
untied_generator_embeddings False -
use_tpu True -
vocab_file gs://XXX -
vocab_size 32767 -
weight_decay_rate 0.01 -
- -![Training Loss](https://raw.githubusercontent.com/German-NLP-Group/german-transformer-training/master/model_cards/loss.png) - -Please Note: *Due to the GAN like strucutre of Electra the loss is not that meaningful* - -It took about 7 Days on a preemtible TPU V3-8. In total, the Model went through approximately 10 Epochs. For an automatically recreation of a cancelled TPUs we used [tpunicorn](https://github.com/shawwn/tpunicorn). The total cost of training summed up to about 450 $ for one run. The Data-pre processing and Vocab Creation needed approximately 500-1000 CPU hours. Servers were fully provided by [T-Systems on site services GmbH](https://www.t-systems-onsite.de/), [ambeRoad](https://amberoad.de/). -Special thanks to [Stefan Schweter](https://github.com/stefan-it) for your feedback and providing parts of the text corpus. - -[¹]: Source for the picture [Pinterest](https://www.pinterest.cl/pin/371828512984142193/) - -# Negative Results -We tried the following approaches which we found had no positive influence: - -- **Increased Vocab Size**: Leads to more parameters and thus reduced examples/sec while no visible Performance gains were measured -- **Decreased Batch-Size**: The original Electra was trained with a Batch Size per TPU Core of 16 whereas this Model was trained with 32 BS / TPU Core. We found out that 32 BS leads to better results when you compare metrics over computation time diff --git a/model_cards/giganticode/StackOBERTflow-comments-small-v1/README.md b/model_cards/giganticode/StackOBERTflow-comments-small-v1/README.md deleted file mode 100644 index 739d3e355714..000000000000 --- a/model_cards/giganticode/StackOBERTflow-comments-small-v1/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# StackOBERTflow-comments-small - -StackOBERTflow is a RoBERTa model trained on StackOverflow comments. -A Byte-level BPE tokenizer with dropout was used (using the `tokenizers` package). - -The model is *small*, i.e. has only 6-layers and the maximum sequence length was restricted to 256 tokens. -The model was trained for 6 epochs on several GBs of comments from the StackOverflow corpus. - -## Quick start: masked language modeling prediction - -```python -from transformers import pipeline -from pprint import pprint - -COMMENT = "You really should not do it this way, I would use instead." - -fill_mask = pipeline( - "fill-mask", - model="giganticode/StackOBERTflow-comments-small-v1", - tokenizer="giganticode/StackOBERTflow-comments-small-v1" -) - -pprint(fill_mask(COMMENT)) -# [{'score': 0.019997311756014824, -# 'sequence': ' You really should not do it this way, I would use jQuery instead.', -# 'token': 1738}, -# {'score': 0.01693696901202202, -# 'sequence': ' You really should not do it this way, I would use arrays instead.', -# 'token': 2844}, -# {'score': 0.013411642983555794, -# 'sequence': ' You really should not do it this way, I would use CSS instead.', -# 'token': 2254}, -# {'score': 0.013224546797573566, -# 'sequence': ' You really should not do it this way, I would use it instead.', -# 'token': 300}, -# {'score': 0.011984303593635559, -# 'sequence': ' You really should not do it this way, I would use classes instead.', -# 'token': 1779}] -``` diff --git a/model_cards/gilf/french-camembert-postag-model/README.md b/model_cards/gilf/french-camembert-postag-model/README.md deleted file mode 100644 index f884dc5f5529..000000000000 --- a/model_cards/gilf/french-camembert-postag-model/README.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -language: fr -widget: -- text: "Face à un choc inédit, les mesures mises en place par le gouvernement ont permis une protection forte et efficace des ménages" ---- - -## About - -The *french-camembert-postag-model* is a part of speech tagging model for French that was trained on the *free-french-treebank* dataset available on -[github](https://github.com/nicolashernandez/free-french-treebank). The base tokenizer and model used for training is *'camembert-base'*. - -## Supported Tags - -It uses the following tags: - -| Tag | Category | Extra Info | -|----------|:------------------------------:|------------:| -| ADJ | adjectif | | -| ADJWH | adjectif | | -| ADV | adverbe | | -| ADVWH | adverbe | | -| CC | conjonction de coordination | | -| CLO | pronom | obj | -| CLR | pronom | refl | -| CLS | pronom | suj | -| CS | conjonction de subordination | | -| DET | déterminant | | -| DETWH | déterminant | | -| ET | mot étranger | | -| I | interjection | | -| NC | nom commun | | -| NPP | nom propre | | -| P | préposition | | -| P+D | préposition + déterminant | | -| PONCT | signe de ponctuation | | -| PREF | préfixe | | -| PRO | autres pronoms | | -| PROREL | autres pronoms | rel | -| PROWH | autres pronoms | int | -| U | ? | | -| V | verbe | | -| VIMP | verbe imperatif | | -| VINF | verbe infinitif | | -| VPP | participe passé | | -| VPR | participe présent | | -| VS | subjonctif | | - -More information on the tags can be found here: - -http://alpage.inria.fr/statgram/frdep/Publications/crabbecandi-taln2008-final.pdf - -## Usage - -The usage of this model follows the common transformers patterns. Here is a short example of its usage: - -```python -from transformers import AutoTokenizer, AutoModelForTokenClassification - -tokenizer = AutoTokenizer.from_pretrained("gilf/french-camembert-postag-model") -model = AutoModelForTokenClassification.from_pretrained("gilf/french-camembert-postag-model") - -from transformers import pipeline - -nlp_token_class = pipeline('ner', model=model, tokenizer=tokenizer, grouped_entities=True) - -nlp_token_class('Face à un choc inédit, les mesures mises en place par le gouvernement ont permis une protection forte et efficace des ménages') -``` - -The lines above would display something like this on a Jupyter notebook: - -``` -[{'entity_group': 'NC', 'score': 0.5760144591331482, 'word': ''}, - {'entity_group': 'U', 'score': 0.9946700930595398, 'word': 'Face'}, - {'entity_group': 'P', 'score': 0.999615490436554, 'word': 'à'}, - {'entity_group': 'DET', 'score': 0.9995906352996826, 'word': 'un'}, - {'entity_group': 'NC', 'score': 0.9995531439781189, 'word': 'choc'}, - {'entity_group': 'ADJ', 'score': 0.999183714389801, 'word': 'inédit'}, - {'entity_group': 'P', 'score': 0.3710663616657257, 'word': ','}, - {'entity_group': 'DET', 'score': 0.9995903968811035, 'word': 'les'}, - {'entity_group': 'NC', 'score': 0.9995649456977844, 'word': 'mesures'}, - {'entity_group': 'VPP', 'score': 0.9988670349121094, 'word': 'mises'}, - {'entity_group': 'P', 'score': 0.9996246099472046, 'word': 'en'}, - {'entity_group': 'NC', 'score': 0.9995329976081848, 'word': 'place'}, - {'entity_group': 'P', 'score': 0.9996233582496643, 'word': 'par'}, - {'entity_group': 'DET', 'score': 0.9995935559272766, 'word': 'le'}, - {'entity_group': 'NC', 'score': 0.9995369911193848, 'word': 'gouvernement'}, - {'entity_group': 'V', 'score': 0.9993771314620972, 'word': 'ont'}, - {'entity_group': 'VPP', 'score': 0.9991101026535034, 'word': 'permis'}, - {'entity_group': 'DET', 'score': 0.9995885491371155, 'word': 'une'}, - {'entity_group': 'NC', 'score': 0.9995636343955994, 'word': 'protection'}, - {'entity_group': 'ADJ', 'score': 0.9991781711578369, 'word': 'forte'}, - {'entity_group': 'CC', 'score': 0.9991298317909241, 'word': 'et'}, - {'entity_group': 'ADJ', 'score': 0.9992275238037109, 'word': 'efficace'}, - {'entity_group': 'P+D', 'score': 0.9993300437927246, 'word': 'des'}, - {'entity_group': 'NC', 'score': 0.8353511393070221, 'word': 'ménages'}] -``` diff --git a/model_cards/gilf/french-postag-model/README.md b/model_cards/gilf/french-postag-model/README.md deleted file mode 100644 index 613b7e22229c..000000000000 --- a/model_cards/gilf/french-postag-model/README.md +++ /dev/null @@ -1,90 +0,0 @@ -## About - -The *french-postag-model* is a part of speech tagging model for French that was trained on the *free-french-treebank* dataset available on -[github](https://github.com/nicolashernandez/free-french-treebank). The base tokenizer and model used for training is *'bert-base-multilingual-cased'*. - -## Supported Tags - -It uses the following tags: - -| Tag | Category | Extra Info | -|----------|:------------------------------:|------------:| -| ADJ | adjectif | | -| ADJWH | adjectif | | -| ADV | adverbe | | -| ADVWH | adverbe | | -| CC | conjonction de coordination | | -| CLO | pronom | obj | -| CLR | pronom | refl | -| CLS | pronom | suj | -| CS | conjonction de subordination | | -| DET | déterminant | | -| DETWH | déterminant | | -| ET | mot étranger | | -| I | interjection | | -| NC | nom commun | | -| NPP | nom propre | | -| P | préposition | | -| P+D | préposition + déterminant | | -| PONCT | signe de ponctuation | | -| PREF | préfixe | | -| PRO | autres pronoms | | -| PROREL | autres pronoms | rel | -| PROWH | autres pronoms | int | -| U | ? | | -| V | verbe | | -| VIMP | verbe imperatif | | -| VINF | verbe infinitif | | -| VPP | participe passé | | -| VPR | participe présent | | -| VS | subjonctif | | - -More information on the tags can be found here: - -http://alpage.inria.fr/statgram/frdep/Publications/crabbecandi-taln2008-final.pdf - -## Usage - -The usage of this model follows the common transformers patterns. Here is a short example of its usage: - -```python -from transformers import AutoTokenizer, AutoModelForTokenClassification - -tokenizer = AutoTokenizer.from_pretrained("gilf/french-postag-model") -model = AutoModelForTokenClassification.from_pretrained("gilf/french-postag-model") - -from transformers import pipeline - -nlp_token_class = pipeline('ner', model=model, tokenizer=tokenizer, grouped_entities=True) - -nlp_token_class('Face à un choc inédit, les mesures mises en place par le gouvernement ont permis une protection forte et efficace des ménages') -``` - -The lines above would display something like this on a Jupyter notebook: - -``` -[{'entity_group': 'PONCT', 'score': 0.0742340236902237, 'word': '[CLS]'}, - {'entity_group': 'U', 'score': 0.9995399713516235, 'word': 'Face'}, - {'entity_group': 'P', 'score': 0.9999609589576721, 'word': 'à'}, - {'entity_group': 'DET', 'score': 0.9999597072601318, 'word': 'un'}, - {'entity_group': 'NC', 'score': 0.9998948276042938, 'word': 'choc'}, - {'entity_group': 'ADJ', 'score': 0.995318204164505, 'word': 'inédit'}, - {'entity_group': 'PONCT', 'score': 0.9999793171882629, 'word': ','}, - {'entity_group': 'DET', 'score': 0.999964714050293, 'word': 'les'}, - {'entity_group': 'NC', 'score': 0.999936580657959, 'word': 'mesures'}, - {'entity_group': 'VPP', 'score': 0.9995776414871216, 'word': 'mises'}, - {'entity_group': 'P', 'score': 0.99996417760849, 'word': 'en'}, - {'entity_group': 'NC', 'score': 0.999882161617279, 'word': 'place'}, - {'entity_group': 'P', 'score': 0.9999671578407288, 'word': 'par'}, - {'entity_group': 'DET', 'score': 0.9999637603759766, 'word': 'le'}, - {'entity_group': 'NC', 'score': 0.9999350309371948, 'word': 'gouvernement'}, - {'entity_group': 'V', 'score': 0.9999298453330994, 'word': 'ont'}, - {'entity_group': 'VPP', 'score': 0.9998740553855896, 'word': 'permis'}, - {'entity_group': 'DET', 'score': 0.9999625086784363, 'word': 'une'}, - {'entity_group': 'NC', 'score': 0.9999420046806335, 'word': 'protection'}, - {'entity_group': 'ADJ', 'score': 0.9998913407325745, 'word': 'forte'}, - {'entity_group': 'CC', 'score': 0.9998615980148315, 'word': 'et'}, - {'entity_group': 'ADJ', 'score': 0.9998483657836914, 'word': 'efficace'}, - {'entity_group': 'P+D', 'score': 0.9987645149230957, 'word': 'des'}, - {'entity_group': 'NC', 'score': 0.8720395267009735, 'word': 'ménages [SEP]'}] -``` diff --git a/model_cards/google/bert2bert_L-24_wmt_de_en/README.md b/model_cards/google/bert2bert_L-24_wmt_de_en/README.md deleted file mode 100644 index af86e42e3200..000000000000 --- a/model_cards/google/bert2bert_L-24_wmt_de_en/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -language: -- en -- de -license: apache-2.0 -datasets: -- wmt14 -tags: -- translation ---- - -# bert2bert_L-24_wmt_de_en EncoderDecoder model - -The model was introduced in -[this paper](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn and first released in [this repository](https://tfhub.dev/google/bertseq2seq/bert24_de_en/1). - -The model is an encoder-decoder model that was initialized on the `bert-large` checkpoints for both the encoder -and decoder and fine-tuned on German to English translation on the WMT dataset, which is linked above. - -Disclaimer: The model card has been written by the Hugging Face team. - -## How to use - -You can use this model for translation, *e.g.* - -```python -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - -tokenizer = AutoTokenizer.from_pretrained("google/bert2bert_L-24_wmt_de_en", pad_token="", eos_token="", bos_token="") -model = AutoModelForSeq2SeqLM.from_pretrained("google/bert2bert_L-24_wmt_de_en") - -sentence = "Willst du einen Kaffee trinken gehen mit mir?" - -input_ids = tokenizer(sentence, return_tensors="pt", add_special_tokens=False).input_ids -output_ids = model.generate(input_ids)[0] -print(tokenizer.decode(output_ids, skip_special_tokens=True)) -# should output -# Want to drink a kaffee go with me? . -``` diff --git a/model_cards/google/bert2bert_L-24_wmt_en_de/README.md b/model_cards/google/bert2bert_L-24_wmt_en_de/README.md deleted file mode 100644 index ab17b7ffa9e2..000000000000 --- a/model_cards/google/bert2bert_L-24_wmt_en_de/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -language: -- en -- de -license: apache-2.0 -datasets: -- wmt14 -tags: -- translation ---- - -# bert2bert_L-24_wmt_en_de EncoderDecoder model - -The model was introduced in -[this paper](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn and first released in [this repository](https://tfhub.dev/google/bertseq2seq/bert24_en_de/1). - -The model is an encoder-decoder model that was initialized on the `bert-large` checkpoints for both the encoder -and decoder and fine-tuned on English to German translation on the WMT dataset, which is linked above. - -Disclaimer: The model card has been written by the Hugging Face team. - -## How to use - -You can use this model for translation, *e.g.* - -```python -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - -tokenizer = AutoTokenizer.from_pretrained("google/bert2bert_L-24_wmt_en_de", pad_token="", eos_token="", bos_token="") -model = AutoModelForSeq2SeqLM.from_pretrained("google/bert2bert_L-24_wmt_en_de") - -sentence = "Would you like to grab a coffee with me this week?" - -input_ids = tokenizer(sentence, return_tensors="pt", add_special_tokens=False).input_ids -output_ids = model.generate(input_ids)[0] -print(tokenizer.decode(output_ids, skip_special_tokens=True)) -# should output -# Möchten Sie diese Woche einen Kaffee mit mir schnappen? diff --git a/model_cards/google/bert_uncased_L-10_H-128_A-2/README.md b/model_cards/google/bert_uncased_L-10_H-128_A-2/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-10_H-128_A-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-10_H-256_A-4/README.md b/model_cards/google/bert_uncased_L-10_H-256_A-4/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-10_H-256_A-4/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-10_H-512_A-8/README.md b/model_cards/google/bert_uncased_L-10_H-512_A-8/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-10_H-512_A-8/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-10_H-768_A-12/README.md b/model_cards/google/bert_uncased_L-10_H-768_A-12/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-10_H-768_A-12/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-12_H-128_A-2/README.md b/model_cards/google/bert_uncased_L-12_H-128_A-2/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-12_H-128_A-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-12_H-256_A-4/README.md b/model_cards/google/bert_uncased_L-12_H-256_A-4/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-12_H-256_A-4/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-12_H-512_A-8/README.md b/model_cards/google/bert_uncased_L-12_H-512_A-8/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-12_H-512_A-8/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-12_H-768_A-12/README.md b/model_cards/google/bert_uncased_L-12_H-768_A-12/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-12_H-768_A-12/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-2_H-128_A-2/README.md b/model_cards/google/bert_uncased_L-2_H-128_A-2/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-2_H-128_A-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-2_H-256_A-4/README.md b/model_cards/google/bert_uncased_L-2_H-256_A-4/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-2_H-256_A-4/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-2_H-512_A-8/README.md b/model_cards/google/bert_uncased_L-2_H-512_A-8/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-2_H-512_A-8/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-2_H-768_A-12/README.md b/model_cards/google/bert_uncased_L-2_H-768_A-12/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-2_H-768_A-12/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-4_H-128_A-2/README.md b/model_cards/google/bert_uncased_L-4_H-128_A-2/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-4_H-128_A-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-4_H-256_A-4/README.md b/model_cards/google/bert_uncased_L-4_H-256_A-4/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-4_H-256_A-4/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-4_H-512_A-8/README.md b/model_cards/google/bert_uncased_L-4_H-512_A-8/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-4_H-512_A-8/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-4_H-768_A-12/README.md b/model_cards/google/bert_uncased_L-4_H-768_A-12/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-4_H-768_A-12/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-6_H-128_A-2/README.md b/model_cards/google/bert_uncased_L-6_H-128_A-2/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-6_H-128_A-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-6_H-256_A-4/README.md b/model_cards/google/bert_uncased_L-6_H-256_A-4/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-6_H-256_A-4/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-6_H-512_A-8/README.md b/model_cards/google/bert_uncased_L-6_H-512_A-8/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-6_H-512_A-8/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-6_H-768_A-12/README.md b/model_cards/google/bert_uncased_L-6_H-768_A-12/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-6_H-768_A-12/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-8_H-128_A-2/README.md b/model_cards/google/bert_uncased_L-8_H-128_A-2/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-8_H-128_A-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-8_H-256_A-4/README.md b/model_cards/google/bert_uncased_L-8_H-256_A-4/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-8_H-256_A-4/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-8_H-512_A-8/README.md b/model_cards/google/bert_uncased_L-8_H-512_A-8/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-8_H-512_A-8/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/bert_uncased_L-8_H-768_A-12/README.md b/model_cards/google/bert_uncased_L-8_H-768_A-12/README.md deleted file mode 120000 index 1ee4bd8fef17..000000000000 --- a/model_cards/google/bert_uncased_L-8_H-768_A-12/README.md +++ /dev/null @@ -1 +0,0 @@ -../../iuliaturc/bert_uncased_L-2_H-128_A-2/README.md \ No newline at end of file diff --git a/model_cards/google/electra-base-discriminator/README.md b/model_cards/google/electra-base-discriminator/README.md deleted file mode 100644 index b24ddfef7c68..000000000000 --- a/model_cards/google/electra-base-discriminator/README.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -language: en -thumbnail: https://huggingface.co/front/thumbnails/google.png - -license: apache-2.0 ---- - -## ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer to our paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -This repository contains code to pre-train ELECTRA, including small ELECTRA models on a single GPU. It also supports fine-tuning ELECTRA on downstream tasks including classification tasks (e.g,. [GLUE](https://gluebenchmark.com/)), QA tasks (e.g., [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/)), and sequence tagging tasks (e.g., [text chunking](https://www.clips.uantwerpen.be/conll2000/chunking/)). - -## How to use the discriminator in `transformers` - -```python -from transformers import ElectraForPreTraining, ElectraTokenizerFast -import torch - -discriminator = ElectraForPreTraining.from_pretrained("google/electra-base-discriminator") -tokenizer = ElectraTokenizerFast.from_pretrained("google/electra-base-discriminator") - -sentence = "The quick brown fox jumps over the lazy dog" -fake_sentence = "The quick brown fox fake over the lazy dog" - -fake_tokens = tokenizer.tokenize(fake_sentence) -fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt") -discriminator_outputs = discriminator(fake_inputs) -predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) - -[print("%7s" % token, end="") for token in fake_tokens] - -[print("%7s" % int(prediction), end="") for prediction in predictions.tolist()] -``` diff --git a/model_cards/google/electra-base-generator/README.md b/model_cards/google/electra-base-generator/README.md deleted file mode 100644 index 8eb2d129cdc0..000000000000 --- a/model_cards/google/electra-base-generator/README.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -language: en -thumbnail: https://huggingface.co/front/thumbnails/google.png - -license: apache-2.0 ---- - -## ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer to our paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -This repository contains code to pre-train ELECTRA, including small ELECTRA models on a single GPU. It also supports fine-tuning ELECTRA on downstream tasks including classification tasks (e.g,. [GLUE](https://gluebenchmark.com/)), QA tasks (e.g., [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/)), and sequence tagging tasks (e.g., [text chunking](https://www.clips.uantwerpen.be/conll2000/chunking/)). - -## How to use the generator in `transformers` - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="google/electra-base-generator", - tokenizer="google/electra-base-generator" -) - -print( - fill_mask(f"HuggingFace is creating a {fill_mask.tokenizer.mask_token} that the community uses to solve NLP tasks.") -) - -``` diff --git a/model_cards/google/electra-large-discriminator/README.md b/model_cards/google/electra-large-discriminator/README.md deleted file mode 100644 index 119b012ace2e..000000000000 --- a/model_cards/google/electra-large-discriminator/README.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -language: en -thumbnail: https://huggingface.co/front/thumbnails/google.png - -license: apache-2.0 ---- - -## ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer to our paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -This repository contains code to pre-train ELECTRA, including small ELECTRA models on a single GPU. It also supports fine-tuning ELECTRA on downstream tasks including classification tasks (e.g,. [GLUE](https://gluebenchmark.com/)), QA tasks (e.g., [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/)), and sequence tagging tasks (e.g., [text chunking](https://www.clips.uantwerpen.be/conll2000/chunking/)). - -## How to use the discriminator in `transformers` - -```python -from transformers import ElectraForPreTraining, ElectraTokenizerFast -import torch - -discriminator = ElectraForPreTraining.from_pretrained("google/electra-large-discriminator") -tokenizer = ElectraTokenizerFast.from_pretrained("google/electra-large-discriminator") - -sentence = "The quick brown fox jumps over the lazy dog" -fake_sentence = "The quick brown fox fake over the lazy dog" - -fake_tokens = tokenizer.tokenize(fake_sentence) -fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt") -discriminator_outputs = discriminator(fake_inputs) -predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) - -[print("%7s" % token, end="") for token in fake_tokens] - -[print("%7s" % int(prediction), end="") for prediction in predictions.tolist()] -``` diff --git a/model_cards/google/electra-large-generator/README.md b/model_cards/google/electra-large-generator/README.md deleted file mode 100644 index 31a033f44b5a..000000000000 --- a/model_cards/google/electra-large-generator/README.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -language: en -thumbnail: https://huggingface.co/front/thumbnails/google.png - -license: apache-2.0 ---- - -## ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer to our paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -This repository contains code to pre-train ELECTRA, including small ELECTRA models on a single GPU. It also supports fine-tuning ELECTRA on downstream tasks including classification tasks (e.g,. [GLUE](https://gluebenchmark.com/)), QA tasks (e.g., [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/)), and sequence tagging tasks (e.g., [text chunking](https://www.clips.uantwerpen.be/conll2000/chunking/)). - -## How to use the generator in `transformers` - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="google/electra-large-generator", - tokenizer="google/electra-large-generator" -) - -print( - fill_mask(f"HuggingFace is creating a {nlp.tokenizer.mask_token} that the community uses to solve NLP tasks.") -) - -``` diff --git a/model_cards/google/electra-small-discriminator/README.md b/model_cards/google/electra-small-discriminator/README.md deleted file mode 100644 index 650d3fd687bb..000000000000 --- a/model_cards/google/electra-small-discriminator/README.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -language: en -thumbnail: https://huggingface.co/front/thumbnails/google.png - -license: apache-2.0 ---- - -## ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer to our paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -This repository contains code to pre-train ELECTRA, including small ELECTRA models on a single GPU. It also supports fine-tuning ELECTRA on downstream tasks including classification tasks (e.g,. [GLUE](https://gluebenchmark.com/)), QA tasks (e.g., [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/)), and sequence tagging tasks (e.g., [text chunking](https://www.clips.uantwerpen.be/conll2000/chunking/)). - -## How to use the discriminator in `transformers` - -```python -from transformers import ElectraForPreTraining, ElectraTokenizerFast -import torch - -discriminator = ElectraForPreTraining.from_pretrained("google/electra-small-discriminator") -tokenizer = ElectraTokenizerFast.from_pretrained("google/electra-small-discriminator") - -sentence = "The quick brown fox jumps over the lazy dog" -fake_sentence = "The quick brown fox fake over the lazy dog" - -fake_tokens = tokenizer.tokenize(fake_sentence) -fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt") -discriminator_outputs = discriminator(fake_inputs) -predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) - -[print("%7s" % token, end="") for token in fake_tokens] - -[print("%7s" % int(prediction), end="") for prediction in predictions.tolist()] -``` diff --git a/model_cards/google/electra-small-generator/README.md b/model_cards/google/electra-small-generator/README.md deleted file mode 100644 index b097328d0c85..000000000000 --- a/model_cards/google/electra-small-generator/README.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -language: en -thumbnail: https://huggingface.co/front/thumbnails/google.png - -license: apache-2.0 ---- - -## ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer to our paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -This repository contains code to pre-train ELECTRA, including small ELECTRA models on a single GPU. It also supports fine-tuning ELECTRA on downstream tasks including classification tasks (e.g,. [GLUE](https://gluebenchmark.com/)), QA tasks (e.g., [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/)), and sequence tagging tasks (e.g., [text chunking](https://www.clips.uantwerpen.be/conll2000/chunking/)). - -## How to use the generator in `transformers` - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="google/electra-small-generator", - tokenizer="google/electra-small-generator" -) - -print( - fill_mask(f"HuggingFace is creating a {nlp.tokenizer.mask_token} that the community uses to solve NLP tasks.") -) - -``` diff --git a/model_cards/google/mobilebert-uncased/README.md b/model_cards/google/mobilebert-uncased/README.md deleted file mode 100644 index 16a4c1fd579d..000000000000 --- a/model_cards/google/mobilebert-uncased/README.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -language: en -thumbnail: https://huggingface.co/front/thumbnails/google.png - -license: apache-2.0 ---- - -## MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices - -MobileBERT is a thin version of BERT_LARGE, while equipped with bottleneck structures and a carefully designed balance -between self-attentions and feed-forward networks. - -This checkpoint is the original MobileBert Optimized Uncased English: -[uncased_L-24_H-128_B-512_A-4_F-4_OPT](https://storage.googleapis.com/cloud-tpu-checkpoints/mobilebert/uncased_L-24_H-128_B-512_A-4_F-4_OPT.tar.gz) -checkpoint. - -## How to use MobileBERT in `transformers` - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="google/mobilebert-uncased", - tokenizer="google/mobilebert-uncased" -) - -print( - fill_mask(f"HuggingFace is creating a {fill_mask.tokenizer.mask_token} that the community uses to solve NLP tasks.") -) - -``` diff --git a/model_cards/google/reformer-crime-and-punishment/README.md b/model_cards/google/reformer-crime-and-punishment/README.md deleted file mode 100644 index bd37073c3c52..000000000000 --- a/model_cards/google/reformer-crime-and-punishment/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## Reformer Model trained on "Crime and Punishment" - -Crime and Punishment is a novel written by Fyodor Dostoevsky and was translated into English. - -Crime and Punishment training data was taken from `gs://trax-ml/reformer/crime-and-punishment-2554.txt` and contains -roughly 0.5M tokens. - -The ReformerLM model was trained in flax using colab notebook proposed by authors: https://colab.research.google.com/github/google/trax/blob/master/trax/models/reformer/text_generation.ipynb and the weights were converted to Hugging Face's PyTorch ReformerLM model `ReformerModelWithLMHead`. - -The model is a language model that operates on small sub-word units. Text can be generated as follows: - -```python -model = ReformerModelWithLMHead.from_pretrained("google/reformer-crime-and-punishment") -tok = ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment") -tok.decode(model.generate(tok.encode("A few months later", return_tensors="pt"), do_sample=True,temperature=0.7, max_length=100)[0]) - -# gives:'A few months later on was more than anything in the flat. -# “I have already.” “That’s not my notion that he had forgotten him. -# What does that matter? And why do you mean? It’s only another fellow,” he said as he went out, as though he want' -``` diff --git a/model_cards/google/reformer-enwik8/README.md b/model_cards/google/reformer-enwik8/README.md deleted file mode 100644 index 998ac33d9c7e..000000000000 --- a/model_cards/google/reformer-enwik8/README.md +++ /dev/null @@ -1,57 +0,0 @@ -## Reformer Language model on character level and trained on enwik8. - -*enwik8* is a dataset based on Wikipedia and is often used to measure the model's ability to *compress* data, *e.g.* in -the scope of the *Hutter prize*: https://en.wikipedia.org/wiki/Hutter_Prize. - -`reformer-enwik8` was pretrained on the first 90M chars of *enwik8* whereas the text was chunked into batches of size 65536 chars (=2^16). -The model's weights were taken from https://console.cloud.google.com/storage/browser/trax-ml/reformer/enwik8 and converted -to Hugging Face's PyTorch ReformerLM model `ReformerModelWithLMHead`. - -The model is a language model that operates on characters. -Therefore, this model does not need a tokenizer. The following function can instead be used for **encoding** and **decoding**: - -```python -import torch - -# Encoding -def encode(list_of_strings, pad_token_id=0): - max_length = max([len(string) for string in list_of_strings]) - - # create emtpy tensors - attention_masks = torch.zeros((len(list_of_strings), max_length), dtype=torch.long) - input_ids = torch.full((len(list_of_strings), max_length), pad_token_id, dtype=torch.long) - - for idx, string in enumerate(list_of_strings): - # make sure string is in byte format - if not isinstance(string, bytes): - string = str.encode(string) - - input_ids[idx, :len(string)] = torch.tensor([x + 2 for x in string]) - attention_masks[idx, :len(string)] = 1 - - return input_ids, attention_masks - -# Decoding -def decode(outputs_ids): - decoded_outputs = [] - for output_ids in outputs_ids.tolist(): - # transform id back to char IDs < 2 are simply transformed to "" - decoded_outputs.append("".join([chr(x - 2) if x > 1 else "" for x in output_ids])) - return decoded_outputs -``` - -Text can be generated as follows: - -```python -from transformers import ReformerModelWithLMHead - -model = ReformerModelWithLMHead.from_pretrained("google/reformer-enwik8") -encoded, attention_masks = encode(["In 1965, Brooks left IBM to found the Department of"]) -decode(model.generate(encoded, do_sample=True, max_length=150)) - -# gives: -# In 1965, Brooks left IBM to found the Department of Journalism in 1968. IBM had jurisdiction himself in 1980, while Brooks resolved, nevertheless thro - -``` - -***Note***: Language generation using `ReformerModelWithLMHead` is not optimized yet and is rather slow. diff --git a/model_cards/google/roberta2roberta_L-24_bbc/README.md b/model_cards/google/roberta2roberta_L-24_bbc/README.md deleted file mode 100644 index 9e0c959f7a32..000000000000 --- a/model_cards/google/roberta2roberta_L-24_bbc/README.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- xsum -tags: -- summarization ---- - -# Roberta2Roberta_L-24_bbc EncoderDecoder model - -The model was introduced in -[this paper](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn and first released in [this repository](https://tfhub.dev/google/bertseq2seq/roberta24_bbc/1). - -The model is an encoder-decoder model that was initialized on the `roberta-large` checkpoints for both the encoder -and decoder and fine-tuned on extreme summarization on the BBC XSum dataset, which is linked above. - -Disclaimer: The model card has been written by the Hugging Face team. - -## How to use - -You can use this model for extreme summarization, *e.g.* - -```python -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - -tokenizer = AutoTokenizer.from_pretrained("google/roberta2roberta_L-24_bbc") -model = AutoModelForSeq2SeqLM.from_pretrained("google/roberta2roberta_L-24_bbc") - -article = """The problem is affecting people using the older -versions of the PlayStation 3, called the "Fat" -model.The problem isn't affecting the newer PS3 -Slim systems that have been on sale since -September last year.Sony have also said they are -aiming to have the problem fixed shortly but is -advising some users to avoid using their console -for the time being."We hope to resolve this -problem within the next 24 hours," a statement -reads. "In the meantime, if you have a model other -than the new slim PS3, we advise that you do not -use your PS3 system, as doing so may result in -errors in some functionality, such as recording -obtained trophies, and not being able to restore -certain data."We believe we have identified that -this problem is being caused by a bug in the clock -functionality incorporated in the system."The -PlayStation Network is used by millions of people -around the world.It allows users to play their -friends at games like Fifa over the internet and -also do things like download software or visit -online stores.""" - -input_ids = tokenizer(article, return_tensors="pt").input_ids -output_ids = model.generate(input_ids)[0] -print(tokenizer.decode(output_ids, skip_special_tokens=True)) -# should output -# Some Sony PlayStation gamers are being advised to stay away from the network because of a problem with the PlayStation 3 network. -``` diff --git a/model_cards/google/roberta2roberta_L-24_cnn_daily_mail/README.md b/model_cards/google/roberta2roberta_L-24_cnn_daily_mail/README.md deleted file mode 100644 index 58ccd1f1d5dc..000000000000 --- a/model_cards/google/roberta2roberta_L-24_cnn_daily_mail/README.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- cnn_dailymail -tags: -- summarization ---- - -# Roberta2Roberta_L-24_cnn_daily_mail EncoderDecoder model - -The model was introduced in -[this paper](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn and first released in [this repository](https://tfhub.dev/google/bertseq2seq/roberta24_cnndm/1). - -The model is an encoder-decoder model that was initialized on the `roberta-large` checkpoints for both the encoder -and decoder and fine-tuned on summarization on the CNN / Dailymail dataset, which is linked above. - -Disclaimer: The model card has been written by the Hugging Face team. - -## How to use - -You can use this model for summarization, *e.g.* - -```python -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - -tokenizer = AutoTokenizer.from_pretrained("google/roberta2roberta_L-24_cnn_daily_mail") -model = AutoModelForSeq2SeqLM.from_pretrained("google/roberta2roberta_L-24_cnn_daily_mail") - -article = """ (The Hollywood Reporter)"The Rocky Horror Picture -Show" is the latest musical getting the small- -screen treatment. Fox is developing a two-hour -remake of the 1975 cult classic to be directed, -executive-produced and choreographed by Kenneth -Ortega ("High School Musical"). The project, -tentatively titled "The Rocky Horror Picture Show -Event," is casting-contingent. The special will be -filmed in advance and not air live, but few -details beyond that are known. In addition to -Ortega, Gail Berman and Lou Adler, who produced -the original film, are also attached as executive -producers. The special will be produced by Fox 21 -Television Studios, and Berman's The Jackal Group. -The special is timed to celebrate the 40th -anniversary of the film, which has grossed more -than $112 million and still plays in theaters -across the country. TV premiere dates: The -complete guide . This isn't the first stab at -adapting "The Rocky Horror Picture Show." In 2002, -Fox unveiled plans for an adaptation timed to the -30th anniversary that never came to fruition. The -faces of pilot season 2015 . Fox's "Glee" covered -several of the show's most popular songs for a -Season 2 episode and even released a special "The -Rocky Horror Glee Show" EP. There is no plan yet -for when the adaptation will air. Fox also has a -live musical production of "Grease", starring -Julianne Hough and Vanessa Hudgens, scheduled to -air on Jan. 31, 2016. Broadcast TV scorecard . -Following in the footsteps of "The Sound of Music" -and "Peter Pan," NBC recently announced plans to -air a live version of The Wiz later this year. -Ortega's credits include "Gilmore Girls," "This Is -It" and "Hocus Pocus." He is repped by Paradigm -and Hanson, Jacobson. ©2015 The Hollywood -Reporter. All rights reserved.""" - -input_ids = tokenizer(article, return_tensors="pt").input_ids -output_ids = model.generate(input_ids)[0] -print(tokenizer.decode(output_ids, skip_special_tokens=True)) -# should output -# Fox is developing a two-hour remake of the 1975 cult classic. The special will be directed, executive-produced and choreographed by Kenneth Ortega. -# The special is timed to celebrate the 40th anniversary of the film, which has grossed more than $112 million. - -``` diff --git a/model_cards/google/roberta2roberta_L-24_discofuse/README.md b/model_cards/google/roberta2roberta_L-24_discofuse/README.md deleted file mode 100644 index f7219725308a..000000000000 --- a/model_cards/google/roberta2roberta_L-24_discofuse/README.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- discofuse ---- - -# Roberta2Roberta_L-24_discofuse EncoderDecoder model - -The model was introduced in -[this paper](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn and first released in [this repository](https://tfhub.dev/google/bertseq2seq/roberta24_discofuse/1). - -The model is an encoder-decoder model that was initialized on the `roberta-large` checkpoints for both the encoder -and decoder and fine-tuned on sentencefusion on the discofuse dataset, which is linked above. - -Disclaimer: The model card has been written by the Hugging Face team. - -## How to use - -You can use this model for sentence fusion, *e.g.* - -IMPORTANT: The model was not trained on the `"` (double quotation mark) character -> so the before tokenizing the text, it is advised to replace all `"` (double quotation marks) with a single `` ` `` (single back tick). - -```python -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - -tokenizer = AutoTokenizer.from_pretrained("google/roberta2roberta_L-24_discofuse") -model = AutoModelForSeq2SeqLM.from_pretrained("google/roberta2roberta_L-24_discofuse") - -discofuse = """As a run-blocker, Zeitler moves relatively well. Zeitler often struggles at the point of contact in space.""" - -input_ids = tokenizer(discofuse, return_tensors="pt").input_ids -output_ids = model.generate(input_ids)[0] -print(tokenizer.decode(output_ids, skip_special_tokens=True)) -# should output -# As a run-blocker, Zeitler moves relatively well. However, Zeitler often struggles at the point of contact in space. -``` diff --git a/model_cards/google/roberta2roberta_L-24_gigaword/README.md b/model_cards/google/roberta2roberta_L-24_gigaword/README.md deleted file mode 100644 index a465a34604c5..000000000000 --- a/model_cards/google/roberta2roberta_L-24_gigaword/README.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- gigaword -tags: -- summarization ---- - -# Roberta2Roberta_L-24_gigaword EncoderDecoder model - -The model was introduced in -[this paper](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn and first released in [this repository](https://tfhub.dev/google/bertseq2seq/roberta24_gigaword/1). - -The model is an encoder-decoder model that was initialized on the `roberta-large` checkpoints for both the encoder -and decoder and fine-tuned on headline generation using the Gigaword dataset, which is linked above. - -Disclaimer: The model card has been written by the Hugging Face team. - -## How to use - -You can use this model for extreme summarization, *e.g.* - -```python -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - -tokenizer = AutoTokenizer.from_pretrained("google/roberta2roberta_L-24_gigaword") -model = AutoModelForSeq2SeqLM.from_pretrained("google/roberta2roberta_L-24_gigaword") - -article = """australian shares closed down #.# percent monday -following a weak lead from the united states and -lower commodity prices , dealers said .""" - -input_ids = tokenizer(article, return_tensors="pt").input_ids -output_ids = model.generate(input_ids)[0] -print(tokenizer.decode(output_ids, skip_special_tokens=True)) -# should output -# australian shares close down #.# percent. -``` diff --git a/model_cards/google/roberta2roberta_L-24_wikisplit/README.md b/model_cards/google/roberta2roberta_L-24_wikisplit/README.md deleted file mode 100644 index 8d4a2b380a7e..000000000000 --- a/model_cards/google/roberta2roberta_L-24_wikisplit/README.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -language: en -license: apache-2.0 ---- - -# Roberta2Roberta_L-24_wikisplit EncoderDecoder model - -The model was introduced in -[this paper](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn and first released in [this repository](https://tfhub.dev/google/bertseq2seq/roberta24_cnndm/1). - -The model is an encoder-decoder model that was initialized on the `roberta-large` checkpoints for both the encoder -and decoder and fine-tuned on sentence splitting on the [WikiSplit](https://github.com/google-research-datasets/wiki-split) dataset. - -Disclaimer: The model card has been written by the Hugging Face team. - -## How to use - -You can use this model for sentence splitting, *e.g.* - -**IMPORTANT**: The model was not trained on the `"` (double quotation mark) character -> so the before tokenizing the text, -it is advised to replace all `"` (double quotation marks) with two single `'` (single quotation mark). - -```python -from transformers import AutoTokenizer, AutoModelForSeq2SeqLM - -tokenizer = AutoTokenizer.from_pretrained("google/roberta2roberta_L-24_wikisplit") -model = AutoModelForSeq2SeqLM.from_pretrained("google/roberta2roberta_L-24_wikisplit") - -long_sentence = """Due to the hurricane, Lobsterfest has been canceled, making Bob very happy about it and he decides to open Bob 's Burgers for customers who were planning on going to Lobsterfest.""" - -input_ids = tokenizer(tokenizer.bos_token + long_sentence + tokenizer.eos_token, return_tensors="pt").input_ids -output_ids = model.generate(input_ids)[0] -print(tokenizer.decode(output_ids, skip_special_tokens=True)) -# should output -# Due to the hurricane, Lobsterfest has been canceled, making Bob very happy about it. He decides to open Bob's Burgers for customers who were planning on going to Lobsterfest. -``` diff --git a/model_cards/google/tapas-base/README.md b/model_cards/google/tapas-base/README.md new file mode 100644 index 000000000000..9685f28566d4 --- /dev/null +++ b/model_cards/google/tapas-base/README.md @@ -0,0 +1,123 @@ +--- +language: en +tags: +- tapas +- masked-lm +license: apache-2.0 +--- + +# TAPAS base model + +This model corresponds to the `tapas_inter_masklm_base_reset` checkpoint of the [original Github repository](https://github.com/google-research/tapas). + +Disclaimer: The team releasing TAPAS did not write a model card for this model so this model card has been written by +the Hugging Face team and contributors. + +## Model description + +TAPAS is a BERT-like transformers model pretrained on a large corpus of English data from Wikipedia in a self-supervised fashion. +This means it was pretrained on the raw tables and associated texts only, with no humans labelling them in any way (which is why it +can use lots of publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it +was pretrained with two objectives: + +- Masked language modeling (MLM): taking a (flattened) table and associated context, the model randomly masks 15% of the words in + the input, then runs the entire (partially masked) sequence through the model. The model then has to predict the masked words. + This is different from traditional recurrent neural networks (RNNs) that usually see the words one after the other, + or from autoregressive models like GPT which internally mask the future tokens. It allows the model to learn a bidirectional + representation of a table and associated text. +- Intermediate pre-training: to encourage numerical reasoning on tables, the authors additionally pre-trained the model by creating + a balanced dataset of millions of syntactically created training examples. Here, the model must predict (classify) whether a sentence + is supported or refuted by the contents of a table. The training examples are created based on synthetic as well as counterfactual statements. + +This way, the model learns an inner representation of the English language used in tables and associated texts, which can then be used +to extract features useful for downstream tasks such as answering questions about a table, or determining whether a sentence is entailed +or refuted by the contents of a table. Fine-tuning is done by adding classification heads on top of the pre-trained model, and then jointly +train the randomly initialized classification heads with the base model on a labelled dataset. + +## Intended uses & limitations + +You can use the raw model for masked language modeling, but it's mostly intended to be fine-tuned on a downstream task. +See the [model hub](https://huggingface.co/models?filter=tapas) to look for fine-tuned versions on a task that interests you. + + +Here is how to use this model to get the features of a given table-text pair in PyTorch: + +```python +from transformers import TapasTokenizer, TapasModel +import pandas as pd +tokenizer = TapasTokenizer.from_pretrained('tapase-base') +model = TapasModel.from_pretrained("tapas-base") +data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], + 'Age': ["56", "45", "59"], + 'Number of movies': ["87", "53", "69"] +} +table = pd.DataFrame.from_dict(data) +queries = ["How many movies has George Clooney played in?"] +text = "Replace me by any text you'd like." +encoded_input = tokenizer(table=table, queries=queries, return_tensors='pt') +output = model(**encoded_input) +``` + +## Training data + +For masked language modeling (MLM), a collection of 6.2 million tables was extracted from English Wikipedia: 3.3M of class [Infobox](https://en.wikipedia.org/wiki/Help:Infobox) +and 2.9M of class WikiTable. The author only considered tables with at most 500 cells. As a proxy for questions that appear in the +downstream tasks, the authros extracted the table caption, article title, article description, segment title and text of the segment +the table occurs in as relevant text snippets. In this way, 21.3M snippets were created. For more info, see the original [TAPAS paper](https://www.aclweb.org/anthology/2020.acl-main.398.pdf). + +For intermediate pre-training, 2 tasks are introduced: one based on synthetic and the other from counterfactual statements. The first one +generates a sentence by sampling from a set of logical expressions that filter, combine and compare the information on the table, which is +required in table entailment (e.g., knowing that Gerald Ford is taller than the average president requires summing +all presidents and dividing by the number of presidents). The second one corrupts sentences about tables appearing on Wikipedia by swapping +entities for plausible alternatives. Examples of the two tasks can be seen in Figure 1. The procedure is described in detail in section 3 of +the [TAPAS follow-up paper](https://www.aclweb.org/anthology/2020.findings-emnlp.27.pdf). + +## Training procedure + +### Preprocessing + +The texts are lowercased and tokenized using WordPiece and a vocabulary size of 30,000. The inputs of the model are +then of the form: + +``` +[CLS] Context [SEP] Flattened table [SEP] +``` + +The details of the masking procedure for each sequence are the following: +- 15% of the tokens are masked. +- In 80% of the cases, the masked tokens are replaced by `[MASK]`. +- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace. +- In the 10% remaining cases, the masked tokens are left as is. + +The details of the creation of the synthetic and counterfactual examples can be found in the [follow-up paper](https://arxiv.org/abs/2010.00571). + +### Pretraining + +The model was trained on 32 Cloud TPU v3 cores for one million steps with maximum sequence length 512 and batch size of 512. +In this setup, pre-training takes around 3 days. The optimizer used is Adam with a learning rate of 5e-5, and a warmup ratio +of 0.10. + + +### BibTeX entry and citation info + +```bibtex +@misc{herzig2020tapas, + title={TAPAS: Weakly Supervised Table Parsing via Pre-training}, + author={Jonathan Herzig and Paweł Krzysztof Nowak and Thomas Müller and Francesco Piccinno and Julian Martin Eisenschlos}, + year={2020}, + eprint={2004.02349}, + archivePrefix={arXiv}, + primaryClass={cs.IR} +} +``` + +```bibtex +@misc{eisenschlos2020understanding, + title={Understanding tables with intermediate pre-training}, + author={Julian Martin Eisenschlos and Syrine Krichene and Thomas Müller}, + year={2020}, + eprint={2010.00571}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +``` \ No newline at end of file diff --git a/model_cards/gpt2-README.md b/model_cards/gpt2-README.md deleted file mode 100644 index 65fe7b5e382d..000000000000 --- a/model_cards/gpt2-README.md +++ /dev/null @@ -1,163 +0,0 @@ ---- -language: en -tags: -- exbert - -license: mit ---- - - -# GPT-2 - -Test the whole generation capabilities here: https://transformer.huggingface.co/doc/gpt2-large - -Pretrained model on English language using a causal language modeling (CLM) objective. It was introduced in -[this paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) -and first released at [this page](https://openai.com/blog/better-language-models/). - -Disclaimer: The team releasing GPT-2 also wrote a -[model card](https://github.com/openai/gpt-2/blob/master/model_card.md) for their model. Content from this model card -has been written by the Hugging Face team to complete the information they provided and give specific examples of bias. - -## Model description - -GPT-2 is a transformers model pretrained on a very large corpus of English data in a self-supervised fashion. This -means it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots -of publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, -it was trained to guess the next word in sentences. - -More precisely, inputs are sequences of continuous text of a certain length and the targets are the same sequence, -shifted one token (word or piece of word) to the right. The model uses internally a mask-mechanism to make sure the -predictions for the token `i` only uses the inputs from `1` to `i` but not the future tokens. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks. The model is best at what it was pretrained for however, which is generating texts from a -prompt. - -## Intended uses & limitations - -You can use the raw model for text generation or fine-tune it to a downstream task. See the -[model hub](https://huggingface.co/models?filter=gpt2) to look for fine-tuned versions on a task that interests you. - -### How to use - -You can use this model directly with a pipeline for text generation. Since the generation relies on some randomness, we -set a seed for reproducibility: - -```python ->>> from transformers import pipeline, set_seed ->>> generator = pipeline('text-generation', model='gpt2') ->>> set_seed(42) ->>> generator("Hello, I'm a language model,", max_length=30, num_return_sequences=5) - -[{'generated_text': "Hello, I'm a language model, a language for thinking, a language for expressing thoughts."}, - {'generated_text': "Hello, I'm a language model, a compiler, a compiler library, I just want to know how I build this kind of stuff. I don"}, - {'generated_text': "Hello, I'm a language model, and also have more than a few of your own, but I understand that they're going to need some help"}, - {'generated_text': "Hello, I'm a language model, a system model. I want to know my language so that it might be more interesting, more user-friendly"}, - {'generated_text': 'Hello, I\'m a language model, not a language model"\n\nThe concept of "no-tricks" comes in handy later with new'}] -``` - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import GPT2Tokenizer, GPT2Model -tokenizer = GPT2Tokenizer.from_pretrained('gpt2') -model = GPT2Model.from_pretrained('gpt2') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import GPT2Tokenizer, TFGPT2Model -tokenizer = GPT2Tokenizer.from_pretrained('gpt2') -model = TFGPT2Model.from_pretrained('gpt2') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -### Limitations and bias - -The training data used for this model has not been released as a dataset one can browse. We know it contains a lot of -unfiltered content from the internet, which is far from neutral. As the openAI team themselves point out in their -[model card](https://github.com/openai/gpt-2/blob/master/model_card.md#out-of-scope-use-cases): - -> Because large-scale language models like GPT-2 do not distinguish fact from fiction, we don’t support use-cases -> that require the generated text to be true. -> -> Additionally, language models like GPT-2 reflect the biases inherent to the systems they were trained on, so we do -> not recommend that they be deployed into systems that interact with humans > unless the deployers first carry out a -> study of biases relevant to the intended use-case. We found no statistically significant difference in gender, race, -> and religious bias probes between 774M and 1.5B, implying all versions of GPT-2 should be approached with similar -> levels of caution around use cases that are sensitive to biases around human attributes. - -Here's an example of how the model can have biased predictions: - -```python ->>> from transformers import pipeline, set_seed ->>> generator = pipeline('text-generation', model='gpt2') ->>> set_seed(42) ->>> generator("The White man worked as a", max_length=10, num_return_sequences=5) - -[{'generated_text': 'The White man worked as a mannequin for'}, - {'generated_text': 'The White man worked as a maniser of the'}, - {'generated_text': 'The White man worked as a bus conductor by day'}, - {'generated_text': 'The White man worked as a plumber at the'}, - {'generated_text': 'The White man worked as a journalist. He had'}] - ->>> set_seed(42) ->>> generator("The Black man worked as a", max_length=10, num_return_sequences=5) - -[{'generated_text': 'The Black man worked as a man at a restaurant'}, - {'generated_text': 'The Black man worked as a car salesman in a'}, - {'generated_text': 'The Black man worked as a police sergeant at the'}, - {'generated_text': 'The Black man worked as a man-eating monster'}, - {'generated_text': 'The Black man worked as a slave, and was'}] -``` - -This bias will also affect all fine-tuned versions of this model. - -## Training data - -The OpenAI team wanted to train this model on a corpus as large as possible. To build it, they scraped all the web -pages from outbound links on Reddit which received at least 3 karma. Note that all Wikipedia pages were removed from -this dataset, so the model was not trained on any part of Wikipedia. The resulting dataset (called WebText) weights -40GB of texts but has not been publicly released. You can find a list of the top 1,000 domains present in WebText -[here](https://github.com/openai/gpt-2/blob/master/domains.txt). - -## Training procedure - -### Preprocessing - -The texts are tokenized using a byte-level version of Byte Pair Encoding (BPE) (for unicode characters) and a -vocabulary size of 50,257. The inputs are sequences of 1024 consecutive tokens. - -The larger model was trained on 256 cloud TPU v3 cores. The training duration was not disclosed, nor were the exact -details of training. - -## Evaluation results - -The model achieves the following results without any fine-tuning (zero-shot): - -| Dataset | LAMBADA | LAMBADA | CBT-CN | CBT-NE | WikiText2 | PTB | enwiki8 | text8 | WikiText103 | 1BW | -|:--------:|:-------:|:-------:|:------:|:------:|:---------:|:------:|:-------:|:------:|:-----------:|:-----:| -| (metric) | (PPL) | (ACC) | (ACC) | (ACC) | (PPL) | (PPL) | (BPB) | (BPC) | (PPL) | (PPL) | -| | 35.13 | 45.99 | 87.65 | 83.4 | 29.41 | 65.85 | 1.16 | 1,17 | 37.50 | 75.20 | - - -### BibTeX entry and citation info - -```bibtex -@article{radford2019language, - title={Language Models are Unsupervised Multitask Learners}, - author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya}, - year={2019} -} -``` - - - - diff --git a/model_cards/gpt2-large-README.md b/model_cards/gpt2-large-README.md deleted file mode 100644 index a019393abfd3..000000000000 --- a/model_cards/gpt2-large-README.md +++ /dev/null @@ -1 +0,0 @@ -Test the full generation capabilities here: https://transformer.huggingface.co/doc/gpt2-large diff --git a/model_cards/gpt2-medium-README.md b/model_cards/gpt2-medium-README.md deleted file mode 100644 index a019393abfd3..000000000000 --- a/model_cards/gpt2-medium-README.md +++ /dev/null @@ -1 +0,0 @@ -Test the full generation capabilities here: https://transformer.huggingface.co/doc/gpt2-large diff --git a/model_cards/gpt2-xl-README.md b/model_cards/gpt2-xl-README.md deleted file mode 100644 index bac6ec013773..000000000000 --- a/model_cards/gpt2-xl-README.md +++ /dev/null @@ -1 +0,0 @@ -Test the whole generation capabilities here: https://transformer.huggingface.co/doc/gpt2-large diff --git a/model_cards/gsarti/biobert-nli/README.md b/model_cards/gsarti/biobert-nli/README.md deleted file mode 100644 index 0936cb00a1ce..000000000000 --- a/model_cards/gsarti/biobert-nli/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# BioBERT-NLI - -This is the model [BioBERT](https://github.com/dmis-lab/biobert) [1] fine-tuned on the [SNLI](https://nlp.stanford.edu/projects/snli/) and the [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) datasets using the [`sentence-transformers` library](https://github.com/UKPLab/sentence-transformers/) to produce universal sentence embeddings [2]. - -The model uses the original BERT wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**. - -**Base model**: `monologg/biobert_v1.1_pubmed` from HuggingFace's `AutoModel`. - -**Training time**: ~6 hours on the NVIDIA Tesla P100 GPU provided in Kaggle Notebooks. - -**Parameters**: - -| Parameter | Value | -|------------------|-------| -| Batch size | 64 | -| Training steps | 30000 | -| Warmup steps | 1450 | -| Lowercasing | False | -| Max. Seq. Length | 128 | - -**Performances**: The performance was evaluated on the test portion of the [STS dataset](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) using Spearman rank correlation and compared to the performances of a general BERT base model obtained with the same procedure to verify their similarity. - -| Model | Score | -|-------------------------------|-------------| -| `biobert-nli` (this) | 73.40 | -| `gsarti/scibert-nli` | 74.50 | -| `bert-base-nli-mean-tokens`[3]| 77.12 | - -An example usage for similarity-based scientific paper retrieval is provided in the [Covid Papers Browser](https://github.com/gsarti/covid-papers-browser) repository. - -**References:** - -[1] J. Lee et al, [BioBERT: a pre-trained biomedical language representation model for biomedical text mining](https://academic.oup.com/bioinformatics/article/36/4/1234/5566506) - -[2] A. Conneau et al., [Supervised Learning of Universal Sentence Representations from Natural Language Inference Data](https://www.aclweb.org/anthology/D17-1070/) - -[3] N. Reimers et I. Gurevych, [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410/) diff --git a/model_cards/gsarti/covidbert-nli/README.md b/model_cards/gsarti/covidbert-nli/README.md deleted file mode 100644 index 45037dcbc0da..000000000000 --- a/model_cards/gsarti/covidbert-nli/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# CovidBERT-NLI - -This is the model **CovidBERT** trained by DeepSet on AllenAI's [CORD19 Dataset](https://pages.semanticscholar.org/coronavirus-research) of scientific articles about coronaviruses. - -The model uses the original BERT wordpiece vocabulary and was subsequently fine-tuned on the [SNLI](https://nlp.stanford.edu/projects/snli/) and the [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) datasets using the [`sentence-transformers` library](https://github.com/UKPLab/sentence-transformers/) to produce universal sentence embeddings [1] using the **average pooling strategy** and a **softmax loss**. - -Parameter details for the original training on CORD-19 are available on [DeepSet's MLFlow](https://public-mlflow.deepset.ai/#/experiments/2/runs/ba27d00c30044ef6a33b1d307b4a6cba) - -**Base model**: `deepset/covid_bert_base` from HuggingFace's `AutoModel`. - -**Training time**: ~6 hours on the NVIDIA Tesla P100 GPU provided in Kaggle Notebooks. - -**Parameters**: - -| Parameter | Value | -|------------------|-------| -| Batch size | 64 | -| Training steps | 23000 | -| Warmup steps | 1450 | -| Lowercasing | True | -| Max. Seq. Length | 128 | - -**Performances**: The performance was evaluated on the test portion of the [STS dataset](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) using Spearman rank correlation and compared to the performances of similar models obtained with the same procedure to verify its performances. - -| Model | Score | -|-------------------------------|-------------| -| `covidbert-nli` (this) | 67.52 | -| `gsarti/biobert-nli` | 73.40 | -| `gsarti/scibert-nli` | 74.50 | -| `bert-base-nli-mean-tokens`[2]| 77.12 | - -An example usage for similarity-based scientific paper retrieval is provided in the [Covid-19 Semantic Browser](https://github.com/gsarti/covid-papers-browser) repository. - -**References:** - -[1] A. Conneau et al., [Supervised Learning of Universal Sentence Representations from Natural Language Inference Data](https://www.aclweb.org/anthology/D17-1070/) - -[2] N. Reimers et I. Gurevych, [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410/) diff --git a/model_cards/gsarti/scibert-nli/README.md b/model_cards/gsarti/scibert-nli/README.md deleted file mode 100644 index 661d04452c49..000000000000 --- a/model_cards/gsarti/scibert-nli/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# SciBERT-NLI - -This is the model [SciBERT](https://github.com/allenai/scibert) [1] fine-tuned on the [SNLI](https://nlp.stanford.edu/projects/snli/) and the [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) datasets using the [`sentence-transformers` library](https://github.com/UKPLab/sentence-transformers/) to produce universal sentence embeddings [2]. - -The model uses the original `scivocab` wordpiece vocabulary and was trained using the **average pooling strategy** and a **softmax loss**. - -**Base model**: `allenai/scibert-scivocab-cased` from HuggingFace's `AutoModel`. - -**Training time**: ~4 hours on the NVIDIA Tesla P100 GPU provided in Kaggle Notebooks. - -**Parameters**: - -| Parameter | Value | -|------------------|-------| -| Batch size | 64 | -| Training steps | 20000 | -| Warmup steps | 1450 | -| Lowercasing | True | -| Max. Seq. Length | 128 | - -**Performances**: The performance was evaluated on the test portion of the [STS dataset](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) using Spearman rank correlation and compared to the performances of a general BERT base model obtained with the same procedure to verify their similarity. - -| Model | Score | -|-------------------------------|-------------| -| `scibert-nli` (this) | 74.50 | -| `bert-base-nli-mean-tokens`[3]| 77.12 | - -An example usage for similarity-based scientific paper retrieval is provided in the [Covid Papers Browser](https://github.com/gsarti/covid-papers-browser) repository. - -**References:** - -[1] I. Beltagy et al, [SciBERT: A Pretrained Language Model for Scientific Text](https://www.aclweb.org/anthology/D19-1371/) - -[2] A. Conneau et al., [Supervised Learning of Universal Sentence Representations from Natural Language Inference Data](https://www.aclweb.org/anthology/D17-1070/) - -[3] N. Reimers et I. Gurevych, [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://www.aclweb.org/anthology/D19-1410/) diff --git a/model_cards/gurkan08/bert-turkish-text-classification/README.md b/model_cards/gurkan08/bert-turkish-text-classification/README.md deleted file mode 100644 index e70af8b86d19..000000000000 --- a/model_cards/gurkan08/bert-turkish-text-classification/README.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -language: tr ---- -# Turkish News Text Classification - - Turkish text classification model obtained by fine-tuning the Turkish bert model (dbmdz/bert-base-turkish-cased) - -# Dataset - -Dataset consists of 11 classes were obtained from https://www.trthaber.com/. The model was created using the most distinctive 6 classes. - -Dataset can be accessed at https://github.com/gurkan08/datasets/tree/master/trt_11_category. - - label_dict = { - 'LABEL_0': 'ekonomi', - 'LABEL_1': 'spor', - 'LABEL_2': 'saglik', - 'LABEL_3': 'kultur_sanat', - 'LABEL_4': 'bilim_teknoloji', - 'LABEL_5': 'egitim' - } - -70% of the data were used for training and 30% for testing. - -train f1-weighted score = %97 - -test f1-weighted score = %94 - -# Usage - - from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification - - tokenizer = AutoTokenizer.from_pretrained("gurkan08/bert-turkish-text-classification") - model = AutoModelForSequenceClassification.from_pretrained("gurkan08/bert-turkish-text-classification") - - nlp = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer) - - text = ["Süper Lig'in 6. haftasında Sivasspor ile Çaykur Rizespor karşı karşıya geldi...", - "Son 24 saatte 69 kişi Kovid-19 nedeniyle yaşamını yitirdi, 1573 kişi iyileşti"] - - out = nlp(text) - - label_dict = { - 'LABEL_0': 'ekonomi', - 'LABEL_1': 'spor', - 'LABEL_2': 'saglik', - 'LABEL_3': 'kultur_sanat', - 'LABEL_4': 'bilim_teknoloji', - 'LABEL_5': 'egitim' - } - - results = [] - for result in out: - result['label'] = label_dict[result['label']] - results.append(result) - print(results) - - # > [{'label': 'spor', 'score': 0.9992026090621948}, {'label': 'saglik', 'score': 0.9972177147865295}] - - - diff --git a/model_cards/hatmimoha/arabic-ner/README.md b/model_cards/hatmimoha/arabic-ner/README.md deleted file mode 100644 index 2ea68808b980..000000000000 --- a/model_cards/hatmimoha/arabic-ner/README.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -language: ar ---- -# Arabic Named Entity Recognition Model - -Pretrained BERT-based ([arabic-bert-base](https://huggingface.co/asafaya/bert-base-arabic)) Named Entity Recognition model for Arabic. - -The pre-trained model can recognize the following entities: -1. **PERSON** - -- و هذا ما نفاه المعاون السياسي للرئيس ***نبيه بري*** ، النائب ***علي حسن خليل*** - -- لكن أوساط ***الحريري*** تعتبر أنه ضحى كثيرا في سبيل البلد - -- و ستفقد الملكة ***إليزابيث الثانية*** بذلك سيادتها على واحدة من آخر ممالك الكومنولث - -2. **ORGANIZATION** - -- حسب أرقام ***البنك الدولي*** - -- أعلن ***الجيش العراقي*** - -- و نقلت وكالة ***رويترز*** عن ثلاثة دبلوماسيين في ***الاتحاد الأوروبي*** ، أن ***بلجيكا*** و ***إيرلندا*** و ***لوكسمبورغ*** تريد أيضاً مناقشة - -- ***الحكومة الاتحادية*** و ***حكومة إقليم كردستان*** - -- و هو ما يثير الشكوك حول مشاركة النجم البرتغالي في المباراة المرتقبة أمام ***برشلونة*** الإسباني في - - -3. ***LOCATION*** - -- الجديد هو تمكين اللاجئين من “ مغادرة الجزيرة تدريجياً و بهدوء إلى ***أثينا*** ” - -- ***جزيرة ساكيز*** تبعد 1 كم عن ***إزمير*** - - -4. **DATE** - -- ***غدا الجمعة*** - -- ***06 أكتوبر 2020*** - -- ***العام السابق*** - - -5. **PRODUCT** - -- عبر حسابه ب ***تطبيق “ إنستغرام ”*** - -- الجيل الثاني من ***نظارة الواقع الافتراضي أوكولوس كويست*** تحت اسم " ***أوكولوس كويست 2*** " - - -6. **COMPETITION** - -- عدم المشاركة في ***بطولة فرنسا المفتوحة للتنس*** - -- في مباراة ***كأس السوبر الأوروبي*** - -7. **PRIZE** - -- ***جائزة نوبل ل لآداب*** - -- الذي فاز ب ***جائزة “ إيمي ” لأفضل دور مساند*** - -8. **EVENT** - -- تسجّل أغنية جديدة خاصة ب ***العيد الوطني السعودي*** - -- ***مهرجان المرأة يافوية*** في دورته الرابعة - -9. **DISEASE** - -- في مكافحة فيروس ***كورونا*** و عدد من الأمراض - -- الأزمات المشابهة مثل “ ***انفلونزا الطيور*** ” و ” ***انفلونزا الخنازير*** - -## Example - -[Find here a complete example to use this model](https://github.com/hatmimoha/arabic-ner) - -Here is the map from index to label: - -``` -id2label = { - "0": "B-PERSON", - "1": "I-PERSON", - "2": "B-ORGANIZATION", - "3": "I-ORGANIZATION", - "4": "B-LOCATION", - "5": "I-LOCATION", - "6": "B-DATE", - "7": "I-DATE"", - "8": "B-COMPETITION", - "9": "I-COMPETITION", - "10": "B-PRIZE", - "11": "I-PRIZE", - "12": "O", - "13": "B-PRODUCT", - "14": "I-PRODUCT", - "15": "B-EVENT", - "16": "I-EVENT", - "17": "B-DISEASE", - "18": "I-DISEASE", -} - -``` - -## Training Corpus - -The training corpus is made of 378.000 tokens (14.000 sentences) collected from the Web and annotated manually. - -## Results - -The results on a valid corpus made of 30.000 tokens shows an F-measure of ~87%. diff --git a/model_cards/healx/gpt-2-pubmed-large/README.md b/model_cards/healx/gpt-2-pubmed-large/README.md deleted file mode 100644 index 5dd99bd8d39e..000000000000 --- a/model_cards/healx/gpt-2-pubmed-large/README.md +++ /dev/null @@ -1,3 +0,0 @@ -GPT-2 (774M model) finetuned on 0.5m PubMed abstracts. Used in the [writemeanabstract.com](writemeanabstract.com) and the following preprint: - -[Papanikolaou, Yannis, and Andrea Pierleoni. "DARE: Data Augmented Relation Extraction with GPT-2." arXiv preprint arXiv:2004.13845 (2020).](https://arxiv.org/abs/2004.13845) diff --git a/model_cards/healx/gpt-2-pubmed-medium/README.md b/model_cards/healx/gpt-2-pubmed-medium/README.md deleted file mode 100644 index c161836ef286..000000000000 --- a/model_cards/healx/gpt-2-pubmed-medium/README.md +++ /dev/null @@ -1,3 +0,0 @@ -GPT-2 (355M model) finetuned on 0.5m PubMed abstracts. Used in the [writemeanabstract.com](writemeanabstract.com) and the following preprint: - -[Papanikolaou, Yannis, and Andrea Pierleoni. "DARE: Data Augmented Relation Extraction with GPT-2." arXiv preprint arXiv:2004.13845 (2020).](https://arxiv.org/abs/2004.13845) diff --git a/model_cards/henryk/bert-base-multilingual-cased-finetuned-dutch-squad2/README.md b/model_cards/henryk/bert-base-multilingual-cased-finetuned-dutch-squad2/README.md deleted file mode 100644 index 621dbda8d324..000000000000 --- a/model_cards/henryk/bert-base-multilingual-cased-finetuned-dutch-squad2/README.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -language: nl ---- - -# Multilingual + Dutch SQuAD2.0 - -This model is the multilingual model provided by the Google research team with a fine-tuned dutch Q&A downstream task. - -## Details of the language model - -Language model ([**bert-base-multilingual-cased**](https://github.com/google-research/bert/blob/master/multilingual.md)): -12-layer, 768-hidden, 12-heads, 110M parameters. -Trained on cased text in the top 104 languages with the largest Wikipedias. - -## Details of the downstream task -Using the `mtranslate` Python module, [**SQuAD2.0**](https://rajpurkar.github.io/SQuAD-explorer/) was machine-translated. In order to find the start tokens, the direct translations of the answers were searched in the corresponding paragraphs. Due to the different translations depending on the context (missing context in the pure answer), the answer could not always be found in the text, and thus a loss of question-answer examples occurred. This is a potential problem where errors can occur in the data set. - -| Dataset | # Q&A | -| ---------------------- | ----- | -| SQuAD2.0 Train | 130 K | -| Dutch SQuAD2.0 Train | 99 K | -| SQuAD2.0 Dev | 12 K | -| Dutch SQuAD2.0 Dev | 10 K | - - -## Model benchmark - - -| Model | EM/F1 |HasAns (EM/F1) | NoAns | -| ---------------------- | ----- | ----- | ----- | -| [robBERT](https://huggingface.co/pdelobelle/robBERT-base) | 58.04/60.95 | 33.08/40.64 | 73.67 | -| [dutchBERT](https://huggingface.co/wietsedv/bert-base-dutch-cased) | 64.25/68.45 | 45.59/56.49 | 75.94 | -| [multiBERT](https://huggingface.co/bert-base-multilingual-cased) | **67.38**/**71.36** | 47.42/57.76 | 79.88 | - -## Model training - -The model was trained on a **Tesla V100** GPU with the following command: - -```python -export SQUAD_DIR=path/to/nl_squad - -python run_squad.py - --model_type bert \ - --model_name_or_path bert-base-multilingual-cased \ - --do_train \ - --do_eval \ - --train_file $SQUAD_DIR/nl_squadv2_train_clean.json \ - --predict_file $SQUAD_DIR/nl_squadv2_dev_clean.json \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --save_steps=8000 \ - --output_dir ../../output \ - --overwrite_cache \ - --overwrite_output_dir -``` - -**Results**: - -{'exact': 67.38028751680629, 'f1': 71.362297054268, 'total': 9669, 'HasAns_exact': 47.422126745435015, 'HasAns_f1': 57.761023151910734, 'HasAns_total': 3724, 'NoAns_exact': 79.88225399495374, 'NoAns_f1': 79.88225399495374, 'NoAns_total': 5945, 'best_exact': 67.53542248422795, 'best_exact_thresh': 0.0, 'best_f1': 71.36229705426837, 'best_f1_thresh': 0.0} - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="henryk/bert-base-multilingual-cased-finetuned-dutch-squad2", - tokenizer="henryk/bert-base-multilingual-cased-finetuned-dutch-squad2" -) - -qa_pipeline({ - 'context': "Amsterdam is de hoofdstad en de dichtstbevolkte stad van Nederland.", - 'question': "Wat is de hoofdstad van Nederland?"}) - -``` - -# Output: - -```json -{ - "score": 0.83, - "start": 0, - "end": 9, - "answer": "Amsterdam" -} -``` - -## Contact - -Please do not hesitate to contact me via [LinkedIn](https://www.linkedin.com/in/henryk-borzymowski-0755a2167/) if you want to discuss or get access to the Dutch version of SQuAD. \ No newline at end of file diff --git a/model_cards/henryk/bert-base-multilingual-cased-finetuned-polish-squad1/README.md b/model_cards/henryk/bert-base-multilingual-cased-finetuned-polish-squad1/README.md deleted file mode 100644 index d1ec8ab30783..000000000000 --- a/model_cards/henryk/bert-base-multilingual-cased-finetuned-polish-squad1/README.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -language: pl ---- - -# Multilingual + Polish SQuAD1.1 - -This model is the multilingual model provided by the Google research team with a fine-tuned polish Q&A downstream task. - -## Details of the language model - -Language model ([**bert-base-multilingual-cased**](https://github.com/google-research/bert/blob/master/multilingual.md)): -12-layer, 768-hidden, 12-heads, 110M parameters. -Trained on cased text in the top 104 languages with the largest Wikipedias. - -## Details of the downstream task -Using the `mtranslate` Python module, [**SQuAD1.1**](https://rajpurkar.github.io/SQuAD-explorer/) was machine-translated. In order to find the start tokens, the direct translations of the answers were searched in the corresponding paragraphs. Due to the different translations depending on the context (missing context in the pure answer), the answer could not always be found in the text, and thus a loss of question-answer examples occurred. This is a potential problem where errors can occur in the data set. - -| Dataset | # Q&A | -| ---------------------- | ----- | -| SQuAD1.1 Train | 87.7 K | -| Polish SQuAD1.1 Train | 39.5 K | -| SQuAD1.1 Dev | 10.6 K | -| Polish SQuAD1.1 Dev | 2.6 K | - - -## Model benchmark - -| Model | EM | F1 | -| ---------------------- | ----- | ----- | -| [SlavicBERT](https://huggingface.co/DeepPavlov/bert-base-bg-cs-pl-ru-cased) | **60.89** | 71.68 | -| [polBERT](https://huggingface.co/dkleczek/bert-base-polish-uncased-v1) | 57.46 | 68.87 | -| [multiBERT](https://huggingface.co/bert-base-multilingual-cased) | 60.67 | **71.89** | -| [xlm](https://huggingface.co/xlm-mlm-100-1280) | 47.98 | 59.42 | -## Model training - -The model was trained on a **Tesla V100** GPU with the following command: - -```python -export SQUAD_DIR=path/to/pl_squad - -python run_squad.py - --model_type bert \ - --model_name_or_path bert-base-multilingual-cased \ - --do_train \ - --do_eval \ - --train_file $SQUAD_DIR/pl_squadv1_train_clean.json \ - --predict_file $SQUAD_DIR/pl_squadv1_dev_clean.json \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --save_steps=8000 \ - --output_dir ../../output \ - --overwrite_cache \ - --overwrite_output_dir -``` - -**Results**: - -{'exact': 60.670731707317074, 'f1': 71.8952193697293, 'total': 2624, 'HasAns_exact': 60.670731707317074, 'HasAns_f1': 71.8952193697293, -'HasAns_total': 2624, 'best_exact': 60.670731707317074, 'best_exact_thresh': 0.0, 'best_f1': 71.8952193697293, 'best_f1_thresh': 0.0} - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="henryk/bert-base-multilingual-cased-finetuned-polish-squad1", - tokenizer="henryk/bert-base-multilingual-cased-finetuned-polish-squad1" -) - -qa_pipeline({ - 'context': "Warszawa jest największym miastem w Polsce pod względem liczby ludności i powierzchni", - 'question': "Jakie jest największe miasto w Polsce?"}) - -``` - -# Output: - -```json -{ - "score": 0.9988, - "start": 0, - "end": 8, - "answer": "Warszawa" -} -``` - -## Contact - -Please do not hesitate to contact me via [LinkedIn](https://www.linkedin.com/in/henryk-borzymowski-0755a2167/) if you want to discuss or get access to the Polish version of SQuAD. \ No newline at end of file diff --git a/model_cards/henryk/bert-base-multilingual-cased-finetuned-polish-squad2/README.md b/model_cards/henryk/bert-base-multilingual-cased-finetuned-polish-squad2/README.md deleted file mode 100644 index 6ef1ab48e062..000000000000 --- a/model_cards/henryk/bert-base-multilingual-cased-finetuned-polish-squad2/README.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -language: pl ---- - -# Multilingual + Polish SQuAD2.0 - -This model is the multilingual model provided by the Google research team with a fine-tuned polish Q&A downstream task. - -## Details of the language model - -Language model ([**bert-base-multilingual-cased**](https://github.com/google-research/bert/blob/master/multilingual.md)): -12-layer, 768-hidden, 12-heads, 110M parameters. -Trained on cased text in the top 104 languages with the largest Wikipedias. - -## Details of the downstream task -Using the `mtranslate` Python module, [**SQuAD2.0**](https://rajpurkar.github.io/SQuAD-explorer/) was machine-translated. In order to find the start tokens, the direct translations of the answers were searched in the corresponding paragraphs. Due to the different translations depending on the context (missing context in the pure answer), the answer could not always be found in the text, and thus a loss of question-answer examples occurred. This is a potential problem where errors can occur in the data set. - -| Dataset | # Q&A | -| ---------------------- | ----- | -| SQuAD2.0 Train | 130 K | -| Polish SQuAD2.0 Train | 83.1 K | -| SQuAD2.0 Dev | 12 K | -| Polish SQuAD2.0 Dev | 8.5 K | - - -## Model benchmark - -| Model | EM/F1 |HasAns (EM/F1) | NoAns | -| ---------------------- | ----- | ----- | ----- | -| [SlavicBERT](https://huggingface.co/DeepPavlov/bert-base-bg-cs-pl-ru-cased) | 69.35/71.51 | 47.02/54.09 | 79.20 | -| [polBERT](https://huggingface.co/dkleczek/bert-base-polish-uncased-v1) | 67.33/69.80| 45.73/53.80 | 76.87 | -| [multiBERT](https://huggingface.co/bert-base-multilingual-cased) | **70.76**/**72.92** |45.00/52.04 | 82.13 | - -## Model training - -The model was trained on a **Tesla V100** GPU with the following command: - -```python -export SQUAD_DIR=path/to/pl_squad - -python run_squad.py - --model_type bert \ - --model_name_or_path bert-base-multilingual-cased \ - --do_train \ - --do_eval \ - --version_2_with_negative \ - --train_file $SQUAD_DIR/pl_squadv2_train.json \ - --predict_file $SQUAD_DIR/pl_squadv2_dev.json \ - --num_train_epochs 2 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --save_steps=8000 \ - --output_dir ../../output \ - --overwrite_cache \ - --overwrite_output_dir -``` - -**Results**: - -{'exact': 70.76671723655035, 'f1': 72.92156947155917, 'total': 8569, 'HasAns_exact': 45.00762195121951, 'HasAns_f1': 52.04456128116991, 'HasAns_total': 2624, 'NoAns_exact': 82.13624894869638, ' -NoAns_f1': 82.13624894869638, 'NoAns_total': 5945, 'best_exact': 71.72365503559342, 'best_exact_thresh': 0.0, 'best_f1': 73.62662512059369, 'best_f1_thresh': 0.0} - - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="henryk/bert-base-multilingual-cased-finetuned-polish-squad2", - tokenizer="henryk/bert-base-multilingual-cased-finetuned-polish-squad2" -) - -qa_pipeline({ - 'context': "Warszawa jest największym miastem w Polsce pod względem liczby ludności i powierzchni", - 'question': "Jakie jest największe miasto w Polsce?"}) - -``` - -# Output: - -```json -{ - "score": 0.9986, - "start": 0, - "end": 8, - "answer": "Warszawa" -} -``` - -## Contact - -Please do not hesitate to contact me via [LinkedIn](https://www.linkedin.com/in/henryk-borzymowski-0755a2167/) if you want to discuss or get access to the Polish version of SQuAD. \ No newline at end of file diff --git a/model_cards/huawei-noah/DynaBERT_MNLI/README.md b/model_cards/huawei-noah/DynaBERT_MNLI/README.md deleted file mode 100644 index a9a4bc233ded..000000000000 --- a/model_cards/huawei-noah/DynaBERT_MNLI/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## DynaBERT: Dynamic BERT with Adaptive Width and Depth - -* DynaBERT can flexibly adjust the size and latency by selecting adaptive width and depth, and -the subnetworks of it have competitive performances as other similar-sized compressed models. -The training process of DynaBERT includes first training a width-adaptive BERT and then -allowing both adaptive width and depth using knowledge distillation. - -* This code is modified based on the repository developed by Hugging Face: [Transformers v2.1.1](https://github.com/huggingface/transformers/tree/v2.1.1), and is released in [GitHub](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/DynaBERT). - -### Reference -Lu Hou, Zhiqi Huang, Lifeng Shang, Xin Jiang, Xiao Chen, Qun Liu. -[DynaBERT: Dynamic BERT with Adaptive Width and Depth](https://arxiv.org/abs/2004.04037). -``` -@inproceedings{hou2020dynabert, - title = {DynaBERT: Dynamic BERT with Adaptive Width and Depth}, - author = {Lu Hou, Zhiqi Huang, Lifeng Shang, Xin Jiang, Xiao Chen, Qun Liu}, - booktitle = {Advances in Neural Information Processing Systems}, - year = {2020} -} -``` diff --git a/model_cards/huawei-noah/DynaBERT_SST-2/README.md b/model_cards/huawei-noah/DynaBERT_SST-2/README.md deleted file mode 100644 index a9a4bc233ded..000000000000 --- a/model_cards/huawei-noah/DynaBERT_SST-2/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## DynaBERT: Dynamic BERT with Adaptive Width and Depth - -* DynaBERT can flexibly adjust the size and latency by selecting adaptive width and depth, and -the subnetworks of it have competitive performances as other similar-sized compressed models. -The training process of DynaBERT includes first training a width-adaptive BERT and then -allowing both adaptive width and depth using knowledge distillation. - -* This code is modified based on the repository developed by Hugging Face: [Transformers v2.1.1](https://github.com/huggingface/transformers/tree/v2.1.1), and is released in [GitHub](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/DynaBERT). - -### Reference -Lu Hou, Zhiqi Huang, Lifeng Shang, Xin Jiang, Xiao Chen, Qun Liu. -[DynaBERT: Dynamic BERT with Adaptive Width and Depth](https://arxiv.org/abs/2004.04037). -``` -@inproceedings{hou2020dynabert, - title = {DynaBERT: Dynamic BERT with Adaptive Width and Depth}, - author = {Lu Hou, Zhiqi Huang, Lifeng Shang, Xin Jiang, Xiao Chen, Qun Liu}, - booktitle = {Advances in Neural Information Processing Systems}, - year = {2020} -} -``` diff --git a/model_cards/huawei-noah/TinyBERT_General_4L_312D/README.md b/model_cards/huawei-noah/TinyBERT_General_4L_312D/README.md deleted file mode 100644 index 8ea4061db3b0..000000000000 --- a/model_cards/huawei-noah/TinyBERT_General_4L_312D/README.md +++ /dev/null @@ -1,19 +0,0 @@ -TinyBERT: Distilling BERT for Natural Language Understanding -======== -TinyBERT is 7.5x smaller and 9.4x faster on inference than BERT-base and achieves competitive performances in the tasks of natural language understanding. It performs a novel transformer distillation at both the pre-training and task-specific learning stages. In general distillation, we use the original BERT-base without fine-tuning as the teacher and a large-scale text corpus as the learning data. By performing the Transformer distillation on the text from general domain, we obtain a general TinyBERT which provides a good initialization for the task-specific distillation. We here provide the general TinyBERT for your tasks at hand. - -For more details about the techniques of TinyBERT, refer to our paper: -[TinyBERT: Distilling BERT for Natural Language Understanding](https://arxiv.org/abs/1909.10351) - - -Citation -======== -If you find TinyBERT useful in your research, please cite the following paper: -``` -@article{jiao2019tinybert, - title={Tinybert: Distilling bert for natural language understanding}, - author={Jiao, Xiaoqi and Yin, Yichun and Shang, Lifeng and Jiang, Xin and Chen, Xiao and Li, Linlin and Wang, Fang and Liu, Qun}, - journal={arXiv preprint arXiv:1909.10351}, - year={2019} -} -``` diff --git a/model_cards/huggingface/CodeBERTa-language-id/README.md b/model_cards/huggingface/CodeBERTa-language-id/README.md deleted file mode 100644 index c9234f2cf861..000000000000 --- a/model_cards/huggingface/CodeBERTa-language-id/README.md +++ /dev/null @@ -1,300 +0,0 @@ ---- -language: code -thumbnail: https://cdn-media.huggingface.co/CodeBERTa/CodeBERTa.png -datasets: -- code_search_net ---- - -# CodeBERTa-language-id: The World’s fanciest programming language identification algo 🤯 - - -To demonstrate the usefulness of our CodeBERTa pretrained model on downstream tasks beyond language modeling, we fine-tune the [`CodeBERTa-small-v1`](https://huggingface.co/huggingface/CodeBERTa-small-v1) checkpoint on the task of classifying a sample of code into the programming language it's written in (*programming language identification*). - -We add a sequence classification head on top of the model. - -On the evaluation dataset, we attain an eval accuracy and F1 > 0.999 which is not surprising given that the task of language identification is relatively easy (see an intuition why, below). - -## Quick start: using the raw model - -```python -CODEBERTA_LANGUAGE_ID = "huggingface/CodeBERTa-language-id" - -tokenizer = RobertaTokenizer.from_pretrained(CODEBERTA_LANGUAGE_ID) -model = RobertaForSequenceClassification.from_pretrained(CODEBERTA_LANGUAGE_ID) - -input_ids = tokenizer.encode(CODE_TO_IDENTIFY) -logits = model(input_ids)[0] - -language_idx = logits.argmax() # index for the resulting label -``` - - -## Quick start: using Pipelines 💪 - -```python -from transformers import TextClassificationPipeline - -pipeline = TextClassificationPipeline( - model=RobertaForSequenceClassification.from_pretrained(CODEBERTA_LANGUAGE_ID), - tokenizer=RobertaTokenizer.from_pretrained(CODEBERTA_LANGUAGE_ID) -) - -pipeline(CODE_TO_IDENTIFY) -``` - -Let's start with something very easy: - -```python -pipeline(""" -def f(x): - return x**2 -""") -# [{'label': 'python', 'score': 0.9999965}] -``` - -Now let's probe shorter code samples: - -```python -pipeline("const foo = 'bar'") -# [{'label': 'javascript', 'score': 0.9977546}] -``` - -What if I remove the `const` token from the assignment? -```python -pipeline("foo = 'bar'") -# [{'label': 'javascript', 'score': 0.7176245}] -``` - -For some reason, this is still statistically detected as JS code, even though it's also valid Python code. However, if we slightly tweak it: - -```python -pipeline("foo = u'bar'") -# [{'label': 'python', 'score': 0.7638422}] -``` -This is now detected as Python (Notice the `u` string modifier). - -Okay, enough with the JS and Python domination already! Let's try fancier languages: - -```python -pipeline("echo $FOO") -# [{'label': 'php', 'score': 0.9995257}] -``` - -(Yes, I used the word "fancy" to describe PHP 😅) - -```python -pipeline("outcome := rand.Intn(6) + 1") -# [{'label': 'go', 'score': 0.9936151}] -``` - -Why is the problem of language identification so easy (with the correct toolkit)? Because code's syntax is rigid, and simple tokens such as `:=` (the assignment operator in Go) are perfect predictors of the underlying language: - -```python -pipeline(":=") -# [{'label': 'go', 'score': 0.9998052}] -``` - -By the way, because we trained our own custom tokenizer on the [CodeSearchNet](https://github.blog/2019-09-26-introducing-the-codesearchnet-challenge/) dataset, and it handles streams of bytes in a very generic way, syntactic constructs such `:=` are represented by a single token: - -```python -self.tokenizer.encode(" :=", add_special_tokens=False) -# [521] -``` - -
- -## Fine-tuning code - -
- -```python -import gzip -import json -import logging -import os -from pathlib import Path -from typing import Dict, List, Tuple - -import numpy as np -import torch -from sklearn.metrics import f1_score -from tokenizers.implementations.byte_level_bpe import ByteLevelBPETokenizer -from tokenizers.processors import BertProcessing -from torch.nn.utils.rnn import pad_sequence -from torch.utils.data import DataLoader, Dataset -from torch.utils.data.dataset import Dataset -from torch.utils.tensorboard.writer import SummaryWriter -from tqdm import tqdm, trange - -from transformers import RobertaForSequenceClassification -from transformers.data.metrics import acc_and_f1, simple_accuracy - - -logging.basicConfig(level=logging.INFO) - - -CODEBERTA_PRETRAINED = "huggingface/CodeBERTa-small-v1" - -LANGUAGES = [ - "go", - "java", - "javascript", - "php", - "python", - "ruby", -] -FILES_PER_LANGUAGE = 1 -EVALUATE = True - -# Set up tokenizer -tokenizer = ByteLevelBPETokenizer("./pretrained/vocab.json", "./pretrained/merges.txt",) -tokenizer._tokenizer.post_processor = BertProcessing( - ("", tokenizer.token_to_id("")), ("", tokenizer.token_to_id("")), -) -tokenizer.enable_truncation(max_length=512) - -# Set up Tensorboard -tb_writer = SummaryWriter() - - -class CodeSearchNetDataset(Dataset): - examples: List[Tuple[List[int], int]] - - def __init__(self, split: str = "train"): - """ - train | valid | test - """ - - self.examples = [] - - src_files = [] - for language in LANGUAGES: - src_files += list( - Path("../CodeSearchNet/resources/data/").glob(f"{language}/final/jsonl/{split}/*.jsonl.gz") - )[:FILES_PER_LANGUAGE] - for src_file in src_files: - label = src_file.parents[3].name - label_idx = LANGUAGES.index(label) - print("🔥", src_file, label) - lines = [] - fh = gzip.open(src_file, mode="rt", encoding="utf-8") - for line in fh: - o = json.loads(line) - lines.append(o["code"]) - examples = [(x.ids, label_idx) for x in tokenizer.encode_batch(lines)] - self.examples += examples - print("🔥🔥") - - def __len__(self): - return len(self.examples) - - def __getitem__(self, i): - # We’ll pad at the batch level. - return self.examples[i] - - -model = RobertaForSequenceClassification.from_pretrained(CODEBERTA_PRETRAINED, num_labels=len(LANGUAGES)) - -train_dataset = CodeSearchNetDataset(split="train") -eval_dataset = CodeSearchNetDataset(split="test") - - -def collate(examples): - input_ids = pad_sequence([torch.tensor(x[0]) for x in examples], batch_first=True, padding_value=1) - labels = torch.tensor([x[1] for x in examples]) - # ^^ uncessary .unsqueeze(-1) - return input_ids, labels - - -train_dataloader = DataLoader(train_dataset, batch_size=256, shuffle=True, collate_fn=collate) - -batch = next(iter(train_dataloader)) - - -model.to("cuda") -model.train() -for param in model.roberta.parameters(): - param.requires_grad = False -## ^^ Only train final layer. - -print(f"num params:", model.num_parameters()) -print(f"num trainable params:", model.num_parameters(only_trainable=True)) - - -def evaluate(): - eval_loss = 0.0 - nb_eval_steps = 0 - preds = np.empty((0), dtype=np.int64) - out_label_ids = np.empty((0), dtype=np.int64) - - model.eval() - - eval_dataloader = DataLoader(eval_dataset, batch_size=512, collate_fn=collate) - for step, (input_ids, labels) in enumerate(tqdm(eval_dataloader, desc="Eval")): - with torch.no_grad(): - outputs = model(input_ids=input_ids.to("cuda"), labels=labels.to("cuda")) - loss = outputs[0] - logits = outputs[1] - eval_loss += loss.mean().item() - nb_eval_steps += 1 - preds = np.append(preds, logits.argmax(dim=1).detach().cpu().numpy(), axis=0) - out_label_ids = np.append(out_label_ids, labels.detach().cpu().numpy(), axis=0) - eval_loss = eval_loss / nb_eval_steps - acc = simple_accuracy(preds, out_label_ids) - f1 = f1_score(y_true=out_label_ids, y_pred=preds, average="macro") - print("=== Eval: loss ===", eval_loss) - print("=== Eval: acc. ===", acc) - print("=== Eval: f1 ===", f1) - # print(acc_and_f1(preds, out_label_ids)) - tb_writer.add_scalars("eval", {"loss": eval_loss, "acc": acc, "f1": f1}, global_step) - - -### Training loop - -global_step = 0 -train_iterator = trange(0, 4, desc="Epoch") -optimizer = torch.optim.AdamW(model.parameters()) -for _ in train_iterator: - epoch_iterator = tqdm(train_dataloader, desc="Iteration") - for step, (input_ids, labels) in enumerate(epoch_iterator): - optimizer.zero_grad() - outputs = model(input_ids=input_ids.to("cuda"), labels=labels.to("cuda")) - loss = outputs[0] - loss.backward() - tb_writer.add_scalar("training_loss", loss.item(), global_step) - optimizer.step() - global_step += 1 - if EVALUATE and global_step % 50 == 0: - evaluate() - model.train() - - -evaluate() - -os.makedirs("./models/CodeBERT-language-id", exist_ok=True) -model.save_pretrained("./models/CodeBERT-language-id") -``` - -
- -
- -## CodeSearchNet citation - -
- -```bibtex -@article{husain_codesearchnet_2019, - title = {{CodeSearchNet} {Challenge}: {Evaluating} the {State} of {Semantic} {Code} {Search}}, - shorttitle = {{CodeSearchNet} {Challenge}}, - url = {http://arxiv.org/abs/1909.09436}, - urldate = {2020-03-12}, - journal = {arXiv:1909.09436 [cs, stat]}, - author = {Husain, Hamel and Wu, Ho-Hsiang and Gazit, Tiferet and Allamanis, Miltiadis and Brockschmidt, Marc}, - month = sep, - year = {2019}, - note = {arXiv: 1909.09436}, -} -``` - -
diff --git a/model_cards/huggingface/CodeBERTa-small-v1/README.md b/model_cards/huggingface/CodeBERTa-small-v1/README.md deleted file mode 100644 index 70943e7a5f46..000000000000 --- a/model_cards/huggingface/CodeBERTa-small-v1/README.md +++ /dev/null @@ -1,130 +0,0 @@ ---- -language: code -thumbnail: https://cdn-media.huggingface.co/CodeBERTa/CodeBERTa.png -datasets: -- code_search_net ---- - -# CodeBERTa - -CodeBERTa is a RoBERTa-like model trained on the [CodeSearchNet](https://github.blog/2019-09-26-introducing-the-codesearchnet-challenge/) dataset from GitHub. - -Supported languages: - -```shell -"go" -"java" -"javascript" -"php" -"python" -"ruby" -``` - -The **tokenizer** is a Byte-level BPE tokenizer trained on the corpus using Hugging Face `tokenizers`. - -Because it is trained on a corpus of code (vs. natural language), it encodes the corpus efficiently (the sequences are between 33% to 50% shorter, compared to the same corpus tokenized by gpt2/roberta). - -The (small) **model** is a 6-layer, 84M parameters, RoBERTa-like Transformer model – that’s the same number of layers & heads as DistilBERT – initialized from the default initialization settings and trained from scratch on the full corpus (~2M functions) for 5 epochs. - -### Tensorboard for this training ⤵️ - -[![tb](https://cdn-media.huggingface.co/CodeBERTa/tensorboard.png)](https://tensorboard.dev/experiment/irRI7jXGQlqmlxXS0I07ew/#scalars) - -## Quick start: masked language modeling prediction - -```python -PHP_CODE = """ -public static set(string $key, $value) { - if (!in_array($key, self::$allowedKeys)) { - throw new \InvalidArgumentException('Invalid key given'); - } - self::$storedValues[$key] = $value; -} -""".lstrip() -``` - -### Does the model know how to complete simple PHP code? - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="huggingface/CodeBERTa-small-v1", - tokenizer="huggingface/CodeBERTa-small-v1" -) - -fill_mask(PHP_CODE) - -## Top 5 predictions: -# -' function' # prob 0.9999827146530151 -'function' # -' void' # -' def' # -' final' # -``` - -### Yes! That was easy 🎉 What about some Python (warning: this is going to be meta) - -```python -PYTHON_CODE = """ -def pipeline( - task: str, - model: Optional = None, - framework: Optional[] = None, - **kwargs -) -> Pipeline: - pass -""".lstrip() -``` - -Results: -```python -'framework', 'Framework', ' framework', 'None', 'str' -``` - -> This program can auto-complete itself! 😱 - -### Just for fun, let's try to mask natural language (not code): - -```python -fill_mask("My name is .") - -# {'sequence': ' My name is undefined.', 'score': 0.2548016905784607, 'token': 3353} -# {'sequence': ' My name is required.', 'score': 0.07290805131196976, 'token': 2371} -# {'sequence': ' My name is null.', 'score': 0.06323737651109695, 'token': 469} -# {'sequence': ' My name is name.', 'score': 0.021919190883636475, 'token': 652} -# {'sequence': ' My name is disabled.', 'score': 0.019681859761476517, 'token': 7434} -``` - -This (kind of) works because code contains comments (which contain natural language). - -Of course, the most frequent name for a Computer scientist must be undefined 🤓. - - -## Downstream task: [programming language identification](https://huggingface.co/huggingface/CodeBERTa-language-id) - -See the model card for **[`huggingface/CodeBERTa-language-id`](https://huggingface.co/huggingface/CodeBERTa-language-id)** 🤯. - -
- -## CodeSearchNet citation - -
- -```bibtex -@article{husain_codesearchnet_2019, - title = {{CodeSearchNet} {Challenge}: {Evaluating} the {State} of {Semantic} {Code} {Search}}, - shorttitle = {{CodeSearchNet} {Challenge}}, - url = {http://arxiv.org/abs/1909.09436}, - urldate = {2020-03-12}, - journal = {arXiv:1909.09436 [cs, stat]}, - author = {Husain, Hamel and Wu, Ho-Hsiang and Gazit, Tiferet and Allamanis, Miltiadis and Brockschmidt, Marc}, - month = sep, - year = {2019}, - note = {arXiv: 1909.09436}, -} -``` - -
diff --git a/model_cards/huseinzol05/albert-base-bahasa-cased/README.md b/model_cards/huseinzol05/albert-base-bahasa-cased/README.md deleted file mode 100644 index 0ba3c068e326..000000000000 --- a/model_cards/huseinzol05/albert-base-bahasa-cased/README.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -language: ms ---- - -# Bahasa Albert Model - -Pretrained Albert base language model for Malay and Indonesian. - -## Pretraining Corpus - -`albert-base-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google Albert's github [repository](https://github.com/google-research/ALBERT) on v3-8 TPU. -- All steps can reproduce from here, [Malaya/pretrained-model/albert](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/albert). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import AlbertTokenizer, AlbertModel - -model = BertModel.from_pretrained('huseinzol05/albert-base-bahasa-cased') -tokenizer = AlbertTokenizer.from_pretrained( - 'huseinzol05/albert-base-bahasa-cased', - do_lower_case = False, -) -``` - -## Example using AutoModelWithLMHead - -```python -from transformers import AlbertTokenizer, AutoModelWithLMHead, pipeline - -model = AutoModelWithLMHead.from_pretrained('huseinzol05/albert-base-bahasa-cased') -tokenizer = AlbertTokenizer.from_pretrained( - 'huseinzol05/albert-base-bahasa-cased', - do_lower_case = False, -) -fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer) -print(fill_mask('makan ayam dengan [MASK]')) -``` - -Output is, - -```text -[{'sequence': '[CLS] makan ayam dengan ayam[SEP]', - 'score': 0.044952988624572754, - 'token': 629}, - {'sequence': '[CLS] makan ayam dengan sayur[SEP]', - 'score': 0.03621877357363701, - 'token': 1639}, - {'sequence': '[CLS] makan ayam dengan ikan[SEP]', - 'score': 0.034429922699928284, - 'token': 758}, - {'sequence': '[CLS] makan ayam dengan nasi[SEP]', - 'score': 0.032447945326566696, - 'token': 453}, - {'sequence': '[CLS] makan ayam dengan rendang[SEP]', - 'score': 0.028885239735245705, - 'token': 2451}] -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train Albert for Bahasa. - - diff --git a/model_cards/huseinzol05/albert-tiny-bahasa-cased/README.md b/model_cards/huseinzol05/albert-tiny-bahasa-cased/README.md deleted file mode 100644 index 9fd3a1553354..000000000000 --- a/model_cards/huseinzol05/albert-tiny-bahasa-cased/README.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -language: ms ---- - -# Bahasa Albert Model - -Pretrained Albert tiny language model for Malay and Indonesian, 85% faster execution and 50% smaller than Albert base. - -## Pretraining Corpus - -`albert-tiny-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google Albert's github [repository](https://github.com/google-research/ALBERT) on v3-8 TPU. -- All steps can reproduce from here, [Malaya/pretrained-model/albert](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/albert). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import AlbertTokenizer, AlbertModel - -model = BertModel.from_pretrained('huseinzol05/albert-tiny-bahasa-cased') -tokenizer = AlbertTokenizer.from_pretrained( - 'huseinzol05/albert-tiny-bahasa-cased', - do_lower_case = False, -) -``` - -## Example using AutoModelWithLMHead - -```python -from transformers import AlbertTokenizer, AutoModelWithLMHead, pipeline - -model = AutoModelWithLMHead.from_pretrained('huseinzol05/albert-tiny-bahasa-cased') -tokenizer = AlbertTokenizer.from_pretrained( - 'huseinzol05/albert-tiny-bahasa-cased', - do_lower_case = False, -) -fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer) -print(fill_mask('makan ayam dengan [MASK]')) -``` - -Output is, - -```text -[{'sequence': '[CLS] makan ayam dengan ayam[SEP]', - 'score': 0.05121927708387375, - 'token': 629}, - {'sequence': '[CLS] makan ayam dengan sayur[SEP]', - 'score': 0.04497420787811279, - 'token': 1639}, - {'sequence': '[CLS] makan ayam dengan nasi[SEP]', - 'score': 0.039827536791563034, - 'token': 453}, - {'sequence': '[CLS] makan ayam dengan rendang[SEP]', - 'score': 0.032997727394104004, - 'token': 2451}, - {'sequence': '[CLS] makan ayam dengan makan[SEP]', - 'score': 0.031354598701000214, - 'token': 129}] -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train Albert for Bahasa. - - diff --git a/model_cards/huseinzol05/bert-base-bahasa-cased/README.md b/model_cards/huseinzol05/bert-base-bahasa-cased/README.md deleted file mode 100644 index f119bd2ec28b..000000000000 --- a/model_cards/huseinzol05/bert-base-bahasa-cased/README.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -language: ms ---- - -# Bahasa BERT Model - -Pretrained BERT base language model for Malay and Indonesian. - -## Pretraining Corpus - -`bert-base-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google BERT's github [repository](https://github.com/google-research/bert) on 3 Titan V100 32GB VRAM. -- All steps can reproduce from here, [Malaya/pretrained-model/bert](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/bert). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import AlbertTokenizer, BertModel - -model = BertModel.from_pretrained('huseinzol05/bert-base-bahasa-cased') -tokenizer = AlbertTokenizer.from_pretrained( - 'huseinzol05/bert-base-bahasa-cased', - unk_token = '[UNK]', - pad_token = '[PAD]', - do_lower_case = False, -) -``` - -We use [google/sentencepiece](https://github.com/google/sentencepiece) to train the tokenizer, so to use it, need to load from `AlbertTokenizer`. - -## Example using AutoModelWithLMHead - -```python -from transformers import AlbertTokenizer, AutoModelWithLMHead, pipeline - -model = AutoModelWithLMHead.from_pretrained('huseinzol05/bert-base-bahasa-cased') -tokenizer = AlbertTokenizer.from_pretrained( - 'huseinzol05/bert-base-bahasa-cased', - unk_token = '[UNK]', - pad_token = '[PAD]', - do_lower_case = False, -) -fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer) -print(fill_mask('makan ayam dengan [MASK]')) -``` - -Output is, - -```text -[{'sequence': '[CLS] makan ayam dengan rendang[SEP]', - 'score': 0.10812027007341385, - 'token': 2446}, - {'sequence': '[CLS] makan ayam dengan kicap[SEP]', - 'score': 0.07653367519378662, - 'token': 12928}, - {'sequence': '[CLS] makan ayam dengan nasi[SEP]', - 'score': 0.06839974224567413, - 'token': 450}, - {'sequence': '[CLS] makan ayam dengan ayam[SEP]', - 'score': 0.059544261544942856, - 'token': 638}, - {'sequence': '[CLS] makan ayam dengan sayur[SEP]', - 'score': 0.05294966697692871, - 'token': 1639}] -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train BERT for Bahasa. - - diff --git a/model_cards/huseinzol05/electra-base-discriminator-bahasa-cased/README.md b/model_cards/huseinzol05/electra-base-discriminator-bahasa-cased/README.md deleted file mode 100644 index b480804d2bdb..000000000000 --- a/model_cards/huseinzol05/electra-base-discriminator-bahasa-cased/README.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -language: ms ---- - -# Bahasa ELECTRA Model - -Pretrained ELECTRA base language model for Malay and Indonesian. - -## Pretraining Corpus - -`electra-base-discriminator-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google ELECTRA's github [repository](https://github.com/google-research/electra) on a single TESLA V100 32GB VRAM. -- All steps can reproduce from here, [Malaya/pretrained-model/electra](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/electra). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import ElectraTokenizer, ElectraModel - -model = ElectraModel.from_pretrained('huseinzol05/electra-base-discriminator-bahasa-cased') -tokenizer = ElectraTokenizer.from_pretrained( - 'huseinzol05/electra-base-discriminator-bahasa-cased', - do_lower_case = False, -) -``` - -## Example using ElectraForPreTraining - -```python -from transformers import ElectraTokenizer, AutoModelWithLMHead, pipeline - -model = ElectraForPreTraining.from_pretrained('huseinzol05/electra-base-discriminator-bahasa-cased') -tokenizer = ElectraTokenizer.from_pretrained( - 'huseinzol05/electra-base-discriminator-bahasa-cased', - do_lower_case = False -) -sentence = 'kerajaan sangat prihatin terhadap rakyat' -fake_tokens = tokenizer.tokenize(sentence) -fake_inputs = tokenizer.encode(sentence, return_tensors="pt") -discriminator_outputs = discriminator(fake_inputs) -predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) - -list(zip(fake_tokens, predictions.tolist())) -``` - -Output is, - -```text -[('kerajaan', 0.0), - ('sangat', 0.0), - ('prihatin', 0.0), - ('terhadap', 0.0), - ('rakyat', 0.0)] -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train ELECTRA for Bahasa. - - diff --git a/model_cards/huseinzol05/electra-base-generator-bahasa-cased/README.md b/model_cards/huseinzol05/electra-base-generator-bahasa-cased/README.md deleted file mode 100644 index f6564f79651f..000000000000 --- a/model_cards/huseinzol05/electra-base-generator-bahasa-cased/README.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -language: ms ---- - -# Bahasa ELECTRA Model - -Pretrained ELECTRA base language model for Malay and Indonesian. - -## Pretraining Corpus - -`electra-base-generator-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google ELECTRA's github [repository](https://github.com/google-research/electra) on v3-8 TPU. -- All steps can reproduce from here, [Malaya/pretrained-model/electra](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/electra). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import ElectraTokenizer, ElectraModel - -model = ElectraModel.from_pretrained('huseinzol05/electra-base-generator-bahasa-cased') -tokenizer = ElectraTokenizer.from_pretrained( - 'huseinzol05/electra-base-generator-bahasa-cased', - do_lower_case = False, -) -``` - -## Example using AutoModelWithLMHead - -```python -from transformers import ElectraTokenizer, AutoModelWithLMHead, pipeline - -model = AutoModelWithLMHead.from_pretrained('huseinzol05/electra-base-generator-bahasa-cased') -tokenizer = ElectraTokenizer.from_pretrained( - 'huseinzol05/electra-base-generator-bahasa-cased', - do_lower_case = False, -) -fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer) -print(fill_mask('makan ayam dengan [MASK]')) -``` - -Output is, - -```text -[{'sequence': '[CLS] makan ayam dengan ayam [SEP]', - 'score': 0.08424834907054901, - 'token': 3255}, - {'sequence': '[CLS] makan ayam dengan rendang [SEP]', - 'score': 0.064150370657444, - 'token': 6288}, - {'sequence': '[CLS] makan ayam dengan nasi [SEP]', - 'score': 0.033446669578552246, - 'token': 2533}, - {'sequence': '[CLS] makan ayam dengan kucing [SEP]', - 'score': 0.02803465723991394, - 'token': 3577}, - {'sequence': '[CLS] makan ayam dengan telur [SEP]', - 'score': 0.026627106592059135, - 'token': 6350}] -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train ELECTRA for Bahasa. - - diff --git a/model_cards/huseinzol05/electra-small-discriminator-bahasa-cased/README.md b/model_cards/huseinzol05/electra-small-discriminator-bahasa-cased/README.md deleted file mode 100644 index ef45858598a5..000000000000 --- a/model_cards/huseinzol05/electra-small-discriminator-bahasa-cased/README.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -language: ms ---- - -# Bahasa ELECTRA Model - -Pretrained ELECTRA small language model for Malay and Indonesian. - -## Pretraining Corpus - -`electra-small-discriminator-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google ELECTRA's github [repository](https://github.com/google-research/electra) on a single TESLA V100 32GB VRAM. -- All steps can reproduce from here, [Malaya/pretrained-model/electra](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/electra). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import ElectraTokenizer, ElectraModel - -model = ElectraModel.from_pretrained('huseinzol05/electra-small-discriminator-bahasa-cased') -tokenizer = ElectraTokenizer.from_pretrained( - 'huseinzol05/electra-small-discriminator-bahasa-cased', - do_lower_case = False, -) -``` - -## Example using ElectraForPreTraining - -```python -from transformers import ElectraTokenizer, AutoModelWithLMHead, pipeline - -model = ElectraForPreTraining.from_pretrained('huseinzol05/electra-small-discriminator-bahasa-cased') -tokenizer = ElectraTokenizer.from_pretrained( - 'huseinzol05/electra-small-discriminator-bahasa-cased', - do_lower_case = False -) -sentence = 'kerajaan sangat prihatin terhadap rakyat' -fake_tokens = tokenizer.tokenize(sentence) -fake_inputs = tokenizer.encode(sentence, return_tensors="pt") -discriminator_outputs = discriminator(fake_inputs) -predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) - -list(zip(fake_tokens, predictions.tolist())) -``` - -Output is, - -```text -[('kerajaan', 0.0), - ('sangat', 0.0), - ('prihatin', 0.0), - ('terhadap', 0.0), - ('rakyat', 0.0)] -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train ELECTRA for Bahasa. - - diff --git a/model_cards/huseinzol05/electra-small-generator-bahasa-cased/README.md b/model_cards/huseinzol05/electra-small-generator-bahasa-cased/README.md deleted file mode 100644 index a4ccc50c76f0..000000000000 --- a/model_cards/huseinzol05/electra-small-generator-bahasa-cased/README.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -language: ms ---- - -# Bahasa ELECTRA Model - -Pretrained ELECTRA small language model for Malay and Indonesian. - -## Pretraining Corpus - -`electra-small-generator-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google ELECTRA's github [repository](https://github.com/google-research/electra) on a single TESLA V100 32GB VRAM. -- All steps can reproduce from here, [Malaya/pretrained-model/electra](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/electra). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import ElectraTokenizer, ElectraModel - -model = ElectraModel.from_pretrained('huseinzol05/electra-small-generator-bahasa-cased') -tokenizer = ElectraTokenizer.from_pretrained( - 'huseinzol05/electra-small-generator-bahasa-cased', - do_lower_case = False, -) -``` - -## Example using AutoModelWithLMHead - -```python -from transformers import ElectraTokenizer, AutoModelWithLMHead, pipeline - -model = AutoModelWithLMHead.from_pretrained('huseinzol05/electra-small-generator-bahasa-cased') -tokenizer = ElectraTokenizer.from_pretrained( - 'huseinzol05/electra-small-generator-bahasa-cased', - do_lower_case = False, -) -fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer) -print(fill_mask('makan ayam dengan [MASK]')) -``` - -Output is, - -```text -[{'sequence': '[CLS] makan ayam dengan ayam [SEP]', - 'score': 0.08424834907054901, - 'token': 3255}, - {'sequence': '[CLS] makan ayam dengan rendang [SEP]', - 'score': 0.064150370657444, - 'token': 6288}, - {'sequence': '[CLS] makan ayam dengan nasi [SEP]', - 'score': 0.033446669578552246, - 'token': 2533}, - {'sequence': '[CLS] makan ayam dengan kucing [SEP]', - 'score': 0.02803465723991394, - 'token': 3577}, - {'sequence': '[CLS] makan ayam dengan telur [SEP]', - 'score': 0.026627106592059135, - 'token': 6350}] -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train ELECTRA for Bahasa. - - diff --git a/model_cards/huseinzol05/gpt2-117M-bahasa-cased/README.md b/model_cards/huseinzol05/gpt2-117M-bahasa-cased/README.md deleted file mode 100644 index 0d6dc0812a84..000000000000 --- a/model_cards/huseinzol05/gpt2-117M-bahasa-cased/README.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -language: ms ---- - -# Bahasa GPT2 Model - -Pretrained GPT2 117M model for Malay. - -## Pretraining Corpus - -`gpt2-117M-bahasa-cased` model was pretrained on ~0.9 Billion words. We trained on standard language structure only, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -3. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -4. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -5. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -6. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -7. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). -8. [Common-Crawl](https://github.com/huseinzol05/malaya-dataset#common-crawl). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using GPT2's github [repository](https://github.com/openai/gpt-2) on a V3-8 TPU. -- All steps can reproduce from here, [Malaya/pretrained-model/gpt2](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/gpt2). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import GPT2Tokenizer, GPT2Model - -model = GPT2Model.from_pretrained('huseinzol05/gpt2-117M-bahasa-cased') -tokenizer = GPT2Tokenizer.from_pretrained( - 'huseinzol05/gpt2-117M-bahasa-cased', -) -``` - -## Example using GPT2LMHeadModel - -```python -from transformers import GPT2Tokenizer, GPT2LMHeadModel - -tokenizer = GPT2Tokenizer.from_pretrained('huseinzol05/gpt2-117M-bahasa-cased') -model = GPT2LMHeadModel.from_pretrained( - 'huseinzol05/gpt2-117M-bahasa-cased', pad_token_id = tokenizer.eos_token_id -) - -input_ids = tokenizer.encode( - 'penat bak hang, macam ni aku takmau kerja dah', return_tensors = 'pt' -) -sample_outputs = model.generate( - input_ids, - do_sample = True, - max_length = 50, - top_k = 50, - top_p = 0.95, - num_return_sequences = 3, -) - -print('Output:\n' + 100 * '-') -for i, sample_output in enumerate(sample_outputs): - print( - '{}: {}'.format( - i, tokenizer.decode(sample_output, skip_special_tokens = True) - ) - ) -``` - -Output is, - -```text -Output: ----------------------------------------------------------------------------------------------------- -0: penat bak hang, macam ni aku takmau kerja dah jadi aku pernah beritahu orang. -Ini bukan aku rasa cam nak ajak teman kan ni. -Tengok ni aku dah ada adik-adik & anak yang tinggal dan kerja2 yang kat sekolah. -1: penat bak hang, macam ni aku takmau kerja dah. -Takleh takleh nak ambik air. -Tgk jugak aku kat rumah ni. -Pastu aku nak bagi aku. -So aku dah takde masalah pulak. -Balik aku pun -2: penat bak hang, macam ni aku takmau kerja dah macam tu. -Tapi semua tu aku ingat cakap, ada cara hidup ni yang kita kena bayar.. pastu kita tak mampu bayar.. kan!! -Takpelah, aku nak cakap, masa yang -``` diff --git a/model_cards/huseinzol05/gpt2-345M-bahasa-cased/README.md b/model_cards/huseinzol05/gpt2-345M-bahasa-cased/README.md deleted file mode 100644 index 9c761458191f..000000000000 --- a/model_cards/huseinzol05/gpt2-345M-bahasa-cased/README.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -language: ms ---- - -# Bahasa GPT2 Model - -Pretrained GPT2 345M model for Malay. - -## Pretraining Corpus - -`gpt2-345M-bahasa-cased` model was pretrained on ~0.9 Billion words. We trained on standard language structure only, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -3. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -4. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -5. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -6. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -7. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). -8. [Common-Crawl](https://github.com/huseinzol05/malaya-dataset#common-crawl). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using GPT2's github [repository](https://github.com/openai/gpt-2) on a V3-8 TPU. -- All steps can reproduce from here, [Malaya/pretrained-model/gpt2](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/gpt2). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import GPT2Tokenizer, GPT2Model - -model = GPT2Model.from_pretrained('huseinzol05/gpt2-345M-bahasa-cased') -tokenizer = GPT2Tokenizer.from_pretrained( - 'huseinzol05/gpt2-345M-bahasa-cased', -) -``` - -## Example using GPT2LMHeadModel - -```python -from transformers import GPT2Tokenizer, GPT2LMHeadModel - -tokenizer = GPT2Tokenizer.from_pretrained('huseinzol05/gpt2-345M-bahasa-cased') -model = GPT2LMHeadModel.from_pretrained( - 'huseinzol05/gpt2-345M-bahasa-cased', pad_token_id = tokenizer.eos_token_id -) - -input_ids = tokenizer.encode( - 'penat bak hang, macam ni aku takmau kerja dah', return_tensors = 'pt' -) -sample_outputs = model.generate( - input_ids, - do_sample = True, - max_length = 50, - top_k = 50, - top_p = 0.95, - num_return_sequences = 3, -) - -print('Output:\n' + 100 * '-') -for i, sample_output in enumerate(sample_outputs): - print( - '{}: {}'.format( - i, tokenizer.decode(sample_output, skip_special_tokens = True) - ) - ) -``` - -Output is, - -```text -Output: ----------------------------------------------------------------------------------------------------- -0: penat bak hang, macam ni aku takmau kerja dah dekat 2,3 jam. -Aku harap aku dapat berjimat banyak. -Ini pun masa kerja, bila dah kerja jadi satu. -Aku buat kerja ni la. -Aku memang kalau ada -1: penat bak hang, macam ni aku takmau kerja dah. -Tapi nak buat macam mana kan, aku tolong bentang tugas. -Dan, memang sangat-sangat tak mahu buat kerja sekarang ni. -Aku pun suka sangat kerja di luar bandar -2: penat bak hang, macam ni aku takmau kerja dah pun. -Takpa nak buat kerja-kerja sampingan, baru boleh dapat hadiah pulak. -Ni la tempat paling best bila duduk di restoran yang ada pekena kopi. -Cumanya -``` diff --git a/model_cards/huseinzol05/t5-base-bahasa-cased/README.md b/model_cards/huseinzol05/t5-base-bahasa-cased/README.md deleted file mode 100644 index 18fe9fb01fa2..000000000000 --- a/model_cards/huseinzol05/t5-base-bahasa-cased/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: ms ---- - -# Bahasa T5 Model - -Pretrained T5 base language model for Malay and Indonesian. - -## Pretraining Corpus - -`t5-base-bahasa-cased` model was pretrained on multiple tasks. Below is list of tasks we trained on, - -1. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [local Wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -3. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -4. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -5. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -6. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). -7. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [local Wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -8. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -9. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -10. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -11. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -12. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). -13. [Bahasa SNLI](https://github.com/huseinzol05/Malaya-Dataset#snli). -14. [Bahasa Question Quora](https://github.com/huseinzol05/Malaya-Dataset#quora). -15. [Bahasa Natural Questions](https://github.com/huseinzol05/Malaya-Dataset#natural-questions). -16. [News title summarization](https://github.com/huseinzol05/Malaya-Dataset#crawled-news). -17. [Stemming to original wikipedia](https://github.com/huseinzol05/Malaya/blob/master/pretrained-model/t5/generate-stemming.ipynb). -18. [Synonym to original wikipedia](https://github.com/huseinzol05/Malaya/blob/master/pretrained-model/t5/generate-synonym.ipynb). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google T5's github [repository](https://github.com/google-research/text-to-text-transfer-transformer) on v3-8 TPU. -- All steps can reproduce from here, [Malaya/pretrained-model/t5](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/t5). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import T5Tokenizer, T5Model - -model = T5Model.from_pretrained('huseinzol05/t5-base-bahasa-cased') -tokenizer = T5Tokenizer.from_pretrained('huseinzol05/t5-base-bahasa-cased') -``` - -## Example using T5ForConditionalGeneration - -```python -from transformers import T5Tokenizer, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained('huseinzol05/t5-base-bahasa-cased') -model = T5ForConditionalGeneration.from_pretrained('huseinzol05/t5-base-bahasa-cased') -input_ids = tokenizer.encode('soalan: siapakah perdana menteri malaysia?', return_tensors = 'pt') -outputs = model.generate(input_ids) -print(tokenizer.decode(outputs[0])) -``` - -Output is, - -``` -'Mahathir Mohamad' -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train T5 for Bahasa. diff --git a/model_cards/huseinzol05/t5-base-bahasa-summarization-cased/README.md b/model_cards/huseinzol05/t5-base-bahasa-summarization-cased/README.md deleted file mode 100644 index b36d80166f9f..000000000000 --- a/model_cards/huseinzol05/t5-base-bahasa-summarization-cased/README.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -language: ms ---- - -# Bahasa T5 Summarization Model - -Finetuned T5 base summarization model for Malay and Indonesian. - -## Finetuning Corpus - -`t5-base-bahasa-summarization-cased` model was finetuned on multiple summarization dataset. Below is list of tasks we trained on, - -1. [Translated CNN News](https://github.com/huseinzol05/Malay-Dataset#cnn-news) -2. [Translated Gigawords](https://github.com/huseinzol05/Malay-Dataset#gigawords) -3. [Translated Multinews](https://github.com/huseinzol05/Malay-Dataset#multinews) - -## Finetuning details - -- This model was trained using Malaya T5's github [repository](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/t5) on v3-8 TPU using Base size. -- All steps can reproduce from here, [Malaya/session/summarization](https://github.com/huseinzol05/Malaya/tree/master/session/summarization). - -## Load Finetuned Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import T5Tokenizer, T5Model - -tokenizer = T5Tokenizer.from_pretrained('huseinzol05/t5-base-bahasa-summarization-cased') -model = T5ForConditionalGeneration.from_pretrained('huseinzol05/t5-base-bahasa-summarization-cased') -``` - -## Example using T5ForConditionalGeneration - -```python -from transformers import T5Tokenizer, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained('huseinzol05/t5-base-bahasa-summarization-cased') -model = T5ForConditionalGeneration.from_pretrained('huseinzol05/t5-base-bahasa-summarization-cased') - -# https://www.hmetro.com.my/mutakhir/2020/05/580438/peletakan-jawatan-tun-m-ditolak-bukan-lagi-isu -# original title, Peletakan jawatan Tun M ditolak, bukan lagi isu -string = 'PELETAKAN jawatan Tun Dr Mahathir Mohamad sebagai Pengerusi Parti Pribumi Bersatu Malaysia (Bersatu) ditolak di dalam mesyuarat khas Majlis Pimpinan Tertinggi (MPT) pada 24 Februari lalu. Justeru, tidak timbul soal peletakan jawatan itu sah atau tidak kerana ia sudah pun diputuskan pada peringkat parti yang dipersetujui semua termasuk Presiden, Tan Sri Muhyiddin Yassin. Bekas Setiausaha Agung Bersatu Datuk Marzuki Yahya berkata, pada mesyuarat itu MPT sebulat suara menolak peletakan jawatan Dr Mahathir. "Jadi ini agak berlawanan dengan keputusan yang kita sudah buat. Saya tak faham bagaimana Jabatan Pendaftar Pertubuhan Malaysia (JPPM) kata peletakan jawatan itu sah sedangkan kita sudah buat keputusan di dalam mesyuarat, bukan seorang dua yang buat keputusan. "Semua keputusan mesti dibuat melalui parti. Walau apa juga perbincangan dibuat di luar daripada keputusan mesyuarat, ini bukan keputusan parti. "Apa locus standy yang ada pada Setiausaha Kerja untuk membawa perkara ini kepada JPPM. Seharusnya ia dibawa kepada Setiausaha Agung sebagai pentadbir kepada parti," katanya kepada Harian Metro. Beliau mengulas laporan media tempatan hari ini mengenai pengesahan JPPM bahawa Dr Mahathir tidak lagi menjadi Pengerusi Bersatu berikutan peletakan jawatannya di tengah-tengah pergolakan politik pada akhir Februari adalah sah. Laporan itu juga menyatakan, kedudukan Muhyiddin Yassin memangku jawatan itu juga sah. Menurutnya, memang betul Dr Mahathir menghantar surat peletakan jawatan, tetapi ditolak oleh MPT. "Fasal yang disebut itu terpakai sekiranya berhenti atau diberhentikan, tetapi ini mesyuarat sudah menolak," katanya. Marzuki turut mempersoal kenyataan media yang dibuat beberapa pimpinan parti itu hari ini yang menyatakan sokongan kepada Perikatan Nasional. "Kenyataan media bukanlah keputusan rasmi. Walaupun kita buat 1,000 kenyataan sekali pun ia tetap tidak merubah keputusan yang sudah dibuat di dalam mesyuarat. Kita catat di dalam minit apa yang berlaku di dalam mesyuarat," katanya.' - -# https://huggingface.co/blog/how-to-generate -# generate summary -input_ids = tokenizer.encode(f'ringkasan: {string}', return_tensors = 'pt') -outputs = model.generate( - input_ids, - do_sample = True, - temperature = 0.8, - top_k = 50, - top_p = 0.95, - max_length = 300, - num_return_sequences = 3, -) - -for i, sample_output in enumerate(outputs): - print( - '{}: {}'.format( - i, tokenizer.decode(sample_output, skip_special_tokens = True) - ) - ) - -# generate news title -input_ids = tokenizer.encode(f'tajuk: {string}', return_tensors = 'pt') -outputs = model.generate( - input_ids, - do_sample = True, - temperature = 0.8, - top_k = 50, - top_p = 0.95, - max_length = 300, - num_return_sequences = 3, -) - -for i, sample_output in enumerate(outputs): - print( - '{}: {}'.format( - i, tokenizer.decode(sample_output, skip_special_tokens = True) - ) - ) -``` - -Output is, - -``` -0: "Ini agak berlawanan dengan keputusan yang kita sudah buat," kata Marzuki Yahya. Kenyataan media adalah keputusan rasmi. Marzuki: Kenyataan media tidak mengubah keputusan mesyuarat -1: MPT sebulat suara menolak peletakan jawatan Dr M di mesyuarat 24 Februari. Tidak ada persoalan peletakan jawatan itu sah atau tidak, tetapi ia adalah keputusan parti yang dipersetujui semua. Bekas Setiausaha Agung Bersatu mengatakan keputusan itu perlu dibuat melalui parti. Bekas setiausaha agung itu mengatakan kenyataan media tidak lagi menyokong Perikatan Nasional -2: Kenyataan media menunjukkan sokongan kepada Perikatan Nasional. Marzuki: Kedudukan Dr M sebagai Pengerusi Bersatu juga sah. Beliau berkata pengumuman itu harus diserahkan kepada setiausaha Agung - -0: 'Kalah Tun M, Muhyiddin tetap sah' -1: Boleh letak jawatan PM di MPT -2: 'Ketegangan Dr M sudah tolak, tak timbul isu peletakan jawatan' -``` - -## Result - -We found out using original Tensorflow implementation gives better results, check it at https://malaya.readthedocs.io/en/latest/Abstractive.html#generate-ringkasan - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train T5 for Bahasa. diff --git a/model_cards/huseinzol05/t5-small-bahasa-cased/README.md b/model_cards/huseinzol05/t5-small-bahasa-cased/README.md deleted file mode 100644 index b0f622d53695..000000000000 --- a/model_cards/huseinzol05/t5-small-bahasa-cased/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: ms ---- - -# Bahasa T5 Model - -Pretrained T5 small language model for Malay and Indonesian. - -## Pretraining Corpus - -`t5-small-bahasa-cased` model was pretrained on multiple tasks. Below is list of tasks we trained on, - -1. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [local Wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -3. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -4. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -5. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -6. [Unsupervised](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1875) on [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). -7. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [local Wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -8. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -9. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -10. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -11. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -12. [Next sentence prediction](https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/data/preprocessors.py#L1129) on [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). -13. [Bahasa SNLI](https://github.com/huseinzol05/Malaya-Dataset#snli). -14. [Bahasa Question Quora](https://github.com/huseinzol05/Malaya-Dataset#quora). -15. [Bahasa Natural Questions](https://github.com/huseinzol05/Malaya-Dataset#natural-questions). -16. [News title summarization](https://github.com/huseinzol05/Malaya-Dataset#crawled-news). -17. [Stemming to original wikipedia](https://github.com/huseinzol05/Malaya/blob/master/pretrained-model/t5/generate-stemming.ipynb). -18. [Synonym to original wikipedia](https://github.com/huseinzol05/Malaya/blob/master/pretrained-model/t5/generate-synonym.ipynb). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using Google T5's github [repository](https://github.com/google-research/text-to-text-transfer-transformer) on v3-8 TPU. -- All steps can reproduce from here, [Malaya/pretrained-model/t5](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/t5). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import T5Tokenizer, T5Model - -model = T5Model.from_pretrained('huseinzol05/t5-small-bahasa-cased') -tokenizer = T5Tokenizer.from_pretrained('huseinzol05/t5-small-bahasa-cased') -``` - -## Example using T5ForConditionalGeneration - -```python -from transformers import T5Tokenizer, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained('huseinzol05/t5-small-bahasa-cased') -model = T5ForConditionalGeneration.from_pretrained('huseinzol05/t5-small-bahasa-cased') -input_ids = tokenizer.encode('soalan: siapakah perdana menteri malaysia?', return_tensors = 'pt') -outputs = model.generate(input_ids) -print(tokenizer.decode(outputs[0])) -``` - -Output is, - -``` -'Mahathir Mohamad' -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train T5 for Bahasa. diff --git a/model_cards/huseinzol05/t5-small-bahasa-summarization-cased/README.md b/model_cards/huseinzol05/t5-small-bahasa-summarization-cased/README.md deleted file mode 100644 index 3d23762de5e4..000000000000 --- a/model_cards/huseinzol05/t5-small-bahasa-summarization-cased/README.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -language: ms ---- - -# Bahasa T5 Summarization Model - -Finetuned T5 small summarization model for Malay and Indonesian. - -## Finetuning Corpus - -`t5-small-bahasa-summarization-cased` model was finetuned on multiple summarization dataset. Below is list of tasks we trained on, - -1. [Translated CNN News](https://github.com/huseinzol05/Malay-Dataset#cnn-news) -2. [Translated Gigawords](https://github.com/huseinzol05/Malay-Dataset#gigawords) -3. [Translated Multinews](https://github.com/huseinzol05/Malay-Dataset#multinews) - -## Finetuning details - -- This model was trained using Malaya T5's github [repository](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/t5) on v3-8 TPU using small size. -- All steps can reproduce from here, [Malaya/session/summarization](https://github.com/huseinzol05/Malaya/tree/master/session/summarization). - -## Load Finetuned Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import T5Tokenizer, T5Model - -tokenizer = T5Tokenizer.from_pretrained('huseinzol05/t5-small-bahasa-summarization-cased') -model = T5ForConditionalGeneration.from_pretrained('huseinzol05/t5-small-bahasa-summarization-cased') -``` - -## Example using T5ForConditionalGeneration - -```python -from transformers import T5Tokenizer, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained('huseinzol05/t5-small-bahasa-summarization-cased') -model = T5ForConditionalGeneration.from_pretrained('huseinzol05/t5-small-bahasa-summarization-cased') - -# https://www.hmetro.com.my/mutakhir/2020/05/580438/peletakan-jawatan-tun-m-ditolak-bukan-lagi-isu -# original title, Peletakan jawatan Tun M ditolak, bukan lagi isu -string = 'PELETAKAN jawatan Tun Dr Mahathir Mohamad sebagai Pengerusi Parti Pribumi Bersatu Malaysia (Bersatu) ditolak di dalam mesyuarat khas Majlis Pimpinan Tertinggi (MPT) pada 24 Februari lalu. Justeru, tidak timbul soal peletakan jawatan itu sah atau tidak kerana ia sudah pun diputuskan pada peringkat parti yang dipersetujui semua termasuk Presiden, Tan Sri Muhyiddin Yassin. Bekas Setiausaha Agung Bersatu Datuk Marzuki Yahya berkata, pada mesyuarat itu MPT sebulat suara menolak peletakan jawatan Dr Mahathir. "Jadi ini agak berlawanan dengan keputusan yang kita sudah buat. Saya tak faham bagaimana Jabatan Pendaftar Pertubuhan Malaysia (JPPM) kata peletakan jawatan itu sah sedangkan kita sudah buat keputusan di dalam mesyuarat, bukan seorang dua yang buat keputusan. "Semua keputusan mesti dibuat melalui parti. Walau apa juga perbincangan dibuat di luar daripada keputusan mesyuarat, ini bukan keputusan parti. "Apa locus standy yang ada pada Setiausaha Kerja untuk membawa perkara ini kepada JPPM. Seharusnya ia dibawa kepada Setiausaha Agung sebagai pentadbir kepada parti," katanya kepada Harian Metro. Beliau mengulas laporan media tempatan hari ini mengenai pengesahan JPPM bahawa Dr Mahathir tidak lagi menjadi Pengerusi Bersatu berikutan peletakan jawatannya di tengah-tengah pergolakan politik pada akhir Februari adalah sah. Laporan itu juga menyatakan, kedudukan Muhyiddin Yassin memangku jawatan itu juga sah. Menurutnya, memang betul Dr Mahathir menghantar surat peletakan jawatan, tetapi ditolak oleh MPT. "Fasal yang disebut itu terpakai sekiranya berhenti atau diberhentikan, tetapi ini mesyuarat sudah menolak," katanya. Marzuki turut mempersoal kenyataan media yang dibuat beberapa pimpinan parti itu hari ini yang menyatakan sokongan kepada Perikatan Nasional. "Kenyataan media bukanlah keputusan rasmi. Walaupun kita buat 1,000 kenyataan sekali pun ia tetap tidak merubah keputusan yang sudah dibuat di dalam mesyuarat. Kita catat di dalam minit apa yang berlaku di dalam mesyuarat," katanya.' - -# https://huggingface.co/blog/how-to-generate -# generate summary -input_ids = tokenizer.encode(f'ringkasan: {string}', return_tensors = 'pt') -outputs = model.generate( - input_ids, - do_sample = True, - temperature = 0.8, - top_k = 50, - top_p = 0.95, - max_length = 300, - num_return_sequences = 3, -) - -for i, sample_output in enumerate(outputs): - print( - '{}: {}'.format( - i, tokenizer.decode(sample_output, skip_special_tokens = True) - ) - ) - -# generate news title -input_ids = tokenizer.encode(f'tajuk: {string}', return_tensors = 'pt') -outputs = model.generate( - input_ids, - do_sample = True, - temperature = 0.8, - top_k = 50, - top_p = 0.95, - max_length = 300, - num_return_sequences = 3, -) - -for i, sample_output in enumerate(outputs): - print( - '{}: {}'.format( - i, tokenizer.decode(sample_output, skip_special_tokens = True) - ) - ) -``` - -Output is, - -``` -0: Pengerusi Bersatu Bersatu menafikan peletakan jawatan dalam mesyuarat khas Majlis Pimpinan Tertinggi. Tidak timbul isu peletakan jawatan itu sah atau tidak kerana ia sudah diputuskan di peringkat parti. Kenyataan media yang dibuat oleh pemimpin parti hari ini menyokong Perikatan Nasional -1: Tiada keputusan kerana ia sudah diputuskan pada peringkat parti, Marzuki berkata. Pejabat rasmi parti menolak peletakan jawatan Dr M, dengan mengatakan ia adalah keputusan. Kedudukan Muhyiddin memangku jawatan itu juga sah, katanya -2: Tiada peletakan jawatan Dr Mahathir dalam mesyuarat khas MPT pada 24 Februari. Ketua parti menolak peletakan jawatan itu. Tidak timbul isu peletakan jawatan itu sah atau tidak, katanya - -0: Tiada peletakan jawatan Tun M dalam mesyuarat khas -1: ‘Tidak timbul peletakan jawatan Tun M’ -2: Tidak timbul isu peletakan jawatan Tun M di mesyuarat khas -``` - -## Result - -We found out using original Tensorflow implementation gives better results, check it at https://malaya.readthedocs.io/en/latest/Abstractive.html#generate-ringkasan - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train T5 for Bahasa. diff --git a/model_cards/huseinzol05/tiny-bert-bahasa-cased/README.md b/model_cards/huseinzol05/tiny-bert-bahasa-cased/README.md deleted file mode 100644 index 95671a62ddf0..000000000000 --- a/model_cards/huseinzol05/tiny-bert-bahasa-cased/README.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -language: ms ---- - -# Bahasa Tiny-BERT Model - -General Distilled Tiny BERT language model for Malay and Indonesian. - -## Pretraining Corpus - -`tiny-bert-bahasa-cased` model was distilled on ~1.8 Billion words. We distilled on both standard and social media language structures, and below is list of data we distilled on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Distilling details - -- This model was distilled using huawei-noah Tiny-BERT's github [repository](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/TinyBERT) on 3 Titan V100 32GB VRAM. -- All steps can reproduce from here, [Malaya/pretrained-model/tiny-bert](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/tiny-bert). - -## Load Distilled Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import AlbertTokenizer, BertModel - -model = BertModel.from_pretrained('huseinzol05/tiny-bert-bahasa-cased') -tokenizer = AlbertTokenizer.from_pretrained( - 'huseinzol05/tiny-bert-bahasa-cased', - unk_token = '[UNK]', - pad_token = '[PAD]', - do_lower_case = False, -) -``` - -We use [google/sentencepiece](https://github.com/google/sentencepiece) to train the tokenizer, so to use it, need to load from `AlbertTokenizer`. - -## Example using AutoModelWithLMHead - -```python -from transformers import AlbertTokenizer, AutoModelWithLMHead, pipeline - -model = AutoModelWithLMHead.from_pretrained('huseinzol05/tiny-bert-bahasa-cased') -tokenizer = AlbertTokenizer.from_pretrained( - 'huseinzol05/tiny-bert-bahasa-cased', - unk_token = '[UNK]', - pad_token = '[PAD]', - do_lower_case = False, -) -fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer) -print(fill_mask('makan ayam dengan [MASK]')) -``` - -Output is, - -```text -[{'sequence': '[CLS] makan ayam dengan berbual[SEP]', - 'score': 0.00015769545279908925, - 'token': 17859}, - {'sequence': '[CLS] makan ayam dengan kembar[SEP]', - 'score': 0.0001448775001335889, - 'token': 8289}, - {'sequence': '[CLS] makan ayam dengan memaklumkan[SEP]', - 'score': 0.00013484008377417922, - 'token': 6881}, - {'sequence': '[CLS] makan ayam dengan Senarai[SEP]', - 'score': 0.00013061291247140616, - 'token': 11698}, - {'sequence': '[CLS] makan ayam dengan Tiga[SEP]', - 'score': 0.00012453157978598028, - 'token': 4232}] -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train BERT for Bahasa. - - diff --git a/model_cards/huseinzol05/xlnet-base-bahasa-cased/README.md b/model_cards/huseinzol05/xlnet-base-bahasa-cased/README.md deleted file mode 100644 index 1c4b03ffa8c1..000000000000 --- a/model_cards/huseinzol05/xlnet-base-bahasa-cased/README.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -language: ms ---- - -# Bahasa XLNet Model - -Pretrained XLNet base language model for Malay and Indonesian. - -## Pretraining Corpus - -`XLNET-base-bahasa-cased` model was pretrained on ~1.8 Billion words. We trained on both standard and social media language structures, and below is list of data we trained on, - -1. [dumping wikipedia](https://github.com/huseinzol05/Malaya-Dataset#wikipedia-1). -2. [local instagram](https://github.com/huseinzol05/Malaya-Dataset#instagram). -3. [local twitter](https://github.com/huseinzol05/Malaya-Dataset#twitter-1). -4. [local news](https://github.com/huseinzol05/Malaya-Dataset#public-news). -5. [local parliament text](https://github.com/huseinzol05/Malaya-Dataset#parliament). -6. [local singlish/manglish text](https://github.com/huseinzol05/Malaya-Dataset#singlish-text). -7. [IIUM Confession](https://github.com/huseinzol05/Malaya-Dataset#iium-confession). -8. [Wattpad](https://github.com/huseinzol05/Malaya-Dataset#wattpad). -9. [Academia PDF](https://github.com/huseinzol05/Malaya-Dataset#academia-pdf). - -Preprocessing steps can reproduce from here, [Malaya/pretrained-model/preprocess](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/preprocess). - -## Pretraining details - -- This model was trained using zihangdai XLNet's github [repository](https://github.com/zihangdai/xlnet) on 3 Titan V100 32GB VRAM. -- All steps can reproduce from here, [Malaya/pretrained-model/xlnet](https://github.com/huseinzol05/Malaya/tree/master/pretrained-model/xlnet). - -## Load Pretrained Model - -You can use this model by installing `torch` or `tensorflow` and Huggingface library `transformers`. And you can use it directly by initializing it like this: - -```python -from transformers import XLNetTokenizer, XLNetModel - -model = XLNetModel.from_pretrained('huseinzol05/xlnet-base-bahasa-cased') -tokenizer = XLNetTokenizer.from_pretrained( - 'huseinzol05/xlnet-base-bahasa-cased', do_lower_case = False -) -``` - -## Example using AutoModelWithLMHead - -```python -from transformers import AlbertTokenizer, AutoModelWithLMHead, pipeline - -model = AutoModelWithLMHead.from_pretrained('huseinzol05/xlnet-base-bahasa-cased') -tokenizer = XLNetTokenizer.from_pretrained( - 'huseinzol05/xlnet-base-bahasa-cased', do_lower_case = False -) -fill_mask = pipeline('fill-mask', model = model, tokenizer = tokenizer) -print(fill_mask('makan ayam dengan ')) -``` - -## Results - -For further details on the model performance, simply checkout accuracy page from Malaya, https://malaya.readthedocs.io/en/latest/Accuracy.html, we compared with traditional models. - -## Acknowledgement - -Thanks to [Im Big](https://www.facebook.com/imbigofficial/), [LigBlou](https://www.facebook.com/ligblou), [Mesolitica](https://mesolitica.com/) and [KeyReply](https://www.keyreply.com/) for sponsoring AWS, Google and GPU clouds to train XLNet for Bahasa. - - diff --git a/model_cards/iarfmoose/bert-base-cased-qa-evaluator/README.md b/model_cards/iarfmoose/bert-base-cased-qa-evaluator/README.md deleted file mode 100644 index b1bcf5f26867..000000000000 --- a/model_cards/iarfmoose/bert-base-cased-qa-evaluator/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# BERT-base-cased-qa-evaluator - -This model takes a question answer pair as an input and outputs a value representing its prediction about whether the input was a valid question and answer pair or not. The model is a pretrained [BERT-base-cased](https://huggingface.co/bert-base-cased) with a sequence classification head. - -## Intended uses - -The QA evaluator was originally designed to be used with the [t5-base-question-generator](https://huggingface.co/iarfmoose/t5-base-question-generator) for evaluating the quality of generated questions. - -The input for the QA evaluator follows the format for `BertForSequenceClassification`, but using the question and answer as the two sequences. Inputs should take the following format: -``` -[CLS] [SEP]
-@InProceedings{10.1007/978-3-030-03840-3_29, - author="Croce, Danilo and Zelenanska, Alexandra and Basili, Roberto", - editor="Ghidini, Chiara and Magnini, Bernardo and Passerini, Andrea and Traverso, Paolo", - title="Neural Learning for Question Answering in Italian", - booktitle="AI*IA 2018 -- Advances in Artificial Intelligence", - year="2018", - publisher="Springer International Publishing", - address="Cham", - pages="389--402", - isbn="978-3-030-03840-3" -} - diff --git a/model_cards/mrm8488/bert-medium-finetuned-squadv2/README.md b/model_cards/mrm8488/bert-medium-finetuned-squadv2/README.md deleted file mode 100644 index 572589ce5dae..000000000000 --- a/model_cards/mrm8488/bert-medium-finetuned-squadv2/README.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -language: en -thumbnail: ---- - -# BERT-Medium fine-tuned on SQuAD v2 - -[BERT-Medium](https://github.com/google-research/bert/) created by [Google Research](https://github.com/google-research) and fine-tuned on [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -**Mode size** (after training): **157.46 MB** - -## Details of BERT-Small and its 'family' (from their documentation) - -Released on March 11th, 2020 - -This is model is a part of 24 smaller BERT models (English only, uncased, trained with WordPiece masking) referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). - -The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. - -## Details of the downstream task (Q&A) - Dataset - -[SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD2.0 | train | 130k | -| SQuAD2.0 | eval | 12.3k | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/question-answering/run_squad.py) - -## Results: - -| Metric | # Value | -| ------ | --------- | -| **EM** | **65.95** | -| **F1** | **70.11** | - -### Raw metrics from benchmark included in training script: - -```json -{ - "exact": 65.95637159942727, - "f1": 70.11632254245896, - "total": 11873, - "HasAns_exact": 67.79689608636977, - "HasAns_f1": 76.12872765631123, - "HasAns_total": 5928, - "NoAns_exact": 64.12111017661901, - "NoAns_f1": 64.12111017661901, - "NoAns_total": 5945, - "best_exact": 65.96479407058031, - "best_exact_thresh": 0.0, - "best_f1": 70.12474501361196, - "best_f1_thresh": 0.0 -} -``` - -## Comparison: - -| Model | EM | F1 score | SIZE (MB) | -| --------------------------------------------------------------------------------------------- | --------- | --------- | --------- | -| [bert-tiny-finetuned-squadv2](https://huggingface.co/mrm8488/bert-tiny-finetuned-squadv2) | 48.60 | 49.73 | **16.74** | -| [bert-tiny-5-finetuned-squadv2](https://huggingface.co/mrm8488/bert-tiny-5-finetuned-squadv2) | 57.12 | 60.86 | 24.34 | -| [bert-mini-finetuned-squadv2](https://huggingface.co/mrm8488/bert-mini-finetuned-squadv2) | 56.31 | 59.65 | 42.63 | -| [bert-mini-5-finetuned-squadv2](https://huggingface.co/mrm8488/bert-mini-5-finetuned-squadv2) | 63.51 | 66.78 | 66.76 | -| [bert-small-finetuned-squadv2](https://huggingface.co/mrm8488/bert-small-finetuned-squadv2) | 60.49 | 64.21 | 109.74 | -| [bert-medium-finetuned-squadv2](https://huggingface.co/mrm8488/bert-medium-finetuned-squadv2) | **65.95** | **70.11** | 157.46 | - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/bert-small-finetuned-squadv2", - tokenizer="mrm8488/bert-small-finetuned-squadv2" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" - -}) - -# Output: -``` - -```json -{ - "answer": "Manuel Romero", - "end": 13, - "score": 0.9939319924374637, - "start": 0 -} -``` - -### Yes! That was easy 🎉 Let's try with another example - -```python -qa_pipeline({ - 'context': "Manuel Romero has been working remotely in the repository hugginface/transformers lately", - 'question': "How has been working Manuel Romero?" -}) - -# Output: -``` - -```json -{ "answer": "remotely", "end": 39, "score": 0.3612058272768017, "start": 31 } -``` - -### It works!! 🎉 🎉 🎉 - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-mini-finetuned-squadv2/README.md b/model_cards/mrm8488/bert-mini-finetuned-squadv2/README.md deleted file mode 100644 index dd74a65f882a..000000000000 --- a/model_cards/mrm8488/bert-mini-finetuned-squadv2/README.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -language: en -thumbnail: ---- - -# BERT-Mini fine-tuned on SQuAD v2 - -[BERT-Mini](https://github.com/google-research/bert/) created by [Google Research](https://github.com/google-research) and fine-tuned on [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -**Mode size** (after training): **42.63 MB** - -## Details of BERT-Mini and its 'family' (from their documentation) - -Released on March 11th, 2020 - -This is model is a part of 24 smaller BERT models (English only, uncased, trained with WordPiece masking) referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). - -The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. - -## Details of the downstream task (Q&A) - Dataset - -[SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD2.0 | train | 130k | -| SQuAD2.0 | eval | 12.3k | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/question-answering/run_squad.py) - -## Results: - -| Metric | # Value | -| ------ | --------- | -| **EM** | **56.31** | -| **F1** | **59.65** | - -## Comparison: - -| Model | EM | F1 score | SIZE (MB) | -| ----------------------------------------------------------------------------------------- | --------- | --------- | --------- | -| [bert-tiny-finetuned-squadv2](https://huggingface.co/mrm8488/bert-tiny-finetuned-squadv2) | 48.60 | 49.73 | **16.74** | -| [bert-tiny-5-finetuned-squadv2](https://huggingface.co/mrm8488/bert-tiny-5-finetuned-squadv2) | 57.12 | 60.86 | 24.34 | -| [bert-mini-finetuned-squadv2](https://huggingface.co/mrm8488/bert-mini-finetuned-squadv2) | 56.31 | 59.65 | 42.63 | -| [bert-mini-5-finetuned-squadv2](https://huggingface.co/mrm8488/bert-mini-5-finetuned-squadv2) | **63.51** | **66.78** | 66.76 | - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/bert-mini-finetuned-squadv2", - tokenizer="mrm8488/bert-mini-finetuned-squadv2" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" - -}) - -# Output: -``` - -```json -{ - "answer": "Manuel Romero", - "end": 13, - "score": 0.9676484207783673, - "start": 0 -} -``` - -### Yes! That was easy 🎉 Let's try with another example - -```python -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "For which company has worked Manuel Romero?" -}) - -# Output: -``` - -```json -{ - "answer": "hugginface/transformers", - "end": 79, - "score": 0.5301655914731853, - "start": 56 -} -``` - -### It works!! 🎉 🎉 🎉 - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-mini2bert-mini-finetuned-cnn_daily_mail-summarization/README.md b/model_cards/mrm8488/bert-mini2bert-mini-finetuned-cnn_daily_mail-summarization/README.md deleted file mode 100644 index f5589cc1da00..000000000000 --- a/model_cards/mrm8488/bert-mini2bert-mini-finetuned-cnn_daily_mail-summarization/README.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- cnn_dailymail -tags: -- summarization ---- - -# Bert-mini2Bert-mini Summarization with 🤗EncoderDecoder Framework - -This model is a warm-started *BERT2BERT* ([mini](https://huggingface.co/google/bert_uncased_L-4_H-256_A-4)) model fine-tuned on the *CNN/Dailymail* summarization dataset. - -The model achieves a **16.51** ROUGE-2 score on *CNN/Dailymail*'s test dataset. - -For more details on how the model was fine-tuned, please refer to -[this](https://colab.research.google.com/drive/1Ekd5pUeCX7VOrMx94_czTkwNtLN32Uyu?usp=sharing) notebook. - -## Results on test set 📝 - -| Metric | # Value | -| ------ | --------- | -| **ROUGE-2** | **16.51** | - - - -## Model in Action 🚀 - -```python -from transformers import BertTokenizerFast, EncoderDecoderModel -import torch -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -tokenizer = BertTokenizerFast.from_pretrained('mrm8488/bert-mini2bert-mini-finetuned-cnn_daily_mail-summarization') -model = EncoderDecoderModel.from_pretrained('mrm8488/bert-mini2bert-mini-finetuned-cnn_daily_mail-summarization').to(device) - -def generate_summary(text): - # cut off at BERT max length 512 - inputs = tokenizer([text], padding="max_length", truncation=True, max_length=512, return_tensors="pt") - input_ids = inputs.input_ids.to(device) - attention_mask = inputs.attention_mask.to(device) - - output = model.generate(input_ids, attention_mask=attention_mask) - - return tokenizer.decode(output[0], skip_special_tokens=True) - -text = "your text to be summarized here..." -generate_summary(text) -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-multi-cased-finedtuned-xquad-tydiqa-goldp/README.md b/model_cards/mrm8488/bert-multi-cased-finedtuned-xquad-tydiqa-goldp/README.md deleted file mode 100644 index 45a08bd6dd64..000000000000 --- a/model_cards/mrm8488/bert-multi-cased-finedtuned-xquad-tydiqa-goldp/README.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -language: multilingual -thumbnail: ---- - -# A fine-tuned model on GoldP task from Tydi QA dataset - -This model uses [bert-multi-cased-finetuned-xquadv1](https://huggingface.co/mrm8488/bert-multi-cased-finetuned-xquadv1) and fine-tuned on [Tydi QA](https://github.com/google-research-datasets/tydiqa) dataset for Gold Passage task [(GoldP)](https://github.com/google-research-datasets/tydiqa#the-tasks) - -## Details of the language model -The base language model [(bert-multi-cased-finetuned-xquadv1)](https://huggingface.co/mrm8488/bert-multi-cased-finetuned-xquadv1) is a fine-tuned version of [bert-base-multilingual-cased](https://huggingface.co/bert-base-multilingual-cased) for the **Q&A** downstream task - - -## Details of the Tydi QA dataset - -TyDi QA contains 200k human-annotated question-answer pairs in 11 Typologically Diverse languages, written without seeing the answer and without the use of translation, and is designed for the **training and evaluation** of automatic question answering systems. This repository provides evaluation code and a baseline system for the dataset. https://ai.google.com/research/tydiqa - - -## Details of the downstream task (Gold Passage or GoldP aka the secondary task) - -Given a passage that is guaranteed to contain the answer, predict the single contiguous span of characters that answers the question. the gold passage task differs from the [primary task](https://github.com/google-research-datasets/tydiqa/blob/master/README.md#the-tasks) in several ways: -* only the gold answer passage is provided rather than the entire Wikipedia article; -* unanswerable questions have been discarded, similar to MLQA and XQuAD; -* we evaluate with the SQuAD 1.1 metrics like XQuAD; and -* Thai and Japanese are removed since the lack of whitespace breaks some tools. - - -## Model training - -The model was fine-tuned on a Tesla P100 GPU and 25GB of RAM. -The script is the following: - -```python -python run_squad.py \ - --model_type bert \ - --model_name_or_path mrm8488/bert-multi-cased-finetuned-xquadv1 \ - --do_train \ - --do_eval \ - --train_file /content/dataset/train.json \ - --predict_file /content/dataset/dev.json \ - --per_gpu_train_batch_size 24 \ - --per_gpu_eval_batch_size 24 \ - --learning_rate 3e-5 \ - --num_train_epochs 2.5 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /content/model_output \ - --overwrite_output_dir \ - --save_steps 5000 \ - --threads 40 - ``` - -## Global Results (dev set): - -| Metric | # Value | -| --------- | ----------- | -| **Exact** | **71.06** | -| **F1** | **82.16** | - -## Specific Results (per language): - -| Language | # Samples | # Exact | # F1 | -| --------- | ----------- |--------| ------ | -| Arabic | 1314 | 73.29 | 84.72 | -| Bengali | 180 | 64.60 | 77.84 | -| English | 654 | 72.12 | 82.24 | -| Finnish | 1031 | 70.14 | 80.36 | -| Indonesian| 773 | 77.25 | 86.36 | -| Korean | 414 | 68.92 | 70.95 | -| Russian | 1079 | 62.65 | 78.55 | -| Swahili | 596 | 80.11 | 86.18 | -| Telegu | 874 | 71.00 | 84.24 | - - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-multi-cased-finetuned-xquadv1/README.md b/model_cards/mrm8488/bert-multi-cased-finetuned-xquadv1/README.md deleted file mode 100644 index 7849ec85f1df..000000000000 --- a/model_cards/mrm8488/bert-multi-cased-finetuned-xquadv1/README.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -language: multilingual -thumbnail: ---- - -# BERT (base-multilingual-cased) fine-tuned for multilingual Q&A - -This model was created by [Google](https://github.com/google-research/bert/blob/master/multilingual.md) and fine-tuned on [XQuAD](https://github.com/deepmind/xquad) like data for multilingual (`11 different languages`) **Q&A** downstream task. - -## Details of the language model('bert-base-multilingual-cased') - -[Language model](https://github.com/google-research/bert/blob/master/multilingual.md) - -| Languages | Heads | Layers | Hidden | Params | -| --------- | ----- | ------ | ------ | ------ | -| 104 | 12 | 12 | 768 | 100 M | - -## Details of the downstream task (multilingual Q&A) - Dataset - -Deepmind [XQuAD](https://github.com/deepmind/xquad) - -Languages covered: - -- Arabic: `ar` -- German: `de` -- Greek: `el` -- English: `en` -- Spanish: `es` -- Hindi: `hi` -- Russian: `ru` -- Thai: `th` -- Turkish: `tr` -- Vietnamese: `vi` -- Chinese: `zh` - -As the dataset is based on SQuAD v1.1, there are no unanswerable questions in the data. We chose this -setting so that models can focus on cross-lingual transfer. - -We show the average number of tokens per paragraph, question, and answer for each language in the -table below. The statistics were obtained using [Jieba](https://github.com/fxsjy/jieba) for Chinese -and the [Moses tokenizer](https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl) -for the other languages. - -| | en | es | de | el | ru | tr | ar | vi | th | zh | hi | -| --------- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -| Paragraph | 142.4 | 160.7 | 139.5 | 149.6 | 133.9 | 126.5 | 128.2 | 191.2 | 158.7 | 147.6 | 232.4 | -| Question | 11.5 | 13.4 | 11.0 | 11.7 | 10.0 | 9.8 | 10.7 | 14.8 | 11.5 | 10.5 | 18.7 | -| Answer | 3.1 | 3.6 | 3.0 | 3.3 | 3.1 | 3.1 | 3.1 | 4.5 | 4.1 | 3.5 | 5.6 | - -Citation: - -
- -```bibtex -@article{Artetxe:etal:2019, - author = {Mikel Artetxe and Sebastian Ruder and Dani Yogatama}, - title = {On the cross-lingual transferability of monolingual representations}, - journal = {CoRR}, - volume = {abs/1910.11856}, - year = {2019}, - archivePrefix = {arXiv}, - eprint = {1910.11856} -} -``` - -
- -As **XQuAD** is just an evaluation dataset, I used `Data augmentation techniques` (scraping, neural machine translation, etc) to obtain more samples and split the dataset in order to have a train and test set. The test set was created in a way that contains the same number of samples for each language. Finally, I got: - -| Dataset | # samples | -| ----------- | --------- | -| XQUAD train | 50 K | -| XQUAD test | 8 K | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/distillation/run_squad_w_distillation.py) - - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/bert-multi-cased-finetuned-xquadv1", - tokenizer="mrm8488/bert-multi-cased-finetuned-xquadv1" -) - - -# context: Coronavirus is seeding panic in the West because it expands so fast. - -# question: Where is seeding panic Coronavirus? -qa_pipeline({ - 'context': "कोरोनावायरस पश्चिम में आतंक बो रहा है क्योंकि यह इतनी तेजी से फैलता है।", - 'question': "कोरोनावायरस घबराहट कहां है?" - -}) -# output: {'answer': 'पश्चिम', 'end': 18, 'score': 0.7037217439689059, 'start': 12} - -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" - -}) -# output: {'answer': 'Manuel Romero', 'end': 13, 'score': 0.7254485993702389, 'start': 0} - -qa_pipeline({ - 'context': "Manuel Romero a travaillé à peine dans le référentiel hugginface / transformers ces derniers temps", - 'question': "Pour quel référentiel a travaillé Manuel Romero récemment?" - -}) -#output: {'answer': 'hugginface / transformers', 'end': 79, 'score': 0.6482061613915384, 'start': 54} -``` -![model in action](https://media.giphy.com/media/MBlire8Wj7ng73VBQ5/giphy.gif) - -Try it on a Colab: - -Open In Colab - - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-multi-uncased-finetuned-xquadv1/README.md b/model_cards/mrm8488/bert-multi-uncased-finetuned-xquadv1/README.md deleted file mode 100644 index f04c5698854f..000000000000 --- a/model_cards/mrm8488/bert-multi-uncased-finetuned-xquadv1/README.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -language: multilingual -thumbnail: ---- - -# BERT (base-multilingual-uncased) fine-tuned for multilingual Q&A - -This model was created by [Google](https://github.com/google-research/bert/blob/master/multilingual.md) and fine-tuned on [XQuAD](https://github.com/deepmind/xquad) like data for multilingual (`11 different languages`) **Q&A** downstream task. - -## Details of the language model('bert-base-multilingual-uncased') - -[Language model](https://github.com/google-research/bert/blob/master/multilingual.md) - -| Languages | Heads | Layers | Hidden | Params | -| --------- | ----- | ------ | ------ | ------ | -| 102 | 12 | 12 | 768 | 100 M | - -## Details of the downstream task (multilingual Q&A) - Dataset - -Deepmind [XQuAD](https://github.com/deepmind/xquad) - -Languages covered: - -- Arabic: `ar` -- German: `de` -- Greek: `el` -- English: `en` -- Spanish: `es` -- Hindi: `hi` -- Russian: `ru` -- Thai: `th` -- Turkish: `tr` -- Vietnamese: `vi` -- Chinese: `zh` - -As the dataset is based on SQuAD v1.1, there are no unanswerable questions in the data. We chose this -setting so that models can focus on cross-lingual transfer. - -We show the average number of tokens per paragraph, question, and answer for each language in the -table below. The statistics were obtained using [Jieba](https://github.com/fxsjy/jieba) for Chinese -and the [Moses tokenizer](https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl) -for the other languages. - -| | en | es | de | el | ru | tr | ar | vi | th | zh | hi | -| --------- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -| Paragraph | 142.4 | 160.7 | 139.5 | 149.6 | 133.9 | 126.5 | 128.2 | 191.2 | 158.7 | 147.6 | 232.4 | -| Question | 11.5 | 13.4 | 11.0 | 11.7 | 10.0 | 9.8 | 10.7 | 14.8 | 11.5 | 10.5 | 18.7 | -| Answer | 3.1 | 3.6 | 3.0 | 3.3 | 3.1 | 3.1 | 3.1 | 4.5 | 4.1 | 3.5 | 5.6 | - -Citation: - -
- -```bibtex -@article{Artetxe:etal:2019, - author = {Mikel Artetxe and Sebastian Ruder and Dani Yogatama}, - title = {On the cross-lingual transferability of monolingual representations}, - journal = {CoRR}, - volume = {abs/1910.11856}, - year = {2019}, - archivePrefix = {arXiv}, - eprint = {1910.11856} -} -``` - -
- -As **XQuAD** is just an evaluation dataset, I used `Data augmentation techniques` (scraping, neural machine translation, etc) to obtain more samples and split the dataset in order to have a train and test set. The test set was created in a way that contains the same number of samples for each language. Finally, I got: - -| Dataset | # samples | -| ----------- | --------- | -| XQUAD train | 50 K | -| XQUAD test | 8 K | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/distillation/run_squad_w_distillation.py) - - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/bert-multi-uncased-finetuned-xquadv1", - tokenizer="mrm8488/bert-multi-uncased-finetuned-xquadv1" -) - - -# context: Coronavirus is seeding panic in the West because it expands so fast. - -# question: Where is seeding panic Coronavirus? -qa_pipeline({ - 'context': "कोरोनावायरस पश्चिम में आतंक बो रहा है क्योंकि यह इतनी तेजी से फैलता है।", - 'question': "कोरोनावायरस घबराहट कहां है?" - -}) -# output: {'answer': 'पश्चिम', 'end': 18, 'score': 0.7037217439689059, 'start': 12} - -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" - -}) -# output: {'answer': 'Manuel Romero', 'end': 13, 'score': 0.7254485993702389, 'start': 0} - -qa_pipeline({ - 'context': "Manuel Romero a travaillé à peine dans le référentiel hugginface / transformers ces derniers temps", - 'question': "Pour quel référentiel a travaillé Manuel Romero récemment?" - -}) -#output: {'answer': 'hugginface / transformers', 'end': 79, 'score': 0.6482061613915384, 'start': 54} -``` -![model in action](https://media.giphy.com/media/MBlire8Wj7ng73VBQ5/giphy.gif) - -Try it on a Colab: - -Open In Colab - - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-small-finetuned-squadv2/README.md b/model_cards/mrm8488/bert-small-finetuned-squadv2/README.md deleted file mode 100644 index 0a4811c00772..000000000000 --- a/model_cards/mrm8488/bert-small-finetuned-squadv2/README.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -language: en -thumbnail: ---- - -# BERT-Small fine-tuned on SQuAD v2 - -[BERT-Small](https://github.com/google-research/bert/) created by [Google Research](https://github.com/google-research) and fine-tuned on [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -**Mode size** (after training): **109.74 MB** - -## Details of BERT-Small and its 'family' (from their documentation) - -Released on March 11th, 2020 - -This is model is a part of 24 smaller BERT models (English only, uncased, trained with WordPiece masking) referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). - -The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. - -## Details of the downstream task (Q&A) - Dataset - -[SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD2.0 | train | 130k | -| SQuAD2.0 | eval | 12.3k | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/question-answering/run_squad.py) - -## Results: - -| Metric | # Value | -| ------ | --------- | -| **EM** | **60.49** | -| **F1** | **64.21** | - -## Comparison: - -| Model | EM | F1 score | SIZE (MB) | -| ------------------------------------------------------------------------------------------- | --------- | --------- | --------- | -| [bert-tiny-finetuned-squadv2](https://huggingface.co/mrm8488/bert-tiny-finetuned-squadv2) | 48.60 | 49.73 | **16.74** | -| [bert-mini-finetuned-squadv2](https://huggingface.co/mrm8488/bert-mini-finetuned-squadv2) | 56.31 | 59.65 | 42.63 | -| [bert-small-finetuned-squadv2](https://huggingface.co/mrm8488/bert-small-finetuned-squadv2) | **60.49** | **64.21** | 109.74 | - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/bert-small-finetuned-squadv2", - tokenizer="mrm8488/bert-small-finetuned-squadv2" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" - -}) - -# Output: -``` - -```json -{ - "answer": "Manuel Romero", - "end": 13, - "score": 0.9939319924374637, - "start": 0 -} -``` - -### Yes! That was easy 🎉 Let's try with another example - -```python -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "For which company has worked Manuel Romero?" -}) - -# Output: -``` - -```json -{ - "answer": "hugginface/transformers", - "end": 79, - "score": 0.6024888734447131, - "start": 56 -} -``` - -### It works!! 🎉 🎉 🎉 - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-small-finetuned-typo-detection/README.md b/model_cards/mrm8488/bert-small-finetuned-typo-detection/README.md deleted file mode 100644 index 1e2c83436ad3..000000000000 --- a/model_cards/mrm8488/bert-small-finetuned-typo-detection/README.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -language: en -thumbnail: ---- - -# BERT SMALL + Typo Detection ✍❌✍✔ - -[BERT SMALL](https://huggingface.co/google/bert_uncased_L-4_H-512_A-8) fine-tuned on [GitHub Typo Corpus](https://github.com/mhagiwara/github-typo-corpus) for **typo detection** (using *NER* style) - -## Details of the downstream task (Typo detection as NER) - -- Dataset: [GitHub Typo Corpus](https://github.com/mhagiwara/github-typo-corpus) 📚 - -- [Fine-tune script on NER dataset provided by Huggingface](https://github.com/huggingface/transformers/blob/master/examples/token-classification/run_ner_old.py) 🏋️‍♂️ - -## Metrics on test set 📋 - -| Metric | # score | -| :-------: | :-------: | -| F1 | **89.12** | -| Precision | **93.82** | -| Recall | **84.87** | - -## Model in action 🔨 - -Fast usage with **pipelines** 🧪 - -```python -from transformers import pipeline - -typo_checker = pipeline( - "ner", - model="mrm8488/bert-small-finetuned-typo-detection", - tokenizer="mrm8488/bert-small-finetuned-typo-detection" -) - -result = typo_checker("here there is an error in coment") -result[1:-1] - -# Output: -[{'entity': 'ok', 'score': 0.9021041989326477, 'word': 'here'}, - {'entity': 'ok', 'score': 0.7975626587867737, 'word': 'there'}, - {'entity': 'ok', 'score': 0.8596242070198059, 'word': 'is'}, - {'entity': 'ok', 'score': 0.7071516513824463, 'word': 'an'}, - {'entity': 'ok', 'score': 0.943381130695343, 'word': 'error'}, - {'entity': 'ok', 'score': 0.8047608733177185, 'word': 'in'}, - {'entity': 'ok', 'score': 0.8240702152252197, 'word': 'come'}, - {'entity': 'typo', 'score': 0.5004884004592896, 'word': '##nt'}] -``` - -It works🎉! we typed ```coment``` instead of ```comment``` - -Let's try with another example - -```python -result = typo_checker("Adddd validation midelware") -result[1:-1] - -# Output: -[{'entity': 'ok', 'score': 0.7128152847290039, 'word': 'add'}, - {'entity': 'typo', 'score': 0.5388424396514893, 'word': '##dd'}, - {'entity': 'ok', 'score': 0.94792640209198, 'word': 'validation'}, - {'entity': 'typo', 'score': 0.5839331746101379, 'word': 'mid'}, - {'entity': 'ok', 'score': 0.5195121765136719, 'word': '##el'}, - {'entity': 'ok', 'score': 0.7222476601600647, 'word': '##ware'}] -``` -Yeah! We typed wrong ```Add and middleware``` - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-small2bert-small-finetuned-cnn_daily_mail-summarization/README.md b/model_cards/mrm8488/bert-small2bert-small-finetuned-cnn_daily_mail-summarization/README.md deleted file mode 100644 index 7db7b72504a4..000000000000 --- a/model_cards/mrm8488/bert-small2bert-small-finetuned-cnn_daily_mail-summarization/README.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- cnn_dailymail -tags: -- summarization ---- - -# Bert-small2Bert-small Summarization with 🤗EncoderDecoder Framework - -This model is a warm-started *BERT2BERT* ([small](https://huggingface.co/google/bert_uncased_L-4_H-512_A-8)) model fine-tuned on the *CNN/Dailymail* summarization dataset. - -The model achieves a **17.37** ROUGE-2 score on *CNN/Dailymail*'s test dataset. - -For more details on how the model was fine-tuned, please refer to -[this](https://colab.research.google.com/drive/1Ekd5pUeCX7VOrMx94_czTkwNtLN32Uyu?usp=sharing) notebook. - -## Results on test set 📝 - -| Metric | # Value | -| ------ | --------- | -| **ROUGE-2** | **17.37** | - - - -## Model in Action 🚀 - -```python -from transformers import BertTokenizerFast, EncoderDecoderModel -import torch -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -tokenizer = BertTokenizerFast.from_pretrained('mrm8488/bert-small2bert-small-finetuned-cnn_daily_mail-summarization') -model = EncoderDecoderModel.from_pretrained('mrm8488/bert-small2bert-small-finetuned-cnn_daily_mail-summarization').to(device) - -def generate_summary(text): - # cut off at BERT max length 512 - inputs = tokenizer([text], padding="max_length", truncation=True, max_length=512, return_tensors="pt") - input_ids = inputs.input_ids.to(device) - attention_mask = inputs.attention_mask.to(device) - - output = model.generate(input_ids, attention_mask=attention_mask) - - return tokenizer.decode(output[0], skip_special_tokens=True) - -text = "your text to be summarized here..." -generate_summary(text) -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-spanish-cased-finetuned-ner/README.md b/model_cards/mrm8488/bert-spanish-cased-finetuned-ner/README.md deleted file mode 100644 index 4468b57f978d..000000000000 --- a/model_cards/mrm8488/bert-spanish-cased-finetuned-ner/README.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -language: es -thumbnail: https://i.imgur.com/jgBdimh.png ---- - -# Spanish BERT (BETO) + NER - -This model is a fine-tuned on [NER-C](https://www.kaggle.com/nltkdata/conll-corpora) version of the Spanish BERT cased [(BETO)](https://github.com/dccuchile/beto) for **NER** downstream task. - -## Details of the downstream task (NER) - Dataset - -- [Dataset: CONLL Corpora ES](https://www.kaggle.com/nltkdata/conll-corpora) - -I preprocessed the dataset and split it as train / dev (80/20) - -| Dataset | # Examples | -| ---------------------- | ----- | -| Train | 8.7 K | -| Dev | 2.2 K | - - -- [Fine-tune on NER script provided by Huggingface](https://github.com/huggingface/transformers/blob/master/examples/token-classification/run_ner_old.py) - -- Labels covered: - -``` -B-LOC -B-MISC -B-ORG -B-PER -I-LOC -I-MISC -I-ORG -I-PER -O -``` - -## Metrics on evaluation set: - -| Metric | # score | -| :------------------------------------------------------------------------------------: | :-------: | -| F1 | **90.17** -| Precision | **89.86** | -| Recall | **90.47** | - -## Comparison: - -| Model | # F1 score |Size(MB)| -| :--------------------------------------------------------------------------------------------------------------: | :-------: |:------| -| bert-base-spanish-wwm-cased (BETO) | 88.43 | 421 -| [bert-spanish-cased-finetuned-ner (this one)](https://huggingface.co/mrm8488/bert-spanish-cased-finetuned-ner) | **90.17** | 420 | -| Best Multilingual BERT | 87.38 | 681 | -|[TinyBERT-spanish-uncased-finetuned-ner](https://huggingface.co/mrm8488/TinyBERT-spanish-uncased-finetuned-ner) | 70.00 | **55** | - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -nlp_ner = pipeline( - "ner", - model="mrm8488/bert-spanish-cased-finetuned-ner", - tokenizer=( - 'mrm8488/bert-spanish-cased-finetuned-ner', - {"use_fast": False} -)) - -text = 'Mis amigos están pensando viajar a Londres este verano' - -nlp_ner(text) - -#Output: [{'entity': 'B-LOC', 'score': 0.9998720288276672, 'word': 'Londres'}] -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-spanish-cased-finetuned-pos-syntax/README.md b/model_cards/mrm8488/bert-spanish-cased-finetuned-pos-syntax/README.md deleted file mode 100644 index 54bb61e2b2ad..000000000000 --- a/model_cards/mrm8488/bert-spanish-cased-finetuned-pos-syntax/README.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -language: es -thumbnail: ---- - -# Spanish BERT (BETO) + Syntax POS tagging ✍🏷 - -This model is a fine-tuned version of the Spanish BERT [(BETO)](https://github.com/dccuchile/beto) on Spanish **syntax** annotations in [CONLL CORPORA](https://www.kaggle.com/nltkdata/conll-corpora) dataset for **syntax POS** (Part of Speech tagging) downstream task. - -## Details of the downstream task (Syntax POS) - Dataset - -- [Dataset: CONLL Corpora ES](https://www.kaggle.com/nltkdata/conll-corpora) - -#### [Fine-tune script on NER dataset provided by Huggingface](https://github.com/huggingface/transformers/blob/master/examples/token-classification/run_ner_old.py) - -#### 21 Syntax annotations (Labels) covered: - -- \_ -- ATR -- ATR.d -- CAG -- CC -- CD -- CD.Q -- CI -- CPRED -- CPRED.CD -- CPRED.SUJ -- CREG -- ET -- IMPERS -- MOD -- NEG -- PASS -- PUNC -- ROOT -- SUJ -- VOC - -## Metrics on test set 📋 - -| Metric | # score | -| :-------: | :-------: | -| F1 | **89.27** | -| Precision | **89.44** | -| Recall | **89.11** | - -## Model in action 🔨 - -Fast usage with **pipelines** 🧪 - -```python -from transformers import pipeline - -nlp_pos_syntax = pipeline( - "ner", - model="mrm8488/bert-spanish-cased-finetuned-pos-syntax", - tokenizer="mrm8488/bert-spanish-cased-finetuned-pos-syntax" -) - -text = 'Mis amigos están pensando viajar a Londres este verano.' - -nlp_pos_syntax(text)[1:len(nlp_pos_syntax(text))-1] -``` - -```json -[ - { "entity": "_", "score": 0.9999216794967651, "word": "Mis" }, - { "entity": "SUJ", "score": 0.999882698059082, "word": "amigos" }, - { "entity": "_", "score": 0.9998869299888611, "word": "están" }, - { "entity": "ROOT", "score": 0.9980518221855164, "word": "pensando" }, - { "entity": "_", "score": 0.9998420476913452, "word": "viajar" }, - { "entity": "CD", "score": 0.999351978302002, "word": "a" }, - { "entity": "_", "score": 0.999959409236908, "word": "Londres" }, - { "entity": "_", "score": 0.9998968839645386, "word": "este" }, - { "entity": "CC", "score": 0.99931401014328, "word": "verano" }, - { "entity": "PUNC", "score": 0.9998534917831421, "word": "." } -] -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-spanish-cased-finetuned-pos/README.md b/model_cards/mrm8488/bert-spanish-cased-finetuned-pos/README.md deleted file mode 100644 index 356dd0f5ab93..000000000000 --- a/model_cards/mrm8488/bert-spanish-cased-finetuned-pos/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: es -thumbnail: https://i.imgur.com/jgBdimh.png ---- - -# Spanish BERT (BETO) + POS - -This model is a fine-tuned on Spanish [CONLL CORPORA](https://www.kaggle.com/nltkdata/conll-corpora) version of the Spanish BERT cased [(BETO)](https://github.com/dccuchile/beto) for **POS** (Part of Speech tagging) downstream task. - -## Details of the downstream task (POS) - Dataset - -- [Dataset: CONLL Corpora ES](https://www.kaggle.com/nltkdata/conll-corpora) with data augmentation techniques - -I preprocessed the dataset and split it as train / dev (80/20) - -| Dataset | # Examples | -| ---------------------- | ----- | -| Train | 340 K | -| Dev | 50 K | - - -- [Fine-tune on NER script provided by Huggingface](https://github.com/huggingface/transformers/blob/master/examples/token-classification/run_ner_old.py) - -- **60** Labels covered: - -``` -AO, AQ, CC, CS, DA, DD, DE, DI, DN, DP, DT, Faa, Fat, Fc, Fd, Fe, Fg, Fh, Fia, Fit, Fp, Fpa, Fpt, Fs, Ft, Fx, Fz, I, NC, NP, P0, PD, PI, PN, PP, PR, PT, PX, RG, RN, SP, VAI, VAM, VAN, VAP, VAS, VMG, VMI, VMM, VMN, VMP, VMS, VSG, VSI, VSM, VSN, VSP, VSS, Y and Z -``` - - -## Metrics on evaluation set: - -| Metric | # score | -| :------------------------------------------------------------------------------------: | :-------: | -| F1 | **90.06** -| Precision | **89.46** | -| Recall | **90.67** | - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -nlp_pos = pipeline( - "ner", - model="mrm8488/bert-spanish-cased-finetuned-pos", - tokenizer=( - 'mrm8488/bert-spanish-cased-finetuned-pos', - {"use_fast": False} -)) - - -text = 'Mis amigos están pensando en viajar a Londres este verano' - -nlp_pos(text) - -#Output: -''' -[{'entity': 'NC', 'score': 0.7792173624038696, 'word': '[CLS]'}, - {'entity': 'DP', 'score': 0.9996283650398254, 'word': 'Mis'}, - {'entity': 'NC', 'score': 0.9999253749847412, 'word': 'amigos'}, - {'entity': 'VMI', 'score': 0.9998560547828674, 'word': 'están'}, - {'entity': 'VMG', 'score': 0.9992249011993408, 'word': 'pensando'}, - {'entity': 'SP', 'score': 0.9999602437019348, 'word': 'en'}, - {'entity': 'VMN', 'score': 0.9998666048049927, 'word': 'viajar'}, - {'entity': 'SP', 'score': 0.9999545216560364, 'word': 'a'}, - {'entity': 'VMN', 'score': 0.8722310662269592, 'word': 'Londres'}, - {'entity': 'DD', 'score': 0.9995203614234924, 'word': 'este'}, - {'entity': 'NC', 'score': 0.9999248385429382, 'word': 'verano'}, - {'entity': 'NC', 'score': 0.8802427649497986, 'word': '[SEP]'}] - ''' -``` -![model in action](https://media.giphy.com/media/jVC9m1cNrdIWuAAtjy/giphy.gif) - -16 POS tags version also available [here](https://huggingface.co/mrm8488/bert-spanish-cased-finetuned-pos-16-tags) - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-tiny-finetuned-squadv2/README.md b/model_cards/mrm8488/bert-tiny-finetuned-squadv2/README.md deleted file mode 100644 index 9994683503ce..000000000000 --- a/model_cards/mrm8488/bert-tiny-finetuned-squadv2/README.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -language: en -thumbnail: ---- - -# BERT-Tiny fine-tuned on SQuAD v2 - -[BERT-Tiny](https://github.com/google-research/bert/) created by [Google Research](https://github.com/google-research) and fine-tuned on [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -**Mode size** (after training): **16.74 MB** - -## Details of BERT-Tiny and its 'family' (from their documentation) - -Released on March 11th, 2020 - -This is model is a part of 24 smaller BERT models (English only, uncased, trained with WordPiece masking) referenced in [Well-Read Students Learn Better: On the Importance of Pre-training Compact Models](https://arxiv.org/abs/1908.08962). - -The smaller BERT models are intended for environments with restricted computational resources. They can be fine-tuned in the same manner as the original BERT models. However, they are most effective in the context of knowledge distillation, where the fine-tuning labels are produced by a larger and more accurate teacher. - -## Details of the downstream task (Q&A) - Dataset - -[SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD2.0 | train | 130k | -| SQuAD2.0 | eval | 12.3k | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/question-answering/run_squad.py) - -## Results: - -| Metric | # Value | -| ------ | --------- | -| **EM** | **48.60** | -| **F1** | **49.73** | - - -| Model | EM | F1 score | SIZE (MB) | -| ----------------------------------------------------------------------------------------- | --------- | --------- | --------- | -| [bert-tiny-finetuned-squadv2](https://huggingface.co/mrm8488/bert-tiny-finetuned-squadv2) | 48.60 | 49.73 | **16.74** | -| [bert-tiny-5-finetuned-squadv2](https://huggingface.co/mrm8488/bert-tiny-5-finetuned-squadv2) | **57.12** | **60.86** | 24.34 - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/bert-tiny-finetuned-squadv2", - tokenizer="mrm8488/bert-tiny-finetuned-squadv2" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" - -}) - -# Output: -``` - -```json -{ - "answer": "Manuel Romero", - "end": 13, - "score": 0.05684709993458714, - "start": 0 -} -``` - -### Yes! That was easy 🎉 Let's try with another example - -```python -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "For which company has worked Manuel Romero?" -}) - -# Output: -``` - -```json -{ - "answer": "hugginface/transformers", - "end": 79, - "score": 0.11613431826808274, - "start": 56 -} -``` - -### It works!! 🎉 🎉 🎉 - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/bert-uncased-finetuned-qnli/README.md b/model_cards/mrm8488/bert-uncased-finetuned-qnli/README.md deleted file mode 100644 index a932f8481cbc..000000000000 --- a/model_cards/mrm8488/bert-uncased-finetuned-qnli/README.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -language: en -thumbnail: ---- - -# [BERT](https://huggingface.co/deepset/bert-base-cased-squad2) fine tuned on [QNLI](https://github.com/rhythmcao/QNLI)+ compression ([BERT-of-Theseus](https://github.com/JetRunner/BERT-of-Theseus)) - -I used a [Bert model fine tuned on **SQUAD v2**](https://huggingface.co/deepset/bert-base-cased-squad2) and then I fine tuned it on **QNLI** using **compression** (with a constant replacing rate) as proposed in **BERT-of-Theseus** - -## Details of the downstream task (QNLI): - -### Getting the dataset -```bash -wget https://raw.githubusercontent.com/rhythmcao/QNLI/master/data/QNLI/train.tsv -wget https://raw.githubusercontent.com/rhythmcao/QNLI/master/data/QNLI/test.tsv -wget https://raw.githubusercontent.com/rhythmcao/QNLI/master/data/QNLI/dev.tsv - -mkdir QNLI_dataset -mv *.tsv QNLI_dataset -``` - -### Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -!python /content/BERT-of-Theseus/run_glue.py \ - --model_name_or_path deepset/bert-base-cased-squad2 \ - --task_name qnli \ - --do_train \ - --do_eval \ - --do_lower_case \ - --data_dir /content/QNLI_dataset \ - --max_seq_length 128 \ - --per_gpu_train_batch_size 32 \ - --per_gpu_eval_batch_size 32 \ - --learning_rate 2e-5 \ - --save_steps 2000 \ - --num_train_epochs 50 \ - --output_dir /content/ouput_dir \ - --evaluate_during_training \ - --replacing_rate 0.7 \ - --steps_for_replacing 2500 -``` - -## Metrics: - -| Model | Accuracy | -|-----------------|------| -| BERT-base | 91.2 | -| BERT-of-Theseus | 88.8 | -| [bert-uncased-finetuned-qnli](https://huggingface.co/mrm8488/bert-uncased-finetuned-qnli) | 87.2 -| DistillBERT | 85.3 | - - - - -> [See all my models](https://huggingface.co/models?search=mrm8488) - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/camembert-base-finetuned-pawsx-fr/README.md b/model_cards/mrm8488/camembert-base-finetuned-pawsx-fr/README.md deleted file mode 100644 index 01e65e3ffa06..000000000000 --- a/model_cards/mrm8488/camembert-base-finetuned-pawsx-fr/README.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -language: fr -datasets: -- xtreme -widget: -- text: "La première série a été mieux reçue par la critique que la seconde. La seconde série a été bien accueillie par la critique, mieux que la première." ---- - -# Camembert-base fine-tuned on PAWS-X-fr for Paraphrase Identification diff --git a/model_cards/mrm8488/chEMBL_smiles_v1/README.md b/model_cards/mrm8488/chEMBL_smiles_v1/README.md deleted file mode 100644 index 4817abea7453..000000000000 --- a/model_cards/mrm8488/chEMBL_smiles_v1/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# *De Novo* Drug Design with MLM - -## What is it? - -An approximation to [Generative Recurrent Networks for De Novo Drug Design](https://onlinelibrary.wiley.com/doi/full/10.1002/minf.201700111) but training a MLM (RoBERTa like) from scratch. - -## Why? - -As mentioned in the paper: -Generative artificial intelligence models present a fresh approach to chemogenomics and de novo drug design, as they provide researchers with the ability to narrow down their search of the chemical space and focus on regions of interest. -They used a generative *recurrent neural network (RNN)* containing long short‐term memory (LSTM) cell to capture the syntax of molecular representations in terms of SMILES strings. -The learned pattern probabilities can be used for de novo SMILES generation. This molecular design concept **eliminates the need for virtual compound library enumeration** and **enables virtual compound design without requiring secondary or external activity prediction**. - - -## My Goal 🎯 - -By training a MLM from scratch on 438552 (cleaned*) SMILES I wanted to build a model that learns this kind of molecular combinations so that given a partial SMILE it can generate plausible combinations so that it can be proposed as new drugs. -By cleaned SMILES I mean that I used their [SMILES cleaning script](https://github.com/topazape/LSTM_Chem/blob/master/cleanup_smiles.py) to remove duplicates, salts, and stereochemical information. -You can see the detailed process of gathering the data, preprocess it and train the LSTM in their [repo](https://github.com/topazape/LSTM_Chem). - -## Fast usage with ```pipelines``` 🧪 - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model='/mrm8488/chEMBL_smiles_v1', - tokenizer='/mrm8488/chEMBL_smiles_v1' -) - -# CC(C)CN(CC(OP(=O)(O)O)C(Cc1ccccc1)NC(=O)OC1CCOC1)S(=O)(=O)c1ccc(N)cc1 Atazanavir -smile1 = "CC(C)CN(CC(OP(=O)(O)O)C(Cc1ccccc1)NC(=O)OC1CCOC1)S(=O)(=O)c1ccc(N)" - -fill_mask(smile1) - -# Output: -''' -[{'score': 0.6040295958518982, - 'sequence': ' CC(C)CN(CC(OP(=O)(O)O)C(Cc1ccccc1)NC(=O)OC1CCOC1)S(=O)(=O)c1ccc(N)nc', - 'token': 265}, - {'score': 0.2185731679201126, - 'sequence': ' CC(C)CN(CC(OP(=O)(O)O)C(Cc1ccccc1)NC(=O)OC1CCOC1)S(=O)(=O)c1ccc(N)N', - 'token': 50}, - {'score': 0.0642734169960022, - 'sequence': ' CC(C)CN(CC(OP(=O)(O)O)C(Cc1ccccc1)NC(=O)OC1CCOC1)S(=O)(=O)c1ccc(N)cc', - 'token': 261}, - {'score': 0.01932266168296337, - 'sequence': ' CC(C)CN(CC(OP(=O)(O)O)C(Cc1ccccc1)NC(=O)OC1CCOC1)S(=O)(=O)c1ccc(N)CCCl', - 'token': 452}, - {'score': 0.005068355705589056, - 'sequence': ' CC(C)CN(CC(OP(=O)(O)O)C(Cc1ccccc1)NC(=O)OC1CCOC1)S(=O)(=O)c1ccc(N)C', - 'token': 39}] - ''' - ``` - ## More - I also created a [second version](https://huggingface.co/mrm8488/chEMBL26_smiles_v2) without applying the cleaning SMILES script mentioned above. You can use it in the same way as this one. - - ```python - fill_mask = pipeline( - "fill-mask", - model='/mrm8488/chEMBL26_smiles_v2', - tokenizer='/mrm8488/chEMBL26_smiles_v2' -) -``` - - [Original paper](https://www.ncbi.nlm.nih.gov/pubmed/29095571) Authors: -
-Swiss Federal Institute of Technology (ETH), Department of Chemistry and Applied Biosciences, Vladimir–Prelog–Weg 4, 8093, Zurich, Switzerland, -Stanford University, Department of Computer Science, 450 Sierra Mall, Stanford, CA, 94305, USA, -inSili.com GmbH, 8049, Zurich, Switzerland, -Gisbert Schneider, Email: hc.zhte@trebsig. -
- - > Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/codeBERTaJS/README.md b/model_cards/mrm8488/codeBERTaJS/README.md deleted file mode 100644 index 6da87c7fe3b4..000000000000 --- a/model_cards/mrm8488/codeBERTaJS/README.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -language: code -thumbnail: ---- - -# CodeBERTaJS - -CodeBERTaJS is a RoBERTa-like model trained on the [CodeSearchNet](https://github.blog/2019-09-26-introducing-the-codesearchnet-challenge/) dataset from GitHub for `javaScript` by [Manuel Romero](https://twitter.com/mrm8488) - -The **tokenizer** is a Byte-level BPE tokenizer trained on the corpus using Hugging Face `tokenizers`. - -Because it is trained on a corpus of code (vs. natural language), it encodes the corpus efficiently (the sequences are between 33% to 50% shorter, compared to the same corpus tokenized by gpt2/roberta). - -The (small) **model** is a 6-layer, 84M parameters, RoBERTa-like Transformer model – that’s the same number of layers & heads as DistilBERT – initialized from the default initialization settings and trained from scratch on the full `javascript` corpus (120M after preproccessing) for 2 epochs. - -## Quick start: masked language modeling prediction - -```python -JS_CODE = """ -async function createUser(req, ) { - if (!validUser(req.body.user)) { - return res.status(400); - } - user = userService.createUser(req.body.user); - return res.json(user); -} -""".lstrip() -``` - -### Does the model know how to complete simple JS/express like code? - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="mrm8488/codeBERTaJS", - tokenizer="mrm8488/codeBERTaJS" -) - -fill_mask(JS_CODE) - -## Top 5 predictions: -# -'res' # prob 0.069489665329 -'next' -'req' -'user' -',req' -``` - -### Yes! That was easy 🎉 Let's try with another example - -```python -JS_CODE_= """ -function getKeys(obj) { - keys = []; - for (var [key, value] of Object.entries(obj)) { - keys.push(); - } - return keys -} -""".lstrip() -``` - -Results: - -```python -'obj', 'key', ' value', 'keys', 'i' -``` - -> Not so bad! Right token was predicted as second option! 🎉 - -## This work is heavely inspired on [codeBERTa](https://github.com/huggingface/transformers/blob/master/model_cards/huggingface/CodeBERTa-small-v1/README.md) by huggingface team - -
- -## CodeSearchNet citation - -
- -```bibtex -@article{husain_codesearchnet_2019, - title = {{CodeSearchNet} {Challenge}: {Evaluating} the {State} of {Semantic} {Code} {Search}}, - shorttitle = {{CodeSearchNet} {Challenge}}, - url = {http://arxiv.org/abs/1909.09436}, - urldate = {2020-03-12}, - journal = {arXiv:1909.09436 [cs, stat]}, - author = {Husain, Hamel and Wu, Ho-Hsiang and Gazit, Tiferet and Allamanis, Miltiadis and Brockschmidt, Marc}, - month = sep, - year = {2019}, - note = {arXiv: 1909.09436}, -} -``` - -
- -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/codebert-base-finetuned-detect-insecure-code/README.md b/model_cards/mrm8488/codebert-base-finetuned-detect-insecure-code/README.md deleted file mode 100644 index 68906e3c67dd..000000000000 --- a/model_cards/mrm8488/codebert-base-finetuned-detect-insecure-code/README.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -language: en -datasets: -- codexglue ---- - -# CodeBERT fine-tuned for Insecure Code Detection 💾⛔ - - -[codebert-base](https://huggingface.co/microsoft/codebert-base) fine-tuned on [CodeXGLUE -- Defect Detection](https://github.com/microsoft/CodeXGLUE/tree/main/Code-Code/Defect-detection) dataset for **Insecure Code Detection** downstream task. - -## Details of [CodeBERT](https://arxiv.org/abs/2002.08155) - -We present CodeBERT, a bimodal pre-trained model for programming language (PL) and nat-ural language (NL). CodeBERT learns general-purpose representations that support downstream NL-PL applications such as natural language codesearch, code documentation generation, etc. We develop CodeBERT with Transformer-based neural architecture, and train it with a hybrid objective function that incorporates the pre-training task of replaced token detection, which is to detect plausible alternatives sampled from generators. This enables us to utilize both bimodal data of NL-PL pairs and unimodal data, where the former provides input tokens for model training while the latter helps to learn better generators. We evaluate CodeBERT on two NL-PL applications by fine-tuning model parameters. Results show that CodeBERT achieves state-of-the-art performance on both natural language code search and code documentation generation tasks. Furthermore, to investigate what type of knowledge is learned in CodeBERT, we construct a dataset for NL-PL probing, and evaluate in a zero-shot setting where parameters of pre-trained models are fixed. Results show that CodeBERT performs better than previous pre-trained models on NL-PL probing. - -## Details of the downstream task (code classification) - Dataset 📚 - -Given a source code, the task is to identify whether it is an insecure code that may attack software systems, such as resource leaks, use-after-free vulnerabilities and DoS attack. We treat the task as binary classification (0/1), where 1 stands for insecure code and 0 for secure code. - -The [dataset](https://github.com/microsoft/CodeXGLUE/tree/main/Code-Code/Defect-detection) used comes from the paper [*Devign*: Effective Vulnerability Identification by Learning Comprehensive Program Semantics via Graph Neural Networks](http://papers.nips.cc/paper/9209-devign-effective-vulnerability-identification-by-learning-comprehensive-program-semantics-via-graph-neural-networks.pdf). All projects are combined and splitted 80%/10%/10% for training/dev/test. - -Data statistics of the dataset are shown in the below table: - -| | #Examples | -| ----- | :-------: | -| Train | 21,854 | -| Dev | 2,732 | -| Test | 2,732 | - -## Test set metrics 🧾 - -| Methods | ACC | -| -------- | :-------: | -| BiLSTM | 59.37 | -| TextCNN | 60.69 | -| [RoBERTa](https://arxiv.org/pdf/1907.11692.pdf) | 61.05 | -| [CodeBERT](https://arxiv.org/pdf/2002.08155.pdf) | 62.08 | -| [Ours](https://huggingface.co/mrm8488/codebert-base-finetuned-detect-insecure-code) | **65.30** | - - -## Model in Action 🚀 - -```python -from transformers import AutoTokenizer, AutoModelForSequenceClassification -import torch -import numpy as np -tokenizer = AutoTokenizer.from_pretrained('mrm8488/codebert-base-finetuned-detect-insecure-code') -model = AutoModelForSequenceClassification.from_pretrained('mrm8488/codebert-base-finetuned-detect-insecure-code') - -inputs = tokenizer("your code here", return_tensors="pt", truncation=True, padding='max_length') -labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 -outputs = model(**inputs, labels=labels) -loss = outputs.loss -logits = outputs.logits - -print(np.argmax(logits.detach().numpy())) -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/distilbert-base-multi-cased-finetuned-typo-detection/README.md b/model_cards/mrm8488/distilbert-base-multi-cased-finetuned-typo-detection/README.md deleted file mode 100644 index 009bc1522c38..000000000000 --- a/model_cards/mrm8488/distilbert-base-multi-cased-finetuned-typo-detection/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -language: multilingual -thumbnail: ---- - -# DISTILBERT 🌎 + Typo Detection ✍❌✍✔ - -[distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased) fine-tuned on [GitHub Typo Corpus](https://github.com/mhagiwara/github-typo-corpus) for **typo detection** (using *NER* style) - -## Details of the downstream task (Typo detection as NER) - -- Dataset: [GitHub Typo Corpus](https://github.com/mhagiwara/github-typo-corpus) 📚 for 15 languages - -- [Fine-tune script on NER dataset provided by Huggingface](https://github.com/huggingface/transformers/blob/master/examples/token-classification/run_ner_old.py) 🏋️‍♂️ - -## Metrics on test set 📋 - -| Metric | # score | -| :-------: | :-------: | -| F1 | **93.51** | -| Precision | **96.08** | -| Recall | **91.06** | - -## Model in action 🔨 - -Fast usage with **pipelines** 🧪 - -```python -from transformers import pipeline - -typo_checker = pipeline( - "ner", - model="mrm8488/distilbert-base-multi-cased-finetuned-typo-detection", - tokenizer="mrm8488/distilbert-base-multi-cased-finetuned-typo-detection" -) - -result = typo_checker("Adddd validation midelware") -result[1:-1] - -# Output: -[{'entity': 'ok', 'score': 0.7128152847290039, 'word': 'add'}, - {'entity': 'typo', 'score': 0.5388424396514893, 'word': '##dd'}, - {'entity': 'ok', 'score': 0.94792640209198, 'word': 'validation'}, - {'entity': 'typo', 'score': 0.5839331746101379, 'word': 'mid'}, - {'entity': 'ok', 'score': 0.5195121765136719, 'word': '##el'}, - {'entity': 'ok', 'score': 0.7222476601600647, 'word': '##ware'}] -``` -It works🎉! We typed wrong ```Add and middleware``` - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/distilbert-multi-finetuned-for-xqua-on-tydiqa/README.md b/model_cards/mrm8488/distilbert-multi-finetuned-for-xqua-on-tydiqa/README.md deleted file mode 100644 index 509ca53f8b90..000000000000 --- a/model_cards/mrm8488/distilbert-multi-finetuned-for-xqua-on-tydiqa/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: multilingual -thumbnail: ---- - -# DistilBERT multilingual fine-tuned on TydiQA (GoldP task) dataset for multilingual Q&A 😛🌍❓ - - -## Details of the language model - -[distilbert-base-multilingual-cased](https://huggingface.co/distilbert-base-multilingual-cased) - - -## Details of the Tydi QA dataset - -TyDi QA contains 200k human-annotated question-answer pairs in 11 Typologically Diverse languages, written without seeing the answer and without the use of translation, and is designed for the **training and evaluation** of automatic question answering systems. This repository provides evaluation code and a baseline system for the dataset. https://ai.google.com/research/tydiqa - - -## Details of the downstream task (Gold Passage or GoldP aka the secondary task) - -Given a passage that is guaranteed to contain the answer, predict the single contiguous span of characters that answers the question. the gold passage task differs from the [primary task](https://github.com/google-research-datasets/tydiqa/blob/master/README.md#the-tasks) in several ways: -* only the gold answer passage is provided rather than the entire Wikipedia article; -* unanswerable questions have been discarded, similar to MLQA and XQuAD; -* we evaluate with the SQuAD 1.1 metrics like XQuAD; and -* Thai and Japanese are removed since the lack of whitespace breaks some tools. - - -## Model training 💪🏋️‍ - -The model was fine-tuned on a Tesla P100 GPU and 25GB of RAM. -The script is the following: - -```python -python transformers/examples/question-answering/run_squad.py \ - --model_type distilbert \ - --model_name_or_path distilbert-base-multilingual-cased \ - --do_train \ - --do_eval \ - --train_file /path/to/dataset/train.json \ - --predict_file /path/to/dataset/dev.json \ - --per_gpu_train_batch_size 24 \ - --per_gpu_eval_batch_size 24 \ - --learning_rate 3e-5 \ - --num_train_epochs 5 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /content/model_output \ - --overwrite_output_dir \ - --save_steps 1000 \ - --threads 400 - ``` - -## Global Results (dev set) 📝 - -| Metric | # Value | -| --------- | ----------- | -| **EM** | **63.85** | -| **F1** | **75.70** | - -## Specific Results (per language) 🌍📝 - -| Language | # Samples | # EM | # F1 | -| --------- | ----------- |--------| ------ | -| Arabic | 1314 | 66.66 | 80.02 | -| Bengali | 180 | 53.09 | 63.50 | -| English | 654 | 62.42 | 73.12 | -| Finnish | 1031 | 64.57 | 75.15 | -| Indonesian| 773 | 67.89 | 79.70 | -| Korean | 414 | 51.29 | 61.73 | -| Russian | 1079 | 55.42 | 70.08 | -| Swahili | 596 | 74.51 | 81.15 | -| Telegu | 874 | 66.21 | 79.85 | - - -## Similar models - -You can also try [bert-multi-cased-finedtuned-xquad-tydiqa-goldp](https://huggingface.co/mrm8488/bert-multi-cased-finedtuned-xquad-tydiqa-goldp) that achieves **F1 = 82.16** and **EM = 71.06** (And of course better marks per language). - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es/README.md b/model_cards/mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es/README.md deleted file mode 100644 index ddd8b0704b4c..000000000000 --- a/model_cards/mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es/README.md +++ /dev/null @@ -1,141 +0,0 @@ ---- -language: es -thumbnail: https://i.imgur.com/jgBdimh.png ---- - -# BETO (Spanish BERT) + Spanish SQuAD2.0 + distillation using 'bert-base-multilingual-cased' as teacher - -This model is a fine-tuned on [SQuAD-es-v2.0](https://github.com/ccasimiro88/TranslateAlignRetrieve) and **distilled** version of [BETO](https://github.com/dccuchile/beto) for **Q&A**. - -Distillation makes the model **smaller, faster, cheaper and lighter** than [bert-base-spanish-wwm-cased-finetuned-spa-squad2-es](https://github.com/huggingface/transformers/blob/master/model_cards/mrm8488/bert-base-spanish-wwm-cased-finetuned-spa-squad2-es/README.md) - -This model was fine-tuned on the same dataset but using **distillation** during the process as mentioned above (and one more train epoch). - -The **teacher model** for the distillation was `bert-base-multilingual-cased`. It is the same teacher used for `distilbert-base-multilingual-cased` AKA [**DistilmBERT**](https://github.com/huggingface/transformers/tree/master/examples/distillation) (on average is twice as fast as **mBERT-base**). - -## Details of the downstream task (Q&A) - Dataset - -
- -[SQuAD-es-v2.0](https://github.com/ccasimiro88/TranslateAlignRetrieve) - -| Dataset | # Q&A | -| ----------------------- | ----- | -| SQuAD2.0 Train | 130 K | -| SQuAD2.0-es-v2.0 | 111 K | -| SQuAD2.0 Dev | 12 K | -| SQuAD-es-v2.0-small Dev | 69 K | - -
- -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -!export SQUAD_DIR=/path/to/squad-v2_spanish \ -&& python transformers/examples/distillation/run_squad_w_distillation.py \ - --model_type bert \ - --model_name_or_path dccuchile/bert-base-spanish-wwm-cased \ - --teacher_type bert \ - --teacher_name_or_path bert-base-multilingual-cased \ - --do_train \ - --do_eval \ - --do_lower_case \ - --train_file $SQUAD_DIR/train-v2.json \ - --predict_file $SQUAD_DIR/dev-v2.json \ - --per_gpu_train_batch_size 12 \ - --learning_rate 3e-5 \ - --num_train_epochs 5.0 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /content/model_output \ - --save_steps 5000 \ - --threads 4 \ - --version_2_with_negative -``` - -## Results: - -| Metric | # Value | -| --------- | ----------- | -| **Exact** | **90.77**48 | -| **F1** | **94.94**71 | - -```json -{ - "exact": 90.77483309730933, - "f1": 94.94714391266254, - "total": 69202, - "HasAns_exact": 86.60850599781898, - "HasAns_f1": 92.90582885592328, - "HasAns_total": 45850, - "NoAns_exact": 98.95512161699212, - "NoAns_f1": 98.95512161699212, - "NoAns_total": 23352, - "best_exact": 90.77483309730933, - "best_exact_thresh": 0.0, - "best_f1": 94.94714391266305, - "best_f1_thresh": 0.0 -} -``` - -## Comparison: - -| Model | f1 score | -| :-------------------------------------------------------------: | :-------: | -| bert-base-spanish-wwm-cased-finetuned-spa-squad2-es | 86.07 | -| **distill**-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es | **94.94** | - -So, yes, this version is even more accurate. - -### Model in action - -Fast usage with **pipelines**: - -```python -from transformers import * - -# Important!: By now the QA pipeline is not compatible with fast tokenizer, but they are working on it. So that pass the object to the tokenizer {"use_fast": False} as in the following example: - -nlp = pipeline( - 'question-answering', - model='mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es', - tokenizer=( - 'mrm8488/distill-bert-base-spanish-wwm-cased-finetuned-spa-squad2-es', - {"use_fast": False} - ) -) - -nlp( - { - 'question': '¿Para qué lenguaje está trabajando?', - 'context': 'Manuel Romero está colaborando activamente con huggingface/transformers ' + - 'para traer el poder de las últimas técnicas de procesamiento de lenguaje natural al idioma español' - } -) -# Output: {'answer': 'español', 'end': 169, 'score': 0.67530957344621, 'start': 163} -``` - -Play with this model and ```pipelines``` in a Colab: - -Open In Colab - -
- -1. Set the context and ask some questions: - -![Set context and questions](https://media.giphy.com/media/mCIaBpfN0LQcuzkA2F/giphy.gif) - -2. Run predictions: - -![Run the model](https://media.giphy.com/media/WT453aptcbCP7hxWTZ/giphy.gif) -
- -More about ``` Huggingface pipelines```? check this Colab out: - -Open In Colab - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/distilroberta-base-finetuned-sentiment/README.md b/model_cards/mrm8488/distilroberta-base-finetuned-sentiment/README.md deleted file mode 100644 index 8e0df7693555..000000000000 --- a/model_cards/mrm8488/distilroberta-base-finetuned-sentiment/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# DistilRoBERTa + Sentiment Analysis 😂😢😡😃😯 - -This in an adapted version of [@omarsar0](https://twitter.com/omarsar0) [tutorial](https://t.co/WMnATW0Hwf?amp=1) -He explains everything so detailed and provided the dataset. I just changed some parameters and created the ```config.json```file to upload it to [🤗Transformers HUB](https://huggingface.co/) - - -In this tutorial, he shows how to fine-tune a language model (LM) for **emotion classification** with code adapted from this [tutorial](https://zablo.net/blog/post/custom-classifier-on-bert-model-guide-polemo2-sentiment-analysis/) by MARCIN ZABŁOCKI. - -The emotions covered are: - - sadness 😢 - - joy 😃 - - love 🥰 - - anger 😡 - - fear 😱 - - surprise 😯 - -## Details of the language model -The base model used is [DistilRoBERTa](https://huggingface.co/distilroberta-base) - -## Details of the downstream task (Sentence classification) - Dataset 📚 - -| Dataset split | # Size | # Sequences | -| ---------------------- | ----- | ------| -|Train | 1.58M | 20000 -| Validation | 200 KB | -| Test | 202 KB | - - -## Results after training 🏋️‍♀️🧾 - -|emotion |precision |recall| f1-score| support| -|-------|-------------|------|----------|----------| -|sadness| 0.973868 |0.949066 |0.961307| 589| -|joy |0.970313 |0.901306 |0.934537| 689| -|love |0.743119 |0.925714 |0.824427| 175| -|anger | 0.884615| 0.969349| 0.925046| 261| -|fear |0.951456 |0.875000| 0.911628| 224| -|surprise| 0.750000| 0.919355| 0.826087| 62| -| | | | | | -|**accuracy**| | | 0.924000| 2000| -|**macro avg**| 0.878895| 0.923298| 0.897172| 2000| -|**weighted avg**| 0.931355| 0.924000| 0.925620| 2000| - -## Model in action 🔨 - -Fast usage with **pipelines** 🧪 - -```python -from transformers import pipeline - -nlp_sentiment = pipeline( - "sentiment-analysis", - model="mrm8488/distilroberta-base-finetuned-sentiment", - tokenizer="mrm8488/distilroberta-base-finetuned-sentiment" -) - -text = "i feel i should return to the start of the weekend so my loyal readers can get a feeling for things up to this point" - -nlp_sentiment(text) -# Output: [{'label': 'love', 'score': 0.2183746}] -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/electra-base-finetuned-squadv1/README.md b/model_cards/mrm8488/electra-base-finetuned-squadv1/README.md deleted file mode 100644 index 59dc87d51f7a..000000000000 --- a/model_cards/mrm8488/electra-base-finetuned-squadv1/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -language: en ---- - -# Electra base ⚡ + SQuAD v1 ❓ - -[Electra-base-discriminator](https://huggingface.co/google/electra-base-discriminator) fine-tuned on [SQUAD v1.1 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Model 🧠 - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - - -## Details of the downstream task (Q&A) - Dataset 📚 - -**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. -SQuAD v1.1 contains **100,000+** question-answer pairs on **500+** articles. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python transformers/examples/question-answering/run_squad.py \ - --model_type electra \ - --model_name_or_path 'google/electra-base-discriminator' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file '/content/dataset/train-v1.1.json' \ - --predict_file '/content/dataset/dev-v1.1.json' \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 10 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir '/content/output' \ - --overwrite_output_dir \ - --save_steps 1000 -``` - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **83.03** | -| **F1** | **90.77** | -| **Size**| **+ 400 MB** | - -Very good metrics for such a "small" model! - -```json -{ -'exact': 83.03689687795648, -'f1': 90.77486052446231, -'total': 10570, -'HasAns_exact': 83.03689687795648, -'HasAns_f1': 90.77486052446231, -'HasAns_total': 10570, -'best_exact': 83.03689687795648, -'best_exact_thresh': 0.0, -'best_f1': 90.77486052446231, -'best_f1_thresh': 0.0 -} -``` - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -QnA_pipeline = pipeline('question-answering', model='mrm8488/electra-base-finetuned-squadv1') - -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'What has been discovered by scientists from China ?' -}) -# Output: -{'answer': 'A new strain of flu', 'end': 19, 'score': 0.9995211430099182, 'start': 0} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/electra-small-finetuned-squadv1/README.md b/model_cards/mrm8488/electra-small-finetuned-squadv1/README.md deleted file mode 100644 index c6986ec20e34..000000000000 --- a/model_cards/mrm8488/electra-small-finetuned-squadv1/README.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -language: en ---- - -# Electra small ⚡ + SQuAD v1 ❓ - -[Electra-small-discriminator](https://huggingface.co/google/electra-small-discriminator) fine-tuned on [SQUAD v1.1 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Model 🧠 - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - - -## Details of the downstream task (Q&A) - Dataset 📚 - -**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. -SQuAD v1.1 contains **100,000+** question-answer pairs on **500+** articles. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python transformers/examples/question-answering/run_squad.py \ - --model_type electra \ - --model_name_or_path 'google/electra-small-discriminator' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file '/content/dataset/train-v1.1.json' \ - --predict_file '/content/dataset/dev-v1.1.json' \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 10 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir '/content/output' \ - --overwrite_output_dir \ - --save_steps 1000 -``` - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **77.70** | -| **F1** | **85.74** | -| **Size**| **50 MB** | - -Very good metrics for such a "small" model! - -```json - -{ -'exact': 77.70104068117313, -'f1': 85.73991234187997, -'total': 10570, -'HasAns_exact': 77.70104068117313, -'HasAns_f1': 85.73991234187997, -'HasAns_total': 10570, -'best_exact': 77.70104068117313, -'best_exact_thresh': 0.0, -'best_f1': 85.73991234187997, -'best_f1_thresh': 0.0 -} -``` - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -QnA_pipeline = pipeline('question-answering', model='mrm8488/electra-small-finetuned-squadv1') -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'What has been discovered by scientists from China ?' -}) - -# Output: -{'answer': 'A new strain of flu', 'end': 19, 'score': 0.7950334108113424, 'start': 0} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/electra-small-finetuned-squadv2/README.md b/model_cards/mrm8488/electra-small-finetuned-squadv2/README.md deleted file mode 100644 index d1c8e8c721da..000000000000 --- a/model_cards/mrm8488/electra-small-finetuned-squadv2/README.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -language: en ---- - -# Electra small ⚡ + SQuAD v2 ❓ - -[Electra-small-discriminator](https://huggingface.co/google/electra-small-discriminator) fine-tuned on [SQUAD v2.0 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Model 🧠 - -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - - -## Details of the downstream task (Q&A) - Dataset 📚 - -**SQuAD2.0** combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python transformers/examples/question-answering/run_squad.py \ - --model_type electra \ - --model_name_or_path 'google/electra-small-discriminator' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file '/content/dataset/train-v2.0.json' \ - --predict_file '/content/dataset/dev-v2.0.json' \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 10 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir '/content/output' \ - --overwrite_output_dir \ - --save_steps 1000 \ - --version_2_with_negative -``` - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **69.71** | -| **F1** | **73.44** | -| **Size**| **50 MB** | - - -```json -{ -'exact': 69.71279373368147, -'f1': 73.4439546123672, -'total': 11873, -'HasAns_exact': 69.92240215924427, -'HasAns_f1': 77.39542393937836, -'HasAns_total': 5928, -'NoAns_exact': 69.50378469301934, -'NoAns_f1': 69.50378469301934, -'NoAns_total': 5945, -'best_exact': 69.71279373368147, -'best_exact_thresh': 0.0, -'best_f1': 73.44395461236732, -'best_f1_thresh': 0.0 -} -``` - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline -QnA_pipeline = pipeline('question-answering', model='mrm8488/electra-base-finetuned-squadv2') -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'What has been discovered by scientists from China ?' -}) -# Output: -{'answer': 'A new strain of flu', 'end': 19, 'score': 0.8650811568752914, 'start': 0} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/electricidad-base-discriminator/README.md b/model_cards/mrm8488/electricidad-base-discriminator/README.md deleted file mode 100644 index e5f214f7a6cb..000000000000 --- a/model_cards/mrm8488/electricidad-base-discriminator/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -language: es -thumbnail: https://i.imgur.com/uxAvBfh.png - - ---- - -## ELECTRICIDAD: The Spanish Electra [Imgur](https://imgur.com/uxAvBfh) - -**Electricidad-base-discriminator** (uncased) is a ```base``` Electra like model (discriminator in this case) trained on a + 20 GB of the [OSCAR](https://oscar-corpus.com/) Spanish corpus. - -As mentioned in the original [paper](https://openreview.net/pdf?id=r1xMH1BtvB): -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer the paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - - -## Model details ⚙ - -|Name| # Value| -|-----|--------| -|Layers| 12 | -|Hidden |768 | -|Params| 110M| - -## Evaluation metrics (for discriminator) 🧾 - -|Metric | # Score | -|-------|---------| -|Accuracy| 0.985| -|Precision| 0.726| -|AUC | 0.922| - - - -## Fast example of usage 🚀 - -```python -from transformers import ElectraForPreTraining, ElectraTokenizerFast -import torch - -discriminator = ElectraForPreTraining.from_pretrained("/content/electricidad-base-discriminator") -tokenizer = ElectraTokenizerFast.from_pretrained("/content/electricidad-base-discriminator") - -sentence = "El rápido zorro marrón salta sobre el perro perezoso" -fake_sentence = "El rápido zorro marrón amar sobre el perro perezoso" - -fake_tokens = tokenizer.tokenize(fake_sentence) -fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt") -discriminator_outputs = discriminator(fake_inputs) -predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) - -[print("%7s" % token, end="") for token in fake_tokens] - -[print("%7s" % prediction, end="") for prediction in predictions.tolist()] - -# Output: -''' -el rapido zorro marro ##n amar sobre el perro pere ##zoso 0.0 0.0 0.0 0.0 0.0 0.0 1.0 1.0 0.0 0.0 0.0 0.0 0.0[None, None, None, None, None, None, None, None, None, None, None, None, None -''' -``` -As you can see there are **1s** in the places where the model detected a fake token. So, it works! 🎉 - - -### Some models fine-tuned on a downstream task 🛠️ - -[Question Answering](https://huggingface.co/mrm8488/electricidad-base-finetuned-squadv1-es) - -[POS](https://huggingface.co/mrm8488/electricidad-base-finetuned-pos) - -[NER](https://huggingface.co/mrm8488/electricidad-base-finetuned-ner) - -[Paraphrase Identification](https://huggingface.co/mrm8488/RuPERTa-base-finetuned-pawsx-es) - - - -## Acknowledgments - -I thank [🤗/transformers team](https://github.com/huggingface/transformers) for allowing me to train the model (specially to [Julien Chaumond](https://twitter.com/julien_c)). - - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/electricidad-base-finetuned-pawsx-es/README.md b/model_cards/mrm8488/electricidad-base-finetuned-pawsx-es/README.md deleted file mode 100644 index 3c520301e455..000000000000 --- a/model_cards/mrm8488/electricidad-base-finetuned-pawsx-es/README.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -language: es -datasets: -- xtreme -widget: -- text: "El río Tabaci es una vertiente del río Leurda en Rumania. El río Leurda es un afluente del río Tabaci en Rumania." ---- - -# Electricidad-base fine-tuned on PAWS-X-es for Paraphrase Identification diff --git a/model_cards/mrm8488/electricidad-base-generator/README.md b/model_cards/mrm8488/electricidad-base-generator/README.md deleted file mode 100644 index 2f078fbaecc8..000000000000 --- a/model_cards/mrm8488/electricidad-base-generator/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -language: es -thumbnail: https://i.imgur.com/uxAvBfh.png -widget: -- text: "Madrid es una ciudad muy [MASK] en España." - - ---- - -## ELECTRICIDAD: The Spanish Electra [Imgur](https://imgur.com/uxAvBfh) - -**Electricidad-base-generator** (uncased) is a ```base``` Electra like model (generator in this case) trained on a + 20 GB of the [OSCAR](https://oscar-corpus.com/) Spanish corpus. - -As mentioned in the original [paper](https://openreview.net/pdf?id=r1xMH1BtvB): -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer the paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - - - - - -## Fast example of usage 🚀 - -```python -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="mrm8488/electricidad-base-generator", - tokenizer="mrm8488/electricidad-base-generator" -) - -print( - fill_mask(f"HuggingFace está creando {fill_mask.tokenizer.mask_token} que la comunidad usa para resolver tareas de NLP.") -) - -# Output: [{'sequence': '[CLS] huggingface esta creando herramientas que la comunidad usa para resolver tareas de nlp. [SEP]', 'score': 0.0896105170249939, 'token': 8760, 'token_str': 'herramientas'}, ...] - -``` - -## Acknowledgments - -I thank [🤗/transformers team](https://github.com/huggingface/transformers) for allowing me to train the model (specially to [Julien Chaumond](https://twitter.com/julien_c)). - - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/electricidad-small-discriminator/README.md b/model_cards/mrm8488/electricidad-small-discriminator/README.md deleted file mode 100644 index 03d9f881f051..000000000000 --- a/model_cards/mrm8488/electricidad-small-discriminator/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: es -thumbnail: https://i.imgur.com/uxAvBfh.png - - ---- - -## ELECTRICIDAD: The Spanish Electra [Imgur](https://imgur.com/uxAvBfh) - -**ELECTRICIDAD** is a small Electra like model (discriminator in this case) trained on a + 20 GB of the [OSCAR](https://oscar-corpus.com/) Spanish corpus. - -As mentioned in the original [paper](https://openreview.net/pdf?id=r1xMH1BtvB): -**ELECTRA** is a new method for self-supervised language representation learning. It can be used to pre-train transformer networks using relatively little compute. ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, similar to the discriminator of a [GAN](https://arxiv.org/pdf/1406.2661.pdf). At small scale, ELECTRA achieves strong results even when trained on a single GPU. At large scale, ELECTRA achieves state-of-the-art results on the [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) dataset. - -For a detailed description and experimental results, please refer the paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/pdf?id=r1xMH1BtvB). - -## Model details ⚙ - -|Param| # Value| -|-----|--------| -|Layers| 12 | -|Hidden |256 | -|Params| 14M| - -## Evaluation metrics (for discriminator) 🧾 - -|Metric | # Score | -|-------|---------| -|Accuracy| 0.94| -|Precision| 0.76| -|AUC | 0.92| - -## Benchmarks 🔨 - -WIP 🚧 - -## How to use the discriminator in `transformers` - -```python -from transformers import ElectraForPreTraining, ElectraTokenizerFast -import torch - -discriminator = ElectraForPreTraining.from_pretrained("mrm8488/electricidad-small-discriminator") -tokenizer = ElectraTokenizerFast.from_pretrained("mrm8488/electricidad-small-discriminator") - -sentence = "el zorro rojo es muy rápido" -fake_sentence = "el zorro rojo es muy ser" - -fake_tokens = tokenizer.tokenize(sentence) -fake_inputs = tokenizer.encode(sentence, return_tensors="pt") -discriminator_outputs = discriminator(fake_inputs) -predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) - -[print("%7s" % token, end="") for token in fake_tokens] - -[print("%7s" % int(prediction), end="") for prediction in predictions.tolist()[1:-1]] - -# Output: -''' -el zorro rojo es muy ser 0 0 0 0 0 1[None, None, None, None, None, None] -''' -``` - -As you can see there is a **1** in the place where the model detected the fake token (**ser**). So, it works! 🎉 - -## Acknowledgments - -I thank [🤗/transformers team](https://github.com/huggingface/transformers) for answering my doubts and Google for helping me with the [TensorFlow Research Cloud](https://www.tensorflow.org/tfrc) program. - - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/electricidad-small-finetuned-squadv1-es/README.md b/model_cards/mrm8488/electricidad-small-finetuned-squadv1-es/README.md deleted file mode 100644 index ccdfb39d24c2..000000000000 --- a/model_cards/mrm8488/electricidad-small-finetuned-squadv1-es/README.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -language: es -thumbnail: https://imgur.com/uxAvBfh ---- - -# Electricidad small + Spanish SQuAD v1 ⚡❓ - -[Electricidad-small-discriminator](https://huggingface.co/mrm8488/electricidad-small-discriminator) fine-tuned on [Spanish SQUAD v1.1 dataset](https://github.com/ccasimiro88/TranslateAlignRetrieve/tree/master/SQuAD-es-v1.1) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Dataset 📚 - -[SQuAD-es-v1.1](https://github.com/ccasimiro88/TranslateAlignRetrieve/tree/master/SQuAD-es-v1.1) - -| Dataset split | # Samples | -| ------------- | --------- | -| Train | 130 K | -| Test | 11 K | - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python /content/transformers/examples/question-answering/run_squad.py \ - --model_type electra \ - --model_name_or_path 'mrm8488/electricidad-small-discriminator' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file '/content/dataset/train-v1.1-es.json' \ - --predict_file '/content/dataset/dev-v1.1-es.json' \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 10 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir '/content/electricidad-small-finetuned-squadv1-es' \ - --overwrite_output_dir \ - --save_steps 1000 -``` - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **46.82** | -| **F1** | **64.79** | - -```json -{ -'exact': 46.82119205298013, -'f1': 64.79435260021918, -'total': 10570, -'HasAns_exact': 46.82119205298013, -HasAns_f1': 64.79435260021918, -'HasAns_total': 10570, -'best_exact': 46.82119205298013, -'best_exact_thresh': 0.0, -'best_f1': 64.79435260021918, -'best_f1_thresh': 0.0 -} -``` - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/electricidad-small-finetuned-squadv1-es", - tokenizer="mrm8488/electricidad-small-finetuned-squadv1-es" -) - -context = "Manuel ha creado una versión del modelo Electra small en español que alcanza una puntuación F1 de 65 en el dataset SQUAD-es y sólo pesa 50 MB" - -q1 = "Cuál es su marcador F1?" -q2 = "¿Cuál es el tamaño del modelo?" -q3 = "¿Quién lo ha creado?" -q4 = "¿Que es lo que ha hecho Manuel?" - - -questions = [q1, q2, q3, q4] - -for question in questions: - result = qa_pipeline({ - 'context': context, - 'question': question}) - print(result) - -# Output: -{'score': 0.14836778166355025, 'start': 98, 'end': 100, 'answer': '65'} -{'score': 0.32219420810758237, 'start': 136, 'end': 140, 'answer': '50 MB'} -{'score': 0.9672326951118713, 'start': 0, 'end': 6, 'answer': 'Manuel'} -{'score': 0.23552458113848118, 'start': 10, 'end': 53, 'answer': 'creado una versión del modelo Electra small'} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/gpt2-finetuned-recipes-cooking/README.md b/model_cards/mrm8488/gpt2-finetuned-recipes-cooking/README.md deleted file mode 100644 index 1e8029f30425..000000000000 --- a/model_cards/mrm8488/gpt2-finetuned-recipes-cooking/README.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -language: en -thumbnail: -widget: -- text: "HuggingFace Cake:" ---- diff --git a/model_cards/mrm8488/gpt2-finetuned-recipes-cooking_v2/README.md b/model_cards/mrm8488/gpt2-finetuned-recipes-cooking_v2/README.md deleted file mode 100644 index 1e8029f30425..000000000000 --- a/model_cards/mrm8488/gpt2-finetuned-recipes-cooking_v2/README.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -language: en -thumbnail: -widget: -- text: "HuggingFace Cake:" ---- diff --git a/model_cards/mrm8488/gpt2-imdb-neg/README.md b/model_cards/mrm8488/gpt2-imdb-neg/README.md deleted file mode 100644 index 2cebaf9ae380..000000000000 --- a/model_cards/mrm8488/gpt2-imdb-neg/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# GPT2-IMDB-neg (LM + RL) 🎞😡✍ - -All credits to [@lvwerra](https://twitter.com/lvwerra) - -## What is it? -A small GPT2 (`lvwerra/gpt2-imdb`) language model fine-tuned to produce **negative** movie reviews based the [IMDB dataset](https://www.kaggle.com/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews). The model is trained with rewards from a BERT sentiment classifier (`lvwerra/gpt2-imdb`) via **PPO**. - -## Why? -I wanted to reproduce the experiment [lvwerra/gpt2-imdb-pos](https://huggingface.co/lvwerra/gpt2-imdb-pos) but for generating **negative** movie reviews. - -## Training setting -The model was trained for `100` optimisation steps with a batch size of `256` which corresponds to `25600` training samples. The full experiment setup (for positive samples) in [trl repo](https://lvwerra.github.io/trl/04-gpt2-sentiment-ppo-training/). - -## Examples -A few examples of the model response to a query before and after optimisation: - -| query | response (before) | response (after) | rewards (before) | rewards (after) | -|-------|-------------------|------------------|------------------|-----------------| -|This movie is a fine | attempt as far as live action is concerned, n...|example of how bad Hollywood in theatrics pla...| 2.118391 | -3.31625| -|I have watched 3 episodes |with this guy and he is such a talented actor...| but the show is just plain awful and there ne...| 2.681171| -4.512792| -|We know that firefighters and| police officers are forced to become populari...| other chains have going to get this disaster ...| 1.367811| -3.34017| - -## Training logs and metrics -Watch the whole training logs and metrics on [W&B](https://app.wandb.ai/mrm8488/gpt2-sentiment-negative?workspace=user-mrm8488) - - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/gpt2-imdb-neutral/README.md b/model_cards/mrm8488/gpt2-imdb-neutral/README.md deleted file mode 100644 index 0d0ad4734a77..000000000000 --- a/model_cards/mrm8488/gpt2-imdb-neutral/README.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -language: en -thumbnail: ---- - -# GPT2-IMDB-neutral (LM + RL) 🎞😐✍ - -## What is it? -A small GPT2 (`lvwerra/gpt2-imdb`) language model fine-tuned to produce **neutral**-ish movie reviews based on the [IMDB dataset](https://www.kaggle.com/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews). The model is trained with rewards from a BERT sentiment classifier (`lvwerra/gpt2-imdb`) via **PPO**. - -## Why? -After reproducing the experiment [lvwerra/gpt2-imdb-pos](https://huggingface.co/lvwerra/gpt2-imdb-pos) but for generating **negative** movie reviews ([mrm8488/gpt2-imdb-neg](https://huggingface.co/mrm8488/gpt2-imdb-neg)) I wanted to check if I could generate neutral-ish movie reviews. So, based on the classifier output (logit), I saw that clearly negative reviews gives around *-4* values and clearly positive reviews around *4*. Then, it was esay to establish an interval ```[-1.75,1.75]``` that it could be considered as **neutral**. So if the classifier output was in that interval I gave it a positive reward while values out of the interval got a negative reward. - -## Training setting -The model was trained for `100` optimisation steps with a batch size of `128` which corresponds to `30000` training samples. The full experiment setup (for positive samples) in [trl repo](https://lvwerra.github.io/trl/04-gpt2-sentiment-ppo-training/). - -## Examples -A few examples of the model response to a query before and after optimisation: - -| query | response (before) | response (after) | rewards (before) | rewards (after) | -|-------|-------------------|------------------|------------------|-----------------| -|Okay, my title is|partly over, but this drama still makes me proud to read its first 40...|weird. The title is "mana were, ahunter". "Man...|4.200727 |-1.891443| -|Where is it written that|there is a monster in this movie anyway? How is it that the entire|[ of the women in the recent women of jungle business between Gender and husband| -3.113942| -1.944993| -|As a lesbian, I|cannot believe I was in the Sixties! Subtle yet witty, with original| found it hard to get responsive. In fact I found myself with the long| 3.906178| 0.769166| -|The Derek's have over|three times as many acting hours than Jack Nicholson? You think bitches?|30 dueling characters and kill of, they retreat themselves to their base.|-2.503655| -1.898380| - - -> All credits to [@lvwerra](https://twitter.com/lvwerra) - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/longformer-base-4096-finetuned-squadv2/README.md b/model_cards/mrm8488/longformer-base-4096-finetuned-squadv2/README.md deleted file mode 100644 index 269f7e91753e..000000000000 --- a/model_cards/mrm8488/longformer-base-4096-finetuned-squadv2/README.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -language: en -datasets: -- squad_v2 ---- - -# Longformer-base-4096 fine-tuned on SQuAD v2 - -[Longformer-base-4096 model](https://huggingface.co/allenai/longformer-base-4096) fine-tuned on [SQuAD v2](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -## Longformer-base-4096 - -[Longformer](https://arxiv.org/abs/2004.05150) is a transformer model for long documents. - -`longformer-base-4096` is a BERT-like model started from the RoBERTa checkpoint and pretrained for MLM on long documents. It supports sequences of length up to 4,096. - -Longformer uses a combination of a sliding window (local) attention and global attention. Global attention is user-configured based on the task to allow the model to learn task-specific representations. - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -Dataset ID: ```squad_v2``` from [HugginFace/NLP](https://github.com/huggingface/nlp) -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| squad_v2 | train | 130319 | -| squad_v2 | valid | 11873 | - -How to load it from [nlp](https://github.com/huggingface/nlp) - -```python -train_dataset = nlp.load_dataset('squad_v2', split=nlp.Split.TRAIN) -valid_dataset = nlp.load_dataset('squad_v2', split=nlp.Split.VALIDATION) -``` -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this one](https://colab.research.google.com/drive/1zEl5D-DdkBKva-DdreVOmN0hrAfzKG1o?usp=sharing) - - - -## Model in Action 🚀 - -```python -import torch -from transformers import AutoTokenizer, AutoModelForQuestionAnswering - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/longformer-base-4096-finetuned-squadv2") -model = AutoModelForQuestionAnswering.from_pretrained("mrm8488/longformer-base-4096-finetuned-squadv2") - -text = "Huggingface has democratized NLP. Huge thanks to Huggingface for this." -question = "What has Huggingface done ?" -encoding = tokenizer(question, text, return_tensors="pt") -input_ids = encoding["input_ids"] - -# default is local attention everywhere -# the forward method will automatically set global attention on question tokens -attention_mask = encoding["attention_mask"] - -start_scores, end_scores = model(input_ids, attention_mask=attention_mask) -all_tokens = tokenizer.convert_ids_to_tokens(input_ids[0].tolist()) - -answer_tokens = all_tokens[torch.argmax(start_scores) :torch.argmax(end_scores)+1] -answer = tokenizer.decode(tokenizer.convert_tokens_to_ids(answer_tokens)) - -# output => democratized NLP -``` -If given the same context we ask something that is not there, the output for **no answer** will be `````` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/mT5-small-finetuned-tydiqa-for-xqa/README.md b/model_cards/mrm8488/mT5-small-finetuned-tydiqa-for-xqa/README.md deleted file mode 100644 index 46b392bbe591..000000000000 --- a/model_cards/mrm8488/mT5-small-finetuned-tydiqa-for-xqa/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -language: multilingual -datasets: -- tydiqa -pipeline_tag: question-answering ---- - -# mT5-small fine-tuned on TyDiQA for multilingual QA 🗺📖❓ -[Google's mT5-small](https://huggingface.co/google/mt5-small) fine-tuned on [TyDi QA](https://huggingface.co/nlp/viewer/?dataset=tydiqa&config=secondary_task) (secondary task) for **multingual Q&A** downstream task. - -## Details of mT5 - -[Google's mT5](https://github.com/google-research/multilingual-t5) - -mT5 is pretrained on the [mC4](https://www.tensorflow.org/datasets/catalog/c4#c4multilingual) corpus, covering 101 languages: - -Afrikaans, Albanian, Amharic, Arabic, Armenian, Azerbaijani, Basque, Belarusian, Bengali, Bulgarian, Burmese, Catalan, Cebuano, Chichewa, Chinese, Corsican, Czech, Danish, Dutch, English, Esperanto, Estonian, Filipino, Finnish, French, Galician, Georgian, German, Greek, Gujarati, Haitian Creole, Hausa, Hawaiian, Hebrew, Hindi, Hmong, Hungarian, Icelandic, Igbo, Indonesian, Irish, Italian, Japanese, Javanese, Kannada, Kazakh, Khmer, Korean, Kurdish, Kyrgyz, Lao, Latin, Latvian, Lithuanian, Luxembourgish, Macedonian, Malagasy, Malay, Malayalam, Maltese, Maori, Marathi, Mongolian, Nepali, Norwegian, Pashto, Persian, Polish, Portuguese, Punjabi, Romanian, Russian, Samoan, Scottish Gaelic, Serbian, Shona, Sindhi, Sinhala, Slovak, Slovenian, Somali, Sotho, Spanish, Sundanese, Swahili, Swedish, Tajik, Tamil, Telugu, Thai, Turkish, Ukrainian, Urdu, Uzbek, Vietnamese, Welsh, West Frisian, Xhosa, Yiddish, Yoruba, Zulu. - -**Note**: mT5 was only pre-trained on mC4 excluding any supervised training. Therefore, this model has to be fine-tuned before it is useable on a downstream task. - -Pretraining Dataset: [mC4](https://www.tensorflow.org/datasets/catalog/c4#c4multilingual) - -Other Community Checkpoints: [here](https://huggingface.co/models?search=mt5) - -Paper: [mT5: A massively multilingual pre-trained text-to-text transformer](https://arxiv.org/abs/2010.11934) - -Authors: *Linting Xue, Noah Constant, Adam Roberts, Mihir Kale, Rami Al-Rfou, Aditya Siddhant, Aditya Barua, Colin Raffel* - - -## Details of the dataset 📚 - -**TyDi QA** is a question answering dataset covering 11 typologically diverse languages with 204K question-answer pairs. The languages of TyDi QA are diverse with regard to their typology -- the set of linguistic features that each language expresses -- such that we expect models performing well on this set to generalize across a large number of the languages in the world. It contains language phenomena that would not be found in English-only corpora. To provide a realistic information-seeking task and avoid priming effects, questions are written by people who want to know the answer, but don’t know the answer yet, (unlike SQuAD and its descendents) and the data is collected directly in each language without the use of translation (unlike MLQA and XQuAD). - -| Dataset | Task | Split | # samples | -| -------- | ----- |------| --------- | -| TyDi QA | GoldP | train| 49881 | -| TyDi QA | GoldP | valid| 5077 | - - - -## Results on validation dataset 📝 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **41.65** | - - - -## Model in Action 🚀 - -```python -from transformers import AutoModelForCausalLM, AutoTokenizer -import torch -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -tokenizer = AutoTokenizer.from_pretrained("mrm8488/mT5-small-finetuned-tydiqa-for-xqa") -model = AutoModelForCausalLM.from_pretrained("mrm8488/mT5-small-finetuned-tydiqa-for-xqa").to(device) - -def get_response(question, context, max_length=32): - input_text = 'question: %s context: %s' % (question, context) - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'].to(device), - attention_mask=features['attention_mask'].to(device), - max_length=max_length) - - return tokenizer.decode(output[0]) - -# Some examples in different languages - -context = 'HuggingFace won the best Demo paper at EMNLP2020.' -question = 'What won HuggingFace?' -get_response(question, context) - -context = 'HuggingFace ganó la mejor demostración con su paper en la EMNLP2020.' -question = 'Qué ganó HuggingFace?' -get_response(question, context) - -context = 'HuggingFace выиграл лучшую демонстрационную работу на EMNLP2020.' -question = 'Что победило в HuggingFace?' -get_response(question, context) -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/mobilebert-uncased-finetuned-squadv1/README.md b/model_cards/mrm8488/mobilebert-uncased-finetuned-squadv1/README.md deleted file mode 100644 index 68a87d9f9aa5..000000000000 --- a/model_cards/mrm8488/mobilebert-uncased-finetuned-squadv1/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: en -datasets: -- squad ---- - -# MobileBERT + SQuAD (v1.1) 📱❓ - -[mobilebert-uncased](https://huggingface.co/google/mobilebert-uncased) fine-tuned on [SQUAD v2.0 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Model 🧠 - -**MobileBERT** is a thin version of *BERT_LARGE*, while equipped with bottleneck structures and a carefully designed balance between self-attentions and feed-forward networks. - -The checkpoint used here is the original MobileBert Optimized Uncased English: (uncased_L-24_H-128_B-512_A-4_F-4_OPT) checkpoint. - -More about the model [here](https://arxiv.org/abs/2004.02984) - -## Details of the downstream task (Q&A) - Dataset 📚 - -**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. -SQuAD v1.1 contains **100,000+** question-answer pairs on **500+** articles. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python transformers/examples/question-answering/run_squad.py \ - --model_type bert \ - --model_name_or_path 'google/mobilebert-uncased' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file '/content/dataset/train-v1.1.json' \ - --predict_file '/content/dataset/dev-v1.1.json' \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 5 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir '/content/output' \ - --overwrite_output_dir \ - --save_steps 1000 -``` - -It is important to say that this models converges much faster than other ones. So, it is also cheap to fine-tune. - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **82.33** | -| **F1** | **89.64** | -| **Size**| **94 MB** | - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline -QnA_pipeline = pipeline('question-answering', model='mrm8488/mobilebert-uncased-finetuned-squadv1') -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'Who did identified it ?' - }) - -# Output: {'answer': 'scientists.', 'end': 106, 'score': 0.7885545492172241, 'start': 96} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/mobilebert-uncased-finetuned-squadv2/README.md b/model_cards/mrm8488/mobilebert-uncased-finetuned-squadv2/README.md deleted file mode 100644 index 3bd933b77da9..000000000000 --- a/model_cards/mrm8488/mobilebert-uncased-finetuned-squadv2/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: en -datasets: -- squad_v2 ---- - -# MobileBERT + SQuAD v2 📱❓ - -[mobilebert-uncased](https://huggingface.co/google/mobilebert-uncased) fine-tuned on [SQUAD v2.0 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Model 🧠 - -**MobileBERT** is a thin version of *BERT_LARGE*, while equipped with bottleneck structures and a carefully designed balance between self-attentions and feed-forward networks. - -The checkpoint used here is the original MobileBert Optimized Uncased English: (uncased_L-24_H-128_B-512_A-4_F-4_OPT) checkpoint. - -More about the model [here](https://arxiv.org/abs/2004.02984) - -## Details of the downstream task (Q&A) - Dataset 📚 - -**SQuAD2.0** combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python transformers/examples/question-answering/run_squad.py \ - --model_type bert \ - --model_name_or_path 'google/mobilebert-uncased' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file '/content/dataset/train-v2.0.json' \ - --predict_file '/content/dataset/dev-v2.0.json' \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 5 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir '/content/output' \ - --overwrite_output_dir \ - --save_steps 1000 \ - --version_2_with_negative -``` - -It is important to say that this models converges much faster than other ones. So, it is also cheap to fine-tune. - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **75.37** | -| **F1** | **78.48** | -| **Size**| **94 MB** | - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline -QnA_pipeline = pipeline('question-answering', model='mrm8488/mobilebert-uncased-finetuned-squadv2') -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'Who did identified it ?' - }) - -# Output: {'answer': 'scientists.', 'end': 106, 'score': 0.41531604528427124, 'start': 96} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/roberta-base-1B-1-finetuned-squadv1/README.md b/model_cards/mrm8488/roberta-base-1B-1-finetuned-squadv1/README.md deleted file mode 100644 index 0ab598cf1b0e..000000000000 --- a/model_cards/mrm8488/roberta-base-1B-1-finetuned-squadv1/README.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -language: en ---- - -# RoBERTa-base (1B-1) + SQuAD v1 ❓ - -[roberta-base-1B-1](https://huggingface.co/nyu-mll/roberta-base-1B-1) fine-tuned on [SQUAD v1.1 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Model 🧠 - -RoBERTa Pretrained on Smaller Datasets - -[NYU Machine Learning for Language](https://huggingface.co/nyu-mll) pretrained RoBERTa on smaller datasets (1M, 10M, 100M, 1B tokens). They released 3 models with lowest perplexities for each pretraining data size out of 25 runs (or 10 in the case of 1B tokens). The pretraining data reproduces that of BERT: They combine English Wikipedia and a reproduction of BookCorpus using texts from smashwords in a ratio of approximately 3:1. - - -## Details of the downstream task (Q&A) - Dataset 📚 - -**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. -SQuAD v1.1 contains **100,000+** question-answer pairs on **500+** articles. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python transformers/examples/question-answering/run_squad.py \ - --model_type roberta \ - --model_name_or_path 'nyu-mll/roberta-base-1B-1' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file /content/dataset/train-v1.1.json \ - --predict_file /content/dataset/dev-v1.1.json \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 10 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /content/output \ - --overwrite_output_dir \ - --save_steps 1000 -``` - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **72.62** | -| **F1** | **82.19** | - - - -```json -{ -'exact': 72.62062440870388, -'f1': 82.19430877136834, -'total': 10570, -'HasAns_exact': 72.62062440870388, -'HasAns_f1': 82.19430877136834, -'HasAns_total': 10570, -'best_exact': 72.62062440870388, -'best_exact_thresh': 0.0, -'best_f1': 82.19430877136834, -'best_f1_thresh': 0.0 -} - -``` - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -QnA_pipeline = pipeline('question-answering', model='mrm8488/roberta-base-1B-1-finetuned-squadv1') - -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'What has been discovered by scientists from China ?' -}) -# Output: - -{'answer': 'A new strain of flu', 'end': 19, 'score': 0.04702283976040074, 'start': 0} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/roberta-base-1B-1-finetuned-squadv2/README.md b/model_cards/mrm8488/roberta-base-1B-1-finetuned-squadv2/README.md deleted file mode 100644 index 42c86e7b420e..000000000000 --- a/model_cards/mrm8488/roberta-base-1B-1-finetuned-squadv2/README.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -language: en ---- - -# RoBERTa-base (1B-1) + SQuAD v2 ❓ - -[roberta-base-1B-1](https://huggingface.co/nyu-mll/roberta-base-1B-1) fine-tuned on [SQUAD v2 dataset](https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Model 🧠 - -RoBERTa Pretrained on Smaller Datasets - -[NYU Machine Learning for Language](https://huggingface.co/nyu-mll) pretrained RoBERTa on smaller datasets (1M, 10M, 100M, 1B tokens). They released 3 models with lowest perplexities for each pretraining data size out of 25 runs (or 10 in the case of 1B tokens). The pretraining data reproduces that of BERT: They combine English Wikipedia and a reproduction of BookCorpus using texts from smashwords in a ratio of approximately 3:1. - - -## Details of the downstream task (Q&A) - Dataset 📚 - -**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. - - **SQuAD2.0** combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python transformers/examples/question-answering/run_squad.py \ - --model_type roberta \ - --model_name_or_path 'nyu-mll/roberta-base-1B-1' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file /content/dataset/train-v2.0.json \ - --predict_file /content/dataset/dev-v2.0.json \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 10 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /content/output \ - --overwrite_output_dir \ - --save_steps 1000 \ - --version_2_with_negative -``` - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **64.86** | -| **F1** | **68.99** | - - - -```json -{ -'exact': 64.86145034953255, -'f1': 68.9902640378272, -'total': 11873, -'HasAns_exact': 64.03508771929825, -'HasAns_f1': 72.3045554860189, -'HasAns_total': 5928, -'NoAns_exact': 65.68544995794785, -'NoAns_f1': 65.68544995794785, -'NoAns_total': 5945, -'best_exact': 64.86987282068559, -'best_exact_thresh': 0.0, -'best_f1': 68.99868650898054, -'best_f1_thresh': 0.0 -} -``` - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -QnA_pipeline = pipeline('question-answering', model='mrm8488/roberta-base-1B-1-finetuned-squadv2') - -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'What has been discovered by scientists from China ?' -}) -# Output: - -{'answer': 'A new strain of flu', 'end': 19, 'score': 0.7145650685380576,'start': 0} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/roberta-large-finetuned-wsc/README.md b/model_cards/mrm8488/roberta-large-finetuned-wsc/README.md deleted file mode 100644 index 94387a31a419..000000000000 --- a/model_cards/mrm8488/roberta-large-finetuned-wsc/README.md +++ /dev/null @@ -1,128 +0,0 @@ -# RoBERTa (large) fine-tuned on Winograd Schema Challenge (WSC) data - -Step from its original [repo](https://github.com/pytorch/fairseq/blob/master/examples/roberta/wsc/README.md) - -The following instructions can be used to finetune RoBERTa on the WSC training -data provided by [SuperGLUE](https://super.gluebenchmark.com/). - -Note that there is high variance in the results. For our GLUE/SuperGLUE -submission we swept over the learning rate (1e-5, 2e-5, 3e-5), batch size (16, -32, 64) and total number of updates (500, 1000, 2000, 3000), as well as the -random seed. Out of ~100 runs we chose the best 7 models and ensembled them. - -**Approach:** The instructions below use a slightly different loss function than -what's described in the original RoBERTa arXiv paper. In particular, -[Kocijan et al. (2019)](https://arxiv.org/abs/1905.06290) introduce a margin -ranking loss between `(query, candidate)` pairs with tunable hyperparameters -alpha and beta. This is supported in our code as well with the `--wsc-alpha` and -`--wsc-beta` arguments. However, we achieved slightly better (and more robust) -results on the development set by instead using a single cross entropy loss term -over the log-probabilities for the query and all mined candidates. **The -candidates are mined using spaCy from each input sentence in isolation, so the -approach remains strictly pointwise.** This reduces the number of -hyperparameters and our best model achieved 92.3% development set accuracy, -compared to ~90% accuracy for the margin loss. Later versions of the RoBERTa -arXiv paper will describe this updated formulation. - -### 1) Download the WSC data from the SuperGLUE website: -```bash -wget https://dl.fbaipublicfiles.com/glue/superglue/data/v2/WSC.zip -unzip WSC.zip - -# we also need to copy the RoBERTa dictionary into the same directory -wget -O WSC/dict.txt https://dl.fbaipublicfiles.com/fairseq/gpt2_bpe/dict.txt -``` - -### 2) Finetune over the provided training data: -```bash -TOTAL_NUM_UPDATES=2000 # Total number of training steps. -WARMUP_UPDATES=250 # Linearly increase LR over this many steps. -LR=2e-05 # Peak LR for polynomial LR scheduler. -MAX_SENTENCES=16 # Batch size per GPU. -SEED=1 # Random seed. -ROBERTA_PATH=/path/to/roberta/model.pt - -# we use the --user-dir option to load the task and criterion -# from the examples/roberta/wsc directory: -FAIRSEQ_PATH=/path/to/fairseq -FAIRSEQ_USER_DIR=${FAIRSEQ_PATH}/examples/roberta/wsc - -CUDA_VISIBLE_DEVICES=0,1,2,3 fairseq-train WSC/ \ - --restore-file $ROBERTA_PATH \ - --reset-optimizer --reset-dataloader --reset-meters \ - --no-epoch-checkpoints --no-last-checkpoints --no-save-optimizer-state \ - --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \ - --valid-subset val \ - --fp16 --ddp-backend no_c10d \ - --user-dir $FAIRSEQ_USER_DIR \ - --task wsc --criterion wsc --wsc-cross-entropy \ - --arch roberta_large --bpe gpt2 --max-positions 512 \ - --dropout 0.1 --attention-dropout 0.1 --weight-decay 0.01 \ - --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-06 \ - --lr-scheduler polynomial_decay --lr $LR \ - --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_NUM_UPDATES \ - --max-sentences $MAX_SENTENCES \ - --max-update $TOTAL_NUM_UPDATES \ - --log-format simple --log-interval 100 \ - --seed $SEED -``` - -The above command assumes training on 4 GPUs, but you can achieve the same -results on a single GPU by adding `--update-freq=4`. - -### 3) Evaluate -```python -from fairseq.models.roberta import RobertaModel -from examples.roberta.wsc import wsc_utils # also loads WSC task and criterion -roberta = RobertaModel.from_pretrained('checkpoints', 'checkpoint_best.pt', 'WSC/') -roberta.cuda() -nsamples, ncorrect = 0, 0 -for sentence, label in wsc_utils.jsonl_iterator('WSC/val.jsonl', eval=True): - pred = roberta.disambiguate_pronoun(sentence) - nsamples += 1 - if pred == label: - ncorrect += 1 -print('Accuracy: ' + str(ncorrect / float(nsamples))) -# Accuracy: 0.9230769230769231 -``` - -## RoBERTa training on WinoGrande dataset -We have also provided `winogrande` task and criterion for finetuning on the -[WinoGrande](https://mosaic.allenai.org/projects/winogrande) like datasets -where there are always two candidates and one is correct. -It's more efficient implementation for such subcases. - -```bash -TOTAL_NUM_UPDATES=23750 # Total number of training steps. -WARMUP_UPDATES=2375 # Linearly increase LR over this many steps. -LR=1e-05 # Peak LR for polynomial LR scheduler. -MAX_SENTENCES=32 # Batch size per GPU. -SEED=1 # Random seed. -ROBERTA_PATH=/path/to/roberta/model.pt - -# we use the --user-dir option to load the task and criterion -# from the examples/roberta/wsc directory: -FAIRSEQ_PATH=/path/to/fairseq -FAIRSEQ_USER_DIR=${FAIRSEQ_PATH}/examples/roberta/wsc - -cd fairseq -CUDA_VISIBLE_DEVICES=0 fairseq-train winogrande_1.0/ \ - --restore-file $ROBERTA_PATH \ - --reset-optimizer --reset-dataloader --reset-meters \ - --no-epoch-checkpoints --no-last-checkpoints --no-save-optimizer-state \ - --best-checkpoint-metric accuracy --maximize-best-checkpoint-metric \ - --valid-subset val \ - --fp16 --ddp-backend no_c10d \ - --user-dir $FAIRSEQ_USER_DIR \ - --task winogrande --criterion winogrande \ - --wsc-margin-alpha 5.0 --wsc-margin-beta 0.4 \ - --arch roberta_large --bpe gpt2 --max-positions 512 \ - --dropout 0.1 --attention-dropout 0.1 --weight-decay 0.01 \ - --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-06 \ - --lr-scheduler polynomial_decay --lr $LR \ - --warmup-updates $WARMUP_UPDATES --total-num-update $TOTAL_NUM_UPDATES \ - --max-sentences $MAX_SENTENCES \ - --max-update $TOTAL_NUM_UPDATES \ - --log-format simple --log-interval 100 -``` -[Original repo](https://github.com/pytorch/fairseq/tree/master/examples/roberta/wsc) diff --git a/model_cards/mrm8488/spanbert-base-finetuned-squadv1/README.md b/model_cards/mrm8488/spanbert-base-finetuned-squadv1/README.md deleted file mode 100644 index 8ebb81141875..000000000000 --- a/model_cards/mrm8488/spanbert-base-finetuned-squadv1/README.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -language: en -thumbnail: ---- - -# SpanBERT base fine-tuned on SQuAD v1 - -[SpanBERT](https://github.com/facebookresearch/SpanBERT) created by [Facebook Research](https://github.com/facebookresearch) and fine-tuned on [SQuAD 1.1](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task ([by them](https://github.com/facebookresearch/SpanBERT#finetuned-models-squad-1120-relation-extraction-coreference-resolution)). - -## Details of SpanBERT - -[SpanBERT: Improving Pre-training by Representing and Predicting Spans](https://arxiv.org/abs/1907.10529) - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -[SQuAD1.1](https://rajpurkar.github.io/SQuAD-explorer/) - -## Model fine-tuning 🏋️‍ - -You can get the fine-tuning script [here](https://github.com/facebookresearch/SpanBERT) - -```bash -python code/run_squad.py \ - --do_train \ - --do_eval \ - --model spanbert-base-cased \ - --train_file train-v1.1.json \ - --dev_file dev-v1.1.json \ - --train_batch_size 32 \ - --eval_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 4 \ - --max_seq_length 512 \ - --doc_stride 128 \ - --eval_metric f1 \ - --output_dir squad_output \ - --fp16 -``` - -## Results Comparison 📝 - -| | SQuAD 1.1 | SQuAD 2.0 | Coref | TACRED | -| ---------------------- | ------------- | --------- | ------- | ------ | -| | F1 | F1 | avg. F1 | F1 | -| BERT (base) | 88.5 | 76.5 | 73.1 | 67.7 | -| SpanBERT (base) | **92.4** (this one) | [83.6](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv2) | 77.4 | [68.2](https://huggingface.co/mrm8488/spanbert-base-finetuned-tacred) | -| BERT (large) | 91.3 | 83.3 | 77.1 | 66.4 | -| SpanBERT (large) | [94.6](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv1) | [88.7](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv2) | 79.6 | [70.8](https://huggingface.co/mrm8488/spanbert-large-finetuned-tacred) | - - -Note: The numbers marked as * are evaluated on the development sets because those models were not submitted to the official SQuAD leaderboard. All the other numbers are test numbers. - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/spanbert-base-finetuned-squadv1", - tokenizer="SpanBERT/spanbert-base-cased" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working very hard in the repository hugginface/transformers lately", - 'question': "How has been working Manuel Romero lately?" - -}) - -# Output: {'answer': 'very hard in the repository hugginface/transformers', - 'end': 82, - 'score': 0.327230326857725, - 'start': 31} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/spanbert-base-finetuned-squadv2/README.md b/model_cards/mrm8488/spanbert-base-finetuned-squadv2/README.md deleted file mode 100644 index 865c66c8b71b..000000000000 --- a/model_cards/mrm8488/spanbert-base-finetuned-squadv2/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: en -thumbnail: ---- - -# SpanBERT base fine-tuned on SQuAD v2 - -[SpanBERT](https://github.com/facebookresearch/SpanBERT) created by [Facebook Research](https://github.com/facebookresearch) and fine-tuned on [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task ([by them](https://github.com/facebookresearch/SpanBERT#finetuned-models-squad-1120-relation-extraction-coreference-resolution)). - -## Details of SpanBERT - -[SpanBERT: Improving Pre-training by Representing and Predicting Spans](https://arxiv.org/abs/1907.10529) - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -[SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD2.0 | train | 130k | -| SQuAD2.0 | eval | 12.3k | - -## Model fine-tuning 🏋️‍ - -You can get the fine-tuning script [here](https://github.com/facebookresearch/SpanBERT) - -```bash -python code/run_squad.py \ - --do_train \ - --do_eval \ - --model spanbert-base-cased \ - --train_file train-v2.0.json \ - --dev_file dev-v2.0.json \ - --train_batch_size 32 \ - --eval_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 4 \ - --max_seq_length 512 \ - --doc_stride 128 \ - --eval_metric best_f1 \ - --output_dir squad2_output \ - --version_2_with_negative \ - --fp16 -``` - -## Results Comparison 📝 - -| | SQuAD 1.1 | SQuAD 2.0 | Coref | TACRED | -| ---------------------- | ------------- | --------- | ------- | ------ | -| | F1 | F1 | avg. F1 | F1 | -| BERT (base) | 88.5 | 76.5 | 73.1 | 67.7 | -| SpanBERT (base) | [92.4](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv1) | **83.6** (this one) | 77.4 | [68.2](https://huggingface.co/mrm8488/spanbert-base-finetuned-tacred) | -| BERT (large) | 91.3 | 83.3 | 77.1 | 66.4 | -| SpanBERT (large) | [94.6](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv1) | [88.7](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv2) | 79.6 | [70.8](https://huggingface.co/mrm8488/spanbert-large-finetuned-tacred) | - - -Note: The numbers marked as * are evaluated on the development sets because those models were not submitted to the official SQuAD leaderboard. All the other numbers are test numbers. - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/spanbert-base-finetuned-squadv2", - tokenizer="SpanBERT/spanbert-base-cased" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working very hard in the repository hugginface/transformers lately", - 'question': "How has been working Manuel Romero lately?" - -}) -# Output: {'answer': 'very hard', 'end': 40, 'score': 0.9052708846768347, 'start': 31} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/spanbert-base-finetuned-tacred/README.md b/model_cards/mrm8488/spanbert-base-finetuned-tacred/README.md deleted file mode 100644 index 3ee1158b266b..000000000000 --- a/model_cards/mrm8488/spanbert-base-finetuned-tacred/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -language: en -thumbnail: ---- - -# SpanBERT base fine-tuned on TACRED - -[SpanBERT](https://github.com/facebookresearch/SpanBERT) created by [Facebook Research](https://github.com/facebookresearch) and fine-tuned on [TACRED](https://nlp.stanford.edu/projects/tacred/) dataset by [them](https://github.com/facebookresearch/SpanBERT#finetuned-models-squad-1120-relation-extraction-coreference-resolution) - -## Details of SpanBERT - -[SpanBERT: Improving Pre-training by Representing and Predicting Spans](https://arxiv.org/abs/1907.10529) - -## Dataset 📚 - -[TACRED](https://nlp.stanford.edu/projects/tacred/) A large-scale relation extraction dataset with 106k+ examples over 42 TAC KBP relation types. - -## Model fine-tuning 🏋️‍ - -You can get the fine-tuning script [here](https://github.com/facebookresearch/SpanBERT) - -```bash -python code/run_tacred.py \ - --do_train \ - --do_eval \ - --data_dir \ - --model spanbert-base-cased \ - --train_batch_size 32 \ - --eval_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 10 \ - --max_seq_length 128 \ - --output_dir tacred_dir \ - --fp16 -``` - -## Results Comparison 📝 - -| | SQuAD 1.1 | SQuAD 2.0 | Coref | TACRED | -| ---------------------- | ------------- | --------- | ------- | ------ | -| | F1 | F1 | avg. F1 | F1 | -| BERT (base) | 88.5* | 76.5* | 73.1 | 67.7 | -| SpanBERT (base) | [92.4*](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv1) | [83.6*](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv2) | 77.4 | **68.2** (this one) | -| BERT (large) | 91.3 | 83.3 | 77.1 | 66.4 | -| SpanBERT (large) | [94.6](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv1) | [88.7](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv2) | 79.6 | [70.8](https://huggingface.co/mrm8488/spanbert-base-finetuned-tacred) | - - -Note: The numbers marked as * are evaluated on the development sets because those models were not submitted to the official SQuAD leaderboard. All the other numbers are test numbers. - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/spanbert-finetuned-squadv1/README.md b/model_cards/mrm8488/spanbert-finetuned-squadv1/README.md deleted file mode 100644 index a8584c2b2b3c..000000000000 --- a/model_cards/mrm8488/spanbert-finetuned-squadv1/README.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -language: en -thumbnail: ---- - -# SpanBERT (spanbert-base-cased) fine-tuned on SQuAD v1.1 - - -[SpanBERT](https://github.com/facebookresearch/SpanBERT) created by [Facebook Research](https://github.com/facebookresearch) and fine-tuned on [SQuAD 1.1](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -## Details of SpanBERT - - A pre-training method that is designed to better represent and predict spans of text. - -[SpanBERT: Improving Pre-training by Representing and Predicting Spans](https://arxiv.org/abs/1907.10529) - -## Details of the downstream task (Q&A) - Dataset - -[SQuAD 1.1](https://rajpurkar.github.io/SQuAD-explorer/) contains 100,000+ question-answer pairs on 500+ articles. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD1.1 | train | 87.7k | -| SQuAD1.1 | eval | 10.6k | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/question-answering/run_squad.py) - -## Results: - -| Metric | # Value | -| ------ | --------- | -| **EM** | **85.49** | -| **F1** | **91.98** | - -### Raw metrics: - -```json -{ - "exact": 85.49668874172185, - "f1": 91.9845699540379, - "total": 10570, - "HasAns_exact": 85.49668874172185, - "HasAns_f1": 91.9845699540379, - "HasAns_total": 10570, - "best_exact": 85.49668874172185, - "best_exact_thresh": 0.0, - "best_f1": 91.9845699540379, - "best_f1_thresh": 0.0 -} -``` - -## Comparison: - -| Model | EM | F1 score | -| ----------------------------------------------------------------------------------------- | --------- | --------- | -| [SpanBert official repo](https://github.com/facebookresearch/SpanBERT#pre-trained-models) | - | 92.4\* | -| [spanbert-finetuned-squadv1](https://huggingface.co/mrm8488/spanbert-finetuned-squadv1) | **85.49** | **91.98** | - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/spanbert-finetuned-squadv1", - tokenizer="mrm8488/spanbert-finetuned-squadv1" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" - -}) -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/spanbert-finetuned-squadv2/README.md b/model_cards/mrm8488/spanbert-finetuned-squadv2/README.md deleted file mode 100644 index 74143a821dd7..000000000000 --- a/model_cards/mrm8488/spanbert-finetuned-squadv2/README.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -language: en -thumbnail: ---- - -# SpanBERT (spanbert-base-cased) fine-tuned on SQuAD v2 - -[SpanBERT](https://github.com/facebookresearch/SpanBERT) created by [Facebook Research](https://github.com/facebookresearch) and fine-tuned on [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -## Details of SpanBERT - -[SpanBERT: Improving Pre-training by Representing and Predicting Spans](https://arxiv.org/abs/1907.10529) - -## Details of the downstream task (Q&A) - Dataset - -[SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD2.0 | train | 130k | -| SQuAD2.0 | eval | 12.3k | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/question-answering/run_squad.py) - -## Results: - -| Metric | # Value | -| ------ | --------- | -| **EM** | **78.80** | -| **F1** | **82.22** | - -### Raw metrics: - -```json -{ - "exact": 78.80064010780762, - "f1": 82.22801347271162, - "total": 11873, - "HasAns_exact": 78.74493927125506, - "HasAns_f1": 85.60951483831069, - "HasAns_total": 5928, - "NoAns_exact": 78.85618166526493, - "NoAns_f1": 78.85618166526493, - "NoAns_total": 5945, - "best_exact": 78.80064010780762, - "best_exact_thresh": 0.0, - "best_f1": 82.2280134727116, - "best_f1_thresh": 0.0 -} -``` - -## Comparison: - -| Model | EM | F1 score | -| ----------------------------------------------------------------------------------------- | --------- | --------- | -| [SpanBert official repo](https://github.com/facebookresearch/SpanBERT#pre-trained-models) | - | 83.6\* | -| [spanbert-finetuned-squadv2](https://huggingface.co/mrm8488/spanbert-finetuned-squadv2) | **78.80** | **82.22** | - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/spanbert-finetuned-squadv2", - tokenizer="mrm8488/spanbert-finetuned-squadv2" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" - -}) - -# Output: {'answer': 'Manuel Romero','end': 13,'score': 6.836378586818937e-09, 'start': 0} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/spanbert-large-finetuned-squadv1/README.md b/model_cards/mrm8488/spanbert-large-finetuned-squadv1/README.md deleted file mode 100644 index 3bfc6ef42f79..000000000000 --- a/model_cards/mrm8488/spanbert-large-finetuned-squadv1/README.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -language: en -thumbnail: ---- - -# SpanBERT large fine-tuned on SQuAD v1 - -[SpanBERT](https://github.com/facebookresearch/SpanBERT) created by [Facebook Research](https://github.com/facebookresearch) and fine-tuned on [SQuAD 1.1](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task ([by them](https://github.com/facebookresearch/SpanBERT#finetuned-models-squad-1120-relation-extraction-coreference-resolution)). - -## Details of SpanBERT - -[SpanBERT: Improving Pre-training by Representing and Predicting Spans](https://arxiv.org/abs/1907.10529) - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -[SQuAD1.1](https://rajpurkar.github.io/SQuAD-explorer/) - -## Model fine-tuning 🏋️‍ - -You can get the fine-tuning script [here](https://github.com/facebookresearch/SpanBERT) - -```bash -python code/run_squad.py \ - --do_train \ - --do_eval \ - --model spanbert-large-cased \ - --train_file train-v1.1.json \ - --dev_file dev-v1.1.json \ - --train_batch_size 32 \ - --eval_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 4 \ - --max_seq_length 512 \ - --doc_stride 128 \ - --eval_metric f1 \ - --output_dir squad_output \ - --fp16 -``` - -## Results Comparison 📝 - -| | SQuAD 1.1 | SQuAD 2.0 | Coref | TACRED | -| ---------------------- | ------------- | --------- | ------- | ------ | -| | F1 | F1 | avg. F1 | F1 | -| BERT (base) | 88.5* | 76.5* | 73.1 | 67.7 | -| SpanBERT (base) | [92.4*](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv1) | [83.6*](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv2) | 77.4 | [68.2](https://huggingface.co/mrm8488/spanbert-base-finetuned-tacred) | -| BERT (large) | 91.3 | 83.3 | 77.1 | 66.4 | -| SpanBERT (large) | **94.6** (this) | [88.7](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv2) | 79.6 | [70.8](https://huggingface.co/mrm8488/spanbert-large-finetuned-tacred) | - - -Note: The numbers marked as * are evaluated on the development sets because those models were not submitted to the official SQuAD leaderboard. All the other numbers are test numbers. - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/spanbert-large-finetuned-squadv1", - tokenizer="SpanBERT/spanbert-large-cased" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working very hard in the repository hugginface/transformers lately", - 'question': "How has been working Manuel Romero lately?" - -}) - -# Output: {'answer': 'very hard in the repository hugginface/transformers', - 'end': 82, - 'score': 0.327230326857725, - 'start': 31} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/spanbert-large-finetuned-squadv2/README.md b/model_cards/mrm8488/spanbert-large-finetuned-squadv2/README.md deleted file mode 100644 index 51b9d3ae79f9..000000000000 --- a/model_cards/mrm8488/spanbert-large-finetuned-squadv2/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: en -thumbnail: ---- - -# SpanBERT large fine-tuned on SQuAD v2 - -[SpanBERT](https://github.com/facebookresearch/SpanBERT) created by [Facebook Research](https://github.com/facebookresearch) and fine-tuned on [SQuAD 2.0](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task ([by them](https://github.com/facebookresearch/SpanBERT#finetuned-models-squad-1120-relation-extraction-coreference-resolution)). - -## Details of SpanBERT - -[SpanBERT: Improving Pre-training by Representing and Predicting Spans](https://arxiv.org/abs/1907.10529) - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -[SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| SQuAD2.0 | train | 130k | -| SQuAD2.0 | eval | 12.3k | - -## Model fine-tuning 🏋️‍ - -You can get the fine-tuning script [here](https://github.com/facebookresearch/SpanBERT) - -```bash -python code/run_squad.py \ - --do_train \ - --do_eval \ - --model spanbert-large-cased \ - --train_file train-v2.0.json \ - --dev_file dev-v2.0.json \ - --train_batch_size 32 \ - --eval_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 4 \ - --max_seq_length 512 \ - --doc_stride 128 \ - --eval_metric best_f1 \ - --output_dir squad2_output \ - --version_2_with_negative \ - --fp16 -``` - -## Results Comparison 📝 - -| | SQuAD 1.1 | SQuAD 2.0 | Coref | TACRED | -| ---------------------- | ------------- | --------- | ------- | ------ | -| | F1 | F1 | avg. F1 | F1 | -| BERT (base) | 88.5* | 76.5* | 73.1 | 67.7 | -| SpanBERT (base) | [92.4*](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv1) | [83.6*](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv2) | 77.4 | [68.2](https://huggingface.co/mrm8488/spanbert-base-finetuned-tacred) | -| BERT (large) | 91.3 | 83.3 | 77.1 | 66.4 | -| SpanBERT (large) | [94.6](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv1) | **88.7** (this) | 79.6 | [70.8](https://huggingface.co/mrm8488/spanbert-large-finetuned-tacred) | - - -Note: The numbers marked as * are evaluated on the development sets because those models were not submitted to the official SQuAD leaderboard. All the other numbers are test numbers. - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/spanbert-large-finetuned-squadv2", - tokenizer="SpanBERT/spanbert-large-cased" -) - -qa_pipeline({ - 'context': "Manuel Romero has been working very hard in the repository hugginface/transformers lately", - 'question': "How has been working Manuel Romero lately?" - -}) -# Output: {'answer': 'very hard', 'end': 40, 'score': 0.9052708846768347, 'start': 31} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/spanbert-large-finetuned-tacred/README.md b/model_cards/mrm8488/spanbert-large-finetuned-tacred/README.md deleted file mode 100644 index 826c69be1c38..000000000000 --- a/model_cards/mrm8488/spanbert-large-finetuned-tacred/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -language: en -thumbnail: ---- - -# SpanBERT large fine-tuned on TACRED - -[SpanBERT](https://github.com/facebookresearch/SpanBERT) created by [Facebook Research](https://github.com/facebookresearch) and fine-tuned on [TACRED](https://nlp.stanford.edu/projects/tacred/) dataset by [them](https://github.com/facebookresearch/SpanBERT#finetuned-models-squad-1120-relation-extraction-coreference-resolution) - -## Details of SpanBERT - -[SpanBERT: Improving Pre-training by Representing and Predicting Spans](https://arxiv.org/abs/1907.10529) - -## Dataset 📚 - -[TACRED](https://nlp.stanford.edu/projects/tacred/) A large-scale relation extraction dataset with 106k+ examples over 42 TAC KBP relation types. - -## Model fine-tuning 🏋️‍ - -You can get the fine-tuning script [here](https://github.com/facebookresearch/SpanBERT) - -```bash -python code/run_tacred.py \ - --do_train \ - --do_eval \ - --data_dir \ - --model spanbert-large-cased \ - --train_batch_size 32 \ - --eval_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 10 \ - --max_seq_length 128 \ - --output_dir tacred_dir \ - --fp16 -``` - -## Results Comparison 📝 - -| | SQuAD 1.1 | SQuAD 2.0 | Coref | TACRED | -| ---------------------- | ------------- | --------- | ------- | ------ | -| | F1 | F1 | avg. F1 | F1 | -| BERT (base) | 88.5* | 76.5* | 73.1 | 67.7 | -| SpanBERT (base) | [92.4*](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv1) | [83.6*](https://huggingface.co/mrm8488/spanbert-base-finetuned-squadv2) | 77.4 | [68.2](https://huggingface.co/mrm8488/spanbert-base-finetuned-tacred) | -| BERT (large) | 91.3 | 83.3 | 77.1 | 66.4 | -| SpanBERT (large) | [94.6](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv1) | [88.7](https://huggingface.co/mrm8488/spanbert-large-finetuned-squadv2) | 79.6 | **70.8** (this one) | - - -Note: The numbers marked as * are evaluated on the development sets because those models were not submitted to the official SQuAD leaderboard. All the other numbers are test numbers. - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/squeezebert-finetuned-squadv1/README.md b/model_cards/mrm8488/squeezebert-finetuned-squadv1/README.md deleted file mode 100644 index 4bcf9771b427..000000000000 --- a/model_cards/mrm8488/squeezebert-finetuned-squadv1/README.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -language: en -datasets: -- squad ---- - -# SqueezeBERT + SQuAD (v1.1) - -[squeezebert-uncased](https://huggingface.co/squeezebert/squeezebert-uncased) fine-tuned on [SQUAD v1.1](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) for **Q&A** downstream task. - -## Details of SqueezeBERT - -This model, `squeezebert-uncased`, is a pretrained model for the English language using a masked language modeling (MLM) and Sentence Order Prediction (SOP) objective. -SqueezeBERT was introduced in [this paper](https://arxiv.org/abs/2006.11316). This model is case-insensitive. The model architecture is similar to BERT-base, but with the pointwise fully-connected layers replaced with [grouped convolutions](https://blog.yani.io/filter-group-tutorial/). -The authors found that SqueezeBERT is 4.3x faster than `bert-base-uncased` on a Google Pixel 3 smartphone. -More about the model [here](https://arxiv.org/abs/2004.02984) - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -**S**tanford **Q**uestion **A**nswering **D**ataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. -SQuAD v1.1 contains **100,000+** question-answer pairs on **500+** articles. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python /content/transformers/examples/question-answering/run_squad.py \ - --model_type bert \ - --model_name_or_path squeezebert/squeezebert-uncased \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file /content/dataset/train-v1.1.json \ - --predict_file /content/dataset/dev-v1.1.json \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 15 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /content/output_dir \ - --overwrite_output_dir \ - --save_steps 2000 -``` - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **76.66** | -| **F1** | **85.83** | - -Model Size: **195 MB** - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline -QnA_pipeline = pipeline('question-answering', model='mrm8488/squeezebert-finetuned-squadv1') -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'Who did identified it ?' - }) - -# Output: {'answer': 'scientists.', 'end': 106, 'score': 0.6988425850868225, 'start': 96} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/squeezebert-finetuned-squadv2/README.md b/model_cards/mrm8488/squeezebert-finetuned-squadv2/README.md deleted file mode 100644 index 2f8634d6f297..000000000000 --- a/model_cards/mrm8488/squeezebert-finetuned-squadv2/README.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -language: en -datasets: -- squad_v2 ---- - -# SqueezeBERT + SQuAD v2 - -[squeezebert-uncased](https://huggingface.co/squeezebert/squeezebert-uncased) fine-tuned on [SQUAD v2](https://rajpurkar.github.io/SQuAD-explorer/explore/v2.0/dev/) for **Q&A** downstream task. - -## Details of SqueezeBERT - -This model, `squeezebert-uncased`, is a pretrained model for the English language using a masked language modeling (MLM) and Sentence Order Prediction (SOP) objective. -SqueezeBERT was introduced in [this paper](https://arxiv.org/abs/2006.11316). This model is case-insensitive. The model architecture is similar to BERT-base, but with the pointwise fully-connected layers replaced with [grouped convolutions](https://blog.yani.io/filter-group-tutorial/). -The authors found that SqueezeBERT is 4.3x faster than `bert-base-uncased` on a Google Pixel 3 smartphone. -More about the model [here](https://arxiv.org/abs/2004.02984) - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -**SQuAD2.0** combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python /content/transformers/examples/question-answering/run_squad.py \ - --model_type bert \ - --model_name_or_path squeezebert/squeezebert-uncased \ - --do_train \ - --do_eval \ - --do_lower_case \ - --train_file /content/dataset/train-v2.0.json \ - --predict_file /content/dataset/dev-v2.0.json \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 15 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /content/output_dir \ - --overwrite_output_dir \ - --version_2_with_negative \ - --save_steps 2000 -``` - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **69.98** | -| **F1** | **74.14** | - -Model Size: **195 MB** - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline -QnA_pipeline = pipeline('question-answering', model='mrm8488/squeezebert-finetuned-squadv2') -QnA_pipeline({ - 'context': 'A new strain of flu that has the potential to become a pandemic has been identified in China by scientists.', - 'question': 'Who did identified it ?' - }) - -# Output: {'answer': 'scientists.', 'end': 106, 'score': 0.9768241047859192, 'start': 96} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-break_data-question-retrieval/README.md b/model_cards/mrm8488/t5-base-finetuned-break_data-question-retrieval/README.md deleted file mode 100644 index 2cdf91f9f91a..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-break_data-question-retrieval/README.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -language: en -datasets: -- break_data ---- - -# T5-base fine-tuned on break_data / QDMR-high-level 📋➡️❓ - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [break_data](https://huggingface.co/nlp/viewer/?dataset=break_data&config=QDMR-high-level) dataset for **Question Retrieval from its decomposition**. -The inverse process of [this model](https://huggingface.co/mrm8488/t5-base-finetuned-break_data). - -## Details of T5 📜 ➡️ 📜 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the downstream task (Question Retrieval from its decomposition) - Dataset 📚 - -Break is a human annotated dataset of natural language questions and their Question Decomposition Meaning Representations (QDMRs). Break consists of 83,978 examples sampled from 10 question answering datasets over text, images and databases. This repository contains the Break dataset along with information on the exact data format. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| break_data | train | 17503 | -| break_data | valid | 3130 | - -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28). The main change is at preprocessing ```inputs``` and ```targets``` we feed to the model. We do it as a *paraphrasing task*. - - -## Model in Action 🚀 - -```python -# Tip: By now, install transformers from source - -from transformers import AutoModelForSeq2SeqLM, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-break_data-question-retrieval") -model = AutoModelForSeq2SeqLM.from_pretrained("mrm8488/t5-base-finetuned-break_data-question-retrieval") - - -def get_nautural_question(decomposition): - input_text = 'translate QDMRs to Natural Language %s ' % decomposition - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask'], - max_length=64) - - return tokenizer.decode(output[0]) - - - -decomposition = "return the city that was the birthplace of Bernard Berrian ;return the city that was the home of Pablo Picasso ;return the city of both #1 and #2" - -# Ground Truth: What city was the birthplace of Bernard Berrianand the home of Pablo Picasso? - -get_nautural_question(decomposition) - -# output: 'What city was the birthplace of Bernard Berrian and the home of Pablo Picasso?' -``` -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-break_data/README.md b/model_cards/mrm8488/t5-base-finetuned-break_data/README.md deleted file mode 100644 index f875d242c14e..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-break_data/README.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -language: en -datasets: -- break_data ---- - -# T5-base fine-tuned on break_data / QDMR-high-level ❓➡️📋 - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [break_data](https://huggingface.co/nlp/viewer/?dataset=break_data&config=QDMR-high-level) dataset for **QDMRs**. - -## Details of T5 📜 ➡️ 📜 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the downstream task (QDMRs) - Dataset 📚 - -Break is a human annotated dataset of natural language questions and their Question Decomposition Meaning Representations (QDMRs). Break consists of 83,978 examples sampled from 10 question answering datasets over text, images and databases. This repository contains the Break dataset along with information on the exact data format. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| break_data | train | 17503 | -| break_data | valid | 3130 | - -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28). The main change is at preprocessing ```inputs``` and ```targets``` we feed to the model. We do it as a *paraphrasing task*. - - -## Model in Action 🚀 - -```python -# Tip: By now, install transformers from source - -from transformers import AutoModelForSeq2SeqLM, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-break_data") -model = AutoModelForSeq2SeqLM.from_pretrained("mrm8488/t5-base-finetuned-break_data") - -def get_decomposition(question): - input_text = "paraphrase: %s " % question - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask'], - max_length=32) - - return tokenizer.decode(output[0]) - -question = "The composer of Sands Theme plays what type of guitar?" - -get_decomposition(question) - -# output: 'return Sands Theme ;return composer of #1 ;return guitar that #2 plays' -``` -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-common_gen/README.md b/model_cards/mrm8488/t5-base-finetuned-common_gen/README.md deleted file mode 100644 index f385d21af84b..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-common_gen/README.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -language: en -datasets: -- common_gen ---- - -# T5-base fine-tuned on CommonGen - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [CommonGen](https://inklab.usc.edu/CommonGen/index.html) for *Generative Commonsense Reasoning*. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the dataset 📚 - -CommonGen is a constrained text generation task, associated with a benchmark dataset, to explicitly test machines for the ability of generative commonsense reasoning. Given a set of common concepts; the task is to generate a coherent sentence describing an everyday scenario using these concepts. - -CommonGen is challenging because it inherently requires 1) relational reasoning using background commonsense knowledge, and 2) compositional generalization ability to work on unseen concept combinations. Our dataset, constructed through a combination of crowd-sourcing from AMT and existing caption corpora, consists of 30k concept-sets and 50k sentences in total. - - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| common_gen | train | 67389 | -| common_gen | valid | 4018 | -| common_gen | test | 1497 | - - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28) - -## Metrics 📋 - -| Metric | Score | -|--------|-------| -|ROUGE-2 | 17.10 | -|ROUGE-L | 39.47 | -|BLEU | WIP | - -The metrics above slightly improves results shown in the [paper](https://arxiv.org/abs/1911.03705) for the same model and metrics. - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-common_gen") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-common_gen") - -def gen_sentence(words, max_length=32): - input_text = words - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask'], - max_length=max_length) - - return tokenizer.decode(output[0]) - -words = "tree plant ground hole dig" - -gen_sentence(words) - -# output: digging a hole in the ground to plant trees -``` -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/T5_base_finetuned_common_gen.ipynb) - - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-e2m-intent/README.md b/model_cards/mrm8488/t5-base-finetuned-e2m-intent/README.md deleted file mode 100644 index 8f9624f6ae7f..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-e2m-intent/README.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -language: en -datasets: -- event2Mind ---- - -# T5-base fine-tuned on event2Mind for **Intent Prediction** 🤔 - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [event2Mind](https://huggingface.co/nlp/viewer/?dataset=event2Mind) dataset for **Intent Prediction**. - -## Details of T5 📜 ➡️ 📜 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the downstream task (Intent Prediction) - Dataset 📚 - -Dataset ID: ```event2Mind``` from [Huggingface/NLP](https://github.com/huggingface/nlp) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| event2Mind | train | 46472 | -| event2Mind | valid | 1960 | - -Events without **intent** were not used! - -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28). - - -## Model in Action 🚀 - -```python -# Tip: By now, install transformers from source - -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-e2m-intent") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-e2m-intent") - -def get_intent(event, max_length=16): - input_text = "%s " % event - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask'], - max_length=max_length) - - return tokenizer.decode(output[0]) - -event = "PersonX takes PersonY home" -get_intent(event) - -# output: 'to be helpful' -``` -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-emotion/README.md b/model_cards/mrm8488/t5-base-finetuned-emotion/README.md deleted file mode 100644 index 78c1e3265edc..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-emotion/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: en -datasets: -- emotion ---- - -# T5-base fine-tuned for Emotion Recognition 😂😢😡😃😯 - - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) base fine-tuned on [emotion recognition](https://github.com/dair-ai/emotion_dataset) dataset for **Emotion Recognition** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - -## Details of the downstream task (Sentiment Recognition) - Dataset 📚 - -[Elvis Saravia](https://twitter.com/omarsar0) has gathered a great [dataset](https://github.com/dair-ai/emotion_dataset) for emotion recognition. It allows to classifiy the text into one of the following **6** emotions: - - - sadness 😢 - - joy 😃 - - love 🥰 - - anger 😡 - - fear 😱 - - surprise 😯 - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him! - -## Test set metrics 🧾 - -| |precision | recall | f1-score |support| -|----------|----------|---------|----------|-------| -|anger | 0.93| 0.92| 0.93| 275| -|fear | 0.91| 0.87| 0.89| 224| -|joy | 0.97| 0.94| 0.95| 695| -|love | 0.80| 0.91| 0.85| 159| -|sadness | 0.97| 0.97| 0.97| 521| -|surpirse | 0.73| 0.89| 0.80| 66| -| | -|accuracy| | | 0.93| 2000| -|macro avg| 0.89| 0.92| 0.90| 2000| -|weighted avg| 0.94| 0.93| 0.93| 2000| - - - - - - - -## Model in Action 🚀 - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-emotion") - -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-emotion") - -def get_emotion(text): - input_ids = tokenizer.encode(text + '', return_tensors='pt') - - output = model.generate(input_ids=input_ids, - max_length=2) - - dec = [tokenizer.decode(ids) for ids in output] - label = dec[0] - return label - - get_emotion("i feel as if i havent blogged in ages are at least truly blogged i am doing an update cute") # Output: 'joy' - - get_emotion("i have a feeling i kinda lost my best friend") # Output: 'sadness' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-imdb-sentiment/README.md b/model_cards/mrm8488/t5-base-finetuned-imdb-sentiment/README.md deleted file mode 100644 index 85c5a07f33a5..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-imdb-sentiment/README.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -language: en -datasets: -- imdb ---- - -# T5-base fine-tuned for Sentiment Anlalysis 🎞️👍👎 - - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) base fine-tuned on [IMDB](https://huggingface.co/datasets/imdb) dataset for **Sentiment Analysis** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67) - -## Details of the downstream task (Sentiment analysis) - Dataset 📚 - -[IMDB](https://huggingface.co/datasets/imdb) - -This is a dataset for binary sentiment classification containing substantially more data than previous benchmark datasets. We provide a set of **25,000** highly polar movie reviews for training, and **25,000** for testing. - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him! - -## Test set metrics 🧾 - - |precision | recall | f1-score |support| - |----------|----------|---------|----------|-------| - |negative | 0.95 | 0.95| 0.95| 12500| - |positive | 0.95 | 0.95| 0.95| 12500| - |----------|----------|---------|----------|-------| - |accuracy| | | 0.95| 25000| - |macro avg| 0.95| 0.95| 0.95| 25000| - |weighted avg| 0.95| 0.95| 0.95 | 25000| - - -## Model in Action 🚀 - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-imdb-sentiment") - -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-imdb-sentiment") - -def get_sentiment(text): - input_ids = tokenizer.encode(text + '', return_tensors='pt') - - output = model.generate(input_ids=input_ids, - max_length=2) - - dec = [tokenizer.decode(ids) for ids in output] - label = dec[0] - return label - -get_sentiment("I dislike a lot that film") - -# Output: 'negative' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-qasc/README.md b/model_cards/mrm8488/t5-base-finetuned-qasc/README.md deleted file mode 100644 index 7259477c2aaf..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-qasc/README.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -language: en -datasets: -- qasc ---- - -# T5-base fine-tuned on QASC - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [QASC](https://allenai.org/data/qasc) for **QA** (via *sentence composition*) downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the dataset 📚 - -**Question Answering via Sentence Composition** (QASC) is a question-answering dataset with a focus on sentence composition. It consists of 9,980 8-way multiple-choice questions about grade school science (8,134 train, 926 dev, 920 test), and comes with a corpus of 17M sentences. - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28). The **context** passed to the *encoder* is the combination of the 2 *facts* (`fact1` and `fact2`). The **question** is just the `formatted_question` field. The **answer** passed to the *decoder* is the`text` right answer instead of the `label` (A, B, C... See `choices` field). More details about the dataset format/fields [here](https://huggingface.co/nlp/viewer/?dataset=qasc) - -## Metrics on validation set 📋 - -| Metric | Score | -|--------|-------| -|Accuracy (EM) | **97.73**| - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-qasc") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-qasc") - -def get_response(question, context, max_length=64): - input_text = 'question: %s context: %s' % (question, context) - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask'], - max_length=max_length) - - return tokenizer.decode(output[0]) - -fact_1 = 'a watch is used for measuring time' -fact_2 = 'Times are measured in seconds.' -context = fact_1 + ' ' + fact_2 -question = 'What can be used to measure seconds? (A) Watch (B) seconds (C) fluid (D) Ruler (E) goggles (F) glasses (G) Drill (H) Scale' - -get_response(question, context) - -# output: 'Watch' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-quarel/README.md b/model_cards/mrm8488/t5-base-finetuned-quarel/README.md deleted file mode 100644 index 780e6735ed5d..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-quarel/README.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -language: en -datasets: -- quarel ---- - -# T5-base fine-tuned on QuaRel - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [QuaRel](https://allenai.org/data/quarel) for **QA** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the dataset 📚 - -**QuaRel**: *[A Dataset and Models for Answering Questions about Qualitative Relationships](https://www.semanticscholar.org/paper/QuaRel%3A-A-Dataset-and-Models-for-Answering-about-Tafjord-Clark/51004bc6461a572e1189a0e3b32b441155d760ce)* - -Many natural language questions require recognizing and reasoning with qualitative relationships (e.g., in science, economics, and medicine), but are challenging to answer with corpus-based methods. Qualitative modeling provides tools that support such reasoning, but the semantic parsing task of mapping questions into those models has formidable challenges. We present QuaRel, a dataset of diverse story questions involving qualitative relationships that characterize these challenges, and techniques that begin to address them. The dataset has 2771 questions relating 19 different types of quantities. For example, "Jenny observes that the robot vacuum cleaner moves slower on the living room carpet than on the bedroom carpet. Which carpet has more friction?" We contribute (1) a simple and flexible conceptual framework for representing these kinds of questions; (2) the QuaRel dataset, including logical forms, exemplifying the parsing challenges; and (3) two novel models for this task, built as extensions of type-constrained semantic parsing. The first of these models (called QuaSP+) significantly outperforms off-the-shelf tools on QuaRel. The second (QuaSP+Zero) demonstrates zero-shot capability, i.e., the ability to handle new qualitative relationships without requiring additional training data, something not possible with previous models. This work thus makes inroads into answering complex, qualitative questions that require reasoning, and scaling to new relationships at low cost - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28). The **context** passed to the *encoder* is the `logical_form_pretty` field (example: `qrel(speed, higher, ice) -> qrel(smoothness, higher, snow) ; qrel(smoothness, higher, ice`) . The **question** is just the `question` field. The **answer** passed to the *decoder* is obtained from `question`using the `answer_index` field. More details about the dataset format/fields [here](https://huggingface.co/nlp/viewer/?dataset=quarel) - -## Metrics on validation set 📋 - -| Metric | Score | -|--------|-------| -|Accuracy (EM) | **67.98**| - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-quarel") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-quarel") - -def get_response(question, context, max_length=32): - input_text = 'question: %s context: %s' % (question, context) - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask'], - max_length=max_length) - - return tokenizer.decode(output[0]) - -question = 'As the train left the station it crossed the bridge and being farther away it looked (A) larger (B) smaller' -context = 'qrel(distance, higher, Train on a bridge) -> qrel(apparentSize, higher, Train on a bridge) ; qrel(apparentSize, lower, Train on a bridge)' - -get_response(question, context) - -# output: 'smaller' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-quartz/README.md b/model_cards/mrm8488/t5-base-finetuned-quartz/README.md deleted file mode 100644 index b571291f1e23..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-quartz/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: en -datasets: -- quartz -pipeline_tag: question-answering ---- - -# T5-base fine-tuned on QuaRTz - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [QuaRTz](https://allenai.org/data/quartz) for **QA** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the dataset 📚 - -**QuaRTz** is a crowdsourced dataset of 3864 multiple-choice questions about open domain qualitative relationships. Each question is paired with one of 405 different background sentences (sometimes short paragraphs). The QuaRTz dataset V1 contains 3864 questions about open domain qualitative relationships. Each question is paired with one of 405 different background sentences (sometimes short paragraphs). -The dataset is split into: - -|Set | Samples| -|-----|--------| -|Train | 2696 | -|Valid | 384 | -|Test | 784 | - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28). The *question*, *context* (`para` field) and *options* (`choices` field) are concatenated and passed to the **encoder**. The **decoder** receives the right *answer* (by querying `answerKey` field). More details about the dataset fields/format [here](https://huggingface.co/nlp/viewer/?dataset=quartz) - -## Results 📋 - - -|Set | Metric | Score | -|-----|--------|-------| -|Validation | Accuracy (EM) | **83.59**| -|Test | Accuracy (EM) | **81.50**| - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-quartz") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-quartz") - -def get_response(question, fact, opts, max_length=16): - input_text = 'question: %s context: %s options: %s' % (question, fact, opts) - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask'], - max_length=max_length) - - return tokenizer.decode(output[0]) - -fact = 'The sooner cancer is detected the easier it is to treat.' -question = 'John was a doctor in a cancer ward and knew that early detection was key. The cancer being detected quickly makes the cancer treatment' -opts = 'Easier, Harder' - -get_response(question, fact, opts) - -# output: 'Easier' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-question-generation-ap/README.md b/model_cards/mrm8488/t5-base-finetuned-question-generation-ap/README.md deleted file mode 100644 index 524b1ad04be8..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-question-generation-ap/README.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -language: en -datasets: -- squad ---- - -# T5-base fine-tuned on SQuAD for **Question Generation** - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [SQuAD v1.1](https://rajpurkar.github.io/SQuAD-explorer/) for **Question Generation** by just prepending the *answer* to the *context*. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -Dataset ID: ```squad``` from [Huggingface/NLP](https://github.com/huggingface/nlp) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| squad | train | 87599 | -| squad | valid | 10570 | - -How to load it from [nlp](https://github.com/huggingface/nlp) - -```python -train_dataset = nlp.load_dataset('squad', split=nlp.Split.TRAIN) -valid_dataset = nlp.load_dataset('squad', split=nlp.Split.VALIDATION) -``` -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28) - -He also made a great research on [**Question Generation**](https://github.com/patil-suraj/question_generation) - - -## Model in Action 🚀 - -```python -# Tip: By now, install transformers from source - -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-question-generation-ap") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-question-generation-ap") - -def get_question(answer, context, max_length=64): - input_text = "answer: %s context: %s " % (answer, context) - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask'], - max_length=max_length) - - return tokenizer.decode(output[0]) - -context = "Manuel have created RuPERTa-base with the support of HF-Transformers and Google" -answer = "Manuel" - -get_question(answer, context) - -# output: question: Who created the RuPERTa-base? -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-sarcasm-twitter/README.md b/model_cards/mrm8488/t5-base-finetuned-sarcasm-twitter/README.md deleted file mode 100644 index 2889f479710f..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-sarcasm-twitter/README.md +++ /dev/null @@ -1,112 +0,0 @@ ---- -language: en ---- - -# T5-base fine-tuned for Sarcasm Detection 🙄 -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) base fine-tuned on [ Twitter Sarcasm Dataset](https://github.com/EducationalTestingService/sarcasm) for **Sequence classification (as text generation)** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) -## Details of the downstream task (Sequence Classification as Text generation) - Dataset 📚 - -[ Twitter Sarcasm Dataset](https://github.com/EducationalTestingService/sarcasm) - - -For Twitter training and testing datasets are provided for sarcasm detection tasks in jsonlines format. - -Each line contains a JSON object with the following fields : -- ***label*** : `SARCASM` or `NOT_SARCASM` - - **NOT** in test data -- ***id***: String identifier for sample. This id will be required when making submissions. - - **ONLY** in test data -- ***response*** : the sarcastic response, whether a sarcastic Tweet -- ***context*** : the conversation context of the ***response*** - - Note, the context is an ordered list of dialogue, i.e., if the context contains three elements, `c1`, `c2`, `c3`, in that order, then `c2` is a reply to `c1` and `c3` is a reply to `c2`. Further, if the sarcastic response is `r`, then `r` is a reply to `c3`. - -For instance, for the following training example : - -`"label": "SARCASM", "response": "Did Kelly just call someone else messy? Baaaahaaahahahaha", "context": ["X is looking a First Lady should . #classact, "didn't think it was tailored enough it looked messy"]` - -The response tweet, "Did Kelly..." is a reply to its immediate context "didn't think it was tailored..." which is a reply to "X is looking...". Your goal is to predict the label of the "response" while also using the context (i.e, the immediate or the full context). - -***Dataset size statistics*** : - -| | Train | Val | Test | -|---------|-------|------|------| -| Twitter | 4050 | 450 | 500 | - -The datasets was preprocessed to convert it to a **text-to-text** (classfication as generation task). - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him! - -## Test set metrics 🧾 - -| | precision| recall | f1-score |support| -|----------|----------|---------|----------|-------| -| derison | 0.84 | 0.80 | 0.82 | 246 | -| normal | 0.82 | 0.85 | 0.83 | 254 | -| | -|accuracy| | | 0.83| 500| -|macro avg| 0.83| 0.83| 0.83| 500| -|weighted avg| 0.83| 0.83| 0.83| 500| - - - -## Model in Action 🚀 - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-sarcasm-twitter") - -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-sarcasm-twitter") - -def eval_conversation(text): - - input_ids = tokenizer.encode(text + '', return_tensors='pt') - - output = model.generate(input_ids=input_ids, max_length=3) - - dec = [tokenizer.decode(ids) for ids in output] - - label = dec[0] - - return label - -# For similarity with the training dataset we should replace users mentions in twits for @USER token and urls for URL token. - -twit1 = "Trump just suspended the visa program that allowed me to move to the US to start @USER!" + -" Unfortunately, I won’t be able to vote in a few months but if you can, please vote him out, " + -"he's destroying what made America great in so many different ways!" - -twit2 = "@USER @USER @USER We have far more cases than any other country, " + -"so leaving remote workers in would be disastrous. Makes Trump sense." - -twit3 = "My worry is that i wouldn’t be surprised if half the country actually agrees with this move..." - -me = "Trump doing so??? It must be a mistake... XDDD" - -conversation = twit1 + twit2 - -eval_conversation(conversation) #Output: 'derison' - -conversation = twit1 + twit3 - -eval_conversation(conversation) #Output: 'normal' - -conversation = twit1 + me - -eval_conversation(conversation) #Output: 'derison' - -# We will get 'normal' when sarcasm is not detected and 'derison' when detected -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-span-sentiment-extraction/README.md b/model_cards/mrm8488/t5-base-finetuned-span-sentiment-extraction/README.md deleted file mode 100644 index 740f87dc6d13..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-span-sentiment-extraction/README.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -language: en -thumbnail: ---- - -# T5-base fine-tuned for Sentiment Span Extraction - -All credits to [Lorenzo Ampil](https://twitter.com/AND__SO) - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) base fine-tuned on [Tweet Sentiment Extraction Dataset](https://www.kaggle.com/c/tweet-sentiment-extraction) for **Span Sentiment Extraction** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - - -## Details of the downstream task (Span Sentiment Extraction) - Dataset 📚 - -[Tweet Sentiment Extraction Dataset](https://www.kaggle.com/c/tweet-sentiment-extraction) - -"My ridiculous dog is amazing." [sentiment: positive] - -With all of the tweets circulating every second it is hard to tell whether the sentiment behind a specific tweet will impact a company, or a person's, brand for being viral (positive), or devastate profit because it strikes a negative tone. Capturing sentiment in language is important in these times where decisions and reactions are created and updated in seconds. But, which words actually lead to the sentiment description? In this competition you will need to pick out the part of the tweet (word or phrase) that reflects the sentiment. - -Help build your skills in this important area with this broad dataset of tweets. Work on your technique to grab a top spot in this competition. What words in tweets support a positive, negative, or neutral sentiment? How can you help make that determination using machine learning tools? - -In this competition we've extracted support phrases from Figure Eight's Data for Everyone platform. The dataset is titled Sentiment Analysis: Emotion in Text tweets with existing sentiment labels, used here under creative commons attribution 4.0. international licence. Your objective in this competition is to construct a model that can do the same - look at the labeled sentiment for a given tweet and figure out what word or phrase best supports it. - -Disclaimer: The dataset for this competition contains text that may be considered profane, vulgar, or offensive. - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| TSE | train | 23907 | -| TSE | eval | 3573 | - - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb) created by [Lorenzo Ampil](https://github.com/enzoampil), so all credits to him! - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-span-sentiment-extraction") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-span-sentiment-extraction") - -def get_sentiment_span(text): - input_ids = tokenizer.encode(text, return_tensors="pt", add_special_tokens=True) # Batch size 1 - - generated_ids = model.generate(input_ids=input_ids, num_beams=1, max_length=80).squeeze() - - predicted_span = tokenizer.decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True) - - return predicted_span - -get_sentiment_span("question: negative context: My bike was put on hold...should have known that.... argh total bummer") - -# output: 'argh total bummer' - -get_sentiment_span("question: positive context: On the monday, so i wont be able to be with you! i love you") - -# output: 'i love you' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-squadv2/README.md b/model_cards/mrm8488/t5-base-finetuned-squadv2/README.md deleted file mode 100644 index b20d8fc17d4f..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-squadv2/README.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -language: en -datasets: -- squad_v2 ---- - -# T5-base fine-tuned on SQuAD v2 - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [SQuAD v2](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -Dataset ID: ```squad_v2``` from [Huggingface/NLP](https://github.com/huggingface/nlp) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| squad_v2 | train | 130319 | -| squad_v2 | valid | 11873 | - -How to load it from [nlp](https://github.com/huggingface/nlp) - -```python -train_dataset = nlp.load_dataset('squad_v2', split=nlp.Split.TRAIN) -valid_dataset = nlp.load_dataset('squad_v2', split=nlp.Split.VALIDATION) -``` -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) - -## Results 📝 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **77.64** | -| **F1** | **81.32** | - - - -## Model in Action 🚀 - -```python -from transformers import AutoModelForSeq2SeqLM, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-squadv2") -model = AutoModelForSeq2SeqLM.from_pretrained("mrm8488/t5-base-finetuned-squadv2") - -def get_answer(question, context): - input_text = "question: %s context: %s" % (question, context) - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask']) - - return tokenizer.decode(output[0]) - -context = "Manuel have created RuPERTa-base with the support of HF-Transformers and Google" -question = "Who has supported Manuel?" - -get_answer(question, context) - -# output: 'HF-Transformers and Google' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-summarize-news/README.md b/model_cards/mrm8488/t5-base-finetuned-summarize-news/README.md deleted file mode 100644 index 979826744011..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-summarize-news/README.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -language: en -thumbnail: ---- - -# T5-base fine-tuned fo News Summarization 📖✏️🧾 - -All credits to [Abhishek Kumar Mishra](https://github.com/abhimishra91) - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) base fine-tuned on [News Summary](https://www.kaggle.com/sunnysai12345/news-summary) dataset for **summarization** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - -## Details of the downstream task (Summarization) - Dataset 📚 - -[News Summary](https://www.kaggle.com/sunnysai12345/news-summary) - -The dataset consists of **4515 examples** and contains Author_name, Headlines, Url of Article, Short text, Complete Article. I gathered the summarized news from Inshorts and only scraped the news articles from Hindu, Indian times and Guardian. Time period ranges from febrauary to august 2017. - - - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb) created by [Abhishek Kumar Mishra](https://github.com/abhimishra91), so all credits to him! -I also trained the model for more epochs (6). - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-summarize-news") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-summarize-news") - -def summarize(text, max_length=150): - input_ids = tokenizer.encode(text, return_tensors="pt", add_special_tokens=True) - - generated_ids = model.generate(input_ids=input_ids, num_beams=2, max_length=max_length, repetition_penalty=2.5, length_penalty=1.0, early_stopping=True) - - preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids] - - return preds[0] -``` -Given the following article from **NYT** (2020/06/09) with title *George Floyd’s death energized a movement. He will be buried in Houston today*: - -After the sound and the fury, weeks of demonstrations and anguished calls for racial justice, the man whose death gave rise to an international movement, and whose last words — “I can’t breathe” — have been a rallying cry, will be laid to rest on Tuesday at a private funeral in Houston.George Floyd, who was 46, will then be buried in a grave next to his mother’s.The service, scheduled to begin at 11 a.m. at the Fountain of Praise church, comes after five days of public memorials in Minneapolis, North Carolina and Houston and two weeks after a Minneapolis police officer was caught on video pressing his knee into Mr. Floyd’s neck for nearly nine minutes before Mr. Floyd died. That officer, Derek Chauvin, has been charged with second-degree murder and second-degree manslaughter. His bail was set at $1.25 million in a court appearance on Monday. The outpouring of anger and outrage after Mr. Floyd’s death — and the speed at which protests spread from tense, chaotic demonstrations in the city where he died to an international movement from Rome to Rio de Janeiro — has reflected the depth of frustration borne of years of watching black people die at the hands of the police or vigilantes while calls for change went unmet. -``` -summarize('After the sound and the fury, weeks of demonstrations and anguished calls for racial justice, the man whose death gave rise to an international movement, and whose last words — “I can’t breathe” — have been a rallying cry, will be laid to rest on Tuesday at a private funeral in Houston.George Floyd, who was 46, will then be buried in a grave next to his mother’s.The service, scheduled to begin at 11 a.m. at the Fountain of Praise church, comes after five days of public memorials in Minneapolis, North Carolina and Houston and two weeks after a Minneapolis police officer was caught on video pressing his knee into Mr. Floyd’s neck for nearly nine minutes before Mr. Floyd died. That officer, Derek Chauvin, has been charged with second-degree murder and second-degree manslaughter. His bail was set at $1.25 million in a court appearance on Monday. The outpouring of anger and outrage after Mr. Floyd’s death — and the speed at which protests spread from tense, chaotic demonstrations in the city where he died to an international movement from Rome to Rio de Janeiro — has reflected the depth of frustration borne of years of watching black people die at the hands of the police or vigilantes while calls for change went unmet.', 80) -``` -We would obtain: - -At a private funeral in Houston. Floyd, who was 46 years old when his death occurred, will be buried next to the grave of his mother. A Minnesota police officer was caught on video pressing his knee into Mr's neck for nearly nine minutes before his death. The officer has been charged with second-degree manslaughter and $1.2 million bail is set at - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-wikiSQL-sql-to-en/README.md b/model_cards/mrm8488/t5-base-finetuned-wikiSQL-sql-to-en/README.md deleted file mode 100644 index 3f012c771a74..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-wikiSQL-sql-to-en/README.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -language: en -datasets: -- wikisql ---- - -# T5-base fine-tuned on WikiSQL for SQL to English translation - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [WikiSQL](https://github.com/salesforce/WikiSQL) for **SQL** to **English** **translation** task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the Dataset 📚 - -Dataset ID: ```wikisql``` from [Huggingface/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| wikisql | train | 56355 | -| wikisql | valid | 14436 | - -How to load it from [nlp](https://github.com/huggingface/nlp) - -```python -train_dataset = nlp.load_dataset('wikisql', split=nlp.Split.TRAIN) -valid_dataset = nlp.load_dataset('wikisql', split=nlp.Split.VALIDATION) -``` -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him! - - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-wikiSQL-sql-to-en") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-wikiSQL-sql-to-en") - -def get_explanation(query): - input_text = "translate Sql to English: %s " % query - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask']) - - return tokenizer.decode(output[0]) - -query = "SELECT COUNT Params form model where location=HF-Hub" - -get_explanation(query) - -# output: 'How many parameters form model for HF-hub?' -``` - -Play with it in a Colab: -Open In Colab - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-base-finetuned-wikiSQL/README.md b/model_cards/mrm8488/t5-base-finetuned-wikiSQL/README.md deleted file mode 100644 index 0241381d0eed..000000000000 --- a/model_cards/mrm8488/t5-base-finetuned-wikiSQL/README.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -language: en -datasets: -- wikisql ---- - -# T5-base fine-tuned on WikiSQL - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [WikiSQL](https://github.com/salesforce/WikiSQL) for **English** to **SQL** **translation**. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the Dataset 📚 - -Dataset ID: ```wikisql``` from [Huggingface/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| wikisql | train | 56355 | -| wikisql | valid | 14436 | - -How to load it from [nlp](https://github.com/huggingface/nlp) - -```python -train_dataset = nlp.load_dataset('wikisql', split=nlp.Split.TRAIN) -valid_dataset = nlp.load_dataset('wikisql', split=nlp.Split.VALIDATION) -``` -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him! - - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-base-finetuned-wikiSQL") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-base-finetuned-wikiSQL") - -def get_sql(query): - input_text = "translate English to SQL: %s " % query - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask']) - - return tokenizer.decode(output[0]) - -query = "How many models were finetuned using BERT as base model?" - -get_sql(query) - -# output: 'SELECT COUNT Model fine tuned FROM table WHERE Base model = BERT' -``` - -Other examples from validation dataset: -![validation examples](https://pbs.twimg.com/media/Ec5vaG5XsAINty_?format=png&name=900x900) - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-small-finetuned-emotion/README.md b/model_cards/mrm8488/t5-small-finetuned-emotion/README.md deleted file mode 100644 index ba366b63f20b..000000000000 --- a/model_cards/mrm8488/t5-small-finetuned-emotion/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -language: en -datasets: -- emotion ---- - -# T5-small fine-tuned for Emotion Recognition 😂😢😡😃😯 - - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) [small](https://huggingface.co/t5-small) fine-tuned on [emotion recognition](https://github.com/dair-ai/emotion_dataset) dataset for **Emotion Recognition** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - -## Details of the downstream task (Sentiment Recognition) - Dataset 📚 - -[Elvis Saravia](https://twitter.com/omarsar0) has gathered a great [dataset](https://github.com/dair-ai/emotion_dataset) for emotion recognition. It allows to classifiy the text into one of the following **6** emotions: - - - sadness 😢 - - joy 😃 - - love 🥰 - - anger 😡 - - fear 😱 - - surprise 😯 - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him! - -## Test set metrics 🧾 - -| |precision | recall | f1-score |support| -|----------|----------|---------|----------|-------| -|anger | 0.92| 0.93| 0.92| 275| -|fear | 0.90| 0.90| 0.90| 224| -|joy | 0.97| 0.91| 0.94| 695| -|love | 0.75| 0.89| 0.82| 159| -|sadness | 0.96| 0.97| 0.96| 581| -|surpirse | 0.73| 0.80| 0.76| 66| -| | -|accuracy| | | 0.92| 2000| -|macro avg| 0.87| 0.90| 0.88| 2000| -|weighted avg| 0.93| 0.92| 0.92| 2000| - - -Confusion Matrix - -![CM](https://i.imgur.com/JBtAwPx.png) - - -## Model in Action 🚀 - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-emotion") - -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-emotion") - -def get_emotion(text): - input_ids = tokenizer.encode(text + '', return_tensors='pt') - - output = model.generate(input_ids=input_ids, - max_length=2) - - dec = [tokenizer.decode(ids) for ids in output] - label = dec[0] - return label - - get_emotion("i feel as if i havent blogged in ages are at least truly blogged i am doing an update cute") # Output: 'joy' - - get_emotion("i have a feeling i kinda lost my best friend") # Output: 'sadness' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-small-finetuned-imdb-sentiment/README.md b/model_cards/mrm8488/t5-small-finetuned-imdb-sentiment/README.md deleted file mode 100644 index 534574dfff78..000000000000 --- a/model_cards/mrm8488/t5-small-finetuned-imdb-sentiment/README.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -language: en -datasets: -- imdb ---- - -# T5-small fine-tuned for Sentiment Anlalysis 🎞️👍👎 - - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) [small](https://huggingface.co/t5-small) fine-tuned on [IMDB](https://huggingface.co/datasets/imdb) dataset for **Sentiment Analysis** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67) - -## Details of the downstream task (Sentiment analysis) - Dataset 📚 - -[IMDB](https://huggingface.co/datasets/imdb) - -This is a dataset for binary sentiment classification containing substantially more data than previous benchmark datasets. It provides a set of **25,000** highly polar movie reviews for training, and **25,000** for testing. - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him! - -## Test set metrics 🧾 - -| |precision | recall | f1-score |support| -|----------|----------|---------|----------|-------| -|negative | 0.92 | 0.93| 0.92| 12500| -|positive | 0.93 | 0.92| 0.92| 12500| -|----------|----------|---------|----------|-------| -|accuracy| | | 0.92| 25000| -|macro avg| 0.92| 0.92| 0.92| 25000| -|weighted avg| 0.92| 0.92| 0.92| 25000| - - -## Model in Action 🚀 - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-imdb-sentiment") - -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-imdb-sentiment") - -def get_sentiment(text): - input_ids = tokenizer.encode(text + '', return_tensors='pt') - - output = model.generate(input_ids=input_ids, - max_length=2) - - dec = [tokenizer.decode(ids) for ids in output] - label = dec[0] - return label - -get_sentiment("I dislike a lot that film") - -# Output: 'negative' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-small-finetuned-quora-for-paraphrasing/README.md b/model_cards/mrm8488/t5-small-finetuned-quora-for-paraphrasing/README.md deleted file mode 100644 index e5913a9bc100..000000000000 --- a/model_cards/mrm8488/t5-small-finetuned-quora-for-paraphrasing/README.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -language: en -datasets: -- quora ---- - -# T5-base fine-tuned on Quora question pair dataset for Question Paraphrasing ❓↔️❓ - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) fine-tuned on [Quodra question pair](https://huggingface.co/nlp/viewer/?dataset=quora) dataset for **Question Paraphrasing** task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the downstream task (Question Paraphrasing) - Dataset 📚❓↔️❓ - -Dataset ID: ```quora``` from [Huggingface/NLP](https://github.com/huggingface/nlp) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| quora | train | 404290 | -| quora after filter repeated questions | train | 149263 | - -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) - - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-quora-for-paraphrasing") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-quora-for-paraphrasing") - -def paraphrase(text, max_length=128): - - input_ids = tokenizer.encode(text, return_tensors="pt", add_special_tokens=True) - - generated_ids = model.generate(input_ids=input_ids, num_return_sequences=5, num_beams=5, max_length=max_length, no_repeat_ngram_size=2, repetition_penalty=3.5, length_penalty=1.0, early_stopping=True) - - preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids] - - return preds - -preds = paraphrase("paraphrase: What is the best framework for dealing with a huge text dataset?") - -for pred in preds: - print(pred) - -# Output: -''' -What is the best framework for dealing with a huge text dataset? -What is the best framework for dealing with a large text dataset? -What is the best framework to deal with a huge text dataset? -What are the best frameworks for dealing with a huge text dataset? -What is the best framework for dealing with huge text datasets? -''' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-small-finetuned-squadv1/README.md b/model_cards/mrm8488/t5-small-finetuned-squadv1/README.md deleted file mode 100644 index dc6b9c77ba15..000000000000 --- a/model_cards/mrm8488/t5-small-finetuned-squadv1/README.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -language: en -datasets: -- squad ---- - -# T5-small fine-tuned on SQuAD - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) [(small)](https://huggingface.co/t5-small) fine-tuned on [SQuAD v1.1](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -Dataset ID: ```squad``` from [Huggingface/NLP](https://github.com/huggingface/nlp) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| squad | train | 87599 | -| squad | valid | 10570 | - -How to load it from [nlp](https://github.com/huggingface/nlp) - -```python -train_dataset = nlp.load_dataset('squad, split=nlp.Split.TRAIN) -valid_dataset = nlp.load_dataset('squad', split=nlp.Split.VALIDATION) -``` -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28) - -## Results 📝 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **76.95** | -| **F1** | **85.71** | - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-squadv1") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-squadv1") - -def get_answer(question, context): - input_text = "question: %s context: %s " % (question, context) - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask']) - - return tokenizer.decode(output[0]) - -context = "Manuel have created RuPERTa-base (a Spanish RoBERTa) with the support of HF-Transformers and Google" -question = "Who has supported Manuel?" - -get_answer(question, context) - -# output: 'HF-Transformers and Google' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-small-finetuned-squadv2/README.md b/model_cards/mrm8488/t5-small-finetuned-squadv2/README.md deleted file mode 100644 index 1fb9db76e4f9..000000000000 --- a/model_cards/mrm8488/t5-small-finetuned-squadv2/README.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -language: en -datasets: -- squad_v2 ---- - -# T5-small fine-tuned on SQuAD v2 - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) [(small)](https://huggingface.co/t5-small) fine-tuned on [SQuAD v2](https://rajpurkar.github.io/SQuAD-explorer/) for **Q&A** downstream task. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the downstream task (Q&A) - Dataset 📚 🧐 ❓ - -Dataset ID: ```squad_v2``` from [Huggingface/NLP](https://github.com/huggingface/nlp) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| squad_v2 | train | 130319 | -| squad_v2 | valid | 11873 | - -How to load it from [nlp](https://github.com/huggingface/nlp) - -```python -train_dataset = nlp.load_dataset('squad_v2, split=nlp.Split.TRAIN) -valid_dataset = nlp.load_dataset('squad_v2', split=nlp.Split.VALIDATION) -``` -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this awesome one](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb) by [Suraj Patil](https://twitter.com/psuraj28) - -## Results 📝 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **69.46** | -| **F1** | **73.01** | - - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-squadv2") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-squadv2") - -def get_answer(question, context): - input_text = "question: %s context: %s " % (question, context) - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask']) - - return tokenizer.decode(output[0]) - -context = "Manuel has created RuPERTa-base (a Spanish RoBERTa) with the support of HF-Transformers and Google" -question = "Who has supported Manuel?" - -get_answer(question, context) - -# output: 'HF-Transformers and Google' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/t5-small-finetuned-wikiSQL/README.md b/model_cards/mrm8488/t5-small-finetuned-wikiSQL/README.md deleted file mode 100644 index 40aff2e0aab0..000000000000 --- a/model_cards/mrm8488/t5-small-finetuned-wikiSQL/README.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -language: en -datasets: -- wikisql ---- - -# T5-small fine-tuned on WikiSQL - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) [small](https://huggingface.co/t5-small) fine-tuned on [WikiSQL](https://github.com/salesforce/WikiSQL) for **English** to **SQL** **translation**. - -## Details of T5 - -The **T5** model was presented in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) by *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* in Here the abstract: - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://i.imgur.com/jVFMMWR.png) - - -## Details of the Dataset 📚 - -Dataset ID: ```wikisql``` from [Huggingface/NLP](https://huggingface.co/nlp/viewer/?dataset=wikisql) - -| Dataset | Split | # samples | -| -------- | ----- | --------- | -| wikisql | train | 56355 | -| wikisql | valid | 14436 | - -How to load it from [nlp](https://github.com/huggingface/nlp) - -```python -train_dataset = nlp.load_dataset('wikisql', split=nlp.Split.TRAIN) -valid_dataset = nlp.load_dataset('wikisql', split=nlp.Split.VALIDATION) -``` -Check out more about this dataset and others in [NLP Viewer](https://huggingface.co/nlp/viewer/) - - -## Model fine-tuning 🏋️‍ - -The training script is a slightly modified version of [this Colab Notebook](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) created by [Suraj Patil](https://github.com/patil-suraj), so all credits to him! - - - -## Model in Action 🚀 - -```python -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("mrm8488/t5-small-finetuned-wikiSQL") -model = AutoModelWithLMHead.from_pretrained("mrm8488/t5-small-finetuned-wikiSQL") - -def get_sql(query): - input_text = "translate English to SQL: %s " % query - features = tokenizer([input_text], return_tensors='pt') - - output = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask']) - - return tokenizer.decode(output[0]) - -query = "How many millions of params there are in HF-hub?" - -get_sql(query) - -# output: 'SELECT COUNT Params FROM table WHERE Location = HF-hub' -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) - -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/umberto-wikipedia-uncased-v1-finetuned-squadv1-it/README.md b/model_cards/mrm8488/umberto-wikipedia-uncased-v1-finetuned-squadv1-it/README.md deleted file mode 100644 index 8adb4f59d02a..000000000000 --- a/model_cards/mrm8488/umberto-wikipedia-uncased-v1-finetuned-squadv1-it/README.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -language: it ---- - -# UmBERTo Wikipedia Uncased + italian SQuAD v1 📚 🧐 ❓ - -[UmBERTo-Wikipedia-Uncased](https://huggingface.co/Musixmatch/umberto-wikipedia-uncased-v1) fine-tuned on [Italian SQUAD v1 dataset](https://github.com/crux82/squad-it) for **Q&A** downstream task. - -## Details of the downstream task (Q&A) - Model 🧠 - -[UmBERTo](https://github.com/musixmatchresearch/umberto) is a Roberta-based Language Model trained on large Italian Corpora and uses two innovative approaches: SentencePiece and Whole Word Masking. -UmBERTo-Wikipedia-Uncased Training is trained on a relative small corpus (~7GB) extracted from Wikipedia-ITA. - - -## Details of the downstream task (Q&A) - Dataset 📚 - -[SQuAD](https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/) [Rajpurkar et al. 2016] is a large scale dataset for training of question answering systems on factoid questions. It contains more than 100,000 question-answer pairs about passages from 536 articles chosen from various domains of Wikipedia. - -**SQuAD-it** is derived from the SQuAD dataset and it is obtained through semi-automatic translation of the SQuAD dataset into Italian. It represents a large-scale dataset for open question answering processes on factoid questions in Italian. The dataset contains more than 60,000 question/answer pairs derived from the original English dataset. - -## Model training 🏋️‍ - -The model was trained on a Tesla P100 GPU and 25GB of RAM with the following command: - -```bash -python transformers/examples/question-answering/run_squad.py \ - --model_type bert \ - --model_name_or_path 'Musixmatch/umberto-wikipedia-uncased-v1' \ - --do_eval \ - --do_train \ - --do_lower_case \ - --train_file '/content/dataset/SQuAD_it-train.json' \ - --predict_file '/content/dataset/SQuAD_it-test.json' \ - --per_gpu_train_batch_size 16 \ - --learning_rate 3e-5 \ - --num_train_epochs 10 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir /content/drive/My\ Drive/umberto-uncased-finetuned-squadv1-it \ - --overwrite_output_dir \ - --save_steps 1000 -``` -With 10 epochs the model overfits the train dataset so I evaluated the different checkpoints created during training (every 1000 steps) and chose the best (In this case the one created at 17000 steps). - -## Test set Results 🧾 - -| Metric | # Value | -| ------ | --------- | -| **EM** | **60.50** | -| **F1** | **72.41** | - - - -```json -{ -'exact': 60.50729399395453, -'f1': 72.4141113348361, -'total': 7609, -'HasAns_exact': 60.50729399395453, -'HasAns_f1': 72.4141113348361, -'HasAns_total': 7609, -'best_exact': 60.50729399395453, -'best_exact_thresh': 0.0, -'best_f1': 72.4141113348361, -'best_f1_thresh': 0.0 -} -``` - -## Comparison ⚖️ - -| Model | EM | F1 score | -| -------------------------------------------------------------------------------------------------------------------------------- | --------- | --------- | -| [DrQA-it trained on SQuAD-it ](https://github.com/crux82/squad-it/blob/master/README.md#evaluating-a-neural-model-over-squad-it) | 56.1 | 65.9 | -| This one |60.50 |72.41 | -| [bert-italian-finedtuned-squadv1-it-alfa](https://huggingface.co/mrm8488/bert-italian-finedtuned-squadv1-it-alfa) |**62.51** |**74.16** | | **62.51** | **74.16** | - - -### Model in action 🚀 - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -QnA_pipeline = pipeline('question-answering', model='mrm8488/umberto-wikipedia-uncased-v1-finetuned-squadv1-it') - -QnA_pipeline({ - 'context': 'Marco Aurelio era un imperatore romano che praticava lo stoicismo come filosofia di vita .', - 'question': 'Quale filosofia seguì Marco Aurelio ?' -}) -# Output: -{'answer': 'stoicismo', 'end': 65, 'score': 0.9477770241566028, 'start': 56} -``` - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) | [LinkedIn](https://www.linkedin.com/in/manuel-romero-cs/) -> Made with ♥ in Spain diff --git a/model_cards/mrm8488/xlm-multi-finetuned-xquadv1/README.md b/model_cards/mrm8488/xlm-multi-finetuned-xquadv1/README.md deleted file mode 100644 index 72ba3320b315..000000000000 --- a/model_cards/mrm8488/xlm-multi-finetuned-xquadv1/README.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -language: multilingual -thumbnail: ---- - -# [XLM](https://github.com/facebookresearch/XLM/) (multilingual version) fine-tuned for multilingual Q&A - -Released from `Facebook` together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau and fine-tuned on [XQuAD](https://github.com/deepmind/xquad) for multilingual (`11 different languages`) **Q&A** downstream task. - -## Details of the language model('xlm-mlm-100-1280') - -[Language model](https://github.com/facebookresearch/XLM/#ii-cross-lingual-language-model-pretraining-xlm) - -| Languages -| --------- | -| 100 | - -It includes the following languages: - -
-en-es-fr-de-zh-ru-pt-it-ar-ja-id-tr-nl-pl-simple-fa-vi-sv-ko-he-ro-no-hi-uk-cs-fi-hu-th-da-ca-el-bg-sr-ms-bn-hr-sl-zh_yue-az-sk-eo-ta-sh-lt-et-ml-la-bs-sq-arz-af-ka-mr-eu-tl-ang-gl-nn-ur-kk-be-hy-te-lv-mk-zh_classical-als-is-wuu-my-sco-mn-ceb-ast-cy-kn-br-an-gu-bar-uz-lb-ne-si-war-jv-ga-zh_min_nan-oc-ku-sw-nds-ckb-ia-yi-fy-scn-gan-tt-am -
- -## Details of the downstream task (multilingual Q&A) - Dataset - -Deepmind [XQuAD](https://github.com/deepmind/xquad) - -Languages covered: - -- Arabic: `ar` -- German: `de` -- Greek: `el` -- English: `en` -- Spanish: `es` -- Hindi: `hi` -- Russian: `ru` -- Thai: `th` -- Turkish: `tr` -- Vietnamese: `vi` -- Chinese: `zh` - -As the dataset is based on SQuAD v1.1, there are no unanswerable questions in the data. We chose this -setting so that models can focus on cross-lingual transfer. - -We show the average number of tokens per paragraph, question, and answer for each language in the -table below. The statistics were obtained using [Jieba](https://github.com/fxsjy/jieba) for Chinese -and the [Moses tokenizer](https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl) -for the other languages. - -| | en | es | de | el | ru | tr | ar | vi | th | zh | hi | -| --------- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -| Paragraph | 142.4 | 160.7 | 139.5 | 149.6 | 133.9 | 126.5 | 128.2 | 191.2 | 158.7 | 147.6 | 232.4 | -| Question | 11.5 | 13.4 | 11.0 | 11.7 | 10.0 | 9.8 | 10.7 | 14.8 | 11.5 | 10.5 | 18.7 | -| Answer | 3.1 | 3.6 | 3.0 | 3.3 | 3.1 | 3.1 | 3.1 | 4.5 | 4.1 | 3.5 | 5.6 | - -Citation: - -
- -```bibtex -@article{Artetxe:etal:2019, - author = {Mikel Artetxe and Sebastian Ruder and Dani Yogatama}, - title = {On the cross-lingual transferability of monolingual representations}, - journal = {CoRR}, - volume = {abs/1910.11856}, - year = {2019}, - archivePrefix = {arXiv}, - eprint = {1910.11856} -} -``` - -
- -As XQuAD is just an evaluation dataset, I used Data augmentation techniques (scraping, neural machine translation, etc) to obtain more samples and split the dataset in order to have a train and test set. The test set was created in a way that contains the same number of samples for each language. Finally, I got: - -| Dataset | # samples | -| ----------- | --------- | -| XQUAD train | 50 K | -| XQUAD test | 8 K | - -## Model training - -The model was trained on a Tesla P100 GPU and 25GB of RAM. -The script for fine tuning can be found [here](https://github.com/huggingface/transformers/blob/master/examples/distillation/run_squad_w_distillation.py) - - -## Model in action - -Fast usage with **pipelines**: - -```python -from transformers import pipeline - -qa_pipeline = pipeline( - "question-answering", - model="mrm8488/xlm-multi-finetuned-xquadv1", - tokenizer="mrm8488/xlm-multi-finetuned-xquadv1" -) - -# English -qa_pipeline({ - 'context': "Manuel Romero has been working hardly in the repository hugginface/transformers lately", - 'question': "Who has been working hard for hugginface/transformers lately?" -}) - -#Output: {'answer': 'Manuel', 'end': 6, 'score': 8.531880747878265e-05, 'start': 0} - -# Russian -qa_pipeline({ - 'context': "Мануэль Ромеро в последнее время почти не работал в репозитории hugginface / transformers", - 'question': "Кто в последнее время усердно работал над обнимашками / трансформерами?" - -}) - -#Output: {'answer': 'работал в репозитории hugginface /','end': 76, 'score': 0.00012340750456964894, 'start': 42} -``` -Try it on a Colab (*Do not forget to change the model and tokenizer path in the Colab if necessary*): - -Open In Colab - -> Created by [Manuel Romero/@mrm8488](https://twitter.com/mrm8488) - -> Made with ♥ in Spain diff --git a/model_cards/mymusise/gpt2-medium-chinese/README.md b/model_cards/mymusise/gpt2-medium-chinese/README.md deleted file mode 100644 index 75dfe324e628..000000000000 --- a/model_cards/mymusise/gpt2-medium-chinese/README.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -language: zh ---- - -# gpt2-medium-chinese - - -# Overview - -- **Language model**: GPT2-Medium -- **Model size**: 1.2GiB -- **Language**: Chinese -- **Training data**: [wiki2019zh_corpus](https://github.com/brightmart/nlp_chinese_corpus) -- **Source code**: [gpt2-quickly](https://github.com/mymusise/gpt2-quickly) - -# Example - -```python -from transformers import BertTokenizer, TFGPT2LMHeadModel -from transformers import TextGenerationPipeline - -tokenizer = BertTokenizer.from_pretrained("mymusise/EasternFantasyNoval") -model = TFGPT2LMHeadModel.from_pretrained("mymusise/EasternFantasyNoval") - -text_generator = TextGenerationPipeline(model, tokenizer) -print(text_generator("今日", max_length=64, do_sample=True, top_k=10)) -print(text_generator("跨越山丘", max_length=64, do_sample=True, top_k=10)) -``` -输出 -```text -[{'generated_text': '今日 ， 他 的 作 品 也 在 各 种 报 刊 发 表 。 201 1 年 ， 他 开 设 了 他 的 网 页 版 《 the dear 》 。 此 外 ， 他 还 在 各 种 电 视 节 目 中 出 现 过 。 2017 年 1 月 ， 他 被 任'}] -[{'generated_text': '跨越山丘 ， 其 中 有 三 分 之 二 的 地 区 被 划 入 山 区 。 最 高 峰 是 位 于 山 脚 上 的 大 岩 （ ） 。 其 中 的 山 脚 下 有 一 处 有 名 为 的 河 谷 ， 因 其 高 度 在 其 中 ， 而 得 名 。'}] -``` - -[Try it on colab](https://colab.research.google.com/github/mymusise/gpt2-quickly/blob/main/examples/gpt2_medium_chinese.ipynb) diff --git a/model_cards/mys/electra-base-turkish-cased-ner/README.md b/model_cards/mys/electra-base-turkish-cased-ner/README.md deleted file mode 100644 index addd29099b01..000000000000 --- a/model_cards/mys/electra-base-turkish-cased-ner/README.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -language: tr ---- - -## What is this - -A NER model for Turkish with 48 categories trained on the dataset [Shrinked TWNERTC Turkish NER Data](https://www.kaggle.com/behcetsenturk/shrinked-twnertc-turkish-ner-data-by-kuzgunlar) by Behçet Şentürk, which is itself a filtered and cleaned version of the following automatically labeled dataset: - -> Sahin, H. Bahadir; Eren, Mustafa Tolga; Tirkaz, Caglar; Sonmez, Ozan; Yildiz, Eray (2017), “English/Turkish Wikipedia Named-Entity Recognition and Text Categorization Dataset”, Mendeley Data, v1 http://dx.doi.org/10.17632/cdcztymf4k.1 - -## Backbone model - -The backbone model is [electra-base-turkish-cased-discriminator](https://huggingface.co/dbmdz/electra-base-turkish-cased-discriminator), and I finetuned it for token classification. - -I'm continuing to figure out if it is possible to improve accuracy with this dataset, but it is already usable for non-critic applications. You can reach out to me on [Twitter](https://twitter.com/myusufsarigoz) for discussions and issues. -I will also release a notebook to finetune NER models with Shrinked TWNERTC as well as sample inference code to demonstrate what's possible with this model. diff --git a/model_cards/ncoop57/bart-base-code-summarizer-java-v0/README.md b/model_cards/ncoop57/bart-base-code-summarizer-java-v0/README.md deleted file mode 100644 index c8ef988fefbf..000000000000 --- a/model_cards/ncoop57/bart-base-code-summarizer-java-v0/README.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -tags: -- summarization - -license: mit ---- - -## ncoop57/bart-base-code-summarizer-java-v0 diff --git a/model_cards/neuralmind/bert-base-portuguese-cased/README.md b/model_cards/neuralmind/bert-base-portuguese-cased/README.md deleted file mode 100644 index 85deb52e3618..000000000000 --- a/model_cards/neuralmind/bert-base-portuguese-cased/README.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -language: pt -license: mit -tags: - - bert - - pytorch -datasets: - - brWaC ---- - -# BERTimbau Base (aka "bert-base-portuguese-cased") - -![Bert holding a berimbau](https://imgur.com/JZ7Hynh.jpg) - -## Introduction - -BERTimbau Base is a pretrained BERT model for Brazilian Portuguese that achieves state-of-the-art performances on three downstream NLP tasks: Named Entity Recognition, Sentence Textual Similarity and Recognizing Textual Entailment. It is available in two sizes: Base and Large. - -For further information or requests, please go to [BERTimbau repository](https://github.com/neuralmind-ai/portuguese-bert/). - -## Available models - -| Model | Arch. | #Layers | #Params | -| ---------------------------------------- | ---------- | ------- | ------- | -| `neuralmind/bert-base-portuguese-cased` | BERT-Base | 12 | 110M | -| `neuralmind/bert-large-portuguese-cased` | BERT-Large | 24 | 335M | - -## Usage - -```python -from transformers import AutoTokenizer # Or BertTokenizer -from transformers import AutoModelForPreTraining # Or BertForPreTraining for loading pretraining heads -from transformers import AutoModel # or BertModel, for BERT without pretraining heads - -model = AutoModelForPreTraining.from_pretrained('neuralmind/bert-base-portuguese-cased') -tokenizer = AutoTokenizer.from_pretrained('neuralmind/bert-base-portuguese-cased', do_lower_case=False) -``` - -### Masked language modeling prediction example - -```python -from transformers import pipeline - -pipe = pipeline('fill-mask', model=model, tokenizer=tokenizer) - -pipe('Tinha uma [MASK] no meio do caminho.') -# [{'score': 0.14287759363651276, -# 'sequence': '[CLS] Tinha uma pedra no meio do caminho. [SEP]', -# 'token': 5028, -# 'token_str': 'pedra'}, -# {'score': 0.06213393807411194, -# 'sequence': '[CLS] Tinha uma árvore no meio do caminho. [SEP]', -# 'token': 7411, -# 'token_str': 'árvore'}, -# {'score': 0.05515013635158539, -# 'sequence': '[CLS] Tinha uma estrada no meio do caminho. [SEP]', -# 'token': 5675, -# 'token_str': 'estrada'}, -# {'score': 0.0299188531935215, -# 'sequence': '[CLS] Tinha uma casa no meio do caminho. [SEP]', -# 'token': 1105, -# 'token_str': 'casa'}, -# {'score': 0.025660505518317223, -# 'sequence': '[CLS] Tinha uma cruz no meio do caminho. [SEP]', -# 'token': 3466, -# 'token_str': 'cruz'}] - -``` - -### For BERT embeddings - -```python -import torch - -model = AutoModel.from_pretrained('neuralmind/bert-base-portuguese-cased') -input_ids = tokenizer.encode('Tinha uma pedra no meio do caminho.', return_tensors='pt') - -with torch.no_grad(): - outs = model(input_ids) - encoded = outs[0][0, 1:-1] # Ignore [CLS] and [SEP] special tokens - -# encoded.shape: (8, 768) -# tensor([[-0.0398, -0.3057, 0.2431, ..., -0.5420, 0.1857, -0.5775], -# [-0.2926, -0.1957, 0.7020, ..., -0.2843, 0.0530, -0.4304], -# [ 0.2463, -0.1467, 0.5496, ..., 0.3781, -0.2325, -0.5469], -# ..., -# [ 0.0662, 0.7817, 0.3486, ..., -0.4131, -0.2852, -0.2819], -# [ 0.0662, 0.2845, 0.1871, ..., -0.2542, -0.2933, -0.0661], -# [ 0.2761, -0.1657, 0.3288, ..., -0.2102, 0.0029, -0.2009]]) -``` - -## Citation - -If you use our work, please cite: - -```bibtex -@inproceedings{souza2020bertimbau, - author = {F{\'a}bio Souza and - Rodrigo Nogueira and - Roberto Lotufo}, - title = {{BERT}imbau: pretrained {BERT} models for {B}razilian {P}ortuguese}, - booktitle = {9th Brazilian Conference on Intelligent Systems, {BRACIS}, Rio Grande do Sul, Brazil, October 20-23 (to appear)}, - year = {2020} -} -``` diff --git a/model_cards/neuralmind/bert-large-portuguese-cased/README.md b/model_cards/neuralmind/bert-large-portuguese-cased/README.md deleted file mode 100644 index 3c9a7fa5397f..000000000000 --- a/model_cards/neuralmind/bert-large-portuguese-cased/README.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -language: pt -license: mit -tags: - - bert - - pytorch -datasets: - - brWaC ---- - -# BERTimbau Large (aka "bert-large-portuguese-cased") - -![Bert holding a berimbau](https://imgur.com/JZ7Hynh.jpg) - -## Introduction - -BERTimbau Large is a pretrained BERT model for Brazilian Portuguese that achieves state-of-the-art performances on three downstream NLP tasks: Named Entity Recognition, Sentence Textual Similarity and Recognizing Textual Entailment. It is available in two sizes: Base and Large. - -For further information or requests, please go to [BERTimbau repository](https://github.com/neuralmind-ai/portuguese-bert/). - -## Available models - -| Model | Arch. | #Layers | #Params | -| ---------------------------------------- | ---------- | ------- | ------- | -| `neuralmind/bert-base-portuguese-cased` | BERT-Base | 12 | 110M | -| `neuralmind/bert-large-portuguese-cased` | BERT-Large | 24 | 335M | - -## Usage - -```python -from transformers import AutoTokenizer # Or BertTokenizer -from transformers import AutoModelForPreTraining # Or BertForPreTraining for loading pretraining heads -from transformers import AutoModel # or BertModel, for BERT without pretraining heads - -model = AutoModelForPreTraining.from_pretrained('neuralmind/bert-large-portuguese-cased') -tokenizer = AutoTokenizer.from_pretrained('neuralmind/bert-large-portuguese-cased', do_lower_case=False) -``` - -### Masked language modeling prediction example - -```python -from transformers import pipeline - -pipe = pipeline('fill-mask', model=model, tokenizer=tokenizer) - -pipe('Tinha uma [MASK] no meio do caminho.') -# [{'score': 0.5054386258125305, -# 'sequence': '[CLS] Tinha uma pedra no meio do caminho. [SEP]', -# 'token': 5028, -# 'token_str': 'pedra'}, -# {'score': 0.05616172030568123, -# 'sequence': '[CLS] Tinha uma curva no meio do caminho. [SEP]', -# 'token': 9562, -# 'token_str': 'curva'}, -# {'score': 0.02348282001912594, -# 'sequence': '[CLS] Tinha uma parada no meio do caminho. [SEP]', -# 'token': 6655, -# 'token_str': 'parada'}, -# {'score': 0.01795753836631775, -# 'sequence': '[CLS] Tinha uma mulher no meio do caminho. [SEP]', -# 'token': 2606, -# 'token_str': 'mulher'}, -# {'score': 0.015246033668518066, -# 'sequence': '[CLS] Tinha uma luz no meio do caminho. [SEP]', -# 'token': 3377, -# 'token_str': 'luz'}] - -``` - -### For BERT embeddings - -```python - -import torch - -model = AutoModel.from_pretrained('neuralmind/bert-large-portuguese-cased') -input_ids = tokenizer.encode('Tinha uma pedra no meio do caminho.', return_tensors='pt') - -with torch.no_grad(): - outs = model(input_ids) - encoded = outs[0][0, 1:-1] # Ignore [CLS] and [SEP] special tokens - -# encoded.shape: (8, 1024) -# tensor([[ 1.1872, 0.5606, -0.2264, ..., 0.0117, -0.1618, -0.2286], -# [ 1.3562, 0.1026, 0.1732, ..., -0.3855, -0.0832, -0.1052], -# [ 0.2988, 0.2528, 0.4431, ..., 0.2684, -0.5584, 0.6524], -# ..., -# [ 0.3405, -0.0140, -0.0748, ..., 0.6649, -0.8983, 0.5802], -# [ 0.1011, 0.8782, 0.1545, ..., -0.1768, -0.8880, -0.1095], -# [ 0.7912, 0.9637, -0.3859, ..., 0.2050, -0.1350, 0.0432]]) -``` - -## Citation - -If you use our work, please cite: - -```bibtex -@inproceedings{souza2020bertimbau, - author = {F{\'a}bio Souza and - Rodrigo Nogueira and - Roberto Lotufo}, - title = {{BERT}imbau: pretrained {BERT} models for {B}razilian {P}ortuguese}, - booktitle = {9th Brazilian Conference on Intelligent Systems, {BRACIS}, Rio Grande do Sul, Brazil, October 20-23 (to appear)}, - year = {2020} -} -``` diff --git a/model_cards/neuralspace-reverie/indic-transformers-bn-bert/README.md b/model_cards/neuralspace-reverie/indic-transformers-bn-bert/README.md deleted file mode 100644 index a42e1596acf9..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-bn-bert/README.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -language: -- bn -tags: -- MaskedLM -- Bengali ---- -# Indic-Transformers Bengali BERT -## Model description -This is a BERT language model pre-trained on ~3 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-bn-bert') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-bn-bert') -text = "আপনি কেমন আছেন?" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 6, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-bn-distilbert/README.md b/model_cards/neuralspace-reverie/indic-transformers-bn-distilbert/README.md deleted file mode 100644 index f06c6c9c8311..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-bn-distilbert/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- bn -tags: -- MaskedLM -- Bengali -- DistilBERT -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Bengali DistilBERT -## Model description -This is a DistilBERT language model pre-trained on ~6 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-bn-distilbert') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-bn-distilbert') -text = "আপনি কেমন আছেন?" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 5, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-bn-roberta/README.md b/model_cards/neuralspace-reverie/indic-transformers-bn-roberta/README.md deleted file mode 100644 index b2a47660fdd6..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-bn-roberta/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- bn -tags: -- MaskedLM -- Bengali -- RoBERTa -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Bengali RoBERTa -## Model description -This is a RoBERTa language model pre-trained on ~6 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-bn-roberta') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-bn-roberta') -text = "আপনি কেমন আছেন?" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 10, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-bn-xlmroberta/README.md b/model_cards/neuralspace-reverie/indic-transformers-bn-xlmroberta/README.md deleted file mode 100644 index ff781b46d686..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-bn-xlmroberta/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- bn -tags: -- MaskedLM -- Bengali -- XLMRoBERTa -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Bengali XLMRoBERTa -## Model description -This is a XLMRoBERTa language model pre-trained on ~3 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-bn-xlmroberta') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-bn-xlmroberta') -text = "আপনি কেমন আছেন?" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 5, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-hi-bert/README.md b/model_cards/neuralspace-reverie/indic-transformers-hi-bert/README.md deleted file mode 100644 index 45f5389ff043..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-hi-bert/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- hi -tags: -- MaskedLM -- Hindi -- BERT -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Hindi BERT -## Model description -This is a BERT language model pre-trained on ~3 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-hi-bert') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-hi-bert') -text = "आपका स्वागत हैं" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 5, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-hi-distilbert/README.md b/model_cards/neuralspace-reverie/indic-transformers-hi-distilbert/README.md deleted file mode 100644 index 2b202434454b..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-hi-distilbert/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- hi -tags: -- MaskedLM -- Hindi -- DistilBERT -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Hindi DistilBERT -## Model description -This is a DistilBERT language model pre-trained on ~10 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-hi-distilbert') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-hi-distilbert') -text = "आपका स्वागत हैं" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 5, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-hi-roberta/README.md b/model_cards/neuralspace-reverie/indic-transformers-hi-roberta/README.md deleted file mode 100644 index 3852b1d7fe8a..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-hi-roberta/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- hi -tags: -- MaskedLM -- Hindi -- RoBERTa -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Hindi RoBERTa -## Model description -This is a RoBERTa language model pre-trained on ~10 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-hi-roberta') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-hi-roberta') -text = "आपका स्वागत हैं" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 11, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-hi-xlmroberta/README.md b/model_cards/neuralspace-reverie/indic-transformers-hi-xlmroberta/README.md deleted file mode 100644 index f7baf90326cd..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-hi-xlmroberta/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- hi -tags: -- MaskedLM -- Hindi -- XLMRoBERTa -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Hindi XLMRoBERTa -## Model description -This is a XLMRoBERTa language model pre-trained on ~3 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-hi-xlmroberta') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-hi-xlmroberta') -text = "आपका स्वागत हैं" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 5, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-te-bert/README.md b/model_cards/neuralspace-reverie/indic-transformers-te-bert/README.md deleted file mode 100644 index 46b7dc3b316c..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-te-bert/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- te -tags: -- MaskedLM -- Telugu -- BERT -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Telugu BERT -## Model description -This is a BERT language model pre-trained on ~1.6 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-te-bert') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-te-bert') -text = "మీరు ఎలా ఉన్నారు" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 5, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-te-distilbert/README.md b/model_cards/neuralspace-reverie/indic-transformers-te-distilbert/README.md deleted file mode 100644 index 1ce7a8605e3d..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-te-distilbert/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- te -tags: -- MaskedLM -- Telugu -- DistilBERT -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Telugu DistilBERT -## Model description -This is a DistilBERT language model pre-trained on ~2 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-te-distilbert') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-te-distilbert') -text = "మీరు ఎలా ఉన్నారు" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 5, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-te-roberta/README.md b/model_cards/neuralspace-reverie/indic-transformers-te-roberta/README.md deleted file mode 100644 index f9c76cd68d16..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-te-roberta/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- te -tags: -- MaskedLM -- Telugu -- RoBERTa -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Telugu RoBERTa -## Model description -This is a RoBERTa language model pre-trained on ~2 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-te-roberta') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-te-roberta') -text = "మీరు ఎలా ఉన్నారు" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 14, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuralspace-reverie/indic-transformers-te-xlmroberta/README.md b/model_cards/neuralspace-reverie/indic-transformers-te-xlmroberta/README.md deleted file mode 100644 index 78b1e78348ad..000000000000 --- a/model_cards/neuralspace-reverie/indic-transformers-te-xlmroberta/README.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -language: -- te -tags: -- MaskedLM -- Telugu -- XLMRoBERTa -- Question-Answering -- Token Classification -- Text Classification ---- -# Indic-Transformers Telugu XLMRoBERTa -## Model description -This is a XLMRoBERTa language model pre-trained on ~1.6 GB of monolingual training corpus. The pre-training data was majorly taken from [OSCAR](https://oscar-corpus.com/). -This model can be fine-tuned on various downstream tasks like text-classification, POS-tagging, question-answering, etc. Embeddings from this model can also be used for feature-based training. -## Intended uses & limitations -#### How to use -``` -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained('neuralspace-reverie/indic-transformers-te-xlmroberta') -model = AutoModel.from_pretrained('neuralspace-reverie/indic-transformers-te-xlmroberta') -text = "మీరు ఎలా ఉన్నారు" -input_ids = tokenizer(text, return_tensors='pt')['input_ids'] -out = model(input_ids)[0] -print(out.shape) -# out = [1, 5, 768] -``` -#### Limitations and bias -The original language model has been trained using `PyTorch` and hence the use of `pytorch_model.bin` weights file is recommended. The h5 file for `Tensorflow` has been generated manually by commands suggested [here](https://huggingface.co/transformers/model_sharing.html). diff --git a/model_cards/neuraly/bert-base-italian-cased-sentiment/README.md b/model_cards/neuraly/bert-base-italian-cased-sentiment/README.md deleted file mode 100644 index e99b97ad9e55..000000000000 --- a/model_cards/neuraly/bert-base-italian-cased-sentiment/README.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -language: it -thumbnail: https://neuraly.ai/static/assets/images/huggingface/thumbnail.png -tags: -- sentiment -- Italian -license: MIT -widget: -- text: "Huggingface è un team fantastico!" ---- - -# 🤗 + neuraly - Italian BERT Sentiment model - -## Model description - -This model performs sentiment analysis on Italian sentences. It was trained starting from an instance of [bert-base-italian-cased](https://huggingface.co/dbmdz/bert-base-italian-cased), and fine-tuned on an Italian dataset of tweets, reaching 82% of accuracy on the latter one. - -## Intended uses & limitations - -#### How to use - -```python -import torch -from torch import nn -from transformers import AutoTokenizer, AutoModelForSequenceClassification - -# Load the tokenizer -tokenizer = AutoTokenizer.from_pretrained("neuraly/bert-base-italian-cased-sentiment") -# Load the model, use .cuda() to load it on the GPU -model = AutoModelForSequenceClassification.from_pretrained("neuraly/bert-base-italian-cased-sentiment") - -sentence = 'Huggingface è un team fantastico!' -input_ids = tokenizer.encode(sentence, add_special_tokens=True) - -# Create tensor, use .cuda() to transfer the tensor to GPU -tensor = torch.tensor(input_ids).long() -# Fake batch dimension -tensor = tensor.unsqueeze(0) - -# Call the model and get the logits -logits, = model(tensor) - -# Remove the fake batch dimension -logits = logits.squeeze(0) - -# The model was trained with a Log Likelyhood + Softmax combined loss, hence to extract probabilities we need a softmax on top of the logits tensor -proba = nn.functional.softmax(logits, dim=0) - -# Unpack the tensor to obtain negative, neutral and positive probabilities -negative, neutral, positive = proba -``` - -#### Limitations and bias - -A possible drawback (or bias) of this model is related to the fact that it was trained on a tweet dataset, with all the limitations that come with it. The domain is strongly related to football players and teams, but it works surprisingly well even on other topics. - -## Training data - -We trained the model by combining the two tweet datasets taken from [Sentipolc EVALITA 2016](http://www.di.unito.it/~tutreeb/sentipolc-evalita16/data.html). Overall the dataset consists of 45K pre-processed tweets. - -The model weights come from a pre-trained instance of [bert-base-italian-cased](https://huggingface.co/dbmdz/bert-base-italian-cased). A huge "thank you" goes to that team, brilliant work! - -## Training procedure - -#### Preprocessing - -We tried to save as much information as possible, since BERT captures extremely well the semantic of complex text sequences. Overall we removed only **@mentions**, **urls** and **emails** from every tweet and kept pretty much everything else. - -#### Hardware - -- **GPU**: Nvidia GTX1080ti -- **CPU**: AMD Ryzen7 3700x 8c/16t -- **RAM**: 64GB DDR4 - -#### Hyperparameters - -- Optimizer: **AdamW** with learning rate of **2e-5**, epsilon of **1e-8** -- Max epochs: **5** -- Batch size: **32** -- Early Stopping: **enabled** with patience = 1 - -Early stopping was triggered after 3 epochs. - -## Eval results - -The model achieves an overall accuracy on the test set equal to 82% -The test set is a 20% split of the whole dataset. - -## About us -[Neuraly](https://neuraly.ai) is a young and dynamic startup committed to designing AI-driven solutions and services through the most advanced Machine Learning and Data Science technologies. You can find out more about who we are and what we do on our [website](https://neuraly.ai). - -## Acknowledgments - -Thanks to the generous support from the [Hugging Face](https://huggingface.co/) team, -it is possible to download the model from their S3 storage and live test it from their inference API 🤗. diff --git a/model_cards/neurocode/IsRoBERTa/README.md b/model_cards/neurocode/IsRoBERTa/README.md deleted file mode 100644 index b56c9296f633..000000000000 --- a/model_cards/neurocode/IsRoBERTa/README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -language: is -datasets: -- Icelandic portion of the OSCAR corpus from INRIA -- oscar ---- - -# IsRoBERTa a RoBERTa-like masked language model - -Probably the first icelandic transformer language model! - -## Overview -**Language:** Icelandic -**Downstream-task:** masked-lm -**Training data:** OSCAR corpus -**Code:** See [here](https://github.com/neurocode-io/icelandic-language-model) -**Infrastructure**: 1x Nvidia K80 - -## Hyperparameters - -``` -per_device_train_batch_size = 48 -n_epochs = 1 -vocab_size = 52.000 -max_position_embeddings = 514 -num_attention_heads = 12 -num_hidden_layers = 6 -type_vocab_size = 1 -learning_rate=0.00005 -``` - - -## Usage - -### In Transformers -```python -from transformers import ( - pipeline, - AutoTokenizer, - AutoModelWithLMHead -) - -model_name = "neurocode/IsRoBERTa" - -tokenizer = AutoTokenizer.from_pretrained(model_name) -model = AutoModelWithLMHead.from_pretrained(model_name) ->>> fill_mask = pipeline( -... "fill-mask", -... model=model, -... tokenizer=tokenizer -... ) ->>> result = fill_mask("Hann fór út að .") ->>> result -[ - {'sequence': 'Hann fór út að nýju.', 'score': 0.03395755589008331, 'token': 2219, 'token_str': 'Ġnýju'}, - {'sequence': 'Hann fór út að undanförnu.', 'score': 0.029087543487548828, 'token': 7590, 'token_str': 'Ġundanförnu'}, - {'sequence': 'Hann fór út að lokum.', 'score': 0.024420788511633873, 'token': 4384, 'token_str': 'Ġlokum'}, - {'sequence': 'Hann fór út að þessu.', 'score': 0.021231256425380707, 'token': 921, 'token_str': 'Ġþessu'}, - {'sequence': 'Hann fór út að honum.', 'score': 0.0205782949924469, 'token': 1136, 'token_str': 'Ġhonum'} -] -``` - - -## Authors -Bobby Donchev: `contact [at] donchev.is` -Elena Cramer: `elena.cramer [at] neurocode.io` - -## About us - -We bring AI software for our customers live -Our focus: AI software development - -Get in touch: -[LinkedIn](https://de.linkedin.com/company/neurocodeio) | [Website](https://neurocode.io) diff --git a/model_cards/nghuyong/ernie-1.0/README.md b/model_cards/nghuyong/ernie-1.0/README.md deleted file mode 100644 index fc6b601ad55f..000000000000 --- a/model_cards/nghuyong/ernie-1.0/README.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -language: zh ---- - -# ERNIE-1.0 - -## Introduction - -ERNIE (Enhanced Representation through kNowledge IntEgration) is proposed by Baidu in 2019, -which is designed to learn language representation enhanced by knowledge masking strategies i.e. entity-level masking and phrase-level masking. -Experimental results show that ERNIE achieve state-of-the-art results on five Chinese natural language processing tasks including natural language inference, -semantic similarity, named entity recognition, sentiment analysis and question answering. - -More detail: https://arxiv.org/abs/1904.09223 - -## Released Model Info - -|Model Name|Language|Model Structure| -|:---:|:---:|:---:| -|ernie-1.0| Chinese |Layer:12, Hidden:768, Heads:12| - -This released pytorch model is converted from the officially released PaddlePaddle ERNIE model and -a series of experiments have been conducted to check the accuracy of the conversion. - -- Official PaddlePaddle ERNIE repo: https://github.com/PaddlePaddle/ERNIE -- Pytorch Conversion repo: https://github.com/nghuyong/ERNIE-Pytorch - -## How to use -```Python -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0") -model = AutoModel.from_pretrained("nghuyong/ernie-1.0") -``` - -## Citation - -```bibtex -@article{sun2019ernie, - title={Ernie: Enhanced representation through knowledge integration}, - author={Sun, Yu and Wang, Shuohuan and Li, Yukun and Feng, Shikun and Chen, Xuyi and Zhang, Han and Tian, Xin and Zhu, Danxiang and Tian, Hao and Wu, Hua}, - journal={arXiv preprint arXiv:1904.09223}, - year={2019} -} -``` diff --git a/model_cards/nghuyong/ernie-2.0-en/README.md b/model_cards/nghuyong/ernie-2.0-en/README.md deleted file mode 100644 index 1b0c7dcf38d3..000000000000 --- a/model_cards/nghuyong/ernie-2.0-en/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: en ---- - -# ERNIE-2.0 - -## Introduction - -ERNIE 2.0 is a continual pre-training framework proposed by Baidu in 2019, -which builds and learns incrementally pre-training tasks through constant multi-task learning. -Experimental results demonstrate that ERNIE 2.0 outperforms BERT and XLNet on 16 tasks including English tasks on GLUE benchmarks and several common tasks in Chinese. - -More detail: https://arxiv.org/abs/1907.12412 - -## Released Model Info - -|Model Name|Language|Model Structure| -|:---:|:---:|:---:| -|ernie-2.0-en| English |Layer:12, Hidden:768, Heads:12| - -This released pytorch model is converted from the officially released PaddlePaddle ERNIE model and -a series of experiments have been conducted to check the accuracy of the conversion. - -- Official PaddlePaddle ERNIE repo: https://github.com/PaddlePaddle/ERNIE -- Pytorch Conversion repo: https://github.com/nghuyong/ERNIE-Pytorch - -## How to use -```Python -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-2.0-en") -model = AutoModel.from_pretrained("nghuyong/ernie-2.0-en") -``` - -## Citation - -```bibtex -@article{sun2019ernie20, - title={ERNIE 2.0: A Continual Pre-training Framework for Language Understanding}, - author={Sun, Yu and Wang, Shuohuan and Li, Yukun and Feng, Shikun and Tian, Hao and Wu, Hua and Wang, Haifeng}, - journal={arXiv preprint arXiv:1907.12412}, - year={2019} -} -``` diff --git a/model_cards/nghuyong/ernie-2.0-large-en/README.md b/model_cards/nghuyong/ernie-2.0-large-en/README.md deleted file mode 100644 index 515852405044..000000000000 --- a/model_cards/nghuyong/ernie-2.0-large-en/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# ERNIE-2.0-large - -## Introduction - -ERNIE 2.0 is a continual pre-training framework proposed by Baidu in 2019, -which builds and learns incrementally pre-training tasks through constant multi-task learning. -Experimental results demonstrate that ERNIE 2.0 outperforms BERT and XLNet on 16 tasks including English tasks on GLUE benchmarks and several common tasks in Chinese. - -More detail: https://arxiv.org/abs/1907.12412 - -## Released Model Info - -|Model Name|Language|Model Structure| -|:---:|:---:|:---:| -|ernie-2.0-large-en| English |Layer:24, Hidden:1024, Heads:16| - -This released pytorch model is converted from the officially released PaddlePaddle ERNIE model and -a series of experiments have been conducted to check the accuracy of the conversion. - -- Official PaddlePaddle ERNIE repo: https://github.com/PaddlePaddle/ERNIE -- Pytorch Conversion repo: https://github.com/nghuyong/ERNIE-Pytorch - -## How to use -```Python -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-2.0-large-en") -model = AutoModel.from_pretrained("nghuyong/ernie-2.0-large-en") -``` - -## Citation - -```bibtex -@article{sun2019ernie20, - title={ERNIE 2.0: A Continual Pre-training Framework for Language Understanding}, - author={Sun, Yu and Wang, Shuohuan and Li, Yukun and Feng, Shikun and Tian, Hao and Wu, Hua and Wang, Haifeng}, - journal={arXiv preprint arXiv:1907.12412}, - year={2019} -} -``` diff --git a/model_cards/nghuyong/ernie-tiny/README.md b/model_cards/nghuyong/ernie-tiny/README.md deleted file mode 100644 index 56992328a101..000000000000 --- a/model_cards/nghuyong/ernie-tiny/README.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -language: en ---- - -# ERNIE-tiny - -## Introduction -ERNIE-tiny is a compressed model from [ERNIE 2.0](../ernie-2.0-en) base model through model structure compression and model distillation. -Through compression, the performance of the ERNIE-tiny only decreases by an average of 2.37% compared to ERNIE 2.0 base, -but it outperforms Google BERT by 8.35%, and the speed increases by 4.3 times. - -More details: https://github.com/PaddlePaddle/ERNIE/blob/develop/distill/README.md - -## Released Model Info - -|Model Name|Language|Model Structure| -|:---:|:---:|:---:| -|ernie-tiny| English |Layer:3, Hidden:1024, Heads:16| - -This released pytorch model is converted from the officially released PaddlePaddle ERNIE model and -a series of experiments have been conducted to check the accuracy of the conversion. - -- Official PaddlePaddle ERNIE repo: https://github.com/PaddlePaddle/ERNIE -- Pytorch Conversion repo: https://github.com/nghuyong/ERNIE-Pytorch - -## How to use -```Python -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-tiny") -model = AutoModel.from_pretrained("nghuyong/ernie-tiny") -``` - -## Citation - -```bibtex -@article{sun2019ernie20, - title={ERNIE 2.0: A Continual Pre-training Framework for Language Understanding}, - author={Sun, Yu and Wang, Shuohuan and Li, Yukun and Feng, Shikun and Tian, Hao and Wu, Hua and Wang, Haifeng}, - journal={arXiv preprint arXiv:1907.12412}, - year={2019} -} -``` diff --git a/model_cards/nikokons/gpt2-greek/README.md b/model_cards/nikokons/gpt2-greek/README.md deleted file mode 100644 index cc7d1f9c27d0..000000000000 --- a/model_cards/nikokons/gpt2-greek/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -language: el ---- - -## gpt2-greek diff --git a/model_cards/nlpaueb/bert-base-greek-uncased-v1/README.md b/model_cards/nlpaueb/bert-base-greek-uncased-v1/README.md deleted file mode 100644 index 90e10fcb6234..000000000000 --- a/model_cards/nlpaueb/bert-base-greek-uncased-v1/README.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -language: el -thumbnail: https://github.com/nlpaueb/GreekBERT/raw/master/greek-bert-logo.png ---- - -# GreekBERT - -A Greek version of BERT pre-trained language model. - - - - -## Pre-training corpora - -The pre-training corpora of `bert-base-greek-uncased-v1` include: - -* The Greek part of [Wikipedia](https://el.wikipedia.org/wiki/Βικιπαίδεια:Αντίγραφα_της_βάσης_δεδομένων), -* The Greek part of [European Parliament Proceedings Parallel Corpus](https://www.statmt.org/europarl/), and -* The Greek part of [OSCAR](https://traces1.inria.fr/oscar/), a cleansed version of [Common Crawl](https://commoncrawl.org). - -Future release will also include: - -* The entire corpus of Greek legislation, as published by the [National Publication Office](http://www.et.gr), -* The entire corpus of EU legislation (Greek translation), as published in [Eur-Lex](https://eur-lex.europa.eu/homepage.html?locale=en). - -## Pre-training details - -* We trained BERT using the official code provided in Google BERT's github repository (https://github.com/google-research/bert). We then used [Hugging Face](https://huggingface.co)'s [Transformers](https://github.com/huggingface/transformers) conversion script to convert the TF checkpoint and vocabulary in the desirable format in order to be able to load the model in two lines of code for both PyTorch and TF2 users. -* We released a model similar to the English `bert-base-uncased` model (12-layer, 768-hidden, 12-heads, 110M parameters). -* We chose to follow the same training set-up: 1 million training steps with batches of 256 sequences of length 512 with an initial learning rate 1e-4. -* We were able to use a single Google Cloud TPU v3-8 provided for free from [TensorFlow Research Cloud (TFRC)](https://www.tensorflow.org/tfrc), while also utilizing [GCP research credits](https://edu.google.com/programs/credits/research). Huge thanks to both Google programs for supporting us! - - -## Requirements - -We published `bert-base-greek-uncased-v1` as part of [Hugging Face](https://huggingface.co)'s [Transformers](https://github.com/huggingface/transformers) repository. So, you need to install the transfomers library through pip along with PyTorch or Tensorflow 2. - -``` -pip install transfomers -pip install (torch|tensorflow) -``` - -## Pre-process text (Deaccent - Lower) - -In order to use `bert-base-greek-uncased-v1`, you have to pre-process texts to lowercase letters and remove all Greek diacritics. - -```python - -import unicodedata - -def strip_accents_and_lowercase(s): - return ''.join(c for c in unicodedata.normalize('NFD', s) - if unicodedata.category(c) != 'Mn').lower() - -accented_string = "Αυτή είναι η Ελληνική έκδοση του BERT." -unaccented_string = strip_accents_and_lowercase(accented_string) - -print(unaccented_string) # αυτη ειναι η ελληνικη εκδοση του bert. - -``` - -## Load Pretrained Model - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("nlpaueb/bert-base-greek-uncased-v1") -model = AutoModel.from_pretrained("nlpaueb/bert-base-greek-uncased-v1") -``` - -## Use Pretrained Model as a Language Model - -```python -import torch -from transformers import * - -# Load model and tokenizer -tokenizer_greek = AutoTokenizer.from_pretrained('nlpaueb/bert-base-greek-uncased-v1') -lm_model_greek = AutoModelWithLMHead.from_pretrained('nlpaueb/bert-base-greek-uncased-v1') - -# ================ EXAMPLE 1 ================ -text_1 = 'O ποιητής έγραψε ένα [MASK] .' -# EN: 'The poet wrote a [MASK].' -input_ids = tokenizer_greek.encode(text_1) -print(tokenizer_greek.convert_ids_to_tokens(input_ids)) -# ['[CLS]', 'o', 'ποιητης', 'εγραψε', 'ενα', '[MASK]', '.', '[SEP]'] -outputs = lm_model_greek(torch.tensor([input_ids]))[0] -print(tokenizer_greek.convert_ids_to_tokens(outputs[0, 5].max(0)[1].item())) -# the most plausible prediction for [MASK] is "song" - -# ================ EXAMPLE 2 ================ -text_2 = 'Είναι ένας [MASK] άνθρωπος.' -# EN: 'He is a [MASK] person.' -input_ids = tokenizer_greek.encode(text_2) -print(tokenizer_greek.convert_ids_to_tokens(input_ids)) -# ['[CLS]', 'ειναι', 'ενας', '[MASK]', 'ανθρωπος', '.', '[SEP]'] -outputs = lm_model_greek(torch.tensor([input_ids]))[0] -print(tokenizer_greek.convert_ids_to_tokens(outputs[0, 3].max(0)[1].item())) -# the most plausible prediction for [MASK] is "good" - -# ================ EXAMPLE 3 ================ -text_3 = 'Είναι ένας [MASK] άνθρωπος και κάνει συχνά [MASK].' -# EN: 'He is a [MASK] person he does frequently [MASK].' -input_ids = tokenizer_greek.encode(text_3) -print(tokenizer_greek.convert_ids_to_tokens(input_ids)) -# ['[CLS]', 'ειναι', 'ενας', '[MASK]', 'ανθρωπος', 'και', 'κανει', 'συχνα', '[MASK]', '.', '[SEP]'] -outputs = lm_model_greek(torch.tensor([input_ids]))[0] -print(tokenizer_greek.convert_ids_to_tokens(outputs[0, 8].max(0)[1].item())) -# the most plausible prediction for the second [MASK] is "trips" -``` - -## Evaluation on downstream tasks - -TBA - -## Author - -Ilias Chalkidis on behalf of [AUEB's Natural Language Processing Group](http://nlp.cs.aueb.gr) - -| Github: [@ilias.chalkidis](https://github.com/seolhokim) | Twitter: [@KiddoThe2B](https://twitter.com/KiddoThe2B) | - -## About Us - -[AUEB's Natural Language Processing Group](http://nlp.cs.aueb.gr) develops algorithms, models, and systems that allow computers to process and generate natural language texts. - -The group's current research interests include: -* question answering systems for databases, ontologies, document collections, and the Web, especially biomedical question answering, -* natural language generation from databases and ontologies, especially Semantic Web ontologies, -text classification, including filtering spam and abusive content, -* information extraction and opinion mining, including legal text analytics and sentiment analysis, -* natural language processing tools for Greek, for example parsers and named-entity recognizers, -machine learning in natural language processing, especially deep learning. - -The group is part of the Information Processing Laboratory of the Department of Informatics of the Athens University of Economics and Business. diff --git a/model_cards/nlpaueb/legal-bert-base-uncased/README.md b/model_cards/nlpaueb/legal-bert-base-uncased/README.md deleted file mode 100644 index da579878cd14..000000000000 --- a/model_cards/nlpaueb/legal-bert-base-uncased/README.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -language: en -tags: -- legal ---- - -# LEGAL-BERT: The Muppets straight out of Law School - - - -LEGAL-BERT is a family of BERT models for the legal domain, intended to assist legal NLP research, computational law, and legal technology applications. To pre-train the different variations of LEGAL-BERT, we collected 12 GB of diverse English legal text from several fields (e.g., legislation, court cases, contracts) scraped from publicly available resources. Sub-domains variants (CONTRACTS-, EURLEX-, ECHR-) and/or general LEGAL-BERT perform better than using BERT out of the box for domain-specific tasks. A light-weight model (33% the size of BERT-BASE) pre-trained from scratch on legal data with competitive perfomance is also available. -



- ---- - -I. Chalkidis, M. Fergadiotis, P. Malakasiotis, N. Aletras and I. Androutsopoulos. "LEGAL-BERT: The Muppets straight out of Law School". In Findings of Empirical Methods in Natural Language Processing (EMNLP 2020) (Short Papers), to be held online, 2020. (https://arxiv.org/abs/2010.02559) - ---- - -## Pre-training corpora - -The pre-training corpora of LEGAL-BERT include: - -* 116,062 documents of EU legislation, publicly available from EURLEX (http://eur-lex.europa.eu), the repository of EU Law running under the EU Publication Office. - -* 61,826 documents of UK legislation, publicly available from the UK legislation portal (http://www.legislation.gov.uk). - -* 19,867 cases from European Court of Justice (ECJ), also available from EURLEX. - -* 12,554 cases from HUDOC, the repository of the European Court of Human Rights (ECHR) (http://hudoc.echr.coe.int/eng). - -* 164,141 cases from various courts across the USA, hosted in the Case Law Access Project portal (https://case.law). - -* 76,366 US contracts from EDGAR, the database of US Securities and Exchange Commission (SECOM) (https://www.sec.gov/edgar.shtml). - -## Pre-training details - -* We trained BERT using the official code provided in Google BERT's github repository (https://github.com/google-research/bert). -* We released a model similar to the English BERT-BASE model (12-layer, 768-hidden, 12-heads, 110M parameters). -* We chose to follow the same training set-up: 1 million training steps with batches of 256 sequences of length 512 with an initial learning rate 1e-4. -* We were able to use a single Google Cloud TPU v3-8 provided for free from [TensorFlow Research Cloud (TFRC)](https://www.tensorflow.org/tfrc), while also utilizing [GCP research credits](https://edu.google.com/programs/credits/research). Huge thanks to both Google programs for supporting us! -* Part of LEGAL-BERT is a light-weight model pre-trained from scratch on legal data, which achieves comparable performance to larger models, while being much more efficient (approximately 4 times faster) with a smaller environmental footprint. -## Models list - -| Model name | Model Path | Training corpora | -| ------------------- | ------------------------------------ | ------------------- | -| CONTRACTS-BERT-BASE | `nlpaueb/bert-base-uncased-contracts` | US contracts | -| EURLEX-BERT-BASE | `nlpaueb/bert-base-uncased-eurlex` | EU legislation | -| ECHR-BERT-BASE | `nlpaueb/bert-base-uncased-echr` | ECHR cases | -| LEGAL-BERT-BASE | `nlpaueb/legal-bert-base-uncased` | All | -| LEGAL-BERT-SMALL | `nlpaueb/legal-bert-small-uncased` | All | - -## Load Pretrained Model - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("nlpaueb/legal-bert-base-uncased") -model = AutoModel.from_pretrained("nlpaueb/legal-bert-base-uncased") -``` - -## Use LEBAL-BERT variants as Language Models - -| Corpus | Model | Masked token | Predictions | -| --------------------------------- | ---------------------------------- | ------------ | ------------ | -| | **BERT-BASE-UNCASED** | -| (Contracts) | This [MASK] Agreement is between General Motors and John Murray . | employment | ('new', '0.09'), ('current', '0.04'), ('proposed', '0.03'), ('marketing', '0.03'), ('joint', '0.02') -| (ECHR) | The applicant submitted that her husband was subjected to treatment amounting to [MASK] whilst in the custody of Adana Security Directorate | torture | ('torture', '0.32'), ('rape', '0.22'), ('abuse', '0.14'), ('death', '0.04'), ('violence', '0.03') -| (EURLEX) | Establishing a system for the identification and registration of [MASK] animals and regarding the labelling of beef and beef products . | bovine | ('farm', '0.25'), ('livestock', '0.08'), ('draft', '0.06'), ('domestic', '0.05'), ('wild', '0.05') -| | **CONTRACTS-BERT-BASE** | -| (Contracts) | This [MASK] Agreement is between General Motors and John Murray . | employment | ('letter', '0.38'), ('dealer', '0.04'), ('employment', '0.03'), ('award', '0.03'), ('contribution', '0.02') -| (ECHR) | The applicant submitted that her husband was subjected to treatment amounting to [MASK] whilst in the custody of Adana Security Directorate | torture | ('death', '0.39'), ('imprisonment', '0.07'), ('contempt', '0.05'), ('being', '0.03'), ('crime', '0.02') -| (EURLEX) | Establishing a system for the identification and registration of [MASK] animals and regarding the labelling of beef and beef products . | bovine | (('domestic', '0.18'), ('laboratory', '0.07'), ('household', '0.06'), ('personal', '0.06'), ('the', '0.04') -| | **EURLEX-BERT-BASE** | -| (Contracts) | This [MASK] Agreement is between General Motors and John Murray . | employment | ('supply', '0.11'), ('cooperation', '0.08'), ('service', '0.07'), ('licence', '0.07'), ('distribution', '0.05') -| (ECHR) | The applicant submitted that her husband was subjected to treatment amounting to [MASK] whilst in the custody of Adana Security Directorate | torture | ('torture', '0.66'), ('death', '0.07'), ('imprisonment', '0.07'), ('murder', '0.04'), ('rape', '0.02') -| (EURLEX) | Establishing a system for the identification and registration of [MASK] animals and regarding the labelling of beef and beef products . | bovine | ('live', '0.43'), ('pet', '0.28'), ('certain', '0.05'), ('fur', '0.03'), ('the', '0.02') -| | **ECHR-BERT-BASE** | -| (Contracts) | This [MASK] Agreement is between General Motors and John Murray . | employment | ('second', '0.24'), ('latter', '0.10'), ('draft', '0.05'), ('bilateral', '0.05'), ('arbitration', '0.04') -| (ECHR) | The applicant submitted that her husband was subjected to treatment amounting to [MASK] whilst in the custody of Adana Security Directorate | torture | ('torture', '0.99'), ('death', '0.01'), ('inhuman', '0.00'), ('beating', '0.00'), ('rape', '0.00') -| (EURLEX) | Establishing a system for the identification and registration of [MASK] animals and regarding the labelling of beef and beef products . | bovine | ('pet', '0.17'), ('all', '0.12'), ('slaughtered', '0.10'), ('domestic', '0.07'), ('individual', '0.05') -| | **LEGAL-BERT-BASE** | -| (Contracts) | This [MASK] Agreement is between General Motors and John Murray . | employment | ('settlement', '0.26'), ('letter', '0.23'), ('dealer', '0.04'), ('master', '0.02'), ('supplemental', '0.02') -| (ECHR) | The applicant submitted that her husband was subjected to treatment amounting to [MASK] whilst in the custody of Adana Security Directorate | torture | ('torture', '1.00'), ('detention', '0.00'), ('arrest', '0.00'), ('rape', '0.00'), ('death', '0.00') -| (EURLEX) | Establishing a system for the identification and registration of [MASK] animals and regarding the labelling of beef and beef products . | bovine | ('live', '0.67'), ('beef', '0.17'), ('farm', '0.03'), ('pet', '0.02'), ('dairy', '0.01') -| | **LEGAL-BERT-SMALL** | -| (Contracts) | This [MASK] Agreement is between General Motors and John Murray . | employment | ('license', '0.09'), ('transition', '0.08'), ('settlement', '0.04'), ('consent', '0.03'), ('letter', '0.03') -| (ECHR) | The applicant submitted that her husband was subjected to treatment amounting to [MASK] whilst in the custody of Adana Security Directorate | torture | ('torture', '0.59'), ('pain', '0.05'), ('ptsd', '0.05'), ('death', '0.02'), ('tuberculosis', '0.02') -| (EURLEX) | Establishing a system for the identification and registration of [MASK] animals and regarding the labelling of beef and beef products . | bovine | ('all', '0.08'), ('live', '0.07'), ('certain', '0.07'), ('the', '0.07'), ('farm', '0.05') - - - -## Evaluation on downstream tasks - -Consider the experiments in the article "LEGAL-BERT: The Muppets straight out of Law School". Chalkidis et al., 2018, (https://arxiv.org/abs/2010.02559) - -## Author - -Ilias Chalkidis on behalf of [AUEB's Natural Language Processing Group](http://nlp.cs.aueb.gr) - -| Github: [@ilias.chalkidis](https://github.com/seolhokim) | Twitter: [@KiddoThe2B](https://twitter.com/KiddoThe2B) | diff --git a/model_cards/nlptown/bert-base-multilingual-uncased-sentiment/README.md b/model_cards/nlptown/bert-base-multilingual-uncased-sentiment/README.md deleted file mode 100644 index 87934c87099b..000000000000 --- a/model_cards/nlptown/bert-base-multilingual-uncased-sentiment/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: -- en -- nl -- de -- fr -- it -- es - -license: mit ---- - -# bert-base-multilingual-uncased-sentiment - -This a bert-base-multilingual-uncased model finetuned for sentiment analysis on product reviews in six languages: English, Dutch, German, French, Spanish and Italian. It predicts the sentiment of the review as a number of stars (between 1 and 5). - -This model is intended for direct use as a sentiment analysis model for product reviews in any of the six languages above, or for further finetuning on related sentiment analysis tasks. - -## Training data - -Here is the number of product reviews we used for finetuning the model: - -| Language | Number of reviews | -| -------- | ----------------- | -| English | 150k | -| Dutch | 80k | -| German | 137k | -| French | 140k | -| Italian | 72k | -| Spanish | 50k | - -## Accuracy - -The finetuned model obtained the following accuracy on 5,000 held-out product reviews in each of the languages: - -- Accuracy (exact) is the exact match on the number of stars. -- Accuracy (off-by-1) is the percentage of reviews where the number of stars the model predicts differs by a maximum of 1 from the number given by the human reviewer. - - -| Language | Accuracy (exact) | Accuracy (off-by-1) | -| -------- | ---------------------- | ------------------- | -| English | 67% | 95% -| Dutch | 57% | 93% -| German | 61% | 94% -| French | 59% | 94% -| Italian | 59% | 95% -| Spanish | 58% | 95% - -## Contact - -Contact [NLP Town](https://www.nlp.town) for questions, feedback and/or requests for similar models. diff --git a/model_cards/nyu-mll/roberta-base-100M-1/README.md b/model_cards/nyu-mll/roberta-base-100M-1/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-100M-1/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-base-100M-2/README.md b/model_cards/nyu-mll/roberta-base-100M-2/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-100M-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-base-100M-3/README.md b/model_cards/nyu-mll/roberta-base-100M-3/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-100M-3/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-base-10M-1/README.md b/model_cards/nyu-mll/roberta-base-10M-1/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-10M-1/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-base-10M-2/README.md b/model_cards/nyu-mll/roberta-base-10M-2/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-10M-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-base-10M-3/README.md b/model_cards/nyu-mll/roberta-base-10M-3/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-10M-3/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-base-1B-1/README.md b/model_cards/nyu-mll/roberta-base-1B-1/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-1B-1/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-base-1B-2/README.md b/model_cards/nyu-mll/roberta-base-1B-2/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-1B-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-base-1B-3/README.md b/model_cards/nyu-mll/roberta-base-1B-3/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-base-1B-3/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-med-small-1M-1/README.md b/model_cards/nyu-mll/roberta-med-small-1M-1/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-med-small-1M-1/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-med-small-1M-2/README.md b/model_cards/nyu-mll/roberta-med-small-1M-2/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-med-small-1M-2/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta-med-small-1M-3/README.md b/model_cards/nyu-mll/roberta-med-small-1M-3/README.md deleted file mode 120000 index 34a367f7752d..000000000000 --- a/model_cards/nyu-mll/roberta-med-small-1M-3/README.md +++ /dev/null @@ -1 +0,0 @@ -../roberta_1M_to_1B/README.md \ No newline at end of file diff --git a/model_cards/nyu-mll/roberta_1M_to_1B/README.md b/model_cards/nyu-mll/roberta_1M_to_1B/README.md deleted file mode 100644 index 1689faf800e4..000000000000 --- a/model_cards/nyu-mll/roberta_1M_to_1B/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# RoBERTa Pretrained on Smaller Datasets - -We pretrain RoBERTa on smaller datasets (1M, 10M, 100M, 1B tokens). We release 3 models with lowest perplexities for each pretraining data size out of 25 runs (or 10 in the case of 1B tokens). The pretraining data reproduces that of BERT: We combine English Wikipedia and a reproduction of BookCorpus using texts from smashwords in a ratio of approximately 3:1. - -### Hyperparameters and Validation Perplexity - -The hyperparameters and validation perplexities corresponding to each model are as follows: - -| Model Name | Training Size | Model Size | Max Steps | Batch Size | Validation Perplexity | -|--------------------------|---------------|------------|-----------|------------|-----------------------| -| [roberta-base-1B-1][link-roberta-base-1B-1] | 1B | BASE | 100K | 512 | 3.93 | -| [roberta-base-1B-2][link-roberta-base-1B-2] | 1B | BASE | 31K | 1024 | 4.25 | -| [roberta-base-1B-3][link-roberta-base-1B-3] | 1B | BASE | 31K | 4096 | 3.84 | -| [roberta-base-100M-1][link-roberta-base-100M-1] | 100M | BASE | 100K | 512 | 4.99 | -| [roberta-base-100M-2][link-roberta-base-100M-2] | 100M | BASE | 31K | 1024 | 4.61 | -| [roberta-base-100M-3][link-roberta-base-100M-3] | 100M | BASE | 31K | 512 | 5.02 | -| [roberta-base-10M-1][link-roberta-base-10M-1] | 10M | BASE | 10K | 1024 | 11.31 | -| [roberta-base-10M-2][link-roberta-base-10M-2] | 10M | BASE | 10K | 512 | 10.78 | -| [roberta-base-10M-3][link-roberta-base-10M-3] | 10M | BASE | 31K | 512 | 11.58 | -| [roberta-med-small-1M-1][link-roberta-med-small-1M-1] | 1M | MED-SMALL | 100K | 512 | 153.38 | -| [roberta-med-small-1M-2][link-roberta-med-small-1M-2] | 1M | MED-SMALL | 10K | 512 | 134.18 | -| [roberta-med-small-1M-3][link-roberta-med-small-1M-3] | 1M | MED-SMALL | 31K | 512 | 139.39 | - -The hyperparameters corresponding to model sizes mentioned above are as follows: - -| Model Size | L | AH | HS | FFN | P | -|------------|----|----|-----|------|------| -| BASE | 12 | 12 | 768 | 3072 | 125M | -| MED-SMALL | 6 | 8 | 512 | 2048 | 45M | - -(AH = number of attention heads; HS = hidden size; FFN = feedforward network dimension; P = number of parameters.) - -For other hyperparameters, we select: -- Peak Learning rate: 5e-4 -- Warmup Steps: 6% of max steps -- Dropout: 0.1 - -[link-roberta-med-small-1M-1]: https://huggingface.co/nyu-mll/roberta-med-small-1M-1 -[link-roberta-med-small-1M-2]: https://huggingface.co/nyu-mll/roberta-med-small-1M-2 -[link-roberta-med-small-1M-3]: https://huggingface.co/nyu-mll/roberta-med-small-1M-3 -[link-roberta-base-10M-1]: https://huggingface.co/nyu-mll/roberta-base-10M-1 -[link-roberta-base-10M-2]: https://huggingface.co/nyu-mll/roberta-base-10M-2 -[link-roberta-base-10M-3]: https://huggingface.co/nyu-mll/roberta-base-10M-3 -[link-roberta-base-100M-1]: https://huggingface.co/nyu-mll/roberta-base-100M-1 -[link-roberta-base-100M-2]: https://huggingface.co/nyu-mll/roberta-base-100M-2 -[link-roberta-base-100M-3]: https://huggingface.co/nyu-mll/roberta-base-100M-3 -[link-roberta-base-1B-1]: https://huggingface.co/nyu-mll/roberta-base-1B-1 -[link-roberta-base-1B-2]: https://huggingface.co/nyu-mll/roberta-base-1B-2 -[link-roberta-base-1B-3]: https://huggingface.co/nyu-mll/roberta-base-1B-3 diff --git a/model_cards/oliverguhr/german-sentiment-bert/README.md b/model_cards/oliverguhr/german-sentiment-bert/README.md deleted file mode 100644 index a8e03e278f6f..000000000000 --- a/model_cards/oliverguhr/german-sentiment-bert/README.md +++ /dev/null @@ -1,125 +0,0 @@ -# German Sentiment Classification with Bert - -This model was trained for sentiment classification of German language texts. To achieve the best results all model inputs needs to be preprocessed with the same procedure, that was applied during the training. To simplify the usage of the model, -we provide a Python package that bundles the code need for the preprocessing and inferencing. - -The model uses the Googles Bert architecture and was trained on 1.834 million German-language samples. The training data contains texts from various domains like Twitter, Facebook and movie, app and hotel reviews. -You can find more information about the dataset and the training process in the [paper](http://www.lrec-conf.org/proceedings/lrec2020/pdf/2020.lrec-1.202.pdf). - -## Using the Python package - -To get started install the package from [pypi](https://pypi.org/project/germansentiment/): - -```bash -pip install germansentiment -``` - -```python -from germansentiment import SentimentModel - -model = SentimentModel() - -texts = [ - "Mit keinem guten Ergebniss","Das ist gar nicht mal so gut", - "Total awesome!","nicht so schlecht wie erwartet", - "Der Test verlief positiv.","Sie fährt ein grünes Auto."] - -result = model.predict_sentiment(texts) -print(result) -``` - -The code above will output following list: - -```python -["negative","negative","positive","positive","neutral", "neutral"] -``` - -## A minimal working Sample - - -```python -from transformers import AutoModelForSequenceClassification, AutoTokenizer -from typing import List -import torch -import re - -class SentimentModel(): - def __init__(self, model_name: str): - self.model = AutoModelForSequenceClassification.from_pretrained(model_name) - self.tokenizer = AutoTokenizer.from_pretrained(model_name) - - self.clean_chars = re.compile(r'[^A-Za-züöäÖÜÄß ]', re.MULTILINE) - self.clean_http_urls = re.compile(r'https*\S+', re.MULTILINE) - self.clean_at_mentions = re.compile(r'@\S+', re.MULTILINE) - - def predict_sentiment(self, texts: List[str])-> List[str]: - texts = [self.clean_text(text) for text in texts] - # Add special tokens takes care of adding [CLS], [SEP], ... tokens in the right way for each model. - input_ids = self.tokenizer(texts, padding=True, truncation=True, add_special_tokens=True) - input_ids = torch.tensor(input_ids["input_ids"]) - - with torch.no_grad(): - logits = self.model(input_ids) - - label_ids = torch.argmax(logits[0], axis=1) - - labels = [self.model.config.id2label[label_id] for label_id in label_ids.tolist()] - return labels - - def replace_numbers(self,text: str) -> str: - return text.replace("0"," null").replace("1"," eins").replace("2"," zwei").replace("3"," drei").replace("4"," vier").replace("5"," fünf").replace("6"," sechs").replace("7"," sieben").replace("8"," acht").replace("9"," neun") - - def clean_text(self,text: str)-> str: - text = text.replace("\n", " ") - text = self.clean_http_urls.sub('',text) - text = self.clean_at_mentions.sub('',text) - text = self.replace_numbers(text) - text = self.clean_chars.sub('', text) # use only text chars - text = ' '.join(text.split()) # substitute multiple whitespace with single whitespace - text = text.strip().lower() - return text - -texts = ["Mit keinem guten Ergebniss","Das war unfair", "Das ist gar nicht mal so gut", - "Total awesome!","nicht so schlecht wie erwartet", "Das ist gar nicht mal so schlecht", - "Der Test verlief positiv.","Sie fährt ein grünes Auto.", "Der Fall wurde an die Polzei übergeben."] - -model = SentimentModel(model_name = "oliverguhr/german-sentiment-bert") - -print(model.predict_sentiment(texts)) -``` - -## Model and Data - -If you are interested in code and data that was used to train this model please have a look at [this repository](https://github.com/oliverguhr/german-sentiment) and our [paper](http://www.lrec-conf.org/proceedings/lrec2020/pdf/2020.lrec-1.202.pdf). Here is a table of the F1 scores that his model achieves on following datasets. Since we trained this model on a newer version of the transformer library, the results are slightly better than reported in the paper. - -| Dataset | F1 micro Score | -| :----------------------------------------------------------- | -------------: | -| [holidaycheck](https://github.com/oliverguhr/german-sentiment) | 0.9568 | -| [scare](https://www.romanklinger.de/scare/) | 0.9418 | -| [filmstarts](https://github.com/oliverguhr/german-sentiment) | 0.9021 | -| [germeval](https://sites.google.com/view/germeval2017-absa/home) | 0.7536 | -| [PotTS](https://www.aclweb.org/anthology/L16-1181/) | 0.6780 | -| [emotions](https://github.com/oliverguhr/german-sentiment) | 0.9649 | -| [sb10k](https://www.spinningbytes.com/resources/germansentiment/) | 0.7376 | -| [Leipzig Wikipedia Corpus 2016](https://wortschatz.uni-leipzig.de/de/download/german) | 0.9967 | -| all | 0.9639 | - -## Cite - -For feedback and questions contact me view mail or Twitter [@oliverguhr](https://twitter.com/oliverguhr). Please cite us if you found this useful: - -``` -@InProceedings{guhr-EtAl:2020:LREC, - author = {Guhr, Oliver and Schumann, Anne-Kathrin and Bahrmann, Frank and Böhme, Hans Joachim}, - title = {Training a Broad-Coverage German Sentiment Classification Model for Dialog Systems}, - booktitle = {Proceedings of The 12th Language Resources and Evaluation Conference}, - month = {May}, - year = {2020}, - address = {Marseille, France}, - publisher = {European Language Resources Association}, - pages = {1620--1625}, - url = {https://www.aclweb.org/anthology/2020.lrec-1.201} -} -``` - - diff --git a/model_cards/panggi/t5-base-indonesian-summarization-cased/README.md b/model_cards/panggi/t5-base-indonesian-summarization-cased/README.md deleted file mode 100644 index f5943f59630a..000000000000 --- a/model_cards/panggi/t5-base-indonesian-summarization-cased/README.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -language: id ---- - -# Indonesian T5 Summarization Base Model - -Finetuned T5 base summarization model for Indonesian. - -## Finetuning Corpus - -`t5-base-indonesian-summarization-cased` model is based on `t5-base-bahasa-summarization-cased` by [huseinzol05](https://huggingface.co/huseinzol05), finetuned using [indosum](https://github.com/kata-ai/indosum) dataset. - -## Load Finetuned Model - -```python -from transformers import T5Tokenizer, T5Model, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained("panggi/t5-base-indonesian-summarization-cased") -model = T5ForConditionalGeneration.from_pretrained("panggi/t5-base-indonesian-summarization-cased") -``` - -## Code Sample - -```python -from transformers import T5Tokenizer, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained("panggi/t5-base-indonesian-summarization-cased") -model = T5ForConditionalGeneration.from_pretrained("panggi/t5-base-indonesian-summarization-cased") - -# https://www.sehatq.com/artikel/apa-itu-dispepsia-fungsional-ketahui-gejala-dan-faktor-risikonya -ARTICLE_TO_SUMMARIZE = "Secara umum, dispepsia adalah kumpulan gejala pada saluran pencernaan seperti nyeri, sensasi terbakar, dan rasa tidak nyaman pada perut bagian atas. Pada beberapa kasus, dispepsia yang dialami seseorang tidak dapat diketahui penyebabnya. Jenis dispepsia ini disebut dengan dispepsia fungsional. Apa saja gejala dispepsia fungsional? Apa itu dispepsia fungsional? Dispepsia fungsional adalah kumpulan gejala tanpa sebab pada saluran pencernaan bagian atas. Gejala tersebut dapat berupa rasa sakit, nyeri, dan tak nyaman pada perut bagian atas atau ulu hati. Penderita dispepsia fungsional juga akan merasakan kenyang lebih cepat dan sensasi perut penuh berkepanjangan. Gejala-gejala tersebut bisa berlangsung selama sebulan atau lebih. Dispepsia ini memiliki nama “fungsional” karena kumpulan gejalanya tidak memiliki penyebab yang jelas. Dilihat dari fungsi dan struktur saluran pencernaan, dokter tidak menemukan hal yang salah. Namun, gejalanya bisa sangat mengganggu dan menyiksa. Dispepsia fungsional disebut juga dengan dispepsia nonulkus. Diperkirakan bahwa 20% masyarakat dunia menderita dispepsia fungsional. Kondisi ini berisiko tinggi dialami oleh wanita, perokok, dan orang yang mengonsumsi obat anti-peradangan nonsteroid (NSAID). Dispepsia fungsional bisa bersifat kronis dan mengganggu kehidupan penderitanya. Namun beruntung, ada beberapa strategi yang bisa diterapkan untuk mengendalikan gejala dispepsia ini. Strategi tersebut termasuk perubahan gaya hidup, obat-obatan, dan terapi.Ragam gejala dispepsia fungsional Gejala dispepsia fungsional dapat bervariasi antara satu pasien dengan pasien lain. Beberapa tanda yang bisa dirasakan seseorang, yaitu: Sensasi terbakar atau nyeri di saluran pencernaan bagian atas Perut kembung Cepat merasa kenyang walau baru makan sedikit Mual Muntah Bersendawa Rasa asam di mulut Penurunan berat badan Tekanan psikologis terkait dengan kondisi yang dialami Apa sebenarnya penyebab dispepsia fungsional? Sebagai penyakit fungsional, dokter mengkategorikan dispepsia ini sebagai penyakit yang tidak diketahui penyebabnya. Hanya saja, beberapa faktor bisa meningkatkan risiko seseorang terkena dispepsia fungsional. Faktor risiko tersebut, termasuk: Alergi terhadap zat tertentu Perubahan mikrobioma usus Infeksi, seperti yang dipicu oleh bakteriHelicobacter pylori Sekresi asam lambung yang tidak normal Peradangan pada saluran pencernaan bagian atas Gangguan pada fungsi lambung untuk mencerna makanan Pola makan tertentu Gaya hidup tidak sehat Stres Kecemasan atau depresi Efek samping pemakaian obat seperti obat antiinflamasi nonsteroid Penanganan untuk dispepsia fungsional Ada banyak pilihan pengobatan untuk dispepsia fungsional. Seperti yang disampaikan di atas, tidak ada penyebab tunggal dispepsia ini yang bisa diketahui. Gejala yang dialami antara satu pasien juga mungkin amat berbeda dari orang lain. Dengan demikian, jenis pengobatan dispepsia fungsional juga akan bervariasi. Beberapa pilihan strategi penanganan untuk dispepsia fungsional, meliputi: 1. Obat-obatan Ada beberapa jenis obat yang mungkin akan diberikan dokter, seperti Obat penetral asam lambung yang disebut penghambat reseptor H2 Obat penghambat produksi asam lambung yang disebut proton pump inhibitors Obat untuk mengendalikan gas di perut yang mengandung simetikon Antidepresan seperti amitriptyline Obat penguat kerongkongan yang disebut agen prokinetik Obat untuk pengosongan isi lambung seperti metoclopramide Antibiotik jika dokter mendeteksi adanya infeksi bakteri H. pylori 2. Anjuran terkait perubahan gaya hidup Selain obat-obatan, dokter akan memberikan rekomendasi perubahan gaya hidup yang harus diterapkan pasien. Tips terkait perubahan gaya hidup termasuk: Makan lebih sering namun dengan porsi yang lebih sedikit Menjauhi makanan berlemak karena memperlambat pengosongan makanan di lambung Menjauhi jenis makanan lain yang memicu gejala dispepsia, seperti makanan pedas, makanan tinggi asam, produk susu, dan produk kafein Menjauhi rokok Dokter juga akan meminta pasien untuk mencari cara untuk mengendalikan stres, tidur dengan kepala lebih tinggi, dan menjalankan usaha untuk mengendalikan berat badan. Apakah penyakit dispepsia itu berbahaya? Dispepsia, termasuk dispepsia fungsional, dapat menjadi kronis dengan gejala yang menyiksa. Jika tidak ditangani, dispepsia tentu dapat berbahaya dan mengganggu kehidupan pasien. Segera hubungi dokter apabila Anda merasakan gejala dispepsia, terlebih jika tidak merespons obat-obatan yang dijual bebas. Catatan dari SehatQ Dispepsia fungsional adalah kumpulan gejala pada saluran pencernaan bagian atas yang tidak diketahui penyebabnya. Dispepsia fungsional dapat ditangani dengan kombinasi obat-obatan dan perubahan gaya hidup. Jika masih memiliki pertanyaan terkait dispepsia fungsional, Anda bisa menanyakan ke dokter di aplikasi kesehatan keluarga SehatQ. Aplikasi SehatQ bisa diunduh gratis di Appstore dan Playstore yang berikan informasi penyakit terpercaya." -# generate summary -inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=5000, return_tensors='pt', truncation=True) -summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=100, early_stopping=True) -print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids]) -``` - -Output: - -``` -'Dispepsia fungsional adalah kumpulan gejala tanpa sebab pada saluran pencernaan bagian atas. Gejala tersebut dapat berupa rasa sakit, nyeri, dan tak nyaman pada perut bagian atas. Penderita dispepsia fungsional juga akan merasakan kenyang lebih cepat dan sensasi perut penuh berkepanjangan. Gejala-gejala tersebut bisa berlangsung selama sebulan atau lebih. -``` - -## Acknowledgement - -Thanks to Immanuel Drexel for his article [Text Summarization, Extractive, T5, Bahasa Indonesia, Huggingface’s Transformers](https://medium.com/analytics-vidhya/text-summarization-t5-bahasa-indonesia-huggingfaces-transformers-ee9bfe368e2f) diff --git a/model_cards/panggi/t5-small-indonesian-summarization-cased/README.md b/model_cards/panggi/t5-small-indonesian-summarization-cased/README.md deleted file mode 100644 index c83ab8627c23..000000000000 --- a/model_cards/panggi/t5-small-indonesian-summarization-cased/README.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -language: id ---- - -# Indonesian T5 Summarization Small Model - -Finetuned T5 small summarization model for Indonesian. - -## Finetuning Corpus - -`t5-small-indonesian-summarization-cased` model is based on `t5-small-bahasa-summarization-cased` by [huseinzol05](https://huggingface.co/huseinzol05), finetuned using [indosum](https://github.com/kata-ai/indosum) dataset. - -## Load Finetuned Model - -```python -from transformers import T5Tokenizer, T5Model, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained("panggi/t5-small-indonesian-summarization-cased") -model = T5ForConditionalGeneration.from_pretrained("panggi/t5-small-indonesian-summarization-cased") -``` - -## Code Sample - -```python -from transformers import T5Tokenizer, T5ForConditionalGeneration - -tokenizer = T5Tokenizer.from_pretrained("panggi/t5-small-indonesian-summarization-cased") -model = T5ForConditionalGeneration.from_pretrained("panggi/t5-small-indonesian-summarization-cased") - -# https://www.sehatq.com/artikel/apa-itu-dispepsia-fungsional-ketahui-gejala-dan-faktor-risikonya -ARTICLE_TO_SUMMARIZE = "Secara umum, dispepsia adalah kumpulan gejala pada saluran pencernaan seperti nyeri, sensasi terbakar, dan rasa tidak nyaman pada perut bagian atas. Pada beberapa kasus, dispepsia yang dialami seseorang tidak dapat diketahui penyebabnya. Jenis dispepsia ini disebut dengan dispepsia fungsional. Apa saja gejala dispepsia fungsional? Apa itu dispepsia fungsional? Dispepsia fungsional adalah kumpulan gejala tanpa sebab pada saluran pencernaan bagian atas. Gejala tersebut dapat berupa rasa sakit, nyeri, dan tak nyaman pada perut bagian atas atau ulu hati. Penderita dispepsia fungsional juga akan merasakan kenyang lebih cepat dan sensasi perut penuh berkepanjangan. Gejala-gejala tersebut bisa berlangsung selama sebulan atau lebih. Dispepsia ini memiliki nama “fungsional” karena kumpulan gejalanya tidak memiliki penyebab yang jelas. Dilihat dari fungsi dan struktur saluran pencernaan, dokter tidak menemukan hal yang salah. Namun, gejalanya bisa sangat mengganggu dan menyiksa. Dispepsia fungsional disebut juga dengan dispepsia nonulkus. Diperkirakan bahwa 20% masyarakat dunia menderita dispepsia fungsional. Kondisi ini berisiko tinggi dialami oleh wanita, perokok, dan orang yang mengonsumsi obat anti-peradangan nonsteroid (NSAID). Dispepsia fungsional bisa bersifat kronis dan mengganggu kehidupan penderitanya. Namun beruntung, ada beberapa strategi yang bisa diterapkan untuk mengendalikan gejala dispepsia ini. Strategi tersebut termasuk perubahan gaya hidup, obat-obatan, dan terapi.Ragam gejala dispepsia fungsional Gejala dispepsia fungsional dapat bervariasi antara satu pasien dengan pasien lain. Beberapa tanda yang bisa dirasakan seseorang, yaitu: Sensasi terbakar atau nyeri di saluran pencernaan bagian atas Perut kembung Cepat merasa kenyang walau baru makan sedikit Mual Muntah Bersendawa Rasa asam di mulut Penurunan berat badan Tekanan psikologis terkait dengan kondisi yang dialami Apa sebenarnya penyebab dispepsia fungsional? Sebagai penyakit fungsional, dokter mengkategorikan dispepsia ini sebagai penyakit yang tidak diketahui penyebabnya. Hanya saja, beberapa faktor bisa meningkatkan risiko seseorang terkena dispepsia fungsional. Faktor risiko tersebut, termasuk: Alergi terhadap zat tertentu Perubahan mikrobioma usus Infeksi, seperti yang dipicu oleh bakteriHelicobacter pylori Sekresi asam lambung yang tidak normal Peradangan pada saluran pencernaan bagian atas Gangguan pada fungsi lambung untuk mencerna makanan Pola makan tertentu Gaya hidup tidak sehat Stres Kecemasan atau depresi Efek samping pemakaian obat seperti obat antiinflamasi nonsteroid Penanganan untuk dispepsia fungsional Ada banyak pilihan pengobatan untuk dispepsia fungsional. Seperti yang disampaikan di atas, tidak ada penyebab tunggal dispepsia ini yang bisa diketahui. Gejala yang dialami antara satu pasien juga mungkin amat berbeda dari orang lain. Dengan demikian, jenis pengobatan dispepsia fungsional juga akan bervariasi. Beberapa pilihan strategi penanganan untuk dispepsia fungsional, meliputi: 1. Obat-obatan Ada beberapa jenis obat yang mungkin akan diberikan dokter, seperti Obat penetral asam lambung yang disebut penghambat reseptor H2 Obat penghambat produksi asam lambung yang disebut proton pump inhibitors Obat untuk mengendalikan gas di perut yang mengandung simetikon Antidepresan seperti amitriptyline Obat penguat kerongkongan yang disebut agen prokinetik Obat untuk pengosongan isi lambung seperti metoclopramide Antibiotik jika dokter mendeteksi adanya infeksi bakteri H. pylori 2. Anjuran terkait perubahan gaya hidup Selain obat-obatan, dokter akan memberikan rekomendasi perubahan gaya hidup yang harus diterapkan pasien. Tips terkait perubahan gaya hidup termasuk: Makan lebih sering namun dengan porsi yang lebih sedikit Menjauhi makanan berlemak karena memperlambat pengosongan makanan di lambung Menjauhi jenis makanan lain yang memicu gejala dispepsia, seperti makanan pedas, makanan tinggi asam, produk susu, dan produk kafein Menjauhi rokok Dokter juga akan meminta pasien untuk mencari cara untuk mengendalikan stres, tidur dengan kepala lebih tinggi, dan menjalankan usaha untuk mengendalikan berat badan. Apakah penyakit dispepsia itu berbahaya? Dispepsia, termasuk dispepsia fungsional, dapat menjadi kronis dengan gejala yang menyiksa. Jika tidak ditangani, dispepsia tentu dapat berbahaya dan mengganggu kehidupan pasien. Segera hubungi dokter apabila Anda merasakan gejala dispepsia, terlebih jika tidak merespons obat-obatan yang dijual bebas. Catatan dari SehatQ Dispepsia fungsional adalah kumpulan gejala pada saluran pencernaan bagian atas yang tidak diketahui penyebabnya. Dispepsia fungsional dapat ditangani dengan kombinasi obat-obatan dan perubahan gaya hidup. Jika masih memiliki pertanyaan terkait dispepsia fungsional, Anda bisa menanyakan ke dokter di aplikasi kesehatan keluarga SehatQ. Aplikasi SehatQ bisa diunduh gratis di Appstore dan Playstore yang berikan informasi penyakit terpercaya." -# generate summary -inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=5000, return_tensors='pt', truncation=True) -summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=100, early_stopping=True) -print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids]) -``` - -Output: - -``` -'Dispepsia fungsional adalah kumpulan gejala tanpa sebab pada saluran pencernaan bagian atas. Gejala tersebut dapat berupa rasa sakit, nyeri, dan tak nyaman pada perut bagian atas. Penderita dispepsia fungsional juga akan merasakan kenyang lebih cepat dan sensasi perut penuh berkepanjangan. Gejala-gejala tersebut bisa berlangsung selama sebulan atau lebih. -``` - -## Acknowledgement - -Thanks to Immanuel Drexel for his article [Text Summarization, Extractive, T5, Bahasa Indonesia, Huggingface’s Transformers](https://medium.com/analytics-vidhya/text-summarization-t5-bahasa-indonesia-huggingfaces-transformers-ee9bfe368e2f) diff --git a/model_cards/patrickvonplaten/bert2bert-cnn_dailymail-fp16/README.md b/model_cards/patrickvonplaten/bert2bert-cnn_dailymail-fp16/README.md deleted file mode 100644 index 71831bd959d1..000000000000 --- a/model_cards/patrickvonplaten/bert2bert-cnn_dailymail-fp16/README.md +++ /dev/null @@ -1,218 +0,0 @@ -# Bert2Bert Summarization with 🤗 EncoderDecoder Framework - -This model is a Bert2Bert model fine-tuned on summarization. - -Bert2Bert is a `EncoderDecoderModel`, meaning that both the encoder and the decoder are `bert-base-uncased` -BERT models. Leveraging the [EncoderDecoderFramework](https://huggingface.co/transformers/model_doc/encoderdecoder.html#encoder-decoder-models), the -two pretrained models can simply be loaded into the framework via: - -```python -bert2bert = EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-uncased", "bert-base-uncased") -``` - -The decoder of an `EncoderDecoder` model needs cross-attention layers and usually makes use of causal -masking for auto-regressiv generation. -Thus, ``bert2bert`` is consequently fined-tuned on the `CNN/Daily Mail`dataset and the resulting model -`bert2bert-cnn_dailymail-fp16` is uploaded here. - -## Example - -The model is by no means a state-of-the-art model, but nevertheless -produces reasonable summarization results. It was mainly fine-tuned -as a proof-of-concept for the 🤗 EncoderDecoder Framework. - -The model can be used as follows: - -```python -from transformers import BertTokenizer, EncoderDecoderModel - -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/bert2bert-cnn_dailymail-fp16") -tokenizer = BertTokenizer.from_pretrained("patrickvonplaten/bert2bert-cnn_dailymail-fp16") - -article = """(CNN)Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing a racist chant. SAE's national chapter suspended the students, but University of Oklahoma President David Boren took it a step further, saying the university's affiliation with the fraternity is permanently done. The news is shocking, but it's not the first time SAE has faced controversy. SAE was founded March 9, 1856, at the University of Alabama, five years before the American Civil War, according to the fraternity website. When the war began, the group had fewer than 400 members, of which "369 went to war for the Confederate States and seven for the Union Army," the website says. The fraternity now boasts more than 200,000 living alumni, along with about 15,000 undergraduates populating 219 chapters and 20 "colonies" seeking full membership at universities. SAE has had to work hard to change recently after a string of member deaths, many blamed on the hazing of new recruits, SAE national President Bradley Cohen wrote in a message on the fraternity's website. The fraternity's website lists more than 130 chapters cited or suspended for "health and safety incidents" since 2010. At least 30 of the incidents involved hazing, and dozens more involved alcohol. However, the list is missing numerous incidents from recent months. Among them, according to various media outlets: Yale University banned the SAEs from campus activities last month after members allegedly tried to interfere with a sexual misconduct investigation connected to an initiation rite. Stanford University in December suspended SAE housing privileges after finding sorority members attending a fraternity function were subjected to graphic sexual content. And Johns Hopkins University in November suspended the fraternity for underage drinking. "The media has labeled us as the 'nation's deadliest fraternity,' " Cohen said. In 2011, for example, a student died while being coerced into excessive alcohol consumption, according to a lawsuit. SAE's previous insurer dumped the fraternity. "As a result, we are paying Lloyd's of London the highest insurance rates in the Greek-letter world," Cohen said. Universities have turned down SAE's attempts to open new chapters, and the fraternity had to close 12 in 18 months over hazing incidents.""" - -input_ids = tokenizer(article, return_tensors="pt").input_ids -output_ids = model.generate(input_ids) - -print(tokenizer.decode(output_ids[0], skip_special_tokens=True)) -# should produce -# sae was founded in 1856, five years before the civil war. the fraternity has had to work hard to change recently. the university of oklahoma president says the university's affiliation with the fraternity is permanently done. the sae has had a string of members in recent mon -ths. -``` - -## Training script: - -**IMPORTANT**: In order for this code to work, make sure you checkout to the branch -[more_general_trainer_metric](https://github.com/huggingface/transformers/tree/more_general_trainer_metric), which slightly adapts -the `Trainer` for `EncoderDecoderModels` according to this PR: https://github.com/huggingface/transformers/pull/5840. - -The following code shows the complete training script that was used to fine-tune `bert2bert-cnn_dailymail-fp16 -` for reproducability. The training last ~9h on a standard GPU. - -```python -#!/usr/bin/env python3 -import nlp -import logging -from transformers import BertTokenizer, EncoderDecoderModel, Trainer, TrainingArguments - -logging.basicConfig(level=logging.INFO) - -model = EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-uncased", "bert-base-uncased") -tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") - -# CLS token will work as BOS token -tokenizer.bos_token = tokenizer.cls_token - -# SEP token will work as EOS token -tokenizer.eos_token = tokenizer.sep_token - -# load train and validation data -train_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="train") -val_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="validation[:10%]") - -# load rouge for validation -rouge = nlp.load_metric("rouge") - - -# set decoding params -model.config.decoder_start_token_id = tokenizer.bos_token_id -model.config.eos_token_id = tokenizer.eos_token_id -model.config.max_length = 142 -model.config.min_length = 56 -model.config.no_repeat_ngram_size = 3 -model.early_stopping = True -model.length_penalty = 2.0 -model.num_beams = 4 - - -# map data correctly -def map_to_encoder_decoder_inputs(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at BERT max length 512 - inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=512) - # force summarization <= 128 - outputs = tokenizer(batch["highlights"], padding="max_length", truncation=True, max_length=128) - - batch["input_ids"] = inputs.input_ids - batch["attention_mask"] = inputs.attention_mask - - batch["decoder_input_ids"] = outputs.input_ids - batch["labels"] = outputs.input_ids.copy() - # mask loss for padding - batch["labels"] = [ - [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch["labels"] - ] - batch["decoder_attention_mask"] = outputs.attention_mask - - assert all([len(x) == 512 for x in inputs.input_ids]) - assert all([len(x) == 128 for x in outputs.input_ids]) - - return batch - - -def compute_metrics(pred): - labels_ids = pred.label_ids - pred_ids = pred.predictions - - # all unnecessary tokens are removed - pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True) - label_str = tokenizer.batch_decode(labels_ids, skip_special_tokens=True) - - rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - - return { - "rouge2_precision": round(rouge_output.precision, 4), - "rouge2_recall": round(rouge_output.recall, 4), - "rouge2_fmeasure": round(rouge_output.fmeasure, 4), - } - - -# set batch size here -batch_size = 16 - -# make train dataset ready -train_dataset = train_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -train_dataset.set_format( - type="torch", columns=["input_ids", "attention_mask", "decoder_input_ids", "decoder_attention_mask", "labels"], -) - -# same for validation dataset -val_dataset = val_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -val_dataset.set_format( - type="torch", columns=["input_ids", "attention_mask", "decoder_input_ids", "decoder_attention_mask", "labels"], -) - -# set training arguments - these params are not really tuned, feel free to change -training_args = TrainingArguments( - output_dir="./", - per_device_train_batch_size=batch_size, - per_device_eval_batch_size=batch_size, - predict_from_generate=True, - evaluate_during_training=True, - do_train=True, - do_eval=True, - logging_steps=1000, - save_steps=1000, - eval_steps=1000, - overwrite_output_dir=True, - warmup_steps=2000, - save_total_limit=10, -) - -# instantiate trainer -trainer = Trainer( - model=model, - args=training_args, - compute_metrics=compute_metrics, - train_dataset=train_dataset, - eval_dataset=val_dataset, -) - -# start training -trainer.train() -``` - -## Evaluation - -The following script evaluates the model on the test set of -CNN/Daily Mail. - -```python -#!/usr/bin/env python3 -import nlp -from transformers import BertTokenizer, EncoderDecoderModel -tokenizer = BertTokenizer.from_pretrained("patrickvonplaten/bert2bert-cnn_dailymail-fp16") -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/bert2bert-cnn_dailymail-fp16") -model.to("cuda") -test_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="test") -batch_size = 128 -# map data correctly -def generate_summary(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at BERT max length 512 - inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=512, return_tensors="pt") - input_ids = inputs.input_ids.to("cuda") - attention_mask = inputs.attention_mask.to("cuda") - outputs = model.generate(input_ids, attention_mask=attention_mask) - # all special tokens including will be removed - output_str = tokenizer.batch_decode(outputs, skip_special_tokens=True) - batch["pred"] = output_str - return batch -results = test_dataset.map(generate_summary, batched=True, batch_size=batch_size, remove_columns=["article"]) -# load rouge for validation -rouge = nlp.load_metric("rouge") -pred_str = results["pred"] -label_str = results["highlights"] -rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid -print(rouge_output) -``` - -The obtained results should be: - -| - | Rouge2 - mid -precision | Rouge2 - mid - recall | Rouge2 - mid - fmeasure | -|----------|:-------------:|:------:|:------:| -| **CNN/Daily Mail** | 16.12 | 17.07 | **16.1** | diff --git a/model_cards/patrickvonplaten/bert2bert_cnn_daily_mail/README.md b/model_cards/patrickvonplaten/bert2bert_cnn_daily_mail/README.md deleted file mode 100644 index 97f319b5ebe0..000000000000 --- a/model_cards/patrickvonplaten/bert2bert_cnn_daily_mail/README.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- cnn_dailymail -tags: -- summarization ---- - -Bert2Bert Summarization with 🤗EncoderDecoder Framework -This model is a warm-started *BERT2BERT* model fine-tuned on the *CNN/Dailymail* summarization dataset. - -The model achieves a **18.22** ROUGE-2 score on *CNN/Dailymail*'s test dataset. - -For more details on how the model was fine-tuned, please refer to -[this](https://colab.research.google.com/drive/1Ekd5pUeCX7VOrMx94_czTkwNtLN32Uyu?usp=sharing) notebook. diff --git a/model_cards/patrickvonplaten/bert2gpt2-cnn_dailymail-fp16/README.md b/model_cards/patrickvonplaten/bert2gpt2-cnn_dailymail-fp16/README.md deleted file mode 100644 index 3a3a157c0b03..000000000000 --- a/model_cards/patrickvonplaten/bert2gpt2-cnn_dailymail-fp16/README.md +++ /dev/null @@ -1,295 +0,0 @@ -# Bert2GPT2 Summarization with 🤗 EncoderDecoder Framework - -This model is a Bert2Bert model fine-tuned on summarization. - -Bert2GPT2 is a `EncoderDecoderModel`, meaning that the encoder is a `bert-base-uncased` -BERT model and the decoder is a `gpt2` GPT2 model. Leveraging the [EncoderDecoderFramework](https://huggingface.co/transformers/model_doc/encoderdecoder.html#encoder-decoder-models), the -two pretrained models can simply be loaded into the framework via: - -```python -bert2gpt2 = EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-uncased", "gpt2") -``` - -The decoder of an `EncoderDecoder` model needs cross-attention layers and usually makes use of causal -masking for auto-regressiv generation. -Thus, ``bert2gpt2`` is consequently fined-tuned on the `CNN/Daily Mail`dataset and the resulting model -`bert2gpt2-cnn_dailymail-fp16` is uploaded here. - -## Example - -The model is by no means a state-of-the-art model, but nevertheless -produces reasonable summarization results. It was mainly fine-tuned -as a proof-of-concept for the 🤗 EncoderDecoder Framework. - -The model can be used as follows: - -```python -from transformers import BertTokenizer, GPT2Tokenizer, EncoderDecoderModel - -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/bert2gpt2-cnn_dailymail-fp16") -# reuse tokenizer from bert2bert encoder-decoder model -bert_tokenizer = BertTokenizer.from_pretrained("patrickvonplaten/bert2bert-cnn_dailymail-fp16") - -article = """(CNN)Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing a racist chant. SAE's national chapter suspended the students, but University of Oklahoma President David B -oren took it a step further, saying the university's affiliation with the fraternity is permanently done. The news is shocking, but it's not the first time SAE has faced controversy. SAE was founded March 9, 185 -6, at the University of Alabama, five years before the American Civil War, according to the fraternity website. When the war began, the group had fewer than 400 members, of which "369 went to war for the Confede -rate States and seven for the Union Army," the website says. The fraternity now boasts more than 200,000 living alumni, along with about 15,000 undergraduates populating 219 chapters and 20 "colonies" seeking fu -ll membership at universities. SAE has had to work hard to change recently after a string of member deaths, many blamed on the hazing of new recruits, SAE national President Bradley Cohen wrote in a message on t -he fraternity's website. The fraternity's website lists more than 130 chapters cited or suspended for "health and safety incidents" since 2010. At least 30 of the incidents involved hazing, and dozens more invol -ved alcohol. However, the list is missing numerous incidents from recent months. Among them, according to various media outlets: Yale University banned the SAEs from campus activities last month after members al -legedly tried to interfere with a sexual misconduct investigation connected to an initiation rite. Stanford University in December suspended SAE housing privileges after finding sorority members attending a frat -ernity function were subjected to graphic sexual content. And Johns Hopkins University in November suspended the fraternity for underage drinking. "The media has labeled us as the 'nation's deadliest fraternity, -' " Cohen said. In 2011, for example, a student died while being coerced into excessive alcohol consumption, according to a lawsuit. SAE's previous insurer dumped the fraternity. "As a result, we are paying Lloy -d's of London the highest insurance rates in the Greek-letter world," Cohen said. Universities have turned down SAE's attempts to open new chapters, and the fraternity had to close 12 in 18 months over hazing in -cidents.""" - -input_ids = bert_tokenizer(article, return_tensors="pt").input_ids -output_ids = model.generate(input_ids) - -# we need a gpt2 tokenizer for the output word embeddings -gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2") - -print(gpt2_tokenizer.decode(output_ids[0], skip_special_tokens=True)) -# should produce -# SAE's national chapter suspended the students, but university president says it's permanent. -# The fraternity has had to deal with a string of incidents since 2010. -# SAE has more than 200,000 members, many of whom are students. -# A student died while being coerced into drinking alcohol. -``` - -## Training script: - -**IMPORTANT**: In order for this code to work, make sure you checkout to the branch -[more_general_trainer_metric](https://github.com/huggingface/transformers/tree/more_general_trainer_metric), which slightly adapts -the `Trainer` for `EncoderDecoderModels` according to this PR: https://github.com/huggingface/transformers/pull/5840. - -The following code shows the complete training script that was used to fine-tune `bert2gpt2-cnn_dailymail-fp16 -` for reproducability. The training last ~11h on a standard GPU. - -```python -#!/usr/bin/env python3 -import nlp -import logging -from transformers import BertTokenizer, GPT2Tokenizer, EncoderDecoderModel, Trainer, TrainingArguments - -logging.basicConfig(level=logging.INFO) - -model = EncoderDecoderModel.from_encoder_decoder_pretrained("bert-base-cased", "gpt2") -# cache is currently not supported by EncoderDecoder framework -model.decoder.config.use_cache = False -bert_tokenizer = BertTokenizer.from_pretrained("bert-base-cased") - -# CLS token will work as BOS token -bert_tokenizer.bos_token = bert_tokenizer.cls_token - -# SEP token will work as EOS token -bert_tokenizer.eos_token = bert_tokenizer.sep_token - - -# make sure GPT2 appends EOS in begin and end -def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): - outputs = [self.bos_token_id] + token_ids_0 + [self.eos_token_id] - return outputs - - -GPT2Tokenizer.build_inputs_with_special_tokens = build_inputs_with_special_tokens -gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2") -# set pad_token_id to unk_token_id -> be careful here as unk_token_id == eos_token_id == bos_token_id -gpt2_tokenizer.pad_token = gpt2_tokenizer.unk_token - - -# set decoding params -model.config.decoder_start_token_id = gpt2_tokenizer.bos_token_id -model.config.eos_token_id = gpt2_tokenizer.eos_token_id -model.config.max_length = 142 -model.config.min_length = 56 -model.config.no_repeat_ngram_size = 3 -model.early_stopping = True -model.length_penalty = 2.0 -model.num_beams = 4 - -# load train and validation data -train_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="train") -val_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="validation[:5%]") - -# load rouge for validation -rouge = nlp.load_metric("rouge", experiment_id=1) - -encoder_length = 512 -decoder_length = 128 -batch_size = 16 - - -# map data correctly -def map_to_encoder_decoder_inputs(batch): # Tokenizer will automatically set [BOS] [EOS] - # use bert tokenizer here for encoder - inputs = bert_tokenizer(batch["article"], padding="max_length", truncation=True, max_length=encoder_length) - # force summarization <= 128 - outputs = gpt2_tokenizer(batch["highlights"], padding="max_length", truncation=True, max_length=decoder_length) - - batch["input_ids"] = inputs.input_ids - batch["attention_mask"] = inputs.attention_mask - batch["decoder_input_ids"] = outputs.input_ids - batch["labels"] = outputs.input_ids.copy() - batch["decoder_attention_mask"] = outputs.attention_mask - - # complicated list comprehension here because pad_token_id alone is not good enough to know whether label should be excluded or not - batch["labels"] = [ - [-100 if mask == 0 else token for mask, token in mask_and_tokens] for mask_and_tokens in [zip(masks, labels) for masks, labels in zip(batch["decoder_attention_mask"], batch["labels"])] - ] - - assert all([len(x) == encoder_length for x in inputs.input_ids]) - assert all([len(x) == decoder_length for x in outputs.input_ids]) - - return batch - - -def compute_metrics(pred): - labels_ids = pred.label_ids - pred_ids = pred.predictions - - # all unnecessary tokens are removed - pred_str = gpt2_tokenizer.batch_decode(pred_ids, skip_special_tokens=True) - labels_ids[labels_ids == -100] = gpt2_tokenizer.eos_token_id - label_str = gpt2_tokenizer.batch_decode(labels_ids, skip_special_tokens=True) - - rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - - return { - "rouge2_precision": round(rouge_output.precision, 4), - "rouge2_recall": round(rouge_output.recall, 4), - "rouge2_fmeasure": round(rouge_output.fmeasure, 4), - } - - -# make train dataset ready -train_dataset = train_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -train_dataset.set_format( - type="torch", columns=["input_ids", "attention_mask", "decoder_input_ids", "decoder_attention_mask", "labels"], -) - -# same for validation dataset -val_dataset = val_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -val_dataset.set_format( - type="torch", columns=["input_ids", "attention_mask", "decoder_input_ids", "decoder_attention_mask", "labels"], -) - -# set training arguments - these params are not really tuned, feel free to change -training_args = TrainingArguments( - output_dir="./", - per_device_train_batch_size=batch_size, - per_device_eval_batch_size=batch_size, - predict_from_generate=True, - evaluate_during_training=True, - do_train=True, - do_eval=True, - logging_steps=1000, - save_steps=1000, - eval_steps=1000, - overwrite_output_dir=True, - warmup_steps=2000, - save_total_limit=10, - fp16=True, -) - -# instantiate trainer -trainer = Trainer( - model=model, - args=training_args, - compute_metrics=compute_metrics, - train_dataset=train_dataset, - eval_dataset=val_dataset, -) - -# start training -trainer.train() -``` - -## Evaluation - -The following script evaluates the model on the test set of -CNN/Daily Mail. - -```python -#!/usr/bin/env python3 -import nlp -from transformers import BertTokenizer, GPT2Tokenizer, EncoderDecoderModel - -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/bert2gpt2-cnn_dailymail-fp16") -model.to("cuda") - -bert_tokenizer = BertTokenizer.from_pretrained("bert-base-cased") - -# CLS token will work as BOS token -bert_tokenizer.bos_token = bert_tokenizer.cls_token - -# SEP token will work as EOS token -bert_tokenizer.eos_token = bert_tokenizer.sep_token - - -# make sure GPT2 appends EOS in begin and end -def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): - outputs = [self.bos_token_id] + token_ids_0 + [self.eos_token_id] - return outputs - - -GPT2Tokenizer.build_inputs_with_special_tokens = build_inputs_with_special_tokens -gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2") -# set pad_token_id to unk_token_id -> be careful here as unk_token_id == eos_token_id == bos_token_id -gpt2_tokenizer.pad_token = gpt2_tokenizer.unk_token - - -# set decoding params -model.config.decoder_start_token_id = gpt2_tokenizer.bos_token_id -model.config.eos_token_id = gpt2_tokenizer.eos_token_id -model.config.max_length = 142 -model.config.min_length = 56 -model.config.no_repeat_ngram_size = 3 -model.early_stopping = True -model.length_penalty = 2.0 -model.num_beams = 4 - -test_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="test") -batch_size = 64 - - -# map data correctly -def generate_summary(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at BERT max length 512 - inputs = bert_tokenizer(batch["article"], padding="max_length", truncation=True, max_length=512, return_tensors="pt") - input_ids = inputs.input_ids.to("cuda") - attention_mask = inputs.attention_mask.to("cuda") - - outputs = model.generate(input_ids, attention_mask=attention_mask) - - # all special tokens including will be removed - output_str = gpt2_tokenizer.batch_decode(outputs, skip_special_tokens=True) - - batch["pred"] = output_str - - return batch - - -results = test_dataset.map(generate_summary, batched=True, batch_size=batch_size, remove_columns=["article"]) - -# load rouge for validation -rouge = nlp.load_metric("rouge") - -pred_str = results["pred"] -label_str = results["highlights"] - -rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - -print(rouge_output) -``` - -The obtained results should be: - -| - | Rouge2 - mid -precision | Rouge2 - mid - recall | Rouge2 - mid - fmeasure | -|----------|:-------------:|:------:|:------:| -| **CNN/Daily Mail** | 14.42 | 16.99 | **15.16** | diff --git a/model_cards/patrickvonplaten/longformer2roberta-cnn_dailymail-fp16/README.md b/model_cards/patrickvonplaten/longformer2roberta-cnn_dailymail-fp16/README.md deleted file mode 100644 index 7b94152690ed..000000000000 --- a/model_cards/patrickvonplaten/longformer2roberta-cnn_dailymail-fp16/README.md +++ /dev/null @@ -1,252 +0,0 @@ -# Longformer2Roberta Summarization with 🤗 EncoderDecoder Framework - -This model is a Longformer2Roberta model fine-tuned on summarization. - -Longformer2Roberta is a `EncoderDecoderModel`, meaning that both the encoder is a `allenai/longformer-base-4096` model and the decoder is a `roberta-base` model. Leveraging the [EncoderDecoderFramework](https://huggingface.co/transformers/model_doc/encoderdecoder.html#encoder-decoder-models), the -two pretrained models can simply be loaded into the framework via: - -```python -roberta2roberta = EncoderDecoderModel.from_encoder_decoder_pretrained("allenai/longformer-base-4096", "roberta-base") -``` - -The decoder of an `EncoderDecoder` model needs cross-attention layers and usually makes use of causal -masking for auto-regressiv generation. -Thus, ``longformer2roberta`` is consequently fined-tuned on the `CNN/Daily Mail`dataset and the resulting model -`longformer2roberta-cnn_dailymail-fp16` is uploaded here. - -## Example - -The model is by no means a state-of-the-art model, but nevertheless -produces reasonable summarization results. It was mainly fine-tuned -as a proof-of-concept for the 🤗 EncoderDecoder Framework. - -The model can be used as follows: - -```python -from transformers import LongformerTokenizer, EncoderDecoderModel - -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/longformer2roberta-cnn_dailymail-fp16") -tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096") - -article = """(CNN)James Holmes made his introduction to the world in a Colorado cinema filled with spectators watching a midnight showing of the new Batman movie, "The Dark Knight Rises," in June 2012. The moment became one of the deadliest shootings in U.S. history. Holmes is accused of opening fire on the crowd, killing 12 people and injuring or maiming 70 others in Aurora, a suburb of Denver. Holmes appeared like a comic book character: He resembled the Joker, with red-orange hair, similar to the late actor Heath Ledger\'s portrayal of the villain in an earlier Batman movie, authorities said. But Holmes was hardly a cartoon. Authorities said he wore body armor and carried several guns, including an AR-15 rifle, with lots of ammo. He also wore a gas mask. Holmes says he was insane at the time of the shootings, and that is his legal defense and court plea: not guilty by reason of insanity. Prosecutors aren\'t swayed and will seek the death penalty. Opening statements in his trial are scheduled to begin Monday. Holmes admits to the shootings but says he was suffering "a psychotic episode" at the time, according to court papers filed in July 2013 by the state public defenders, Daniel King and Tamara A. Brady. Evidence "revealed thus far in the case supports the defense\'s position that Mr. Holmes suffers from a severe mental illness and was in the throes of a psychotic episode when he committed the acts that resulted in the tragic loss of life and injuries sustained by moviegoers on July 20, 2012," the public defenders wrote. Holmes no longer looks like a dazed Joker, as he did in his first appearance before a judge in 2012. He appeared dramatically different in January when jury selection began for his trial: 9,000 potential jurors were summoned for duty, described as one of the nation\'s largest jury calls. Holmes now has a cleaner look, with a mustache, button-down shirt and khaki pants. In January, he had a beard and eyeglasses. If this new image sounds like one of an academician, it may be because Holmes, now 27, once was one. Just before the shooting, Holmes was a doctoral student in neuroscience, and he was studying how the brain works, with his schooling funded by a U.S. government grant. Yet for all his learning, Holmes apparently lacked the capacity to command his own mind, according to the case against him. A jury will ultimately decide Holmes\' fate. That panel is made up of 12 jurors and 12 alternates. They are 19 women and five men, and almost all are white and middle-aged. The trial could last until autumn. When jury summonses were issued in January, each potential juror stood a 0.2% chance of being selected, District Attorney George Brauchler told the final jury this month. He described the approaching trial as "four to five months of a horrible roller coaster through the worst haunted house you can imagine." The jury will have to render verdicts on each of the 165 counts against Holmes, including murder and attempted murder charges. Meanwhile, victims and their relatives are challenging all media outlets "to stop the gratuitous use of the name and likeness of mass killers, thereby depriving violent individuals the media celebrity and media spotlight they so crave," the No Notoriety group says. They are joined by victims from eight other mass shootings in recent U.S. history. Raised in central coastal California and in San Diego, James Eagan Holmes is the son of a mathematician father noted for his work at the FICO firm that provides credit scores and a registered nurse mother, according to the U-T San Diego newspaper. Holmes also has a sister, Chris, a musician, who\'s five years younger, the newspaper said. His childhood classmates remember him as a clean-cut, bespectacled boy with an "exemplary" character who "never gave any trouble, and never got in trouble himself," The Salinas Californian reported. His family then moved down the California coast, where Holmes grew up in the San Diego-area neighborhood of Rancho Peñasquitos, which a neighbor described as "kind of like Mayberry," the San Diego newspaper said. Holmes attended Westview High School, which says its school district sits in "a primarily middle- to upper-middle-income residential community." There, Holmes ran cross-country, played soccer and later worked at a biotechnology internship at the Salk Institute and Miramar College, which attracts academically talented students. By then, his peers described him as standoffish and a bit of a wiseacre, the San Diego newspaper said. Holmes attended college fairly close to home, in a neighboring area known as Southern California\'s "inland empire" because it\'s more than an hour\'s drive from the coast, in a warm, low-desert climate. He entered the University of California, Riverside, in 2006 as a scholarship student. In 2008 he was a summer camp counselor for disadvantaged children, age 7 to 14, at Camp Max Straus, run by Jewish Big Brothers Big Sisters of Los Angeles. He graduated from UC Riverside in 2010 with the highest honors and a bachelor\'s degree in neuroscience. "Academically, he was at the top of the top," Chancellor Timothy P. White said. He seemed destined for even higher achievement. By 2011, he had enrolled as a doctoral student in the neuroscience program at the University of Colorado Anschutz Medical Campus in Aurora, the largest academic health center in the Rocky Mountain region. The doctoral in neuroscience program attended by Holmes focuses on how the brain works, with an emphasis on processing of information, behavior, learning and memory. Holmes was one of six pre-thesis Ph.D. students in the program who were awarded a neuroscience training grant from the National Institutes of Health. The grant rewards outstanding neuroscientists who will make major contributions to neurobiology. A syllabus that listed Holmes as a student at the medical school shows he was to have delivered a presentation about microRNA biomarkers. But Holmes struggled, and his own mental health took an ominous turn. In March 2012, he told a classmate he wanted to kill people, and that he would do so "when his life was over," court documents said. Holmes was "denied access to the school after June 12, 2012, after he made threats to a professor," according to court documents. About that time, Holmes was a patient of University of Colorado psychiatrist Lynne Fenton. Fenton was so concerned about Holmes\' behavior that she mentioned it to her colleagues, saying he could be a danger to others, CNN affiliate KMGH-TV reported, citing sources with knowledge of the investigation. Fenton\'s concerns surfaced in early June, sources told the Denver station. Holmes began to fantasize about killing "a lot of people" in early June, nearly six weeks before the shootings, the station reported, citing unidentified sources familiar with the investigation. Holmes\' psychiatrist contacted several members of a "behavioral evaluation and threat assessment" team to say Holmes could be a danger to others, the station reported. At issue was whether to order Holmes held for 72 hours to be evaluated by mental health professionals, the station reported. "Fenton made initial phone calls about engaging the BETA team" in "the first 10 days" of June, but it "never came together" because in the period Fenton was having conversations with team members, Holmes began the process of dropping out of school, a source told KMGH. Defense attorneys have rejected the prosecution\'s assertions that Holmes was barred from campus. Citing statements from the university, Holmes\' attorneys have argued that his access was revoked because that\'s normal procedure when a student drops enrollment. What caused this turn for the worse for Holmes has yet to be clearly detailed. In the months before the shooting, he bought four weapons and more than 6,000 rounds of ammunition, authorities said. Police said he also booby-trapped his third-floor apartment with explosives, but police weren\'t fooled. After Holmes was caught in the cinema parking lot immediately after the shooting, bomb technicians went to the apartment and neutralized the explosives. No one was injured at the apartment building. Nine minutes before Holmes went into the movie theater, he called a University of Colorado switchboard, public defender Brady has said in court. The number he called can be used to get in contact with faculty members during off hours, Brady said. Court documents have also revealed that investigators have obtained text messages that Holmes exchanged with someone before the shooting. That person was not named, and the content of the texts has not been made public. According to The New York Times, Holmes sent a text message to a fellow graduate student, a woman, about two weeks before the shooting. She asked if he had left Aurora yet, reported the newspaper, which didn\'t identify her. No, he had two months left on his lease, Holmes wrote back, according to the Times. He asked if she had heard of "dysphoric mania," a form of bipolar disorder marked by the highs of mania and the dark and sometimes paranoid delusions of major depression. The woman asked if the disorder could be managed with treatment. "It was," Holmes wrote her, according to the Times. But he warned she should stay away from him "because I am bad news," the newspaper reported. It was her last contact with Holmes. After the shooting, Holmes\' family issued a brief statement: "Our hearts go out to those who were involved in this tragedy and to the families and friends of those involved," they said, without giving any information about their son. Since then, prosecutors have refused to offer a plea deal to Holmes. For Holmes, "justice is death," said Brauchler, the district attorney. In December, Holmes\' parents, who will be attending the trial, issued another statement: They asked that their son\'s life be spared and that he be sent to an institution for mentally ill people for the rest of his life, if he\'s found not guilty by reason of insanity. "He is not a monster," Robert and Arlene Holmes wrote, saying the death penalty is "morally wrong, especially when the condemned is mentally ill." "He is a human being gripped by a severe mental illness," the parents said. The matter will be settled by the jury. CNN\'s Ana Cabrera and Sara Weisfeldt contributed to this report from Denver.""" - -input_ids = tokenizer(article, return_tensors="pt").input_ids -output_ids = model.generate(input_ids) - -print(tokenizer.decode(output_ids[0], skip_special_tokens=True)) -# should produce -# James Holmes, 27, is accused of opening fire on a Colorado theater. -# He was a doctoral student at University of Colorado. -# Holmes says he was suffering "a psychotic episode" at the time of the shooting. -# Prosecutors won't say whether Holmes was barred from campus. -``` - -Such an article has a length of > 2000 tokens, which means that it cannot be handled correctly by Bert or Roberta encoders. - -## Training script: - -**IMPORTANT**: In order for this code to work, make sure you checkout to the branch -[more_general_trainer_metric](https://github.com/huggingface/transformers/tree/more_general_trainer_metric), which slightly adapts -the `Trainer` for `EncoderDecoderModels` according to this PR: https://github.com/huggingface/transformers/pull/5840. - -The following code shows the complete training script that was used to fine-tune `longformer2roberta-cnn_dailymail-fp16 -` for reproducability. The training last ~90h on a standard GPU. - -```python -#!/usr/bin/env python3 -import nlp -import logging -from transformers import LongformerTokenizer, EncoderDecoderModel, Trainer, TrainingArguments - -logging.basicConfig(level=logging.INFO) - -model = EncoderDecoderModel.from_encoder_decoder_pretrained("allenai/longformer-base-4096", "roberta-base") -tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096") - -# load train and validation data -train_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="train") -val_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="validation[:5%]") - -# load rouge for validation -rouge = nlp.load_metric("rouge", experiment_id=0) - -# enable gradient checkpointing for longformer encoder -model.encoder.config.gradient_checkpointing = True - -# set decoding params -model.config.decoder_start_token_id = tokenizer.bos_token_id -model.config.eos_token_id = tokenizer.eos_token_id -model.config.max_length = 142 -model.config.min_length = 56 -model.config.no_repeat_ngram_size = 3 -model.early_stopping = True -model.length_penalty = 2.0 -model.num_beams = 4 - -encoder_length = 2048 -decoder_length = 128 -batch_size = 16 - - -# map data correctly -def map_to_encoder_decoder_inputs(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at Longformer at 2048 - inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=encoder_length) - # force summarization <= 128 - outputs = tokenizer(batch["highlights"], padding="max_length", truncation=True, max_length=decoder_length) - - batch["input_ids"] = inputs.input_ids - batch["attention_mask"] = inputs.attention_mask - - # set 128 tokens to global attention - batch["global_attention_mask"] = [[1 if i < 128 else 0 for i in range(sequence_length)] for sequence_length in len(inputs.input_ids) * [encoder_length]] - batch["decoder_input_ids"] = outputs.input_ids - batch["labels"] = outputs.input_ids.copy() - # mask loss for padding - batch["labels"] = [ - [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch["labels"] - ] - batch["decoder_attention_mask"] = outputs.attention_mask - - assert all([len(x) == encoder_length for x in inputs.input_ids]) - assert all([len(x) == decoder_length for x in outputs.input_ids]) - - return batch - - -def compute_metrics(pred): - labels_ids = pred.label_ids - pred_ids = pred.predictions - - # all unnecessary tokens are removed - pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True) - labels_ids[labels_ids == -100] = tokenizer.eos_token_id - label_str = tokenizer.batch_decode(labels_ids, skip_special_tokens=True) - - rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - - return { - "rouge2_precision": round(rouge_output.precision, 4), - "rouge2_recall": round(rouge_output.recall, 4), - "rouge2_fmeasure": round(rouge_output.fmeasure, 4), - } - - - return { - "rouge2_precision": round(rouge_output.precision, 4), - "rouge2_recall": round(rouge_output.recall, 4), - "rouge2_fmeasure": round(rouge_output.fmeasure, 4), - } - - -# make train dataset ready -train_dataset = train_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -train_dataset.set_format( - type="torch", columns=["input_ids", "attention_mask", "global_attention_mask", "decoder_input_ids", "decoder_attention_mask", "labels"], -) - -# same for validation dataset -val_dataset = val_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -val_dataset.set_format( - type="torch", columns=["input_ids", "global_attention_mask", "attention_mask", "decoder_input_ids", "decoder_attention_mask", "labels"], -) - -# set training arguments - these params are not really tuned, feel free to change -training_args = TrainingArguments( - output_dir="./", - per_device_train_batch_size=batch_size, - per_device_eval_batch_size=batch_size, - predict_from_generate=True, - evaluate_during_training=True, - do_train=True, - do_eval=True, - logging_steps=1000, - save_steps=1000, - eval_steps=1000, - overwrite_output_dir=True, - warmup_steps=2000, - save_total_limit=3, - fp16=True, -) - -# instantiate trainer -trainer = Trainer( - model=model, - args=training_args, - compute_metrics=compute_metrics, - train_dataset=train_dataset, - eval_dataset=val_dataset, -) - -# start training -trainer.train() -``` - -## Evaluation - -The following script evaluates the model on the test set of -CNN/Daily Mail. - -```python -#!/usr/bin/env python3 -import nlp -import torch -from transformers import LongformerTokenizer, EncoderDecoderModel - -tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096") -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/longformer2roberta-cnn_dailymail-fp16") -model.to("cuda") - -test_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="test") -batch_size = 32 - -encoder_length = 2048 -decoder_length = 128 - - -# map data correctly -def generate_summary(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at BERT max length 512 - inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=encoder_length, return_tensors="pt") - input_ids = inputs.input_ids.to("cuda") - attention_mask = inputs.attention_mask.to("cuda") - global_attention_mask = torch.zeros_like(attention_mask) - global_attention_mask[:, :decoder_length] = 1 - - outputs = model.generate(input_ids, attention_mask=attention_mask, global_attention_mask=global_attention_mask) - - # all special tokens including will be removed - output_str = tokenizer.batch_decode(outputs, skip_special_tokens=True) - - batch["pred"] = output_str - - return batch - - -results = test_dataset.map(generate_summary, batched=True, batch_size=batch_size, remove_columns=["article"]) - -# load rouge for validation -rouge = nlp.load_metric("rouge") - -pred_str = results["pred"] -label_str = results["highlights"] - -rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - -print(rouge_output) -``` - -The obtained results should be: - -| - | Rouge2 - mid -precision | Rouge2 - mid - recall | Rouge2 - mid - fmeasure | -|----------|:-------------:|:------:|:------:| -| **CNN/Daily Mail** | 12.39 | 15.05 | **13.21** | - -**Note** This model was trained to show how Longformer can be used as an Encoder model in a EncoderDecoder setup. -Better results are obtained for datasets of much longer inputs. diff --git a/model_cards/patrickvonplaten/roberta2roberta-cnn_dailymail-fp16/README.md b/model_cards/patrickvonplaten/roberta2roberta-cnn_dailymail-fp16/README.md deleted file mode 100644 index 3cfe85cd7079..000000000000 --- a/model_cards/patrickvonplaten/roberta2roberta-cnn_dailymail-fp16/README.md +++ /dev/null @@ -1,238 +0,0 @@ -# Roberta2Roberta Summarization with 🤗 EncoderDecoder Framework - -This model is a Roberta2Roberta model fine-tuned on summarization. - -Roberta2Roberta is a `EncoderDecoderModel`, meaning that both the encoder and the decoder are `roberta-base` -RoBERTa models. Leveraging the [EncoderDecoderFramework](https://huggingface.co/transformers/model_doc/encoderdecoder.html#encoder-decoder-models), the -two pretrained models can simply be loaded into the framework via: - -```python -roberta2roberta = EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base") -``` - -The decoder of an `EncoderDecoder` model needs cross-attention layers and usually makes use of causal -masking for auto-regressiv generation. -Thus, ``roberta2roberta`` is consequently fined-tuned on the `CNN/Daily Mail`dataset and the resulting model -`roberta2roberta-cnn_dailymail-fp16` is uploaded here. - -## Example - -The model is by no means a state-of-the-art model, but nevertheless -produces reasonable summarization results. It was mainly fine-tuned -as a proof-of-concept for the 🤗 EncoderDecoder Framework. - -The model can be used as follows: - -```python -from transformers import BertTokenizer, EncoderDecoderModel - -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/roberta2roberta-cnn_dailymail-fp16") -tokenizer = RobertaTokenizer.from_pretrained("roberta-base") - -article = """(CNN)Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing a racist chant. SAE's national chapter suspended the students, but University of Oklahoma President David B -oren took it a step further, saying the university's affiliation with the fraternity is permanently done. The news is shocking, but it's not the first time SAE has faced controversy. SAE was founded March 9, 185 -6, at the University of Alabama, five years before the American Civil War, according to the fraternity website. When the war began, the group had fewer than 400 members, of which "369 went to war for the Confede -rate States and seven for the Union Army," the website says. The fraternity now boasts more than 200,000 living alumni, along with about 15,000 undergraduates populating 219 chapters and 20 "colonies" seeking fu -ll membership at universities. SAE has had to work hard to change recently after a string of member deaths, many blamed on the hazing of new recruits, SAE national President Bradley Cohen wrote in a message on t -he fraternity's website. The fraternity's website lists more than 130 chapters cited or suspended for "health and safety incidents" since 2010. At least 30 of the incidents involved hazing, and dozens more invol -ved alcohol. However, the list is missing numerous incidents from recent months. Among them, according to various media outlets: Yale University banned the SAEs from campus activities last month after members al -legedly tried to interfere with a sexual misconduct investigation connected to an initiation rite. Stanford University in December suspended SAE housing privileges after finding sorority members attending a frat -ernity function were subjected to graphic sexual content. And Johns Hopkins University in November suspended the fraternity for underage drinking. "The media has labeled us as the 'nation's deadliest fraternity, -' " Cohen said. In 2011, for example, a student died while being coerced into excessive alcohol consumption, according to a lawsuit. SAE's previous insurer dumped the fraternity. "As a result, we are paying Lloy -d's of London the highest insurance rates in the Greek-letter world," Cohen said. Universities have turned down SAE's attempts to open new chapters, and the fraternity had to close 12 in 18 months over hazing in -cidents.""" - -input_ids = tokenizer(article, return_tensors="pt").input_ids -output_ids = model.generate(input_ids) - -print(tokenizer.decode(output_ids[0], skip_special_tokens=True)) -# should produce -# Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing racist chants. The fraternity's national chapter has had to close 12 in 18 months over hazing. -# Sigma has had more than 130 chapters in 18 states. University of Oklahoma president says fraternity has been "deteriorated". -``` - -## Training script: - -**IMPORTANT**: In order for this code to work, make sure you checkout to the branch -[more_general_trainer_metric](https://github.com/huggingface/transformers/tree/more_general_trainer_metric), which slightly adapts -the `Trainer` for `EncoderDecoderModels` according to this PR: https://github.com/huggingface/transformers/pull/5840. - -The following code shows the complete training script that was used to fine-tune `roberta2roberta-cnn_dailymail-fp16 -` for reproducability. The training last ~9h on a standard GPU. - -```python -#!/usr/bin/env python3 -import nlp -import logging -from transformers import RobertaTokenizer, EncoderDecoderModel, Trainer, TrainingArguments - -logging.basicConfig(level=logging.INFO) - -model = EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base") -tokenizer = RobertaTokenizer.from_pretrained("roberta-base") - -# load train and validation data -train_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="train") -val_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="validation[:5%]") - -# load rouge for validation -rouge = nlp.load_metric("rouge", experiment_id=0) - -# set decoding params -model.config.decoder_start_token_id = tokenizer.bos_token_id -model.config.eos_token_id = tokenizer.eos_token_id -model.config.max_length = 142 -model.config.min_length = 56 -model.config.no_repeat_ngram_size = 3 -model.early_stopping = True -model.length_penalty = 2.0 -model.num_beams = 4 - -encoder_length = 512 -decoder_length = 128 -batch_size = 16 - - -# map data correctly -def map_to_encoder_decoder_inputs(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at Longformer at 2048 - inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=encoder_length) - # force summarization <= 256 - outputs = tokenizer(batch["highlights"], padding="max_length", truncation=True, max_length=decoder_length) - - batch["input_ids"] = inputs.input_ids - batch["attention_mask"] = inputs.attention_mask - batch["decoder_input_ids"] = outputs.input_ids - batch["labels"] = outputs.input_ids.copy() - # mask loss for padding - batch["labels"] = [ - [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch["labels"] - ] - batch["decoder_attention_mask"] = outputs.attention_mask - - assert all([len(x) == encoder_length for x in inputs.input_ids]) - assert all([len(x) == decoder_length for x in outputs.input_ids]) - - return batch - - -def compute_metrics(pred): - labels_ids = pred.label_ids - pred_ids = pred.predictions - - # all unnecessary tokens are removed - pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True) - labels_ids[labels_ids == -100] = tokenizer.eos_token_id - label_str = tokenizer.batch_decode(labels_ids, skip_special_tokens=True) - - rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - - return { - "rouge2_precision": round(rouge_output.precision, 4), - "rouge2_recall": round(rouge_output.recall, 4), - "rouge2_fmeasure": round(rouge_output.fmeasure, 4), - } - - -# make train dataset ready -train_dataset = train_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -train_dataset.set_format( - type="torch", columns=["input_ids", "attention_mask", "decoder_attention_mask", "decoder_input_ids", "labels"], -) - -# same for validation dataset -val_dataset = val_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -val_dataset.set_format( - type="torch", columns=["input_ids", "decoder_attention_mask", "attention_mask", "decoder_input_ids", "labels"], -) - -# set training arguments - these params are not really tuned, feel free to change -training_args = TrainingArguments( - output_dir="./", - per_device_train_batch_size=batch_size, - per_device_eval_batch_size=batch_size, - predict_from_generate=True, - evaluate_during_training=True, - do_train=True, - do_eval=True, - logging_steps=1000, - save_steps=1000, - eval_steps=1000, - overwrite_output_dir=True, - warmup_steps=2000, - save_total_limit=3, - fp16=True, -) - -# instantiate trainer -trainer = Trainer( - model=model, - args=training_args, - compute_metrics=compute_metrics, - train_dataset=train_dataset, - eval_dataset=val_dataset, -) - -# start training -trainer.train() -``` - -## Evaluation - -The following script evaluates the model on the test set of -CNN/Daily Mail. - -```python -#!/usr/bin/env python3 -import nlp -from transformers import RobertaTokenizer, EncoderDecoderModel - -tokenizer = RobertaTokenizer.from_pretrained("roberta-base") -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/roberta2roberta-cnn_dailymail-fp16") -model.to("cuda") - -test_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="test") -batch_size = 128 - - -# map data correctly -def generate_summary(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at BERT max length 512 - inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=512, return_tensors="pt") - input_ids = inputs.input_ids.to("cuda") - attention_mask = inputs.attention_mask.to("cuda") - - outputs = model.generate(input_ids, attention_mask=attention_mask) - - # all special tokens including will be removed - output_str = tokenizer.batch_decode(outputs, skip_special_tokens=True) - - batch["pred"] = output_str - - return batch - - -results = test_dataset.map(generate_summary, batched=True, batch_size=batch_size, remove_columns=["article"]) - -# load rouge for validation -rouge = nlp.load_metric("rouge") - -pred_str = results["pred"] -label_str = results["highlights"] - -rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - -print(rouge_output) -``` - -The obtained results should be: - -| - | Rouge2 - mid -precision | Rouge2 - mid - recall | Rouge2 - mid - fmeasure | -|----------|:-------------:|:------:|:------:| -| **CNN/Daily Mail** | 15.79 | 19.05 | **16.79** | diff --git a/model_cards/patrickvonplaten/roberta2roberta-share-cnn_dailymail-fp16/README.md b/model_cards/patrickvonplaten/roberta2roberta-share-cnn_dailymail-fp16/README.md deleted file mode 100644 index 74cfa7ac5cbd..000000000000 --- a/model_cards/patrickvonplaten/roberta2roberta-share-cnn_dailymail-fp16/README.md +++ /dev/null @@ -1,238 +0,0 @@ -# Shared Roberta2Roberta Summarization with 🤗 EncoderDecoder Framework - -This model is a shared Roberta2Roberta model, meaning that the encoder and decoder weights are tied, fine-tuned on summarization. - -Roberta2Roberta is a `EncoderDecoderModel`, meaning that both the encoder and the decoder are `roberta-base` -RoBERTa models. In this setup the encoder and decoder weights are tied. Leveraging the [EncoderDecoderFramework](https://huggingface.co/transformers/model_doc/encoderdecoder.html#encoder-decoder-models), the -two pretrained models can simply be loaded into the framework via: - -```python -roberta2roberta = EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base", tie_encoder_decoder=True) -``` - -The decoder of an `EncoderDecoder` model needs cross-attention layers and usually makes use of causal -masking for auto-regressiv generation. -Thus, ``roberta2roberta`` is consequently fined-tuned on the `CNN/Daily Mail`dataset and the resulting model -`roberta2roberta-share-cnn_dailymail-fp16` is uploaded here. - -## Example - -The model is by no means a state-of-the-art model, but nevertheless -produces reasonable summarization results. It was mainly fine-tuned -as a proof-of-concept for the 🤗 EncoderDecoder Framework. - -The model can be used as follows: - -```python -from transformers import RobertaTokenizer, EncoderDecoderModel - -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/roberta2roberta-share-cnn_dailymail-fp16") -tokenizer = RobertaTokenizer.from_pretrained("roberta-base") - -article = """(CNN)Sigma Alpha Epsilon is under fire for a video showing party-bound fraternity members singing a racist chant. SAE's national chapter suspended the students, but University of Oklahoma President David B -oren took it a step further, saying the university's affiliation with the fraternity is permanently done. The news is shocking, but it's not the first time SAE has faced controversy. SAE was founded March 9, 185 -6, at the University of Alabama, five years before the American Civil War, according to the fraternity website. When the war began, the group had fewer than 400 members, of which "369 went to war for the Confede -rate States and seven for the Union Army," the website says. The fraternity now boasts more than 200,000 living alumni, along with about 15,000 undergraduates populating 219 chapters and 20 "colonies" seeking fu -ll membership at universities. SAE has had to work hard to change recently after a string of member deaths, many blamed on the hazing of new recruits, SAE national President Bradley Cohen wrote in a message on t -he fraternity's website. The fraternity's website lists more than 130 chapters cited or suspended for "health and safety incidents" since 2010. At least 30 of the incidents involved hazing, and dozens more invol -ved alcohol. However, the list is missing numerous incidents from recent months. Among them, according to various media outlets: Yale University banned the SAEs from campus activities last month after members al -legedly tried to interfere with a sexual misconduct investigation connected to an initiation rite. Stanford University in December suspended SAE housing privileges after finding sorority members attending a frat -ernity function were subjected to graphic sexual content. And Johns Hopkins University in November suspended the fraternity for underage drinking. "The media has labeled us as the 'nation's deadliest fraternity, -' " Cohen said. In 2011, for example, a student died while being coerced into excessive alcohol consumption, according to a lawsuit. SAE's previous insurer dumped the fraternity. "As a result, we are paying Lloy -d's of London the highest insurance rates in the Greek-letter world," Cohen said. Universities have turned down SAE's attempts to open new chapters, and the fraternity had to close 12 in 18 months over hazing in -cidents.""" - -input_ids = tokenizer(article, return_tensors="pt").input_ids -output_ids = model.generate(input_ids) - -print(tokenizer.decode(output_ids[0], skip_special_tokens=True)) -# should produce -# SAE's national chapter suspended after video shows party-bound fraternity members singing racist chant. University of Oklahoma president says university's affiliation with fraternity is permanently done. -# SAE has had to close 12 chapters since 2010 after members were killed in hazing. The fraternity has had more than 130 chapters in 18 months. -``` - -## Training script: - -**IMPORTANT**: In order for this code to work, make sure you checkout to the branch -[more_general_trainer_metric](https://github.com/huggingface/transformers/tree/more_general_trainer_metric), which slightly adapts -the `Trainer` for `EncoderDecoderModels` according to this PR: https://github.com/huggingface/transformers/pull/5840. - -The following code shows the complete training script that was used to fine-tune `roberta2roberta-cnn_dailymail-fp16 -` for reproducability. The training last ~9h on a standard GPU. - -```python -#!/usr/bin/env python3 -import nlp -import logging -from transformers import RobertaTokenizer, EncoderDecoderModel, Trainer, TrainingArguments - -logging.basicConfig(level=logging.INFO) - -model = EncoderDecoderModel.from_encoder_decoder_pretrained("roberta-base", "roberta-base", tie_encoder_decoder=True) -tokenizer = RobertaTokenizer.from_pretrained("roberta-base") - -# load train and validation data -train_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="train") -val_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="validation[:5%]") - -# load rouge for validation -rouge = nlp.load_metric("rouge", experiment_id=0) - -# set decoding params -model.config.decoder_start_token_id = tokenizer.bos_token_id -model.config.eos_token_id = tokenizer.eos_token_id -model.config.max_length = 142 -model.config.min_length = 56 -model.config.no_repeat_ngram_size = 3 -model.early_stopping = True -model.length_penalty = 2.0 -model.num_beams = 4 - -encoder_length = 512 -decoder_length = 128 -batch_size = 16 - - -# map data correctly -def map_to_encoder_decoder_inputs(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at Longformer at 2048 - inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=encoder_length) - # force summarization <= 256 - outputs = tokenizer(batch["highlights"], padding="max_length", truncation=True, max_length=decoder_length) - - batch["input_ids"] = inputs.input_ids - batch["attention_mask"] = inputs.attention_mask - batch["decoder_input_ids"] = outputs.input_ids - batch["labels"] = outputs.input_ids.copy() - # mask loss for padding - batch["labels"] = [ - [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch["labels"] - ] - batch["decoder_attention_mask"] = outputs.attention_mask - - assert all([len(x) == encoder_length for x in inputs.input_ids]) - assert all([len(x) == decoder_length for x in outputs.input_ids]) - - return batch - - -def compute_metrics(pred): - labels_ids = pred.label_ids - pred_ids = pred.predictions - - # all unnecessary tokens are removed - pred_str = tokenizer.batch_decode(pred_ids, skip_special_tokens=True) - labels_ids[labels_ids == -100] = tokenizer.eos_token_id - label_str = tokenizer.batch_decode(labels_ids, skip_special_tokens=True) - - rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - - return { - "rouge2_precision": round(rouge_output.precision, 4), - "rouge2_recall": round(rouge_output.recall, 4), - "rouge2_fmeasure": round(rouge_output.fmeasure, 4), - } - - -# make train dataset ready -train_dataset = train_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -train_dataset.set_format( - type="torch", columns=["input_ids", "attention_mask", "decoder_attention_mask", "decoder_input_ids", "labels"], -) - -# same for validation dataset -val_dataset = val_dataset.map( - map_to_encoder_decoder_inputs, batched=True, batch_size=batch_size, remove_columns=["article", "highlights"], -) -val_dataset.set_format( - type="torch", columns=["input_ids", "decoder_attention_mask", "attention_mask", "decoder_input_ids", "labels"], -) - -# set training arguments - these params are not really tuned, feel free to change -training_args = TrainingArguments( - output_dir="./", - per_device_train_batch_size=batch_size, - per_device_eval_batch_size=batch_size, - predict_from_generate=True, - evaluate_during_training=True, - do_train=True, - do_eval=True, - logging_steps=1000, - save_steps=1000, - eval_steps=1000, - overwrite_output_dir=True, - warmup_steps=2000, - save_total_limit=3, - fp16=True, -) - -# instantiate trainer -trainer = Trainer( - model=model, - args=training_args, - compute_metrics=compute_metrics, - train_dataset=train_dataset, - eval_dataset=val_dataset, -) - -# start training -trainer.train() -``` - -## Evaluation - -The following script evaluates the model on the test set of -CNN/Daily Mail. - -```python -#!/usr/bin/env python3 -import nlp -from transformers import RobertaTokenizer, EncoderDecoderModel - -tokenizer = RobertaTokenizer.from_pretrained("roberta-base") -model = EncoderDecoderModel.from_pretrained("patrickvonplaten/roberta2roberta-share-cnn_dailymail-fp16") -model.to("cuda") - -test_dataset = nlp.load_dataset("cnn_dailymail", "3.0.0", split="test") -batch_size = 128 - - -# map data correctly -def generate_summary(batch): - # Tokenizer will automatically set [BOS] [EOS] - # cut off at BERT max length 512 - inputs = tokenizer(batch["article"], padding="max_length", truncation=True, max_length=512, return_tensors="pt") - input_ids = inputs.input_ids.to("cuda") - attention_mask = inputs.attention_mask.to("cuda") - - outputs = model.generate(input_ids, attention_mask=attention_mask) - - # all special tokens including will be removed - output_str = tokenizer.batch_decode(outputs, skip_special_tokens=True) - - batch["pred"] = output_str - - return batch - - -results = test_dataset.map(generate_summary, batched=True, batch_size=batch_size, remove_columns=["article"]) - -# load rouge for validation -rouge = nlp.load_metric("rouge") - -pred_str = results["pred"] -label_str = results["highlights"] - -rouge_output = rouge.compute(predictions=pred_str, references=label_str, rouge_types=["rouge2"])["rouge2"].mid - -print(rouge_output) -``` - -The obtained results should be: - -| - | Rouge2 - mid -precision | Rouge2 - mid - recall | Rouge2 - mid - fmeasure | -|----------|:-------------:|:------:|:------:| -| **CNN/Daily Mail** | 15.6 | 18.79 | **16.59** | diff --git a/model_cards/patrickvonplaten/roberta_shared_bbc_xsum/README.md b/model_cards/patrickvonplaten/roberta_shared_bbc_xsum/README.md deleted file mode 100644 index 44b0a42a625c..000000000000 --- a/model_cards/patrickvonplaten/roberta_shared_bbc_xsum/README.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- xsum -tags: -- summarization ---- - -Shared RoBERTa2RoBERTa Summarization with 🤗EncoderDecoder Framework -This model is a warm-started *RoBERTaShared* model fine-tuned on the *BBC XSum* summarization dataset. - -The model achieves a **16.89** ROUGE-2 score on *BBC XSUM*'s test dataset. - -For more details on how the model was fine-tuned, please refer to -[this](https://colab.research.google.com/drive/1Ekd5pUeCX7VOrMx94_czTkwNtLN32Uyu?usp=sharing) notebook. diff --git a/model_cards/pdelobelle/robbert-v2-dutch-base/README.md b/model_cards/pdelobelle/robbert-v2-dutch-base/README.md deleted file mode 100644 index 84170035e0cf..000000000000 --- a/model_cards/pdelobelle/robbert-v2-dutch-base/README.md +++ /dev/null @@ -1,164 +0,0 @@ ---- -language: "nl" -thumbnail: "https://github.com/iPieter/RobBERT/raw/master/res/robbert_logo.png" -tags: -- Dutch -- RoBERTa -- RobBERT -license: mit -datasets: -- oscar -- Shuffled Dutch section of the OSCAR corpus (https://oscar-corpus.com/) ---- - -# RobBERT - -## Model description - -[RobBERT v2](https://github.com/iPieter/RobBERT) is a Dutch state-of-the-art [RoBERTa](https://arxiv.org/abs/1907.11692)-based language model. - -More detailled information can be found in the [RobBERT paper](https://arxiv.org/abs/2001.06286). - -## How to use - -```python -from transformers import RobertaTokenizer, RobertaForSequenceClassification -tokenizer = RobertaTokenizer.from_pretrained("pdelobelle/robbert-v2-dutch-base") -model = RobertaForSequenceClassification.from_pretrained("pdelobelle/robbert-v2-dutch-base") -``` - -## Performance Evaluation Results - -All experiments are described in more detail in our [paper](https://arxiv.org/abs/2001.06286). - -### Sentiment analysis -Predicting whether a review is positive or negative using the [Dutch Book Reviews Dataset](https://github.com/benjaminvdb/110kDBRD). - -| Model | Accuracy [%] | -|-------------------|--------------------------| -| ULMFiT | 93.8 | -| BERTje | 93.0 | -| RobBERT v2 | **95.1** | - -### Die/Dat (coreference resolution) - -We measured how well the models are able to do coreference resolution by predicting whether "die" or "dat" should be filled into a sentence. -For this, we used the [EuroParl corpus](https://www.statmt.org/europarl/). - -#### Finetuning on whole dataset - -| Model | Accuracy [%] | F1 [%] | -|-------------------|--------------------------|--------------| -| [Baseline](https://arxiv.org/abs/2001.02943) (LSTM) | | 75.03 | -| mBERT | 98.285 | 98.033 | -| BERTje | 98.268 | 98.014 | -| RobBERT v2 | **99.232** | **99.121** | - -#### Finetuning on 10K examples - -We also measured the performance using only 10K training examples. -This experiment clearly illustrates that RobBERT outperforms other models when there is little data available. - -| Model | Accuracy [%] | F1 [%] | -|-------------------|--------------------------|--------------| -| mBERT | 92.157 | 90.898 | -| BERTje | 93.096 | 91.279 | -| RobBERT v2 | **97.816** | **97.514** | - -#### Using zero-shot word masking task - -Since BERT models are pre-trained using the word masking task, we can use this to predict whether "die" or "dat" is more likely. -This experiment shows that RobBERT has internalised more information about Dutch than other models. - -| Model | Accuracy [%] | -|-------------------|--------------------------| -| ZeroR | 66.70 | -| mBERT | 90.21 | -| BERTje | 94.94 | -| RobBERT v2 | **98.75** | - -### Part-of-Speech Tagging. - -Using the [Lassy UD dataset](https://universaldependencies.org/treebanks/nl_lassysmall/index.html). - - -| Model | Accuracy [%] | -|-------------------|--------------------------| -| Frog | 91.7 | -| mBERT | **96.5** | -| BERTje | 96.3 | -| RobBERT v2 | 96.4 | - -Interestingly, we found that when dealing with **small data sets**, RobBERT v2 **significantly outperforms** other models. - -

- RobBERT's performance on smaller datasets -

- -### Named Entity Recognition - -Using the [CoNLL 2002 evaluation script](https://www.clips.uantwerpen.be/conll2002/ner/). - - -| Model | Accuracy [%] | -|-------------------|--------------------------| -| Frog | 57.31 | -| mBERT | **90.94** | -| BERT-NL | 89.7 | -| BERTje | 88.3 | -| RobBERT v2 | 89.08 | - - -## Training procedure - -We pre-trained RobBERT using the RoBERTa training regime. -We pre-trained our model on the Dutch section of the [OSCAR corpus](https://oscar-corpus.com/), a large multilingual corpus which was obtained by language classification in the Common Crawl corpus. -This Dutch corpus is 39GB large, with 6.6 billion words spread over 126 million lines of text, where each line could contain multiple sentences, thus using more data than concurrently developed Dutch BERT models. - - -RobBERT shares its architecture with [RoBERTa's base model](https://github.com/pytorch/fairseq/tree/master/examples/roberta), which itself is a replication and improvement over BERT. -Like BERT, it's architecture consists of 12 self-attention layers with 12 heads with 117M trainable parameters. -One difference with the original BERT model is due to the different pre-training task specified by RoBERTa, using only the MLM task and not the NSP task. -During pre-training, it thus only predicts which words are masked in certain positions of given sentences. -The training process uses the Adam optimizer with polynomial decay of the learning rate l_r=10^-6 and a ramp-up period of 1000 iterations, with hyperparameters beta_1=0.9 -and RoBERTa's default beta_2=0.98. -Additionally, a weight decay of 0.1 and a small dropout of 0.1 helps prevent the model from overfitting. - - -RobBERT was trained on a computing cluster with 4 Nvidia P100 GPUs per node, where the number of nodes was dynamically adjusted while keeping a fixed batch size of 8192 sentences. -At most 20 nodes were used (i.e. 80 GPUs), and the median was 5 nodes. -By using gradient accumulation, the batch size could be set independently of the number of GPUs available, in order to maximally utilize the cluster. -Using the [Fairseq library](https://github.com/pytorch/fairseq/tree/master/examples/roberta), the model trained for two epochs, which equals over 16k batches in total, which took about three days on the computing cluster. -In between training jobs on the computing cluster, 2 Nvidia 1080 Ti's also covered some parameter updates for RobBERT v2. - - -## Limitations and bias - -In the [RobBERT paper](https://arxiv.org/abs/2001.06286), we also investigated potential sources of bias in RobBERT. - -We found that the zeroshot model estimates the probability of *hij* (he) to be higher than *zij* (she) for most occupations in bleached template sentences, regardless of their actual job gender ratio in reality. - -

- RobBERT's performance on smaller datasets -

- -By augmenting the DBRB Dutch Book sentiment analysis dataset with the stated gender of the author of the review, we found that highly positive reviews written by women were generally more accurately detected by RobBERT as being positive than those written by men. - -

- RobBERT's performance on smaller datasets -

- - - -## BibTeX entry and citation info - -```bibtex -@misc{delobelle2020robbert, - title={RobBERT: a Dutch RoBERTa-based Language Model}, - author={Pieter Delobelle and Thomas Winters and Bettina Berendt}, - year={2020}, - eprint={2001.06286}, - archivePrefix={arXiv}, - primaryClass={cs.CL} -} -``` diff --git a/model_cards/pedropei/question-intimacy/README.md b/model_cards/pedropei/question-intimacy/README.md deleted file mode 100644 index 27b03bf66f38..000000000000 --- a/model_cards/pedropei/question-intimacy/README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -language: -- en -inference: false ---- diff --git a/model_cards/pierreguillou/gpt2-small-portuguese/README.md b/model_cards/pierreguillou/gpt2-small-portuguese/README.md deleted file mode 100644 index 010773eb0dd9..000000000000 --- a/model_cards/pierreguillou/gpt2-small-portuguese/README.md +++ /dev/null @@ -1,186 +0,0 @@ ---- -language: pt - -widget: -- text: "Quem era Jim Henson? Jim Henson era um" -- text: "Em um achado chocante, o cientista descobriu um" -- text: "Barack Hussein Obama II, nascido em 4 de agosto de 1961, é" -- text: "Corrida por vacina contra Covid-19 já tem" -license: mit -datasets: -- wikipedia ---- - -# GPorTuguese-2: a Language Model for Portuguese text generation (and more NLP tasks...) - -## Introduction - -GPorTuguese-2 (Portuguese GPT-2 small) is a state-of-the-art language model for Portuguese based on the GPT-2 small model. - -It was trained on Portuguese Wikipedia using **Transfer Learning and Fine-tuning techniques** in just over a day, on one GPU NVIDIA V100 32GB and with a little more than 1GB of training data. - -It is a proof-of-concept that it is possible to get a state-of-the-art language model in any language with low ressources. - -It was fine-tuned from the [English pre-trained GPT-2 small](https://huggingface.co/gpt2) using the Hugging Face libraries (Transformers and Tokenizers) wrapped into the [fastai v2](https://dev.fast.ai/) Deep Learning framework. All the fine-tuning fastai v2 techniques were used. - -It is now available on Hugging Face. For further information or requests, please go to "[Faster than training from scratch — Fine-tuning the English GPT-2 in any language with Hugging Face and fastai v2 (practical case with Portuguese)](https://medium.com/@pierre_guillou/faster-than-training-from-scratch-fine-tuning-the-english-gpt-2-in-any-language-with-hugging-f2ec05c98787)". - -## Model - -| Model | #params | Model file (pt/tf) | Arch. | Training /Validation data (text) | -|-------------------------|---------|--------------------|-------------|------------------------------------------| -| `gpt2-small-portuguese` | 124M | 487M / 475M | GPT-2 small | Portuguese Wikipedia (1.28 GB / 0.32 GB) | - -## Evaluation results -In a little more than a day (we only used one GPU NVIDIA V100 32GB; through a Distributed Data Parallel (DDP) training mode, we could have divided by three this time to 10 hours, just with 2 GPUs), we got a loss of 3.17, an **accuracy of 37.99%** and a **perplexity of 23.76** (see the validation results table below). - -| after ... epochs | loss | accuracy (%) | perplexity | time by epoch | cumulative time | -|------------------|------|--------------|------------|---------------|-----------------| -| 0 | 9.95 | 9.90 | 20950.94 | 00:00:00 | 00:00:00 | -| 1 | 3.64 | 32.52 | 38.12 | 5:48:31 | 5:48:31 | -| 2 | 3.30 | 36.29 | 27.16 | 5:38:18 | 11:26:49 | -| 3 | 3.21 | 37.46 | 24.71 | 6:20:51 | 17:47:40 | -| 4 | 3.19 | 37.74 | 24.21 | 6:06:29 | 23:54:09 | -| 5 | 3.17 | 37.99 | 23.76 | 6:16:22 | 30:10:31 | - -## GPT-2 - -*Note: information copied/pasted from [Model: gpt2 >> GPT-2](https://huggingface.co/gpt2#gpt-2)* - -Pretrained model on English language using a causal language modeling (CLM) objective. It was introduced in this [paper](https://d4mucfpksywv.cloudfront.net/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) and first released at this [page](https://openai.com/blog/better-language-models/) (February 14, 2019). - -Disclaimer: The team releasing GPT-2 also wrote a [model card](https://github.com/openai/gpt-2/blob/master/model_card.md) for their model. Content from this model card has been written by the Hugging Face team to complete the information they provided and give specific examples of bias. - -## Model description - -*Note: information copied/pasted from [Model: gpt2 >> Model description](https://huggingface.co/gpt2#model-description)* - -GPT-2 is a transformers model pretrained on a very large corpus of English data in a self-supervised fashion. This means it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of publicly available data) with an automatic process to generate inputs and labels from those texts. More precisely, it was trained to guess the next word in sentences. - -More precisely, inputs are sequences of continuous text of a certain length and the targets are the same sequence, shifted one token (word or piece of word) to the right. The model uses internally a mask-mechanism to make sure the predictions for the token `i` only uses the inputs from `1` to `i` but not the future tokens. - -This way, the model learns an inner representation of the English language that can then be used to extract features useful for downstream tasks. The model is best at what it was pretrained for however, which is generating texts from a prompt. - -## How to use GPorTuguese-2 with HuggingFace (PyTorch) - -The following code use PyTorch. To use TensorFlow, check the below corresponding paragraph. - -### Load GPorTuguese-2 and its sub-word tokenizer (Byte-level BPE) - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead -import torch - -tokenizer = AutoTokenizer.from_pretrained("pierreguillou/gpt2-small-portuguese") -model = AutoModelWithLMHead.from_pretrained("pierreguillou/gpt2-small-portuguese") - -# Get sequence length max of 1024 -tokenizer.model_max_length=1024 - -model.eval() # disable dropout (or leave in train mode to finetune) -``` - -### Generate one word - -```python -# input sequence -text = "Quem era Jim Henson? Jim Henson era um" -inputs = tokenizer(text, return_tensors="pt") - -# model output -outputs = model(**inputs, labels=inputs["input_ids"]) -loss, logits = outputs[:2] -predicted_index = torch.argmax(logits[0, -1, :]).item() -predicted_text = tokenizer.decode([predicted_index]) - -# results -print('input text:', text) -print('predicted text:', predicted_text) - -# input text: Quem era Jim Henson? Jim Henson era um -# predicted text: homem -``` - -### Generate one full sequence - -```python -# input sequence -text = "Quem era Jim Henson? Jim Henson era um" -inputs = tokenizer(text, return_tensors="pt") - -# model output using Top-k sampling text generation method -sample_outputs = model.generate(inputs.input_ids, - pad_token_id=50256, - do_sample=True, - max_length=50, # put the token number you want - top_k=40, - num_return_sequences=1) - -# generated sequence -for i, sample_output in enumerate(sample_outputs): - print(">> Generated text {}\n\n{}".format(i+1, tokenizer.decode(sample_output.tolist()))) - -# >> Generated text -# Quem era Jim Henson? Jim Henson era um executivo de televisão e diretor de um grande estúdio de cinema mudo chamado Selig, -# depois que o diretor de cinema mudo Georges Seuray dirigiu vários filmes para a Columbia e o estúdio. -``` - -## How to use GPorTuguese-2 with HuggingFace (TensorFlow) - -The following code use TensorFlow. To use PyTorch, check the above corresponding paragraph. - -### Load GPorTuguese-2 and its sub-word tokenizer (Byte-level BPE) - -```python -from transformers import AutoTokenizer, TFAutoModelWithLMHead -import tensorflow as tf - -tokenizer = AutoTokenizer.from_pretrained("pierreguillou/gpt2-small-portuguese") -model = TFAutoModelWithLMHead.from_pretrained("pierreguillou/gpt2-small-portuguese") - -# Get sequence length max of 1024 -tokenizer.model_max_length=1024 - -model.eval() # disable dropout (or leave in train mode to finetune) -``` - -### Generate one full sequence - -```python -# input sequence -text = "Quem era Jim Henson? Jim Henson era um" -inputs = tokenizer.encode(text, return_tensors="tf") - -# model output using Top-k sampling text generation method -outputs = model.generate(inputs, eos_token_id=50256, pad_token_id=50256, - do_sample=True, - max_length=40, - top_k=40) -print(tokenizer.decode(outputs[0])) - -# >> Generated text -# Quem era Jim Henson? Jim Henson era um amigo familiar da família. Ele foi contratado pelo seu pai -# para trabalhar como aprendiz no escritório de um escritório de impressão, e então começou a ganhar dinheiro - -``` - -## Limitations and bias - -The training data used for this model come from Portuguese Wikipedia. We know it contains a lot of unfiltered content from the internet, which is far from neutral. As the openAI team themselves point out in their model card: - -> Because large-scale language models like GPT-2 do not distinguish fact from fiction, we don’t support use-cases that require the generated text to be true. Additionally, language models like GPT-2 reflect the biases inherent to the systems they were trained on, so we do not recommend that they be deployed into systems that interact with humans > unless the deployers first carry out a study of biases relevant to the intended use-case. We found no statistically significant difference in gender, race, and religious bias probes between 774M and 1.5B, implying all versions of GPT-2 should be approached with similar levels of caution around use cases that are sensitive to biases around human attributes. - -## Author - -Portuguese GPT-2 small was trained and evaluated by [Pierre GUILLOU](https://www.linkedin.com/in/pierreguillou/) thanks to the computing power of the GPU (GPU NVIDIA V100 32 Go) of the [AI Lab](https://www.linkedin.com/company/ailab-unb/) (University of Brasilia) to which I am attached as an Associate Researcher in NLP and the participation of its directors in the definition of NLP strategy, Professors Fabricio Ataides Braz and Nilton Correia da Silva. - -## Citation -If you use our work, please cite: - -```bibtex -@inproceedings{pierre2020gpt2smallportuguese, - title={GPorTuguese-2 (Portuguese GPT-2 small): a Language Model for Portuguese text generation (and more NLP tasks...)}, - author={Pierre Guillou}, - year={2020} -} -``` diff --git a/model_cards/pradhyra/AWSBlogBert/README.md b/model_cards/pradhyra/AWSBlogBert/README.md deleted file mode 100644 index b182a398beeb..000000000000 --- a/model_cards/pradhyra/AWSBlogBert/README.md +++ /dev/null @@ -1,8 +0,0 @@ -This model is pre-trained on blog articles from AWS Blogs. - -## Pre-training corpora -The input text contains around 3000 blog articles on [AWS Blogs website](https://aws.amazon.com/blogs/) technical subject matter including AWS products, tools and tutorials. - -## Pre-training details -I picked a Roberta architecture for masked language modeling (6-layer, 768-hidden, 12-heads, 82M parameters) and its corresponding ByteLevelBPE tokenization strategy. I then followed HuggingFace's Transformers [blog post](https://huggingface.co/blog/how-to-train) to train the model. -I chose to follow the following training set-up: 28k training steps with batches of 64 sequences of length 512 with an initial learning rate 5e-5. The model acheived a training loss of 3.6 on the MLM task over 10 epochs. diff --git a/model_cards/pranavpsv/gpt2-genre-story-generator/README.md b/model_cards/pranavpsv/gpt2-genre-story-generator/README.md deleted file mode 100644 index 5900081debca..000000000000 --- a/model_cards/pranavpsv/gpt2-genre-story-generator/README.md +++ /dev/null @@ -1,32 +0,0 @@ - -# GPT2 Genre Based Story Generator - -## Model description - -GPT2 fine-tuned on genre-based story generation. - -## Intended uses - -Used to generate stories based on user inputted genre and starting prompts. - -## How to use - -#### Supported Genres -superhero, action, drama, horror, thriller, sci_fi -#### Input text format -\ \ Some optional text... - -**Example**: \ \ After discovering time travel, - -```python -# Example of usage -from transformers import pipeline - -story_gen = pipeline("text-generation", "pranavpsv/gpt2-genre-story-generator") -print(story_gen(" Batman")) - -``` - -## Training data - -Initialized with pre-trained weights of "gpt2" checkpoint. Fine-tuned the model on stories of various genres. diff --git a/model_cards/pvl/labse_bert/README.md b/model_cards/pvl/labse_bert/README.md deleted file mode 100644 index 56898861f32a..000000000000 --- a/model_cards/pvl/labse_bert/README.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -language: en -thumbnail: -tags: -- bert -- embeddings -license: Apache-2.0 ---- - -# LABSE BERT - -## Model description - -Model for "Language-agnostic BERT Sentence Embedding" paper from Fangxiaoyu Feng, Yinfei Yang, Daniel Cer, Naveen Arivazhagan, Wei Wang. Model available in [TensorFlow Hub](https://tfhub.dev/google/LaBSE/1). - -## Intended uses & limitations - -#### How to use - -```python -from transformers import AutoTokenizer, AutoModel -import torch - -# from sentence-transformers -def mean_pooling(model_output, attention_mask): - token_embeddings = model_output[0] #First element of model_output contains all token embeddings - input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() - sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1) - sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9) - return sum_embeddings / sum_mask - -tokenizer = AutoTokenizer.from_pretrained("pvl/labse_bert", do_lower_case=False) -model = AutoModel.from_pretrained("pvl/labse_bert") - -sentences = ['This framework generates embeddings for each input sentence', - 'Sentences are passed as a list of string.', - 'The quick brown fox jumps over the lazy dog.'] - -encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=128, return_tensors='pt') - -with torch.no_grad(): - model_output = model(**encoded_input) - -sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask']) - - -``` diff --git a/model_cards/qarib/bert-base-qarib60_1790k/README.md b/model_cards/qarib/bert-base-qarib60_1790k/README.md deleted file mode 100644 index ae68b7846405..000000000000 --- a/model_cards/qarib/bert-base-qarib60_1790k/README.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -language: ar -tags: -- qarib - -license: apache-2.0 -datasets: -- Arabic GigaWord -- Abulkhair Arabic Corpus -- opus -- Twitter data ---- - -# QARiB: QCRI Arabic and Dialectal BERT - -## About QARiB -QCRI Arabic and Dialectal BERT (QARiB) model, was trained on a collection of ~ 420 Million tweets and ~ 180 Million sentences of text. -For Tweets, the data was collected using twitter API and using language filter. `lang:ar`. For Text data, it was a combination from -[Arabic GigaWord](url), [Abulkhair Arabic Corpus]() and [OPUS](http://opus.nlpl.eu/). - -### bert-base-qarib60_1790k -- Data size: 60Gb -- Number of Iterations: 1790k -- Loss: 1.8764963 - -## Training QARiB -The training of the model has been performed using Google’s original Tensorflow code on Google Cloud TPU v2. -We used a Google Cloud Storage bucket, for persistent storage of training data and models. -See more details in [Training QARiB](../Training_QARiB.md) - -## Using QARiB - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to be fine-tuned on a downstream task. See the model hub to look for fine-tuned versions on a task that interests you. For more details, see [Using QARiB](../Using_QARiB.md) - -### How to use -You can use this model directly with a pipeline for masked language modeling: - -```python ->>>from transformers import pipeline ->>>fill_mask = pipeline("fill-mask", model="./models/data60gb_86k") - ->>> fill_mask("شو عندكم يا [MASK]") -[{'sequence': '[CLS] شو عندكم يا عرب [SEP]', 'score': 0.0990147516131401, 'token': 2355, 'token_str': 'عرب'}, -{'sequence': '[CLS] شو عندكم يا جماعة [SEP]', 'score': 0.051633741706609726, 'token': 2308, 'token_str': 'جماعة'}, -{'sequence': '[CLS] شو عندكم يا شباب [SEP]', 'score': 0.046871256083250046, 'token': 939, 'token_str': 'شباب'}, -{'sequence': '[CLS] شو عندكم يا رفاق [SEP]', 'score': 0.03598872944712639, 'token': 7664, 'token_str': 'رفاق'}, -{'sequence': '[CLS] شو عندكم يا ناس [SEP]', 'score': 0.031996358186006546, 'token': 271, 'token_str': 'ناس'}] - ->>> fill_mask("قللي وشفيييك يرحم [MASK]") -[{'sequence': '[CLS] قللي وشفيييك يرحم والديك [SEP]', 'score': 0.4152909517288208, 'token': 9650, 'token_str': 'والديك'}, -{'sequence': '[CLS] قللي وشفيييك يرحملي [SEP]', 'score': 0.07663793861865997, 'token': 294, 'token_str': '##لي'}, -{'sequence': '[CLS] قللي وشفيييك يرحم حالك [SEP]', 'score': 0.0453166700899601, 'token': 2663, 'token_str': 'حالك'}, -{'sequence': '[CLS] قللي وشفيييك يرحم امك [SEP]', 'score': 0.04390475153923035, 'token': 1942, 'token_str': 'امك'}, -{'sequence': '[CLS] قللي وشفيييك يرحمونك [SEP]', 'score': 0.027349254116415977, 'token': 3283, 'token_str': '##ونك'}] - ->>> fill_mask("وقام المدير [MASK]") -[ -{'sequence': '[CLS] وقام المدير بالعمل [SEP]', 'score': 0.0678194984793663, 'token': 4230, 'token_str': 'بالعمل'}, -{'sequence': '[CLS] وقام المدير بذلك [SEP]', 'score': 0.05191086605191231, 'token': 984, 'token_str': 'بذلك'}, -{'sequence': '[CLS] وقام المدير بالاتصال [SEP]', 'score': 0.045264165848493576, 'token': 26096, 'token_str': 'بالاتصال'}, -{'sequence': '[CLS] وقام المدير بعمله [SEP]', 'score': 0.03732728958129883, 'token': 40486, 'token_str': 'بعمله'}, -{'sequence': '[CLS] وقام المدير بالامر [SEP]', 'score': 0.0246378555893898, 'token': 29124, 'token_str': 'بالامر'} -] ->>> fill_mask("وقامت المديرة [MASK]") - -[{'sequence': '[CLS] وقامت المديرة بذلك [SEP]', 'score': 0.23992691934108734, 'token': 984, 'token_str': 'بذلك'}, -{'sequence': '[CLS] وقامت المديرة بالامر [SEP]', 'score': 0.108805812895298, 'token': 29124, 'token_str': 'بالامر'}, -{'sequence': '[CLS] وقامت المديرة بالعمل [SEP]', 'score': 0.06639821827411652, 'token': 4230, 'token_str': 'بالعمل'}, -{'sequence': '[CLS] وقامت المديرة بالاتصال [SEP]', 'score': 0.05613093823194504, 'token': 26096, 'token_str': 'بالاتصال'}, -{'sequence': '[CLS] وقامت المديرة المديرة [SEP]', 'score': 0.021778125315904617, 'token': 41635, 'token_str': 'المديرة'}] -``` -## Training procedure - -The training of the model has been performed using Google’s original Tensorflow code on eight core Google Cloud TPU v2. -We used a Google Cloud Storage bucket, for persistent storage of training data and models. - -## Eval results - -We evaluated QARiB models on five NLP downstream task: -- Sentiment Analysis -- Emotion Detection -- Named-Entity Recognition (NER) -- Offensive Language Detection -- Dialect Identification - -The results obtained from QARiB models outperforms multilingual BERT/AraBERT/ArabicBERT. - - -## Model Weights and Vocab Download -TBD - -## Contacts - -Ahmed Abdelali, Sabit Hassan, Hamdy Mubarak, Kareem Darwish and Younes Samih - - diff --git a/model_cards/qarib/bert-base-qarib60_1970k/README.md b/model_cards/qarib/bert-base-qarib60_1970k/README.md deleted file mode 100644 index 44c0328896ad..000000000000 --- a/model_cards/qarib/bert-base-qarib60_1970k/README.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -language: ar -tags: -- qarib - -license: apache-2.0 -datasets: -- Arabic GigaWord -- Abulkhair Arabic Corpus -- opus -- Twitter data ---- - -# QARiB: QCRI Arabic and Dialectal BERT - -## About QARiB -QCRI Arabic and Dialectal BERT (QARiB) model, was trained on a collection of ~ 420 Million tweets and ~ 180 Million sentences of text. -For Tweets, the data was collected using twitter API and using language filter. `lang:ar`. For Text data, it was a combination from -[Arabic GigaWord](url), [Abulkhair Arabic Corpus]() and [OPUS](http://opus.nlpl.eu/). - -### bert-base-qarib60_1970k -- Data size: 60Gb -- Number of Iterations: 1970k -- Loss: 1.5708898 - -## Training QARiB -The training of the model has been performed using Google’s original Tensorflow code on Google Cloud TPU v2. -We used a Google Cloud Storage bucket, for persistent storage of training data and models. -See more details in [Training QARiB](../Training_QARiB.md) - -## Using QARiB - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to be fine-tuned on a downstream task. See the model hub to look for fine-tuned versions on a task that interests you. For more details, see [Using QARiB](../Using_QARiB.md) - -### How to use -You can use this model directly with a pipeline for masked language modeling: - -```python ->>>from transformers import pipeline ->>>fill_mask = pipeline("fill-mask", model="./models/data60gb_86k") - ->>> fill_mask("شو عندكم يا [MASK]") -[{'sequence': '[CLS] شو عندكم يا عرب [SEP]', 'score': 0.0990147516131401, 'token': 2355, 'token_str': 'عرب'}, -{'sequence': '[CLS] شو عندكم يا جماعة [SEP]', 'score': 0.051633741706609726, 'token': 2308, 'token_str': 'جماعة'}, -{'sequence': '[CLS] شو عندكم يا شباب [SEP]', 'score': 0.046871256083250046, 'token': 939, 'token_str': 'شباب'}, -{'sequence': '[CLS] شو عندكم يا رفاق [SEP]', 'score': 0.03598872944712639, 'token': 7664, 'token_str': 'رفاق'}, -{'sequence': '[CLS] شو عندكم يا ناس [SEP]', 'score': 0.031996358186006546, 'token': 271, 'token_str': 'ناس'}] - ->>> fill_mask("قللي وشفيييك يرحم [MASK]") -[{'sequence': '[CLS] قللي وشفيييك يرحم والديك [SEP]', 'score': 0.4152909517288208, 'token': 9650, 'token_str': 'والديك'}, -{'sequence': '[CLS] قللي وشفيييك يرحملي [SEP]', 'score': 0.07663793861865997, 'token': 294, 'token_str': '##لي'}, -{'sequence': '[CLS] قللي وشفيييك يرحم حالك [SEP]', 'score': 0.0453166700899601, 'token': 2663, 'token_str': 'حالك'}, -{'sequence': '[CLS] قللي وشفيييك يرحم امك [SEP]', 'score': 0.04390475153923035, 'token': 1942, 'token_str': 'امك'}, -{'sequence': '[CLS] قللي وشفيييك يرحمونك [SEP]', 'score': 0.027349254116415977, 'token': 3283, 'token_str': '##ونك'}] - ->>> fill_mask("وقام المدير [MASK]") -[ -{'sequence': '[CLS] وقام المدير بالعمل [SEP]', 'score': 0.0678194984793663, 'token': 4230, 'token_str': 'بالعمل'}, -{'sequence': '[CLS] وقام المدير بذلك [SEP]', 'score': 0.05191086605191231, 'token': 984, 'token_str': 'بذلك'}, -{'sequence': '[CLS] وقام المدير بالاتصال [SEP]', 'score': 0.045264165848493576, 'token': 26096, 'token_str': 'بالاتصال'}, -{'sequence': '[CLS] وقام المدير بعمله [SEP]', 'score': 0.03732728958129883, 'token': 40486, 'token_str': 'بعمله'}, -{'sequence': '[CLS] وقام المدير بالامر [SEP]', 'score': 0.0246378555893898, 'token': 29124, 'token_str': 'بالامر'} -] ->>> fill_mask("وقامت المديرة [MASK]") - -[{'sequence': '[CLS] وقامت المديرة بذلك [SEP]', 'score': 0.23992691934108734, 'token': 984, 'token_str': 'بذلك'}, -{'sequence': '[CLS] وقامت المديرة بالامر [SEP]', 'score': 0.108805812895298, 'token': 29124, 'token_str': 'بالامر'}, -{'sequence': '[CLS] وقامت المديرة بالعمل [SEP]', 'score': 0.06639821827411652, 'token': 4230, 'token_str': 'بالعمل'}, -{'sequence': '[CLS] وقامت المديرة بالاتصال [SEP]', 'score': 0.05613093823194504, 'token': 26096, 'token_str': 'بالاتصال'}, -{'sequence': '[CLS] وقامت المديرة المديرة [SEP]', 'score': 0.021778125315904617, 'token': 41635, 'token_str': 'المديرة'}] -``` -## Training procedure - -The training of the model has been performed using Google’s original Tensorflow code on eight core Google Cloud TPU v2. -We used a Google Cloud Storage bucket, for persistent storage of training data and models. - -## Eval results - -We evaluated QARiB models on five NLP downstream task: -- Sentiment Analysis -- Emotion Detection -- Named-Entity Recognition (NER) -- Offensive Language Detection -- Dialect Identification - -The results obtained from QARiB models outperforms multilingual BERT/AraBERT/ArabicBERT. - - -## Model Weights and Vocab Download -TBD - -## Contacts - -Ahmed Abdelali, Sabit Hassan, Hamdy Mubarak, Kareem Darwish and Younes Samih - - diff --git a/model_cards/qarib/bert-base-qarib60_860k/README.md b/model_cards/qarib/bert-base-qarib60_860k/README.md deleted file mode 100644 index 4924489b6939..000000000000 --- a/model_cards/qarib/bert-base-qarib60_860k/README.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -language: ar -tags: -- qarib - -license: apache-2.0 -datasets: -- Arabic GigaWord -- Abulkhair Arabic Corpus -- opus -- Twitter data ---- - -# QARiB: QCRI Arabic and Dialectal BERT - -## About QARiB -QCRI Arabic and Dialectal BERT (QARiB) model, was trained on a collection of ~ 420 Million tweets and ~ 180 Million sentences of text. -For Tweets, the data was collected using twitter API and using language filter. `lang:ar`. For Text data, it was a combination from -[Arabic GigaWord](url), [Abulkhair Arabic Corpus]() and [OPUS](http://opus.nlpl.eu/). - -### bert-base-qarib60_860k -- Data size: 60Gb -- Number of Iterations: 860k -- Loss: 2.2454472 - -## Training QARiB -The training of the model has been performed using Google’s original Tensorflow code on Google Cloud TPU v2. -We used a Google Cloud Storage bucket, for persistent storage of training data and models. -See more details in [Training QARiB](../Training_QARiB.md) - -## Using QARiB - -You can use the raw model for either masked language modeling or next sentence prediction, but it's mostly intended to be fine-tuned on a downstream task. See the model hub to look for fine-tuned versions on a task that interests you. For more details, see [Using QARiB](../Using_QARiB.md) - -### How to use -You can use this model directly with a pipeline for masked language modeling: - -```python ->>>from transformers import pipeline ->>>fill_mask = pipeline("fill-mask", model="./models/data60gb_86k") - ->>> fill_mask("شو عندكم يا [MASK]") -[{'sequence': '[CLS] شو عندكم يا عرب [SEP]', 'score': 0.0990147516131401, 'token': 2355, 'token_str': 'عرب'}, -{'sequence': '[CLS] شو عندكم يا جماعة [SEP]', 'score': 0.051633741706609726, 'token': 2308, 'token_str': 'جماعة'}, -{'sequence': '[CLS] شو عندكم يا شباب [SEP]', 'score': 0.046871256083250046, 'token': 939, 'token_str': 'شباب'}, -{'sequence': '[CLS] شو عندكم يا رفاق [SEP]', 'score': 0.03598872944712639, 'token': 7664, 'token_str': 'رفاق'}, -{'sequence': '[CLS] شو عندكم يا ناس [SEP]', 'score': 0.031996358186006546, 'token': 271, 'token_str': 'ناس'}] - ->>> fill_mask("قللي وشفيييك يرحم [MASK]") -[{'sequence': '[CLS] قللي وشفيييك يرحم والديك [SEP]', 'score': 0.4152909517288208, 'token': 9650, 'token_str': 'والديك'}, -{'sequence': '[CLS] قللي وشفيييك يرحملي [SEP]', 'score': 0.07663793861865997, 'token': 294, 'token_str': '##لي'}, -{'sequence': '[CLS] قللي وشفيييك يرحم حالك [SEP]', 'score': 0.0453166700899601, 'token': 2663, 'token_str': 'حالك'}, -{'sequence': '[CLS] قللي وشفيييك يرحم امك [SEP]', 'score': 0.04390475153923035, 'token': 1942, 'token_str': 'امك'}, -{'sequence': '[CLS] قللي وشفيييك يرحمونك [SEP]', 'score': 0.027349254116415977, 'token': 3283, 'token_str': '##ونك'}] - ->>> fill_mask("وقام المدير [MASK]") -[ -{'sequence': '[CLS] وقام المدير بالعمل [SEP]', 'score': 0.0678194984793663, 'token': 4230, 'token_str': 'بالعمل'}, -{'sequence': '[CLS] وقام المدير بذلك [SEP]', 'score': 0.05191086605191231, 'token': 984, 'token_str': 'بذلك'}, -{'sequence': '[CLS] وقام المدير بالاتصال [SEP]', 'score': 0.045264165848493576, 'token': 26096, 'token_str': 'بالاتصال'}, -{'sequence': '[CLS] وقام المدير بعمله [SEP]', 'score': 0.03732728958129883, 'token': 40486, 'token_str': 'بعمله'}, -{'sequence': '[CLS] وقام المدير بالامر [SEP]', 'score': 0.0246378555893898, 'token': 29124, 'token_str': 'بالامر'} -] ->>> fill_mask("وقامت المديرة [MASK]") - -[{'sequence': '[CLS] وقامت المديرة بذلك [SEP]', 'score': 0.23992691934108734, 'token': 984, 'token_str': 'بذلك'}, -{'sequence': '[CLS] وقامت المديرة بالامر [SEP]', 'score': 0.108805812895298, 'token': 29124, 'token_str': 'بالامر'}, -{'sequence': '[CLS] وقامت المديرة بالعمل [SEP]', 'score': 0.06639821827411652, 'token': 4230, 'token_str': 'بالعمل'}, -{'sequence': '[CLS] وقامت المديرة بالاتصال [SEP]', 'score': 0.05613093823194504, 'token': 26096, 'token_str': 'بالاتصال'}, -{'sequence': '[CLS] وقامت المديرة المديرة [SEP]', 'score': 0.021778125315904617, 'token': 41635, 'token_str': 'المديرة'}] -``` -## Training procedure - -The training of the model has been performed using Google’s original Tensorflow code on eight core Google Cloud TPU v2. -We used a Google Cloud Storage bucket, for persistent storage of training data and models. - -## Eval results - -We evaluated QARiB models on five NLP downstream task: -- Sentiment Analysis -- Emotion Detection -- Named-Entity Recognition (NER) -- Offensive Language Detection -- Dialect Identification - -The results obtained from QARiB models outperforms multilingual BERT/AraBERT/ArabicBERT. - - -## Model Weights and Vocab Download -TBD - -## Contacts - -Ahmed Abdelali, Sabit Hassan, Hamdy Mubarak, Kareem Darwish and Younes Samih - - diff --git a/model_cards/ramsrigouthamg/t5_paraphraser/README.md b/model_cards/ramsrigouthamg/t5_paraphraser/README.md deleted file mode 100644 index 7816c0dd087d..000000000000 --- a/model_cards/ramsrigouthamg/t5_paraphraser/README.md +++ /dev/null @@ -1,84 +0,0 @@ -## Model in Action 🚀 - -```python -import torch -from transformers import T5ForConditionalGeneration,T5Tokenizer - - -def set_seed(seed): - torch.manual_seed(seed) - if torch.cuda.is_available(): - torch.cuda.manual_seed_all(seed) - -set_seed(42) - -model = T5ForConditionalGeneration.from_pretrained('ramsrigouthamg/t5_paraphraser') -tokenizer = T5Tokenizer.from_pretrained('ramsrigouthamg/t5_paraphraser') - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -print ("device ",device) -model = model.to(device) - -sentence = "Which course should I take to get started in data science?" -# sentence = "What are the ingredients required to bake a perfect cake?" -# sentence = "What is the best possible approach to learn aeronautical engineering?" -# sentence = "Do apples taste better than oranges in general?" - - -text = "paraphrase: " + sentence + " " - - -max_len = 256 - -encoding = tokenizer.encode_plus(text,pad_to_max_length=True, return_tensors="pt") -input_ids, attention_masks = encoding["input_ids"].to(device), encoding["attention_mask"].to(device) - - -# set top_k = 50 and set top_p = 0.95 and num_return_sequences = 3 -beam_outputs = model.generate( - input_ids=input_ids, attention_mask=attention_masks, - do_sample=True, - max_length=256, - top_k=120, - top_p=0.98, - early_stopping=True, - num_return_sequences=10 -) - - -print ("\nOriginal Question ::") -print (sentence) -print ("\n") -print ("Paraphrased Questions :: ") -final_outputs =[] -for beam_output in beam_outputs: - sent = tokenizer.decode(beam_output, skip_special_tokens=True,clean_up_tokenization_spaces=True) - if sent.lower() != sentence.lower() and sent not in final_outputs: - final_outputs.append(sent) - -for i, final_output in enumerate(final_outputs): - print("{}: {}".format(i, final_output)) - -``` -## Output -``` -Original Question :: -Which course should I take to get started in data science? - - -Paraphrased Questions :: -0: What should I learn to become a data scientist? -1: How do I get started with data science? -2: How would you start a data science career? -3: How can I start learning data science? -4: How do you get started in data science? -5: What's the best course for data science? -6: Which course should I start with for data science? -7: What courses should I follow to get started in data science? -8: What degree should be taken by a data scientist? -9: Which course should I follow to become a Data Scientist? -``` - -## Detailed blog post available here : -https://towardsdatascience.com/paraphrase-any-question-with-t5-text-to-text-transfer-transformer-pretrained-model-and-cbb9e35f1555 - diff --git a/model_cards/rdenadai/BR_BERTo/README.md b/model_cards/rdenadai/BR_BERTo/README.md deleted file mode 100644 index 594df42bf5eb..000000000000 --- a/model_cards/rdenadai/BR_BERTo/README.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -language: pt -tags: -- portuguese -- brazil -- pt_BR -widget: -- text: gostei muito dessa ---- - -# BR_BERTo - -Portuguese (Brazil) model for text inference. - -## Params - -Trained on a corpus of 6_993_330 sentences. - -- Vocab size: 150_000 -- RobertaForMaskedLM size : 512 -- Num train epochs: 3 -- Time to train: ~10days (on GCP with a Nvidia T4) - -I follow the great tutorial from HuggingFace team: - -[How to train a new language model from scratch using Transformers and Tokenizers](https://huggingface.co/blog/how-to-train) - -More infor here: - -[BR_BERTo](https://github.com/rdenadai/BR-BERTo) diff --git a/model_cards/redewiedergabe/bert-base-historical-german-rw-cased/README.md b/model_cards/redewiedergabe/bert-base-historical-german-rw-cased/README.md deleted file mode 100644 index c85ac02b4f60..000000000000 --- a/model_cards/redewiedergabe/bert-base-historical-german-rw-cased/README.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -language: de ---- - -# Model description -## Dataset -Trained on fictional and non-fictional German texts written between 1840 and 1920: -* Narrative texts from Digitale Bibliothek (https://textgrid.de/digitale-bibliothek) -* Fairy tales and sagas from Grimm Korpus (https://www1.ids-mannheim.de/kl/projekte/korpora/archiv/gri.html) -* Newspaper and magazine article from Mannheimer Korpus Historischer Zeitungen und Zeitschriften (https://repos.ids-mannheim.de/mkhz-beschreibung.html) -* Magazine article from the journal „Die Grenzboten“ (http://www.deutschestextarchiv.de/doku/textquellen#grenzboten) -* Fictional and non-fictional texts from Projekt Gutenberg (https://www.projekt-gutenberg.org) - -## Hardware used -1 Tesla P4 GPU - -## Hyperparameters - -| Parameter | Value | -|-------------------------------|----------| -| Epochs | 3 | -| Gradient_accumulation_steps | 1 | -| Train_batch_size | 32 | -| Learning_rate | 0.00003 | -| Max_seq_len | 128 | - -## Evaluation results: Automatic tagging of four forms of speech/thought/writing representation in historical fictional and non-fictional German texts - -The language model was used in the task to tag direct, indirect, reported and free indirect speech/thought/writing representation in fictional and non-fictional German texts. The tagger is available and described in detail at https://github.com/redewiedergabe/tagger. - -The tagging model was trained using the SequenceTagger Class of the Flair framework ([Akbik et al., 2019](https://www.aclweb.org/anthology/N19-4010)) which implements a BiLSTM-CRF architecture on top of a language embedding (as proposed by [Huang et al. (2015)](https://arxiv.org/abs/1508.01991)). - - -Hyperparameters - -| Parameter | Value | -|-------------------------------|------------| -| Hidden_size | 256 | -| Learning_rate | 0.1 | -| Mini_batch_size | 8 | -| Max_epochs | 150 | - -Results are reported below in comparison to a custom trained flair embedding, which was stacked onto a custom trained fastText-model. Both models were trained on the same dataset. - -| | BERT ||| FastText+Flair |||Test data| -|----------------|----------|-----------|----------|------|-----------|--------|--------| -| | F1 | Precision | Recall | F1 | Precision | Recall || -| Direct | 0.80 | 0.86 | 0.74 | 0.84 | 0.90 | 0.79 |historical German, fictional & non-fictional| -| Indirect | **0.76** | **0.79** | **0.73** | 0.73 | 0.78 | 0.68 |historical German, fictional & non-fictional| -| Reported | **0.58** | **0.69** | **0.51** | 0.56 | 0.68 | 0.48 |historical German, fictional & non-fictional| -| Free indirect | **0.57** | **0.80** | **0.44** | 0.47 | 0.78 | 0.34 |modern German, fictional| - -## Intended use: -Historical German Texts (1840 to 1920) - -(Showed good performance with modern German fictional texts as well) - diff --git a/model_cards/rjbownes/Magic-The-Generating/README.md b/model_cards/rjbownes/Magic-The-Generating/README.md deleted file mode 100644 index c20f94715dcb..000000000000 --- a/model_cards/rjbownes/Magic-The-Generating/README.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -widget: -- text: "Even the Dwarves" -- text: "The secrets of" ---- - -# Model name -Magic The Generating - -## Model description - -This is a fine tuned GPT-2 model trained on a corpus of all available English language Magic the Gathering card flavour texts. - -## Intended uses & limitations - -This is intended only for use in generating new, novel, and sometimes surprising, MtG like flavour texts. - -#### How to use - -```python -from transformers import GPT2Tokenizer, GPT2LMHeadModel - -tokenizer = GPT2Tokenizer.from_pretrained("rjbownes/Magic-The-Generating") - -model = GPT2LMHeadModel.from_pretrained("rjbownes/Magic-The-Generating") - -``` - -#### Limitations and bias - -The training corpus was surprisingly small, only ~29000 cards, I had suspected there were more. This might mean there is a real limit to the number of entirely original strings this will generate. -This is also only based on the 117M parameter GPT2, it's a pretty obvious upgrade to retrain with medium, large or XL models. However, despite this, the outputs I tested were very convincing! - -## Training data - -The data was 29222 MtG card flavour texts. The model was based on the "gpt2" pretrained transformer: https://huggingface.co/gpt2. - -## Training procedure - -Only English language MtG flavour texts were scraped from the [Scryfall](https://scryfall.com/) API. Empty strings and any non-UTF-8 encoded tokens were removed leaving 29222 entries. -This was trained using google Colab with a T4 instance. 4 epochs, adamW optimizer with default parameters and a batch size of 32. Token embedding lengths were capped at 98 tokens as this was the longest string and an attention mask was added to the training model to ignore all padding tokens. - -## Eval results - -Average Training Loss: 0.44866578806635815. -Validation loss: 0.5606984243444775. - -Sample model outputs: - -1. "Every branch a crossroads, every vine a swift steed." - —Gwendlyn Di Corci - -2. "The secrets of this world will tell their masters where to strike if need be." - —Noyan Dar, Tazeem roilmage - -3. "The secrets of nature are expensive. You'd be better off just to have more freedom." - -4. "Even the Dwarves knew to leave some stones unturned." - -5. "The wise always keep an ear open to the whispers of power." - -### BibTeX entry and citation info - -```bibtex -@article{BownesLM, - title={Fine Tuning GPT-2 for Magic the Gathering flavour text generation.}, - author={Richard J. Bownes}, - journal={Medium}, - year={2020} -} - -``` diff --git a/model_cards/roberta-base-README.md b/model_cards/roberta-base-README.md deleted file mode 100644 index 3a89cda006f1..000000000000 --- a/model_cards/roberta-base-README.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -language: en -tags: -- exbert -license: mit -datasets: -- bookcorpus -- wikipedia ---- - -# RoBERTa base model - -Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in -[this paper](https://arxiv.org/abs/1907.11692) and first released in -[this repository](https://github.com/pytorch/fairseq/tree/master/examples/roberta). This model is case-sensitive: it -makes a difference between english and English. - -Disclaimer: The team releasing RoBERTa did not write a model card for this model so this model card has been written by -the Hugging Face team. - -## Model description - -RoBERTa is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means -it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, it was pretrained with the Masked language modeling (MLM) objective. Taking a sentence, the model -randomly masks 15% of the words in the input then run the entire masked sentence through the model and has to predict -the masked words. This is different from traditional recurrent neural networks (RNNs) that usually see the words one -after the other, or from autoregressive models like GPT which internally mask the future tokens. It allows the model to -learn a bidirectional representation of the sentence. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model for masked language modeling, but it's mostly intended to be fine-tuned on a downstream task. -See the [model hub](https://huggingface.co/models?filter=roberta) to look for fine-tuned versions on a task that -interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='roberta-base') ->>> unmasker("Hello I'm a model.") - -[{'sequence': "Hello I'm a male model.", - 'score': 0.3306540250778198, - 'token': 2943, - 'token_str': 'Ġmale'}, - {'sequence': "Hello I'm a female model.", - 'score': 0.04655390977859497, - 'token': 2182, - 'token_str': 'Ġfemale'}, - {'sequence': "Hello I'm a professional model.", - 'score': 0.04232972860336304, - 'token': 2038, - 'token_str': 'Ġprofessional'}, - {'sequence': "Hello I'm a fashion model.", - 'score': 0.037216778844594955, - 'token': 2734, - 'token_str': 'Ġfashion'}, - {'sequence': "Hello I'm a Russian model.", - 'score': 0.03253649175167084, - 'token': 1083, - 'token_str': 'ĠRussian'}] -``` - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import RobertaTokenizer, RobertaModel -tokenizer = RobertaTokenizer.from_pretrained('roberta-base') -model = RobertaModel.from_pretrained('roberta-base') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import RobertaTokenizer, TFRobertaModel -tokenizer = RobertaTokenizer.from_pretrained('roberta-base') -model = TFRobertaModel.from_pretrained('roberta-base') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -### Limitations and bias - -The training data used for this model contains a lot of unfiltered content from the internet, which is far from -neutral. Therefore, the model can have biased predictions: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='roberta-base') ->>> unmasker("The man worked as a .") - -[{'sequence': 'The man worked as a mechanic.', - 'score': 0.08702439814805984, - 'token': 25682, - 'token_str': 'Ġmechanic'}, - {'sequence': 'The man worked as a waiter.', - 'score': 0.0819653645157814, - 'token': 38233, - 'token_str': 'Ġwaiter'}, - {'sequence': 'The man worked as a butcher.', - 'score': 0.073323555290699, - 'token': 32364, - 'token_str': 'Ġbutcher'}, - {'sequence': 'The man worked as a miner.', - 'score': 0.046322137117385864, - 'token': 18678, - 'token_str': 'Ġminer'}, - {'sequence': 'The man worked as a guard.', - 'score': 0.040150221437215805, - 'token': 2510, - 'token_str': 'Ġguard'}] - ->>> unmasker("The Black woman worked as a .") - -[{'sequence': 'The Black woman worked as a waitress.', - 'score': 0.22177888453006744, - 'token': 35698, - 'token_str': 'Ġwaitress'}, - {'sequence': 'The Black woman worked as a prostitute.', - 'score': 0.19288744032382965, - 'token': 36289, - 'token_str': 'Ġprostitute'}, - {'sequence': 'The Black woman worked as a maid.', - 'score': 0.06498628109693527, - 'token': 29754, - 'token_str': 'Ġmaid'}, - {'sequence': 'The Black woman worked as a secretary.', - 'score': 0.05375480651855469, - 'token': 2971, - 'token_str': 'Ġsecretary'}, - {'sequence': 'The Black woman worked as a nurse.', - 'score': 0.05245552211999893, - 'token': 9008, - 'token_str': 'Ġnurse'}] -``` - -This bias will also affect all fine-tuned versions of this model. - -## Training data - -The RoBERTa model was pretrained on the reunion of five datasets: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books; -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers) ; -- [CC-News](https://commoncrawl.org/2016/10/news-dataset-available/), a dataset containing 63 millions English news - articles crawled between September 2016 and February 2019. -- [OpenWebText](https://github.com/jcpeterson/openwebtext), an opensource recreation of the WebText dataset used to - train GPT-2, -- [Stories](https://arxiv.org/abs/1806.02847) a dataset containing a subset of CommonCrawl data filtered to match the - story-like style of Winograd schemas. - -Together theses datasets weight 160GB of text. - -## Training procedure - -### Preprocessing - -The texts are tokenized using a byte version of Byte-Pair Encoding (BPE) and a vocabulary size of 50,000. The inputs of -the model take pieces of 512 contiguous token that may span over documents. The beginning of a new document is marked -with `` and the end of one by `` - -The details of the masking procedure for each sentence are the following: -- 15% of the tokens are masked. -- In 80% of the cases, the masked tokens are replaced by ``. -- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace. -- In the 10% remaining cases, the masked tokens are left as is. - -Contrary to BERT, the masking is done dynamically during pretraining (e.g., it changes at each epoch and is not fixed). - -### Pretraining - -The model was trained on 1024 V100 GPUs for 500K steps with a batch size of 8K and a sequence length of 512. The -optimizer used is Adam with a learning rate of 6e-4, \\(\beta_{1} = 0.9\\), \\(\beta_{2} = 0.98\\) and -\\(\epsilon = 1e-6\\), a weight decay of 0.01, learning rate warmup for 24,000 steps and linear decay of the learning -rate after. - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Glue test results: - -| Task | MNLI | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | -|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:| -| | 87.6 | 91.9 | 92.8 | 94.8 | 63.6 | 91.2 | 90.2 | 78.7 | - - -### BibTeX entry and citation info - -```bibtex -@article{DBLP:journals/corr/abs-1907-11692, - author = {Yinhan Liu and - Myle Ott and - Naman Goyal and - Jingfei Du and - Mandar Joshi and - Danqi Chen and - Omer Levy and - Mike Lewis and - Luke Zettlemoyer and - Veselin Stoyanov}, - title = {RoBERTa: {A} Robustly Optimized {BERT} Pretraining Approach}, - journal = {CoRR}, - volume = {abs/1907.11692}, - year = {2019}, - url = {http://arxiv.org/abs/1907.11692}, - archivePrefix = {arXiv}, - eprint = {1907.11692}, - timestamp = {Thu, 01 Aug 2019 08:59:33 +0200}, - biburl = {https://dblp.org/rec/journals/corr/abs-1907-11692.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` - - - - diff --git a/model_cards/roberta-large-README.md b/model_cards/roberta-large-README.md deleted file mode 100644 index 788f15291020..000000000000 --- a/model_cards/roberta-large-README.md +++ /dev/null @@ -1,235 +0,0 @@ ---- -language: en -tags: -- exbert -license: mit -datasets: -- bookcorpus -- wikipedia ---- - -# RoBERTa large model - -Pretrained model on English language using a masked language modeling (MLM) objective. It was introduced in -[this paper](https://arxiv.org/abs/1907.11692) and first released in -[this repository](https://github.com/pytorch/fairseq/tree/master/examples/roberta). This model is case-sensitive: it -makes a difference between english and English. - -Disclaimer: The team releasing RoBERTa did not write a model card for this model so this model card has been written by -the Hugging Face team. - -## Model description - -RoBERTa is a transformers model pretrained on a large corpus of English data in a self-supervised fashion. This means -it was pretrained on the raw texts only, with no humans labelling them in any way (which is why it can use lots of -publicly available data) with an automatic process to generate inputs and labels from those texts. - -More precisely, it was pretrained with the Masked language modeling (MLM) objective. Taking a sentence, the model -randomly masks 15% of the words in the input then run the entire masked sentence through the model and has to predict -the masked words. This is different from traditional recurrent neural networks (RNNs) that usually see the words one -after the other, or from autoregressive models like GPT which internally mask the future tokens. It allows the model to -learn a bidirectional representation of the sentence. - -This way, the model learns an inner representation of the English language that can then be used to extract features -useful for downstream tasks: if you have a dataset of labeled sentences for instance, you can train a standard -classifier using the features produced by the BERT model as inputs. - -## Intended uses & limitations - -You can use the raw model for masked language modeling, but it's mostly intended to be fine-tuned on a downstream task. -See the [model hub](https://huggingface.co/models?filter=roberta) to look for fine-tuned versions on a task that -interests you. - -Note that this model is primarily aimed at being fine-tuned on tasks that use the whole sentence (potentially masked) -to make decisions, such as sequence classification, token classification or question answering. For tasks such as text -generation you should look at model like GPT2. - -### How to use - -You can use this model directly with a pipeline for masked language modeling: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='roberta-large') ->>> unmasker("Hello I'm a model.") - -[{'sequence': "Hello I'm a male model.", - 'score': 0.3317350447177887, - 'token': 2943, - 'token_str': 'Ġmale'}, - {'sequence': "Hello I'm a fashion model.", - 'score': 0.14171843230724335, - 'token': 2734, - 'token_str': 'Ġfashion'}, - {'sequence': "Hello I'm a professional model.", - 'score': 0.04291723668575287, - 'token': 2038, - 'token_str': 'Ġprofessional'}, - {'sequence': "Hello I'm a freelance model.", - 'score': 0.02134818211197853, - 'token': 18150, - 'token_str': 'Ġfreelance'}, - {'sequence': "Hello I'm a young model.", - 'score': 0.021098261699080467, - 'token': 664, - 'token_str': 'Ġyoung'}] -``` - -Here is how to use this model to get the features of a given text in PyTorch: - -```python -from transformers import RobertaTokenizer, RobertaModel -tokenizer = RobertaTokenizer.from_pretrained('roberta-large') -model = RobertaModel.from_pretrained('roberta-large') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: - -```python -from transformers import RobertaTokenizer, TFRobertaModel -tokenizer = RobertaTokenizer.from_pretrained('roberta-large') -model = TFRobertaModel.from_pretrained('roberta-large') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -### Limitations and bias - -The training data used for this model contains a lot of unfiltered content from the internet, which is far from -neutral. Therefore, the model can have biased predictions: - -```python ->>> from transformers import pipeline ->>> unmasker = pipeline('fill-mask', model='roberta-large') ->>> unmasker("The man worked as a .") - -[{'sequence': 'The man worked as a mechanic.', - 'score': 0.08260300755500793, - 'token': 25682, - 'token_str': 'Ġmechanic'}, - {'sequence': 'The man worked as a driver.', - 'score': 0.05736079439520836, - 'token': 1393, - 'token_str': 'Ġdriver'}, - {'sequence': 'The man worked as a teacher.', - 'score': 0.04709019884467125, - 'token': 3254, - 'token_str': 'Ġteacher'}, - {'sequence': 'The man worked as a bartender.', - 'score': 0.04641604796051979, - 'token': 33080, - 'token_str': 'Ġbartender'}, - {'sequence': 'The man worked as a waiter.', - 'score': 0.04239227622747421, - 'token': 38233, - 'token_str': 'Ġwaiter'}] - ->>> unmasker("The woman worked as a .") - -[{'sequence': 'The woman worked as a nurse.', - 'score': 0.2667474150657654, - 'token': 9008, - 'token_str': 'Ġnurse'}, - {'sequence': 'The woman worked as a waitress.', - 'score': 0.12280137836933136, - 'token': 35698, - 'token_str': 'Ġwaitress'}, - {'sequence': 'The woman worked as a teacher.', - 'score': 0.09747499972581863, - 'token': 3254, - 'token_str': 'Ġteacher'}, - {'sequence': 'The woman worked as a secretary.', - 'score': 0.05783602222800255, - 'token': 2971, - 'token_str': 'Ġsecretary'}, - {'sequence': 'The woman worked as a cleaner.', - 'score': 0.05576248839497566, - 'token': 16126, - 'token_str': 'Ġcleaner'}] -``` - -This bias will also affect all fine-tuned versions of this model. - -## Training data - -The RoBERTa model was pretrained on the reunion of five datasets: -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of 11,038 unpublished books; -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) (excluding lists, tables and headers) ; -- [CC-News](https://commoncrawl.org/2016/10/news-dataset-available/), a dataset containing 63 millions English news - articles crawled between September 2016 and February 2019. -- [OpenWebText](https://github.com/jcpeterson/openwebtext), an opensource recreation of the WebText dataset used to - train GPT-2, -- [Stories](https://arxiv.org/abs/1806.02847) a dataset containing a subset of CommonCrawl data filtered to match the - story-like style of Winograd schemas. - -Together theses datasets weight 160GB of text. - -## Training procedure - -### Preprocessing - -The texts are tokenized using a byte version of Byte-Pair Encoding (BPE) and a vocabulary size of 50,000. The inputs of -the model take pieces of 512 contiguous token that may span over documents. The beginning of a new document is marked -with `` and the end of one by `` - -The details of the masking procedure for each sentence are the following: -- 15% of the tokens are masked. -- In 80% of the cases, the masked tokens are replaced by ``. - -- In 10% of the cases, the masked tokens are replaced by a random token (different) from the one they replace. -- In the 10% remaining cases, the masked tokens are left as is. - -Contrary to BERT, the masking is done dynamically during pretraining (e.g., it changes at each epoch and is not fixed). - -### Pretraining - -The model was trained on 1024 V100 GPUs for 500K steps with a batch size of 8K and a sequence length of 512. The -optimizer used is Adam with a learning rate of 4e-4, \\(\beta_{1} = 0.9\\), \\(\beta_{2} = 0.98\\) and -\\(\epsilon = 1e-6\\), a weight decay of 0.01, learning rate warmup for 30,000 steps and linear decay of the learning -rate after. - -## Evaluation results - -When fine-tuned on downstream tasks, this model achieves the following results: - -Glue test results: - -| Task | MNLI | QQP | QNLI | SST-2 | CoLA | STS-B | MRPC | RTE | -|:----:|:----:|:----:|:----:|:-----:|:----:|:-----:|:----:|:----:| -| | 90.2 | 92.2 | 94.7 | 96.4 | 68.0 | 96.4 | 90.9 | 86.6 | - - -### BibTeX entry and citation info - -```bibtex -@article{DBLP:journals/corr/abs-1907-11692, - author = {Yinhan Liu and - Myle Ott and - Naman Goyal and - Jingfei Du and - Mandar Joshi and - Danqi Chen and - Omer Levy and - Mike Lewis and - Luke Zettlemoyer and - Veselin Stoyanov}, - title = {RoBERTa: {A} Robustly Optimized {BERT} Pretraining Approach}, - journal = {CoRR}, - volume = {abs/1907.11692}, - year = {2019}, - url = {http://arxiv.org/abs/1907.11692}, - archivePrefix = {arXiv}, - eprint = {1907.11692}, - timestamp = {Thu, 01 Aug 2019 08:59:33 +0200}, - biburl = {https://dblp.org/rec/journals/corr/abs-1907-11692.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} -} -``` - - - - diff --git a/model_cards/roberta-large-mnli-README.md b/model_cards/roberta-large-mnli-README.md deleted file mode 100644 index 54903275484a..000000000000 --- a/model_cards/roberta-large-mnli-README.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -license: mit -widget: -- text: "I like you. I love you." ---- - - -## roberta-large-mnli - -Trained by Facebook, [original source](https://github.com/pytorch/fairseq/tree/master/examples/roberta) - -```bibtex -@article{liu2019roberta, - title = {RoBERTa: A Robustly Optimized BERT Pretraining Approach}, - author = {Yinhan Liu and Myle Ott and Naman Goyal and Jingfei Du and - Mandar Joshi and Danqi Chen and Omer Levy and Mike Lewis and - Luke Zettlemoyer and Veselin Stoyanov}, - journal={arXiv preprint arXiv:1907.11692}, - year = {2019}, -} -``` - diff --git a/model_cards/rohanrajpal/bert-base-codemixed-uncased-sentiment/README.md b/model_cards/rohanrajpal/bert-base-codemixed-uncased-sentiment/README.md deleted file mode 100644 index 1d25452a7da5..000000000000 --- a/model_cards/rohanrajpal/bert-base-codemixed-uncased-sentiment/README.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -language: -- hi -- en -tags: -- hi -- en -- codemix -datasets: -- SAIL 2017 ---- - -# Model name - -## Model description - -I took a bert-base-multilingual-cased model from huggingface and finetuned it on SAIL 2017 dataset. - -## Intended uses & limitations - -#### How to use - -```python -# You can include sample code which will be formatted -#Coming soon! -``` - -#### Limitations and bias - -Provide examples of latent issues and potential remediations. - -## Training data - -I trained on the SAIL 2017 dataset [link](http://amitavadas.com/SAIL/Data/SAIL_2017.zip) on this [pretrained model](https://huggingface.co/bert-base-multilingual-cased). - - -## Training procedure - -No preprocessing. - -## Eval results - -### BibTeX entry and citation info - -```bibtex -@inproceedings{khanuja-etal-2020-gluecos, - title = "{GLUEC}o{S}: An Evaluation Benchmark for Code-Switched {NLP}", - author = "Khanuja, Simran and - Dandapat, Sandipan and - Srinivasan, Anirudh and - Sitaram, Sunayana and - Choudhury, Monojit", - booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics", - month = jul, - year = "2020", - address = "Online", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/2020.acl-main.329", - pages = "3575--3585" -} -``` diff --git a/model_cards/rohanrajpal/bert-base-en-es-codemix-cased/README.md b/model_cards/rohanrajpal/bert-base-en-es-codemix-cased/README.md deleted file mode 100644 index 151c5f040f97..000000000000 --- a/model_cards/rohanrajpal/bert-base-en-es-codemix-cased/README.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -language: -- es -- en -tags: -- es -- en -- codemix -license: "apache-2.0" -datasets: -- SAIL 2017 -metrics: -- fscore -- accuracy -- precision -- recall ---- - -# BERT codemixed base model for spanglish (cased) - -This model was built using [lingualytics](https://github.com/lingualytics/py-lingualytics), an open-source library that supports code-mixed analytics. - -## Model description - -Input for the model: Any codemixed spanglish text -Output for the model: Sentiment. (0 - Negative, 1 - Neutral, 2 - Positive) - -I took a bert-base-multilingual-cased model from Huggingface and finetuned it on [CS-EN-ES-CORPUS](http://www.grupolys.org/software/CS-CORPORA/cs-en-es-corpus-wassa2015.txt) dataset. - -Performance of this model on the dataset - -| metric | score | -|------------|----------| -| acc | 0.718615 | -| f1 | 0.71759 | -| acc_and_f1 | 0.718103 | -| precision | 0.719302 | -| recall | 0.718615 | - -## Intended uses & limitations - -Make sure to preprocess your data using [these methods](https://github.com/microsoft/GLUECoS/blob/master/Data/Preprocess_Scripts/preprocess_sent_en_es.py) before using this model. - -#### How to use - -Here is how to use this model to get the features of a given text in *PyTorch*: - -```python -# You can include sample code which will be formatted -from transformers import BertTokenizer, BertModelForSequenceClassification -tokenizer = AutoTokenizer.from_pretrained('rohanrajpal/bert-base-en-es-codemix-cased') -model = AutoModelForSequenceClassification.from_pretrained('rohanrajpal/bert-base-en-es-codemix-cased') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in *TensorFlow*: - -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained('rohanrajpal/bert-base-en-es-codemix-cased') -model = TFBertModel.from_pretrained('rohanrajpal/bert-base-en-es-codemix-cased') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -#### Limitations and bias - -Since I dont know spanish, I cant verify the quality of annotations or the dataset itself. This is a very simple transfer learning approach and I'm open to discussions to improve upon this. - -## Training data - -I trained on the dataset on the [bert-base-multilingual-cased model](https://huggingface.co/bert-base-multilingual-cased). - -## Training procedure - -Followed the preprocessing techniques followed [here](https://github.com/microsoft/GLUECoS/blob/master/Data/Preprocess_Scripts/preprocess_sent_en_es.py) - -## Eval results - -### BibTeX entry and citation info - -```bibtex -@inproceedings{khanuja-etal-2020-gluecos, - title = "{GLUEC}o{S}: An Evaluation Benchmark for Code-Switched {NLP}", - author = "Khanuja, Simran and - Dandapat, Sandipan and - Srinivasan, Anirudh and - Sitaram, Sunayana and - Choudhury, Monojit", - booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics", - month = jul, - year = "2020", - address = "Online", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/2020.acl-main.329", - pages = "3575--3585" -} -``` diff --git a/model_cards/rohanrajpal/bert-base-en-hi-codemix-cased/README.md b/model_cards/rohanrajpal/bert-base-en-hi-codemix-cased/README.md deleted file mode 100644 index a407ed9daaaf..000000000000 --- a/model_cards/rohanrajpal/bert-base-en-hi-codemix-cased/README.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -language: -- hi -- en -tags: -- es -- en -- codemix -license: "apache-2.0" -datasets: -- SAIL 2017 -metrics: -- fscore -- accuracy -- precision -- recall ---- - -# BERT codemixed base model for Hinglish (cased) - -This model was built using [lingualytics](https://github.com/lingualytics/py-lingualytics), an open-source library that supports code-mixed analytics. - -## Model description - -Input for the model: Any codemixed Hinglish text -Output for the model: Sentiment. (0 - Negative, 1 - Neutral, 2 - Positive) - -I took a bert-base-multilingual-cased model from Huggingface and finetuned it on [SAIL 2017](http://www.dasdipankar.com/SAILCodeMixed.html) dataset. - -## Eval results - -Performance of this model on the dataset - -| metric | score | -|------------|----------| -| acc | 0.55873 | -| f1 | 0.558369 | -| acc_and_f1 | 0.558549 | -| precision | 0.558075 | -| recall | 0.55873 | - -#### How to use - -Here is how to use this model to get the features of a given text in *PyTorch*: - -```python -# You can include sample code which will be formatted -from transformers import BertTokenizer, BertModelForSequenceClassification -tokenizer = AutoTokenizer.from_pretrained('rohanrajpal/bert-base-en-es-codemix-cased') -model = AutoModelForSequenceClassification.from_pretrained('rohanrajpal/bert-base-en-es-codemix-cased') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in *TensorFlow*: - -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained('rohanrajpal/bert-base-en-es-codemix-cased') -model = TFBertModel.from_pretrained('rohanrajpal/bert-base-en-es-codemix-cased') -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -#### Preprocessing - -Followed standard preprocessing techniques: -- removed digits -- removed punctuation -- removed stopwords -- removed excess whitespace -Here's the snippet - -```python -from pathlib import Path -import pandas as pd -from lingualytics.preprocessing import remove_lessthan, remove_punctuation, remove_stopwords -from lingualytics.stopwords import hi_stopwords,en_stopwords -from texthero.preprocessing import remove_digits, remove_whitespace - -root = Path('') - -for file in 'test','train','validation': - tochange = root / f'{file}.txt' - df = pd.read_csv(tochange,header=None,sep='\t',names=['text','label']) - df['text'] = df['text'].pipe(remove_digits) \ - .pipe(remove_punctuation) \ - .pipe(remove_stopwords,stopwords=en_stopwords.union(hi_stopwords)) \ - .pipe(remove_whitespace) - df.to_csv(tochange,index=None,header=None,sep='\t') -``` - -## Training data - -The dataset and annotations are not good, but this is the best dataset I could find. I am working on procuring my own dataset and will try to come up with a better model! - -## Training procedure - -I trained on the dataset on the [bert-base-multilingual-cased model](https://huggingface.co/bert-base-multilingual-cased). diff --git a/model_cards/rohanrajpal/bert-base-multilingual-codemixed-cased-sentiment/README.md b/model_cards/rohanrajpal/bert-base-multilingual-codemixed-cased-sentiment/README.md deleted file mode 100644 index bac6ca0e1256..000000000000 --- a/model_cards/rohanrajpal/bert-base-multilingual-codemixed-cased-sentiment/README.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -language: -- hi -- en -tags: -- hi -- en -- codemix -license: "apache-2.0" -datasets: -- SAIL 2017 -metrics: -- fscore -- accuracy ---- - -# BERT codemixed base model for hinglish (cased) - -## Model description - -Input for the model: Any codemixed hinglish text -Output for the model: Sentiment. (0 - Negative, 1 - Neutral, 2 - Positive) - -I took a bert-base-multilingual-cased model from Huggingface and finetuned it on [SAIL 2017](http://www.dasdipankar.com/SAILCodeMixed.html) dataset. - -Performance of this model on the SAIL 2017 dataset - -| metric | score | -|------------|----------| -| acc | 0.588889 | -| f1 | 0.582678 | -| acc_and_f1 | 0.585783 | -| precision | 0.586516 | -| recall | 0.588889 | - -## Intended uses & limitations - -#### How to use - -Here is how to use this model to get the features of a given text in *PyTorch*: - -```python -# You can include sample code which will be formatted -from transformers import BertTokenizer, BertModelForSequenceClassification -tokenizer = AutoTokenizer.from_pretrained("rohanrajpal/bert-base-codemixed-uncased-sentiment") -model = AutoModelForSequenceClassification.from_pretrained("rohanrajpal/bert-base-codemixed-uncased-sentiment") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in *TensorFlow*: - -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained('rohanrajpal/bert-base-codemixed-uncased-sentiment') -model = TFBertModel.from_pretrained("rohanrajpal/bert-base-codemixed-uncased-sentiment") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -#### Limitations and bias - -Coming soon! - -## Training data - -I trained on the SAIL 2017 dataset [link](http://amitavadas.com/SAIL/Data/SAIL_2017.zip) on this [pretrained model](https://huggingface.co/bert-base-multilingual-cased). - -## Training procedure - -No preprocessing. - -## Eval results - -### BibTeX entry and citation info - -```bibtex -@inproceedings{khanuja-etal-2020-gluecos, - title = "{GLUEC}o{S}: An Evaluation Benchmark for Code-Switched {NLP}", - author = "Khanuja, Simran and - Dandapat, Sandipan and - Srinivasan, Anirudh and - Sitaram, Sunayana and - Choudhury, Monojit", - booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics", - month = jul, - year = "2020", - address = "Online", - publisher = "Association for Computational Linguistics", - url = "https://www.aclweb.org/anthology/2020.acl-main.329", - pages = "3575--3585" -} -``` diff --git a/model_cards/sachaarbonel/bert-italian-cased-finetuned-pos/README.md b/model_cards/sachaarbonel/bert-italian-cased-finetuned-pos/README.md deleted file mode 100644 index 1fb513c3f860..000000000000 --- a/model_cards/sachaarbonel/bert-italian-cased-finetuned-pos/README.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -language: it -datasets: -- xtreme ---- - -# Italian-Bert (Italian Bert) + POS 🎃🏷 - -This model is a fine-tuned on [xtreme udpos Italian](https://huggingface.co/nlp/viewer/?dataset=xtreme&config=udpos.Italian) version of [Bert Base Italian](https://huggingface.co/dbmdz/bert-base-italian-cased) for **POS** downstream task. - -## Details of the downstream task (POS) - Dataset - -- [Dataset: xtreme udpos Italian](https://huggingface.co/nlp/viewer/?dataset=xtreme&config=udpos.Italian) 📚 - -| Dataset | # Examples | -| ---------------------- | ----- | -| Train | 716 K | -| Dev | 85 K | - -- [Fine-tune on NER script provided by @stefan-it](https://raw.githubusercontent.com/stefan-it/fine-tuned-berts-seq/master/scripts/preprocess.py) - -- Labels covered: - -``` -ADJ -ADP -ADV -AUX -CCONJ -DET -INTJ -NOUN -NUM -PART -PRON -PROPN -PUNCT -SCONJ -SYM -VERB -X -``` - -## Metrics on evaluation set 🧾 - -| Metric | # score | -| :------------------------------------------------------------------------------------: | :-------: | -| F1 | **97.25** -| Precision | **97.15** | -| Recall | **97.36** | - -## Model in action 🔨 - - -Example of usage - -```python -from transformers import pipeline - -nlp_pos = pipeline( - "ner", - model="sachaarbonel/bert-italian-cased-finetuned-pos", - tokenizer=( - 'sachaarbonel/bert-spanish-cased-finetuned-pos', - {"use_fast": False} -)) - - -text = 'Roma è la Capitale d'Italia.' - -nlp_pos(text) - -''' -Output: --------- -[{'entity': 'PROPN', 'index': 1, 'score': 0.9995346665382385, 'word': 'roma'}, - {'entity': 'AUX', 'index': 2, 'score': 0.9966597557067871, 'word': 'e'}, - {'entity': 'DET', 'index': 3, 'score': 0.9994786977767944, 'word': 'la'}, - {'entity': 'NOUN', - 'index': 4, - 'score': 0.9995198249816895, - 'word': 'capitale'}, - {'entity': 'ADP', 'index': 5, 'score': 0.9990894198417664, 'word': 'd'}, - {'entity': 'PART', 'index': 6, 'score': 0.57159024477005, 'word': "'"}, - {'entity': 'PROPN', - 'index': 7, - 'score': 0.9994804263114929, - 'word': 'italia'}, - {'entity': 'PUNCT', 'index': 8, 'score': 0.9772886633872986, 'word': '.'}] -''' -``` -Yeah! Not too bad 🎉 - -> Created by [Sacha Arbonel/@sachaarbonel](https://twitter.com/sachaarbonel) | [LinkedIn](https://www.linkedin.com/in/sacha-arbonel) - -> Made with ♥ in Paris diff --git a/model_cards/sagorsarker/bangla-bert-base/README.md b/model_cards/sagorsarker/bangla-bert-base/README.md deleted file mode 100644 index 859b980cbe67..000000000000 --- a/model_cards/sagorsarker/bangla-bert-base/README.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -language: bn -tags: -- bert -- bengali -- bengali-lm -- bangla -license: MIT -datasets: -- common_crawl -- wikipedia -- oscar ---- - - -# Bangla BERT Base -A long way passed. Here is our **Bangla-Bert**! It is now available in huggingface model hub. - -[Bangla-Bert-Base](https://github.com/sagorbrur/bangla-bert) is a pretrained language model of Bengali language using mask language modeling described in [BERT](https://arxiv.org/abs/1810.04805) and it's github [repository](https://github.com/google-research/bert) - - - -## Pretrain Corpus Details -Corpus was downloaded from two main sources: - -* Bengali commoncrawl copurs downloaded from [OSCAR](https://oscar-corpus.com/) -* [Bengali Wikipedia Dump Dataset](https://dumps.wikimedia.org/bnwiki/latest/) - -After downloading these corpus, we preprocessed it as a Bert format. which is one sentence per line and an extra newline for new documents. - -``` -sentence 1 -sentence 2 - -sentence 1 -sentence 2 - -``` - -## Building Vocab -We used [BNLP](https://github.com/sagorbrur/bnlp) package for training bengali sentencepiece model with vocab size 102025. We preprocess the output vocab file as Bert format. -Our final vocab file availabe at [https://github.com/sagorbrur/bangla-bert](https://github.com/sagorbrur/bangla-bert) and also at [huggingface](https://huggingface.co/sagorsarker/bangla-bert-base) model hub. - -## Training Details -* Bangla-Bert was trained with code provided in Google BERT's github repository (https://github.com/google-research/bert) -* Currently released model follows bert-base-uncased model architecture (12-layer, 768-hidden, 12-heads, 110M parameters) -* Total Training Steps: 1 Million -* The model was trained on a single Google Cloud TPU - -## Evaluation Results - -### LM Evaluation Results -After training 1 millions steps here is the evaluation resutls. - -``` -global_step = 1000000 -loss = 2.2406516 -masked_lm_accuracy = 0.60641736 -masked_lm_loss = 2.201459 -next_sentence_accuracy = 0.98625 -next_sentence_loss = 0.040997364 -perplexity = numpy.exp(2.2406516) = 9.393331287442784 -Loss for final step: 2.426227 - -``` - -### Downstream Task Evaluation Results -Huge Thanks to [Nick Doiron](https://twitter.com/mapmeld) for providing evalution results of classification task. -He used [Bengali Classification Benchmark](https://github.com/rezacsedu/Classification_Benchmarks_Benglai_NLP) datasets for classification task. -Comparing to Nick's [Bengali electra](https://huggingface.co/monsoon-nlp/bangla-electra) and multi-lingual BERT, Bangla BERT Base achieves state of the art result. -Here is the [evaluation script](https://github.com/sagorbrur/bangla-bert/blob/master/notebook/bangla-bert-evaluation-classification-task.ipynb). - - -| Model | Sentiment Analysis | Hate Speech Task | News Topic Task | Average | -| ----- | -------------------| ---------------- | --------------- | ------- | -| mBERT | 68.15 | 52.32 | 72.27 | 64.25 | -| Bengali Electra | 69.19 | 44.84 | 82.33 | 65.45 | -| Bangla BERT Base | 70.37 | 71.83 | 89.19 | 77.13 | - - -**NB: If you use this model for any nlp task please share evaluation results with us. We will add it here.** - - -## How to Use -You can use this model directly with a pipeline for masked language modeling: - -```py -from transformers import BertForMaskedLM, BertTokenizer, pipeline - -model = BertForMaskedLM.from_pretrained("sagorsarker/bangla-bert-base") -tokenizer = BertTokenizer.from_pretrained("sagorsarker/bangla-bert-base") -nlp = pipeline('fill-mask', model=model, tokenizer=tokenizer) -for pred in nlp(f"আমি বাংলায় {nlp.tokenizer.mask_token} গাই।"): - print(pred) - -# {'sequence': '[CLS] আমি বাংলায গান গাই । [SEP]', 'score': 0.13404667377471924, 'token': 2552, 'token_str': 'গান'} - -``` - - -## Author -[Sagor Sarker](https://github.com/sagorbrur) - -## Acknowledgements - -* Thanks to Google [TensorFlow Research Cloud (TFRC)](https://www.tensorflow.org/tfrc) for providing the free TPU credits - thank you! -* Thank to all the people around, who always helping us to build something for Bengali. - -## Reference -* https://github.com/google-research/bert - - - - - diff --git a/model_cards/sagorsarker/bangla-bert-sentiment/README.md b/model_cards/sagorsarker/bangla-bert-sentiment/README.md deleted file mode 100644 index 4a48873f31b7..000000000000 --- a/model_cards/sagorsarker/bangla-bert-sentiment/README.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -language: -- bn -datasets: -- socian -- bangla-sentiment-benchmark -license: mit -tags: -- bengali -- bengali-sentiment -- sentiment-analysis ---- - -# bangla-bert-sentiment -`bangla-bert-sentiment` is a pretrained model for bengali **Sentiment Analysis** using [bangla-bert-base](https://huggingface.co/sagorsarker/bangla-bert-base) model. - -## Datasets Details -This model was trained with two combined datasets -* [socian sentiment data](https://github.com/socian-ai/socian-bangla-sentiment-dataset-labeled) -* [bangla classification dataset](https://github.com/rezacsedu/Classification_Benchmarks_Benglai_NLP) - -||| -|--|--| -|Data Size| 10889 | -|Positive| 4999 | -|Negative| 5890 | -|Train | 8711 | -| Test | 2178 | - -## Training Details -Model trained with [simpletransformers](https://github.com/ThilinaRajapakse/simpletransformers) binary classification script with total of **3 epochs** in `google colab gpu`. - - -## Evaluation Details -Model evaluate with 2178 sentences - -Here is the evaluation result details in table - - -|Eval Loss | TP | TN | FP | FN | F1 Score | -| -------- | -- | -- | -- | -- | -------- | -| 0.3289 | 880 | 1158 | 59 | 81 | 92.63 | - -## Usage - -Calculate sentiment from given sentence - -```py - -from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/bangla-bert-sentiment") - -model = AutoModelForSequenceClassification.from_pretrained("sagorsarker/bangla-bert-sentiment") - -nlp = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer) -sentence = "বাংলার ঘরে ঘরে আজ নবান্নের উৎসব" -nlp(sentence) - -``` - diff --git a/model_cards/sagorsarker/codeswitch-hineng-lid-lince/README.md b/model_cards/sagorsarker/codeswitch-hineng-lid-lince/README.md deleted file mode 100644 index 78ec04340a58..000000000000 --- a/model_cards/sagorsarker/codeswitch-hineng-lid-lince/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -language: -- hi -- en -datasets: -- lince -license: mit -tags: -- codeswitching -- hindi-english -- language-identification ---- - -# codeswitch-hineng-lid-lince -This is a pretrained model for **language identification** of `hindi-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home) - -This model is trained for this below repository. - -[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch) - -To install codeswitch: - -``` -pip install codeswitch -``` - -## Identify Language - -* **Method-1** - -```py - -from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-hineng-lid-lince") - -model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-hineng-lid-lince") -lid_model = pipeline('ner', model=model, tokenizer=tokenizer) - -lid_model("put any hindi english code-mixed sentence") - -``` - -* **Method-2** - -```py -from codeswitch.codeswitch import LanguageIdentification -lid = LanguageIdentification('hin-eng') -text = "" # your code-mixed sentence -result = lid.identify(text) -print(result) -``` - diff --git a/model_cards/sagorsarker/codeswitch-hineng-ner-lince/README.md b/model_cards/sagorsarker/codeswitch-hineng-ner-lince/README.md deleted file mode 100644 index 0d28021249ba..000000000000 --- a/model_cards/sagorsarker/codeswitch-hineng-ner-lince/README.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -language: -- hi -- en -datasets: -- lince -license: mit -tags: -- codeswitching -- hindi-english -- ner ---- - -# codeswitch-hineng-ner-lince -This is a pretrained model for **Name Entity Recognition** of `Hindi-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home) - -This model is trained for this below repository. - -[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch) - -To install codeswitch: - -``` -pip install codeswitch -``` - -## Name Entity Recognition of Code-Mixed Data - -* **Method-1** - -```py - -from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-hineng-ner-lince") - -model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-hineng-ner-lince") - -ner_model = pipeline('ner', model=model, tokenizer=tokenizer) - -ner_model("put any hindi english code-mixed sentence") - -``` - -* **Method-2** - -```py -from codeswitch.codeswitch import NER -ner = NER('hin-eng') -text = "" # your mixed sentence -result = ner.tag(text) -print(result) - -``` diff --git a/model_cards/sagorsarker/codeswitch-hineng-pos-lince/README.md b/model_cards/sagorsarker/codeswitch-hineng-pos-lince/README.md deleted file mode 100644 index e7a382551838..000000000000 --- a/model_cards/sagorsarker/codeswitch-hineng-pos-lince/README.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -language: -- hi -- en -datasets: -- lince -license: mit -tags: -- codeswitching -- hindi-english -- pos ---- - -# codeswitch-hineng-pos-lince -This is a pretrained model for **Part of Speech Tagging** of `hindi-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home) - -This model is trained for this below repository. - -[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch) - -To install codeswitch: - -``` -pip install codeswitch -``` - -## Part-of-Speech Tagging of Hindi-English Mixed Data - -* **Method-1** - -```py - -from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-hineng-pos-lince") - -model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-hineng-pos-lince") -pos_model = pipeline('ner', model=model, tokenizer=tokenizer) - -pos_model("put any hindi english code-mixed sentence") - -``` - -* **Method-2** - -```py -from codeswitch.codeswitch import POS -pos = POS('hin-eng') -text = "" # your mixed sentence -result = pos.tag(text) -print(result) -``` diff --git a/model_cards/sagorsarker/codeswitch-nepeng-lid-lince/README.md b/model_cards/sagorsarker/codeswitch-nepeng-lid-lince/README.md deleted file mode 100644 index df282467082a..000000000000 --- a/model_cards/sagorsarker/codeswitch-nepeng-lid-lince/README.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -language: -- ne -- en -datasets: -- lince -license: mit -tags: -- codeswitching -- nepali-english -- language-identification ---- - -# codeswitch-nepeng-lid-lince -This is a pretrained model for **language identification** of `nepali-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home). - -This model is trained for this below repository. - -[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch) - -To install codeswitch: - -``` -pip install codeswitch -``` - -## Identify Language - -* **Method-1** - -```py - -from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-nepeng-lid-lince") - -model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-nepeng-lid-lince") -lid_model = pipeline('ner', model=model, tokenizer=tokenizer) - -lid_model("put any nepali english code-mixed sentence") - -``` - -* **Method-2** - -```py -from codeswitch.codeswitch import LanguageIdentification -lid = LanguageIdentification('nep-eng') -text = "" # your code-mixed sentence -result = lid.identify(text) -print(result) - -``` - diff --git a/model_cards/sagorsarker/codeswitch-spaeng-lid-lince/README.md b/model_cards/sagorsarker/codeswitch-spaeng-lid-lince/README.md deleted file mode 100644 index d88e4970934e..000000000000 --- a/model_cards/sagorsarker/codeswitch-spaeng-lid-lince/README.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -language: -- es -- en -datasets: -- lince -license: mit -tags: -- codeswitching -- spanish-english -- language-identification ---- - -# codeswitch-spaeng-lid-lince -This is a pretrained model for **language identification** of `spanish-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home) - -This model is trained for this below repository. - -[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch) - -To install codeswitch: - -``` -pip install codeswitch -``` - -## Identify Language - -* **Method-1** - -```py - -from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-spaeng-lid-lince") - -model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-spaeng-lid-lince") -lid_model = pipeline('ner', model=model, tokenizer=tokenizer) - -lid_model("put any spanish english code-mixed sentence") - -``` - -* **Method-2** - -```py -from codeswitch.codeswitch import LanguageIdentification -lid = LanguageIdentification('spa-eng') -text = "" # your code-mixed sentence -result = lid.identify(text) -print(result) -``` diff --git a/model_cards/sagorsarker/codeswitch-spaeng-ner-lince/README.md b/model_cards/sagorsarker/codeswitch-spaeng-ner-lince/README.md deleted file mode 100644 index 27dd3ae0a908..000000000000 --- a/model_cards/sagorsarker/codeswitch-spaeng-ner-lince/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -language: -- es -- en -datasets: -- lince -license: mit -tags: -- codeswitching -- spanish-english -- ner ---- - -# codeswitch-spaeng-ner-lince -This is a pretrained model for **Name Entity Recognition** of `spanish-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home) - -This model is trained for this below repository. - -[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch) - -To install codeswitch: - -``` -pip install codeswitch -``` - -## Name Entity Recognition of Spanish-English Mixed Data - -* **Method-1** - -```py - -from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-spaeng-ner-lince") - -model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-spaeng-ner-lince") - -ner_model = pipeline('ner', model=model, tokenizer=tokenizer) - -ner_model("put any spanish english code-mixed sentence") - -``` - -* **Method-2** - -```py -from codeswitch.codeswitch import NER -ner = NER('spa-eng') -text = "" # your mixed sentence -result = ner.tag(text) -print(result) -``` diff --git a/model_cards/sagorsarker/codeswitch-spaeng-pos-lince/README.md b/model_cards/sagorsarker/codeswitch-spaeng-pos-lince/README.md deleted file mode 100644 index 9ae08df89e9f..000000000000 --- a/model_cards/sagorsarker/codeswitch-spaeng-pos-lince/README.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -language: -- es -- en -datasets: -- lince -license: mit -tags: -- codeswitching -- spanish-english -- pos ---- - -# codeswitch-spaeng-pos-lince -This is a pretrained model for **Part of Speech Tagging** of `spanish-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home) - -This model is trained for this below repository. - -[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch) - -To install codeswitch: - -``` -pip install codeswitch -``` - -## Part-of-Speech Tagging of Spanish-English Mixed Data - -* **Method-1** - -```py - -from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-spaeng-pos-lince") - -model = AutoModelForTokenClassification.from_pretrained("sagorsarker/codeswitch-spaeng-pos-lince") -pos_model = pipeline('ner', model=model, tokenizer=tokenizer) - -pos_model("put any spanish english code-mixed sentence") - -``` - -* **Method-2** - -```py -from codeswitch.codeswitch import POS -pos = POS('spa-eng') -text = "" # your mixed sentence -result = pos.tag(text) -print(result) -``` diff --git a/model_cards/sagorsarker/codeswitch-spaeng-sentiment-analysis-lince/README.md b/model_cards/sagorsarker/codeswitch-spaeng-sentiment-analysis-lince/README.md deleted file mode 100644 index 6a55a953bd1d..000000000000 --- a/model_cards/sagorsarker/codeswitch-spaeng-sentiment-analysis-lince/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -language: -- es -- en -datasets: -- lince -license: mit -tags: -- codeswitching -- spanish-english -- sentiment-analysis ---- - -# codeswitch-spaeng-sentiment-analysis-lince -This is a pretrained model for **Sentiment Analysis** of `spanish-english` code-mixed data used from [LinCE](https://ritual.uh.edu/lince/home) - -This model is trained for this below repository. - -[https://github.com/sagorbrur/codeswitch](https://github.com/sagorbrur/codeswitch) - -To install codeswitch: - -``` -pip install codeswitch -``` - -## Sentiment Analysis of Spanish-English Code-Mixed Data - -* **Method-1** - -```py - -from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline - -tokenizer = AutoTokenizer.from_pretrained("sagorsarker/codeswitch-spaeng-sentiment-analysis-lince") - -model = AutoModelForSequenceClassification.from_pretrained("sagorsarker/codeswitch-spaeng-sentiment-analysis-lince") - -nlp = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer) -sentence = "El perro le ladraba a La Gatita .. .. lol #teamlagatita en las playas de Key Biscayne este Memorial day" -nlp(sentence) - -``` - -* **Method-2** - -```py -from codeswitch.codeswitch import SentimentAnalysis -sa = SentimentAnalysis('spa-eng') -sentence = "El perro le ladraba a La Gatita .. .. lol #teamlagatita en las playas de Key Biscayne este Memorial day" -result = sa.analyze(sentence) -print(result) -``` diff --git a/model_cards/sarahlintang/IndoBERT/README.md b/model_cards/sarahlintang/IndoBERT/README.md deleted file mode 100644 index bb5348ccaa19..000000000000 --- a/model_cards/sarahlintang/IndoBERT/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: id -datasets: -- oscar ---- -# IndoBERT (Indonesian BERT Model) - -## Model description -IndoBERT is a pre-trained language model based on BERT architecture for the Indonesian Language. - -This model is base-uncased version which use bert-base config. - -## Intended uses & limitations - -#### How to use - -```python -from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("sarahlintang/IndoBERT") -model = AutoModel.from_pretrained("sarahlintang/IndoBERT") -tokenizer.encode("hai aku mau makan.") -[2, 8078, 1785, 2318, 1946, 18, 4] -``` - - -## Training data - -This model was pre-trained on 16 GB of raw text ~2 B words from Oscar Corpus (https://oscar-corpus.com/). - -This model is equal to bert-base model which has 32,000 vocabulary size. - -## Training procedure - -The training of the model has been performed using Google’s original Tensorflow code on eight core Google Cloud TPU v2. -We used a Google Cloud Storage bucket, for persistent storage of training data and models. - -## Eval results - -We evaluate this model on three Indonesian NLP downstream task: -- some extractive summarization model -- sentiment analysis -- Part-of-Speech Tagger -it was proven that this model outperforms multilingual BERT for all downstream tasks. diff --git a/model_cards/sarnikowski/electra-small-discriminator-da-256-cased/README.md b/model_cards/sarnikowski/electra-small-discriminator-da-256-cased/README.md deleted file mode 100644 index 649e4344ebd7..000000000000 --- a/model_cards/sarnikowski/electra-small-discriminator-da-256-cased/README.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -language: da -license: cc-by-4.0 ---- - -# Danish ELECTRA small (cased) - -An [ELECTRA](https://arxiv.org/abs/2003.10555) model pretrained on a custom Danish corpus (~17.5gb). -For details regarding data sources and training procedure, along with benchmarks on downstream tasks, go to: https://github.com/sarnikowski/danish_transformers/tree/main/electra - -## Usage - -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("sarnikowski/electra-small-discriminator-da-256-cased") -model = AutoModel.from_pretrained("sarnikowski/electra-small-discriminator-da-256-cased") -``` - -## Questions? - -If you have any questions feel free to open an issue on the [danish_transformers](https://github.com/sarnikowski/danish_transformers) repository, or send an email to p.sarnikowski@gmail.com diff --git a/model_cards/savasy/bert-base-turkish-ner-cased/README.md b/model_cards/savasy/bert-base-turkish-ner-cased/README.md deleted file mode 100644 index 079575985cb1..000000000000 --- a/model_cards/savasy/bert-base-turkish-ner-cased/README.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -language: tr ---- - -# For Turkish language, here is an easy-to-use NER application. - ** Türkçe için kolay bir python NER (Bert + Transfer Learning) (İsim Varlık Tanıma) modeli... - - -Thanks to @stefan-it, I applied the followings for training - - -cd tr-data - -for file in train.txt dev.txt test.txt labels.txt -do - wget https://schweter.eu/storage/turkish-bert-wikiann/$file -done - -cd .. -It will download the pre-processed datasets with training, dev and test splits and put them in a tr-data folder. - -Run pre-training -After downloading the dataset, pre-training can be started. Just set the following environment variables: -``` -export MAX_LENGTH=128 -export BERT_MODEL=dbmdz/bert-base-turkish-cased -export OUTPUT_DIR=tr-new-model -export BATCH_SIZE=32 -export NUM_EPOCHS=3 -export SAVE_STEPS=625 -export SEED=1 -``` -Then run pre-training: -``` -python3 run_ner_old.py --data_dir ./tr-data3 \ ---model_type bert \ ---labels ./tr-data/labels.txt \ ---model_name_or_path $BERT_MODEL \ ---output_dir $OUTPUT_DIR-$SEED \ ---max_seq_length $MAX_LENGTH \ ---num_train_epochs $NUM_EPOCHS \ ---per_gpu_train_batch_size $BATCH_SIZE \ ---save_steps $SAVE_STEPS \ ---seed $SEED \ ---do_train \ ---do_eval \ ---do_predict \ ---fp16 -``` - - -# Usage - -``` -from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer -model = AutoModelForTokenClassification.from_pretrained("savasy/bert-base-turkish-ner-cased") -tokenizer = AutoTokenizer.from_pretrained("savasy/bert-base-turkish-ner-cased") -ner=pipeline('ner', model=model, tokenizer=tokenizer) -ner("Mustafa Kemal Atatürk 19 Mayıs 1919'da Samsun'a ayak bastı.") -``` -# Some results -Data1: For the data above -Eval Results: - -* precision = 0.916400580551524 -* recall = 0.9342309684101502 -* f1 = 0.9252298787412536 -* loss = 0.11335893666411284 - -Test Results: -* precision = 0.9192058759362955 -* recall = 0.9303010230367262 -* f1 = 0.9247201697271198 -* loss = 0.11182546521618497 - - - -Data2: -https://github.com/stefan-it/turkish-bert/files/4558187/nerdata.txt -The performance for the data given by @kemalaraz is as follows - -savas@savas-lenova:~/Desktop/trans/tr-new-model-1$ cat eval_results.txt -* precision = 0.9461980692049029 -* recall = 0.959309358847465 -* f1 = 0.9527086063783312 -* loss = 0.037054269206847804 - -savas@savas-lenova:~/Desktop/trans/tr-new-model-1$ cat test_results.txt -* precision = 0.9458370635631155 -* recall = 0.9588201928530913 -* f1 = 0.952284378344882 -* loss = 0.035431676572445225 - diff --git a/model_cards/savasy/bert-base-turkish-sentiment-cased/README.md b/model_cards/savasy/bert-base-turkish-sentiment-cased/README.md deleted file mode 100644 index da98a3f101bf..000000000000 --- a/model_cards/savasy/bert-base-turkish-sentiment-cased/README.md +++ /dev/null @@ -1,141 +0,0 @@ ---- -language: tr ---- -# Bert-base Turkish Sentiment Model - -https://huggingface.co/savasy/bert-base-turkish-sentiment-cased - -This model is used for Sentiment Analysis, which is based on BERTurk for Turkish Language https://huggingface.co/dbmdz/bert-base-turkish-cased - - -## Dataset - -The dataset is taken from the studies [[2]](#paper-2) and [[3]](#paper-3), and merged. - -* The study [2] gathered movie and product reviews. The products are book, DVD, electronics, and kitchen. -The movie dataset is taken from a cinema Web page ([Beyazperde](www.beyazperde.com)) with -5331 positive and 5331 negative sentences. Reviews in the Web page are marked in -scale from 0 to 5 by the users who made the reviews. The study considered a review -sentiment positive if the rating is equal to or bigger than 4, and negative if it is less -or equal to 2. They also built Turkish product review dataset from an online retailer -Web page. They constructed benchmark dataset consisting of reviews regarding some -products (book, DVD, etc.). Likewise, reviews are marked in the range from 1 to 5, -and majority class of reviews are 5. Each category has 700 positive and 700 negative -reviews in which average rating of negative reviews is 2.27 and of positive reviews -is 4.5. This dataset is also used by the study [[1]](#paper-1). - -* The study [[3]](#paper-3) collected tweet dataset. They proposed a new approach for automatically classifying the sentiment of microblog messages. The proposed approach is based on utilizing robust feature representation and fusion. - -*Merged Dataset* - -| *size* | *data* | -|--------|----| -| 8000 |dev.tsv| -| 8262 |test.tsv| -| 32000 |train.tsv| -| *48290* |*total*| - -### The dataset is used by following papers - -[1] Yildirim, Savaş. (2020). Comparing Deep Neural Networks to Traditional Models for Sentiment Analysis in Turkish Language. 10.1007/978-981-15-1216-2_12. - -[2] Demirtas, Erkin and Mykola Pechenizkiy. 2013. Cross-lingual polarity detection with machine translation. In Proceedings of the Second International Workshop on Issues of Sentiment -Discovery and Opinion Mining (WISDOM ’13) - -[3] Hayran, A., Sert, M. (2017), "Sentiment Analysis on Microblog Data based on Word Embedding and Fusion Techniques", IEEE 25th Signal Processing and Communications Applications Conference (SIU 2017), Belek, Turkey - - -## Training - -```shell -export GLUE_DIR="./sst-2-newall" -export TASK_NAME=SST-2 - -python3 run_glue.py \ - --model_type bert \ - --model_name_or_path dbmdz/bert-base-turkish-uncased\ - --task_name "SST-2" \ - --do_train \ - --do_eval \ - --data_dir "./sst-2-newall" \ - --max_seq_length 128 \ - --per_gpu_train_batch_size 32 \ - --learning_rate 2e-5 \ - --num_train_epochs 3.0 \ - --output_dir "./model" -``` - - -## Results - -> 05/10/2020 17:00:43 - INFO - transformers.trainer - \*\*\*\*\* Running Evaluation \*\*\*\*\* -> 05/10/2020 17:00:43 - INFO - transformers.trainer - Num examples = 7999 -> 05/10/2020 17:00:43 - INFO - transformers.trainer - Batch size = 8 -> Evaluation: 100% 1000/1000 [00:34<00:00, 29.04it/s] -> 05/10/2020 17:01:17 - INFO - \_\_main__ - \*\*\*\*\* Eval results sst-2 \*\*\*\*\* -> 05/10/2020 17:01:17 - INFO - \_\_main__ - acc = 0.9539942492811602 -> 05/10/2020 17:01:17 - INFO - \_\_main__ - loss = 0.16348013816401363 - -Accuracy is about **95.4%** - - -## Code Usage - -```python -from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline - -model = AutoModelForSequenceClassification.from_pretrained("savasy/bert-base-turkish-sentiment-cased") -tokenizer = AutoTokenizer.from_pretrained("savasy/bert-base-turkish-sentiment-cased") -sa= pipeline("sentiment-analysis", tokenizer=tokenizer, model=model) - -p = sa("bu telefon modelleri çok kaliteli , her parçası çok özel bence") -print(p) -# [{'label': 'LABEL_1', 'score': 0.9871089}] -print(p[0]['label'] == 'LABEL_1') -# True - -p = sa("Film çok kötü ve çok sahteydi") -print(p) -# [{'label': 'LABEL_0', 'score': 0.9975505}] -print(p[0]['label'] == 'LABEL_1') -# False -``` - - -## Test -### Data - -Suppose your file has lots of lines of comment and label (1 or 0) at the end (tab seperated) - -> comment1 ... \t label -> comment2 ... \t label -> ... - -### Code - -```python -from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline - -model = AutoModelForSequenceClassification.from_pretrained("savasy/bert-base-turkish-sentiment-cased") -tokenizer = AutoTokenizer.from_pretrained("savasy/bert-base-turkish-sentiment-cased") -sa = pipeline("sentiment-analysis", tokenizer=tokenizer, model=model) - -input_file = "/path/to/your/file/yourfile.tsv" - -i, crr = 0, 0 -for line in open(input_file): - lines = line.strip().split("\t") - if len(lines) == 2: - - i = i + 1 - if i%100 == 0: - print(i) - - pred = sa(lines[0]) - pred = pred[0]["label"].split("_")[1] - - if pred == lines[1]: - crr = crr + 1 - -print(crr, i, crr/i) -``` diff --git a/model_cards/savasy/bert-base-turkish-squad/README.md b/model_cards/savasy/bert-base-turkish-squad/README.md deleted file mode 100644 index 209d5ca66541..000000000000 --- a/model_cards/savasy/bert-base-turkish-squad/README.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -language: tr ---- -# Turkish SQuAD Model : Question Answering - -I fine-tuned Turkish-Bert-Model for Question-Answering problem with Turkish version of SQuAD; TQuAD -* BERT-base: https://huggingface.co/dbmdz/bert-base-turkish-uncased -* TQuAD dataset: https://github.com/TQuad/turkish-nlp-qa-dataset - - -# Training Code - -``` -!python3 run_squad.py \ - --model_type bert \ - --model_name_or_path dbmdz/bert-base-turkish-uncased\ - --do_train \ - --do_eval \ - --train_file trainQ.json \ - --predict_file dev1.json \ - --per_gpu_train_batch_size 12 \ - --learning_rate 3e-5 \ - --num_train_epochs 5.0 \ - --max_seq_length 384 \ - --doc_stride 128 \ - --output_dir "./model" -``` - - -# Example Usage - -> Load Model -``` -from transformers import AutoTokenizer, AutoModelForQuestionAnswering, pipeline -import torch - -tokenizer = AutoTokenizer.from_pretrained("./model") -model = AutoModelForQuestionAnswering.from_pretrained("./model") -nlp=pipeline("question-answering", model=model, tokenizer=tokenizer) -``` - -> Apply the model -``` - -sait="ABASIYANIK, Sait Faik. Hikayeci (Adapazarı 23 Kasım 1906-İstanbul 11 Mayıs 1954). \ -İlk öğrenimine Adapazarı’nda Rehber-i Terakki Mektebi’nde başladı. İki yıl kadar Adapazarı İdadisi’nde okudu.\ -İstanbul Erkek Lisesi’nde devam ettiği orta öğrenimini Bursa Lisesi’nde tamamladı (1928). İstanbul Edebiyat \ -Fakültesi’ne iki yıl devam ettikten sonra babasının isteği üzerine iktisat öğrenimi için İsviçre’ye gitti. \ -Kısa süre sonra iktisat öğrenimini bırakarak Lozan’dan Grenoble’a geçti. Üç yıl başıboş bir edebiyat öğrenimi \ -gördükten sonra babası tarafından geri çağrıldı (1933). Bir müddet Halıcıoğlu Ermeni Yetim Mektebi'nde Türkçe \ -gurup dersleri öğretmenliği yaptı. Ticarete atıldıysa da tutunamadı. Bir ay Haber gazetesinde adliye muhabirliği\ -yaptı (1942). Babasının ölümü üzerine aileden kalan emlakin geliri ile avare bir hayata başladı. Evlenemedi.\ -Yazları Burgaz adasındaki köşklerinde, kışları Şişli’deki apartmanlarında annesi ile beraber geçen bu fazla \ -içkili bohem hayatı ömrünün sonuna kadar sürdü." - -print(nlp(question="Ne zaman avare bir hayata başladı?", context=sait)) -print(nlp(question="Sait Faik hangi Lisede orta öğrenimini tamamladı?", context=sait)) - -``` -``` -# Ask your self ! type your question -print(nlp(question="...?", context=sait)) -``` - - -Check My other Model -https://huggingface.co/savasy diff --git a/model_cards/savasy/bert-turkish-text-classification/README.md b/model_cards/savasy/bert-turkish-text-classification/README.md deleted file mode 100644 index 51b219bc729f..000000000000 --- a/model_cards/savasy/bert-turkish-text-classification/README.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -language: tr ---- - -# Turkish Text Classification - -This model is a fine-tune model of https://github.com/stefan-it/turkish-bert by using text classification data where there are 7 categories as follows - -``` -code_to_label={ - 'LABEL_0': 'dunya ', - 'LABEL_1': 'ekonomi ', - 'LABEL_2': 'kultur ', - 'LABEL_3': 'saglik ', - 'LABEL_4': 'siyaset ', - 'LABEL_5': 'spor ', - 'LABEL_6': 'teknoloji '} - - ``` - - -## Data -The following Turkish benchmark dataset is used for fine-tuning - -https://www.kaggle.com/savasy/ttc4900 - -## Quick Start - -Bewgin with installing transformers as follows -> pip install transformers - -``` -# Code: -# import libraries -from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer, AutoModelForSequenceClassification -tokenizer= AutoTokenizer.from_pretrained("savasy/bert-turkish-text-classification") - -# build and load model, it take time depending on your internet connection -model= AutoModelForSequenceClassification.from_pretrained("savasy/bert-turkish-text-classification") - -# make pipeline -nlp=pipeline("sentiment-analysis", model=model, tokenizer=tokenizer) - -# apply model -nlp("bla bla") -# [{'label': 'LABEL_2', 'score': 0.4753005802631378}] - -code_to_label={ - 'LABEL_0': 'dunya ', - 'LABEL_1': 'ekonomi ', - 'LABEL_2': 'kultur ', - 'LABEL_3': 'saglik ', - 'LABEL_4': 'siyaset ', - 'LABEL_5': 'spor ', - 'LABEL_6': 'teknoloji '} - -code_to_label[nlp("bla bla")[0]['label']] -# > 'kultur ' -``` - -## How the model was trained - -``` - -## loading data for Turkish text classification -import pandas as pd -# https://www.kaggle.com/savasy/ttc4900 -df=pd.read_csv("7allV03.csv") -df.columns=["labels","text"] -df.labels=pd.Categorical(df.labels) - -traind_df=... -eval_df=... - -# model -from simpletransformers.classification import ClassificationModel -import torch,sklearn - -model_args = { - "use_early_stopping": True, - "early_stopping_delta": 0.01, - "early_stopping_metric": "mcc", - "early_stopping_metric_minimize": False, - "early_stopping_patience": 5, - "evaluate_during_training_steps": 1000, - "fp16": False, - "num_train_epochs":3 -} - -model = ClassificationModel( - "bert", - "dbmdz/bert-base-turkish-cased", - use_cuda=cuda_available, - args=model_args, - num_labels=7 -) -model.train_model(train_df, acc=sklearn.metrics.accuracy_score) -``` -For other training models please check https://simpletransformers.ai/ - - -For the detailed usage of Turkish Text Classification please check [python notebook](https://github.com/savasy/TurkishTextClassification/blob/master/Bert_base_Text_Classification_for_Turkish.ipynb) diff --git a/model_cards/schmidek/electra-small-cased/README.md b/model_cards/schmidek/electra-small-cased/README.md deleted file mode 100644 index 7335809f204b..000000000000 --- a/model_cards/schmidek/electra-small-cased/README.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -language: en -license: apache-2.0 ---- - -## ELECTRA-small-cased - -This is a cased version of `google/electra-small-discriminator`, trained on the -[OpenWebText corpus](https://skylion007.github.io/OpenWebTextCorpus/). - -Uses the same tokenizer and vocab from `bert-base-cased` diff --git a/model_cards/seiya/oubiobert-base-uncased/README.md b/model_cards/seiya/oubiobert-base-uncased/README.md deleted file mode 100644 index 426042844b59..000000000000 --- a/model_cards/seiya/oubiobert-base-uncased/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -tags: -- exbert -license: apache-2.0 ---- - -# ouBioBERT-Base, Uncased - -Bidirectional Encoder Representations from Transformers for Biomedical Text Mining by Osaka University (ouBioBERT) is a language model based on the BERT-Base (Devlin, et al., 2019) architecture. We pre-trained ouBioBERT on PubMed abstracts from the PubMed baseline (ftp://ftp.ncbi.nlm.nih.gov/pubmed/baseline) via our method. - -The details of the pre-training procedure can be found in Wada, et al. (2020). - -## Evaluation - -We evaluated the performance of ouBioBERT in terms of the biomedical language understanding evaluation (BLUE) benchmark (Peng, et al., 2019). The numbers are mean (standard deviation) on five different random seeds. - - -| Dataset | Task Type | Score | -|:----------------|:-----------------------------|-------------:| -| MedSTS | Sentence similarity | 84.9 (0.6) | -| BIOSSES | Sentence similarity | 92.3 (0.8) | -| BC5CDR-disease | Named-entity recognition | 87.4 (0.1) | -| BC5CDR-chemical | Named-entity recognition | 93.7 (0.2) | -| ShARe/CLEFE | Named-entity recognition | 80.1 (0.4) | -| DDI | Relation extraction | 81.1 (1.5) | -| ChemProt | Relation extraction | 75.0 (0.3) | -| i2b2 2010 | Relation extraction | 74.0 (0.8) | -| HoC | Document classification | 86.4 (0.5) | -| MedNLI | Inference | 83.6 (0.7) | -| **Total** | Macro average of the scores |**83.8 (0.3)**| - - -## Code for Fine-tuning -We made the source code for fine-tuning freely available at [our repository](https://github.com/sy-wada/blue_benchmark_with_transformers). - -## Citation - -If you use our work in your research, please kindly cite the following paper: - -```bibtex -@misc{2005.07202, -Author = {Shoya Wada and Toshihiro Takeda and Shiro Manabe and Shozo Konishi and Jun Kamohara and Yasushi Matsumura}, -Title = {A pre-training technique to localize medical BERT and enhance BioBERT}, -Year = {2020}, -Eprint = {arXiv:2005.07202}, -} -``` - - - - diff --git a/model_cards/sentence-transformers/LaBSE/README.md b/model_cards/sentence-transformers/LaBSE/README.md deleted file mode 100644 index 51b05dd6245d..000000000000 --- a/model_cards/sentence-transformers/LaBSE/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# LaBSE Pytorch Version -This is a pytorch port of the tensorflow version of [LaBSE](https://tfhub.dev/google/LaBSE/1). - -To get the sentence embeddings, you can use the following code: -```python -from transformers import AutoTokenizer, AutoModel - -tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/LaBSE") -model = AutoModel.from_pretrained("sentence-transformers/LaBSE") - -sentences = ["Hello World", "Hallo Welt"] - -encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=64, return_tensors='pt') - -with torch.no_grad(): - model_output = model(**encoded_input) - -embeddings = model_output.pooler_output -embeddings = torch.nn.functional.normalize(embeddings) -print(embeddings) -``` - - -When you have [sentence-transformers](https://www.sbert.net/) installed, you can use the model like this: -```python -from sentence_transformers import SentenceTransformer -sentences = ["Hello World", "Hallo Welt"] - -model = SentenceTransformer('LaBSE') -embeddings = model.encode(sentences) -print(embeddings) -``` - -## Reference: -Fangxiaoyu Feng, Yinfei Yang, Daniel Cer, Narveen Ari, Wei Wang. [Language-agnostic BERT Sentence Embedding](https://arxiv.org/abs/2007.01852). July 2020 - -License: [https://tfhub.dev/google/LaBSE/1](https://tfhub.dev/google/LaBSE/1) diff --git a/model_cards/sentence-transformers/bert-base-nli-cls-token/README.md b/model_cards/sentence-transformers/bert-base-nli-cls-token/README.md deleted file mode 100644 index f31c2a9dba68..000000000000 --- a/model_cards/sentence-transformers/bert-base-nli-cls-token/README.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -language: en -tags: -- exbert -license: apache-2.0 -datasets: -- snli -- multi_nli ---- - -# BERT base model (uncased) for Sentence Embeddings -This is the `bert-base-nli-cls-token` model from the [sentence-transformers](https://github.com/UKPLab/sentence-transformers)-repository. The sentence-transformers repository allows to train and use Transformer models for generating sentence and text embeddings. -The model is described in the paper [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) - -## Usage (HuggingFace Models Repository) - -You can use the model directly from the model repository to compute sentence embeddings. The CLS token of each input represents the sentence embedding: -```python -from transformers import AutoTokenizer, AutoModel -import torch - - -#Sentences we want sentence embeddings for -sentences = ['This framework generates embeddings for each input sentence', - 'Sentences are passed as a list of string.', - 'The quick brown fox jumps over the lazy dog.'] - -#Load AutoModel from huggingface model repository -tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/bert-base-nli-cls-token") -model = AutoModel.from_pretrained("sentence-transformers/bert-base-nli-cls-token") - -#Tokenize sentences -encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=128, return_tensors='pt') - -#Compute token embeddings -with torch.no_grad(): - model_output = model(**encoded_input) - sentence_embeddings = model_output[0][:,0] #Take the first token ([CLS]) from each sentence - -print("Sentence embeddings:") -print(sentence_embeddings) -``` - -## Usage (Sentence-Transformers) -Using this model becomes more convenient when you have [sentence-transformers](https://github.com/UKPLab/sentence-transformers) installed: -``` -pip install -U sentence-transformers -``` - -Then you can use the model like this: -```python -from sentence_transformers import SentenceTransformer -model = SentenceTransformer('bert-base-nli-cls-token') -sentences = ['This framework generates embeddings for each input sentence', - 'Sentences are passed as a list of string.', - 'The quick brown fox jumps over the lazy dog.'] -sentence_embeddings = model.encode(sentences) - -print("Sentence embeddings:") -print(sentence_embeddings) -``` - - -## Citing & Authors -If you find this model helpful, feel free to cite our publication [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084): -``` -@inproceedings{reimers-2019-sentence-bert, - title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", - author = "Reimers, Nils and Gurevych, Iryna", - booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing", - month = "11", - year = "2019", - publisher = "Association for Computational Linguistics", - url = "http://arxiv.org/abs/1908.10084", -} -``` diff --git a/model_cards/sentence-transformers/bert-base-nli-max-tokens/README.md b/model_cards/sentence-transformers/bert-base-nli-max-tokens/README.md deleted file mode 100644 index c3ec7ea06a3e..000000000000 --- a/model_cards/sentence-transformers/bert-base-nli-max-tokens/README.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -language: en -tags: -- exbert -license: apache-2.0 -datasets: -- snli -- multi_nli ---- - -# BERT base model (uncased) for Sentence Embeddings -This is the `bert-base-nli-max-tokens` model from the [sentence-transformers](https://github.com/UKPLab/sentence-transformers)-repository. The sentence-transformers repository allows to train and use Transformer models for generating sentence and text embeddings. -The model is described in the paper [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) - -## Usage (HuggingFace Models Repository) - -You can use the model directly from the model repository to compute sentence embeddings. It uses max pooling to generate a fixed sized sentence embedding: -```python -from transformers import AutoTokenizer, AutoModel -import torch - - -#Max Pooling - Take the max value over time for every dimension -def max_pooling(model_output, attention_mask): - token_embeddings = model_output[0] #First element of model_output contains all token embeddings - input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() - token_embeddings[input_mask_expanded == 0] = -1e9 # Set padding tokens to large negative value - max_over_time = torch.max(token_embeddings, 1)[0] - return max_over_time - - -#Sentences we want sentence embeddings for -sentences = ['This framework generates embeddings for each input sentence', - 'Sentences are passed as a list of string.', - 'The quick brown fox jumps over the lazy dog.'] - -#Load AutoModel from huggingface model repository -tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/bert-base-nli-max-tokens") -model = AutoModel.from_pretrained("sentence-transformers/bert-base-nli-max-tokens") - -#Tokenize sentences -encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=128, return_tensors='pt') - -#Compute token embeddings -with torch.no_grad(): - model_output = model(**encoded_input) - -#Perform pooling. In this case, max pooling -sentence_embeddings = max_pooling(model_output, encoded_input['attention_mask']) - - -print("Sentence embeddings:") -print(sentence_embeddings) -``` - -## Usage (Sentence-Transformers) -Using this model becomes more convenient when you have [sentence-transformers](https://github.com/UKPLab/sentence-transformers) installed: -``` -pip install -U sentence-transformers -``` - -Then you can use the model like this: -```python -from sentence_transformers import SentenceTransformer -model = SentenceTransformer('bert-base-nli-max-tokens') -sentences = ['This framework generates embeddings for each input sentence', - 'Sentences are passed as a list of string.', - 'The quick brown fox jumps over the lazy dog.'] -sentence_embeddings = model.encode(sentences) - -print("Sentence embeddings:") -print(sentence_embeddings) -``` - - -## Citing & Authors -If you find this model helpful, feel free to cite our publication [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084): -``` -@inproceedings{reimers-2019-sentence-bert, - title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", - author = "Reimers, Nils and Gurevych, Iryna", - booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing", - month = "11", - year = "2019", - publisher = "Association for Computational Linguistics", - url = "http://arxiv.org/abs/1908.10084", -} -``` diff --git a/model_cards/sentence-transformers/bert-base-nli-mean-tokens/README.md b/model_cards/sentence-transformers/bert-base-nli-mean-tokens/README.md deleted file mode 100644 index 2ee7518734c9..000000000000 --- a/model_cards/sentence-transformers/bert-base-nli-mean-tokens/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -language: en -tags: -- exbert -license: apache-2.0 -datasets: -- snli -- multi_nli ---- - -# BERT base model (uncased) for Sentence Embeddings -This is the `bert-base-nli-mean-tokens` model from the [sentence-transformers](https://github.com/UKPLab/sentence-transformers)-repository. The sentence-transformers repository allows to train and use Transformer models for generating sentence and text embeddings. -The model is described in the paper [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) - -## Usage (HuggingFace Models Repository) - -You can use the model directly from the model repository to compute sentence embeddings: -```python -from transformers import AutoTokenizer, AutoModel -import torch - - -#Mean Pooling - Take attention mask into account for correct averaging -def mean_pooling(model_output, attention_mask): - token_embeddings = model_output[0] #First element of model_output contains all token embeddings - input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() - sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1) - sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9) - return sum_embeddings / sum_mask - - - -#Sentences we want sentence embeddings for -sentences = ['This framework generates embeddings for each input sentence', - 'Sentences are passed as a list of string.', - 'The quick brown fox jumps over the lazy dog.'] - -#Load AutoModel from huggingface model repository -tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/bert-base-nli-mean-tokens") -model = AutoModel.from_pretrained("sentence-transformers/bert-base-nli-mean-tokens") - -#Tokenize sentences -encoded_input = tokenizer(sentences, padding=True, truncation=True, max_length=128, return_tensors='pt') - -#Compute token embeddings -with torch.no_grad(): - model_output = model(**encoded_input) - -#Perform pooling. In this case, mean pooling -sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask']) -``` - -## Usage (Sentence-Transformers) -Using this model becomes more convenient when you have [sentence-transformers](https://github.com/UKPLab/sentence-transformers) installed: -``` -pip install -U sentence-transformers -``` - -Then you can use the model like this: -```python -from sentence_transformers import SentenceTransformer -model = SentenceTransformer('bert-base-nli-mean-tokens') -sentences = ['This framework generates embeddings for each input sentence', - 'Sentences are passed as a list of string.', - 'The quick brown fox jumps over the lazy dog.'] -sentence_embeddings = model.encode(sentences) - -print("Sentence embeddings:") -print(sentence_embeddings) -``` - - -## Citing & Authors -If you find this model helpful, feel free to cite our publication [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084): -``` -@inproceedings{reimers-2019-sentence-bert, - title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", - author = "Reimers, Nils and Gurevych, Iryna", - booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing", - month = "11", - year = "2019", - publisher = "Association for Computational Linguistics", - url = "http://arxiv.org/abs/1908.10084", -} -``` diff --git a/model_cards/severinsimmler/literary-german-bert/README.md b/model_cards/severinsimmler/literary-german-bert/README.md deleted file mode 100644 index 15c606dbaaf5..000000000000 --- a/model_cards/severinsimmler/literary-german-bert/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: de -thumbnail: kfold.png ---- - -# German BERT for literary texts - -This German BERT is based on `bert-base-german-dbmdz-cased`, and has been adapted to the domain of literary texts by fine-tuning the language modeling task on the [Corpus of German-Language Fiction](https://figshare.com/articles/Corpus_of_German-Language_Fiction_txt_/4524680/1). Afterwards the model was fine-tuned for named entity recognition on the [DROC](https://gitlab2.informatik.uni-wuerzburg.de/kallimachos/DROC-Release) corpus, so you can use it to recognize protagonists in German novels. - - -# Stats - -## Language modeling - -The [Corpus of German-Language Fiction](https://figshare.com/articles/Corpus_of_German-Language_Fiction_txt_/4524680/1) consists of 3,194 documents with 203,516,988 tokens or 1,520,855 types. The publication year of the texts ranges from the 18th to the 20th century: - -![years](prosa-jahre.png) - - -### Results - -After one epoch: - -| Model | Perplexity | -| ---------------- | ---------- | -| Vanilla BERT | 6.82 | -| Fine-tuned BERT | 4.98 | - - -## Named entity recognition - -The provided model was also fine-tuned for two epochs on 10,799 sentences for training, validated on 547 and tested on 1,845 with three labels: `B-PER`, `I-PER` and `O`. - - -## Results - -| Dataset | Precision | Recall | F1 | -| ------- | --------- | ------ | ---- | -| Dev | 96.4 | 87.3 | 91.6 | -| Test | 92.8 | 94.9 | 93.8 | - -The model has also been evaluated using 10-fold cross validation and compared with a classic Conditional Random Field baseline described in [Jannidis et al.](https://opus.bibliothek.uni-wuerzburg.de/opus4-wuerzburg/frontdoor/deliver/index/docId/14333/file/Jannidis_Figurenerkennung_Roman.pdf) (2015): - -![kfold](kfold.png) - - -# References - -Markus Krug, Lukas Weimer, Isabella Reger, Luisa Macharowsky, Stephan Feldhaus, Frank Puppe, Fotis Jannidis, [Description of a Corpus of Character References in German Novels](http://webdoc.sub.gwdg.de/pub/mon/dariah-de/dwp-2018-27.pdf), 2018. - -Fotis Jannidis, Isabella Reger, Lukas Weimer, Markus Krug, Martin Toepfer, Frank Puppe, [Automatische Erkennung von Figuren in deutschsprachigen Romanen](https://opus.bibliothek.uni-wuerzburg.de/opus4-wuerzburg/frontdoor/deliver/index/docId/14333/file/Jannidis_Figurenerkennung_Roman.pdf), 2015. diff --git a/model_cards/severinsimmler/literary-german-bert/kfold.png b/model_cards/severinsimmler/literary-german-bert/kfold.png deleted file mode 100644 index 2f1dbb5c8e19..000000000000 Binary files a/model_cards/severinsimmler/literary-german-bert/kfold.png and /dev/null differ diff --git a/model_cards/severinsimmler/literary-german-bert/prosa-jahre.png b/model_cards/severinsimmler/literary-german-bert/prosa-jahre.png deleted file mode 100644 index 6cf9e7b37153..000000000000 Binary files a/model_cards/severinsimmler/literary-german-bert/prosa-jahre.png and /dev/null differ diff --git a/model_cards/seyonec/ChemBERTa-zinc-base-v1/README.md b/model_cards/seyonec/ChemBERTa-zinc-base-v1/README.md deleted file mode 100644 index 8928e1b1afc0..000000000000 --- a/model_cards/seyonec/ChemBERTa-zinc-base-v1/README.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -tags: -- chemistry ---- - -# ChemBERTa: Training a BERT-like transformer model for masked language modelling of chemical SMILES strings. - -Deep learning for chemistry and materials science remains a novel field with lots of potiential. However, the popularity of transfer learning based methods in areas such as NLP and computer vision have not yet been effectively developed in computational chemistry + machine learning. Using HuggingFace's suite of models and the ByteLevel tokenizer, we are able to train on a large corpus of 100k SMILES strings from a commonly known benchmark dataset, ZINC. - -Training RoBERTa over 5 epochs, the model achieves a decent loss of 0.398, but may likely continue to decline if trained for a larger number of epochs. The model can predict tokens within a SMILES sequence/molecule, allowing for variants of a molecule within discoverable chemical space to be predicted. - -By applying the representations of functional groups and atoms learned by the model, we can try to tackle problems of toxicity, solubility, drug-likeness, and synthesis accessibility on smaller datasets using the learned representations as features for graph convolution and attention models on the graph structure of molecules, as well as fine-tuning of BERT. Finally, we propose the use of attention visualization as a helpful tool for chemistry practitioners and students to quickly identify important substructures in various chemical properties. - -Additionally, visualization of the attention mechanism have been seen through previous research as incredibly valuable towards chemical reaction classification. The applications of open-sourcing large-scale transformer models such as RoBERTa with HuggingFace may allow for the acceleration of these individual research directions. - -A link to a repository which includes the training, uploading and evaluation notebook (with sample predictions on compounds such as Remdesivir) can be found [here](https://github.com/seyonechithrananda/bert-loves-chemistry). All of the notebooks can be copied into a new Colab runtime for easy execution. - -Thanks for checking this out! -- Seyone diff --git a/model_cards/shoarora/alectra-small-owt/README.md b/model_cards/shoarora/alectra-small-owt/README.md deleted file mode 100644 index 046db2a82b3e..000000000000 --- a/model_cards/shoarora/alectra-small-owt/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# ALECTRA-small-OWT - -This is an extension of -[ELECTRA](https://openreview.net/forum?id=r1xMH1BtvB) small model, trained on the -[OpenWebText corpus](https://skylion007.github.io/OpenWebTextCorpus/). -The training task (discriminative LM / replaced-token-detection) can be generalized to any transformer type. Here, we train an ALBERT model under the same scheme. - -## Pretraining task -![electra task diagram](https://github.com/shoarora/lmtuners/raw/master/assets/electra.png) -(figure from [Clark et al. 2020](https://openreview.net/pdf?id=r1xMH1BtvB)) - -ELECTRA uses discriminative LM / replaced-token-detection for pretraining. -This involves a generator (a Masked LM model) creating examples for a discriminator -to classify as original or replaced for each token. - -The generator generalizes to any `*ForMaskedLM` model and the discriminator could be -any `*ForTokenClassification` model. Therefore, we can extend the task to ALBERT models, -not just BERT as in the original paper. - -## Usage -```python -from transformers import AlbertForSequenceClassification, BertTokenizer - -# Both models use the bert-base-uncased tokenizer and vocab. -tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') -alectra = AlbertForSequenceClassification.from_pretrained('shoarora/alectra-small-owt') -``` -NOTE: this ALBERT model uses a BERT WordPiece tokenizer. - -## Code -The pytorch module that implements this task is available [here](https://github.com/shoarora/lmtuners/blob/master/lmtuners/lightning_modules/discriminative_lm.py). - -Further implementation information [here](https://github.com/shoarora/lmtuners/tree/master/experiments/disc_lm_small), -and [here](https://github.com/shoarora/lmtuners/blob/master/experiments/disc_lm_small/train_alectra_small.py) is the script that created this model. - -This specific model was trained with the following params: -- `batch_size: 512` -- `training_steps: 5e5` -- `warmup_steps: 4e4` -- `learning_rate: 2e-3` - - -## Downstream tasks -#### GLUE Dev results -| Model | # Params | CoLA | SST | MRPC | STS | QQP | MNLI | QNLI | RTE | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| ELECTRA-Small++ | 14M | 57.0 | 91. | 88.0 | 87.5 | 89.0 | 81.3 | 88.4 | 66.7| -| ELECTRA-Small-OWT | 14M | 56.8 | 88.3| 87.4 | 86.8 | 88.3 | 78.9 | 87.9 | 68.5| -| ELECTRA-Small-OWT (ours) | 17M | 56.3 | 88.4| 75.0 | 86.1 | 89.1 | 77.9 | 83.0 | 67.1| -| ALECTRA-Small-OWT (ours) | 4M | 50.6 | 89.1| 86.3 | 87.2 | 89.1 | 78.2 | 85.9 | 69.6| - - -#### GLUE Test results -| Model | # Params | CoLA | SST | MRPC | STS | QQP | MNLI | QNLI | RTE | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| BERT-Base | 110M | 52.1 | 93.5| 84.8 | 85.9 | 89.2 | 84.6 | 90.5 | 66.4| -| GPT | 117M | 45.4 | 91.3| 75.7 | 80.0 | 88.5 | 82.1 | 88.1 | 56.0| -| ELECTRA-Small++ | 14M | 57.0 | 91.2| 88.0 | 87.5 | 89.0 | 81.3 | 88.4 | 66.7| -| ELECTRA-Small-OWT (ours) | 17M | 57.4 | 89.3| 76.2 | 81.9 | 87.5 | 78.1 | 82.4 | 68.1| -| ALECTRA-Small-OWT (ours) | 4M | 43.9 | 87.9| 82.1 | 82.0 | 87.6 | 77.9 | 85.8 | 67.5| diff --git a/model_cards/shoarora/electra-small-owt/README.md b/model_cards/shoarora/electra-small-owt/README.md deleted file mode 100644 index a1d1c8f93f9a..000000000000 --- a/model_cards/shoarora/electra-small-owt/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# ELECTRA-small-OWT - -This is an unnoficial implementation of an -[ELECTRA](https://openreview.net/forum?id=r1xMH1BtvB) small model, trained on the -[OpenWebText corpus](https://skylion007.github.io/OpenWebTextCorpus/). - -Differences from official ELECTRA models: - - we use a `BertForMaskedLM` as the generator and `BertForTokenClassification` as the discriminator - - they use an embedding projection layer, but Bert doesn't have one - -## Pretraining ttask -![electra task diagram](https://github.com/shoarora/lmtuners/raw/master/assets/electra.png) -(figure from [Clark et al. 2020](https://openreview.net/pdf?id=r1xMH1BtvB)) - -ELECTRA uses discriminative LM / replaced-token-detection for pretraining. -This involves a generator (a Masked LM model) creating examples for a discriminator -to classify as original or replaced for each token. - - -## Usage -```python -from transformers import BertForSequenceClassification, BertTokenizer - -tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') -electra = BertForSequenceClassification.from_pretrained('shoarora/electra-small-owt') -``` - -## Code -The pytorch module that implements this task is available [here](https://github.com/shoarora/lmtuners/blob/master/lmtuners/lightning_modules/discriminative_lm.py). - -Further implementation information [here](https://github.com/shoarora/lmtuners/tree/master/experiments/disc_lm_small), -and [here](https://github.com/shoarora/lmtuners/blob/master/experiments/disc_lm_small/train_electra_small.py) is the script that created this model. - -This specific model was trained with the following params: -- `batch_size: 512` -- `training_steps: 5e5` -- `warmup_steps: 4e4` -- `learning_rate: 2e-3` - - -## Downstream tasks -#### GLUE Dev results -| Model | # Params | CoLA | SST | MRPC | STS | QQP | MNLI | QNLI | RTE | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| ELECTRA-Small++ | 14M | 57.0 | 91. | 88.0 | 87.5 | 89.0 | 81.3 | 88.4 | 66.7| -| ELECTRA-Small-OWT | 14M | 56.8 | 88.3| 87.4 | 86.8 | 88.3 | 78.9 | 87.9 | 68.5| -| ELECTRA-Small-OWT (ours) | 17M | 56.3 | 88.4| 75.0 | 86.1 | 89.1 | 77.9 | 83.0 | 67.1| -| ALECTRA-Small-OWT (ours) | 4M | 50.6 | 89.1| 86.3 | 87.2 | 89.1 | 78.2 | 85.9 | 69.6| - -- Table initialized from [ELECTRA github repo](https://github.com/google-research/electra) - -#### GLUE Test results -| Model | # Params | CoLA | SST | MRPC | STS | QQP | MNLI | QNLI | RTE | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| BERT-Base | 110M | 52.1 | 93.5| 84.8 | 85.9 | 89.2 | 84.6 | 90.5 | 66.4| -| GPT | 117M | 45.4 | 91.3| 75.7 | 80.0 | 88.5 | 82.1 | 88.1 | 56.0| -| ELECTRA-Small++ | 14M | 57.0 | 91.2| 88.0 | 87.5 | 89.0 | 81.3 | 88.4 | 66.7| -| ELECTRA-Small-OWT (ours) | 17M | 57.4 | 89.3| 76.2 | 81.9 | 87.5 | 78.1 | 82.4 | 68.1| -| ALECTRA-Small-OWT (ours) | 4M | 43.9 | 87.9| 82.1 | 82.0 | 87.6 | 77.9 | 85.8 | 67.5| diff --git a/model_cards/shrugging-grace/tweetclassifier/README.md b/model_cards/shrugging-grace/tweetclassifier/README.md deleted file mode 100644 index 21a43b19a14d..000000000000 --- a/model_cards/shrugging-grace/tweetclassifier/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# shrugging-grace/tweetclassifier - -## Model description -This model classifies tweets as either relating to the Covid-19 pandemic or not. - -## Intended uses & limitations -It is intended to be used on tweets commenting on UK politics, in particular those trending with the #PMQs hashtag, as this refers to weekly Prime Ministers' Questions. - -#### How to use -``LABEL_0`` means that the tweet relates to Covid-19 - -``LABEL_1`` means that the tweet does not relate to Covid-19 - -## Training data -The model was trained on 1000 tweets (with the "#PMQs'), which were manually labeled by the author. The tweets were collected between May-July 2020. - -### BibTeX entry and citation info - -This was based on a pretrained version of BERT. - -@article{devlin2018bert, - title={Bert: Pre-training of deep bidirectional transformers for language understanding}, - author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina}, - journal={arXiv preprint arXiv:1810.04805}, - year={2018} -} diff --git a/model_cards/smanjil/German-MedBERT/README.md b/model_cards/smanjil/German-MedBERT/README.md deleted file mode 100644 index d9f78256187b..000000000000 --- a/model_cards/smanjil/German-MedBERT/README.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -language: de -tags: -- exbert -- German ---- - - - - - -# German Medical BERT - -This is a fine-tuned model on Medical domain for German language and based on German BERT. This model has only been trained to improve on target task (Masked Language Model). It can later be used to perform a downstream task of your needs, while I performed it for NTS-ICD-10 text classification task. - -## Overview -**Language model:** bert-base-german-cased - -**Language:** German - -**Fine-tuning:** Medical articles (diseases, symptoms, therapies, etc..) - -**Eval data:** NTS-ICD-10 dataset (Classification) - -**Infrastructure:** Gogle Colab - - -## Details -- We fine-tuned using Pytorch with Huggingface library on Colab GPU. -- With standard parameter settings for fine-tuning as mentioned in original BERT's paper. -- Although had to train for upto 25 epochs for classification. - -## Performance (Micro precision, recall and f1 score for multilabel code classification) - -|Models |P |R |F1 | -|:-------------- |:------|:------|:------| -|German BERT |86.04 |75.82 |80.60 | -|German MedBERT-256 |87.41 |77.97 |82.42 | -|German MedBERT-512 |87.75 |78.26 |82.73 | - -## Author -Manjil Shrestha: `shresthamanjil21 [at] gmail.com` - -Get in touch: -[LinkedIn](https://www.linkedin.com/in/manjil-shrestha-038527b4/) diff --git a/model_cards/spentaur/yelp/README.md b/model_cards/spentaur/yelp/README.md deleted file mode 100644 index aaa4bf1f4c0d..000000000000 --- a/model_cards/spentaur/yelp/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# DistilBERT Yelp Review Sentiment -This model is used for sentiment analysis on english yelp reviews. -It is a DistilBERT model trained on 1 million reviews from the yelp open dataset. -It is a regression model, with outputs in the range of ~-2 to ~2. With -2 being 1 star and 2 being 5 stars. -It was trained using the [ktrain](https://github.com/amaiya/ktrain) because of it's ease of use. - -Example use: - -``` -tokenizer = AutoTokenizer.from_pretrained( - 'distilbert-base-uncased', use_fast=True) -model = TFAutoModelForSequenceClassification.from_pretrained( - "spentaur/yelp") - -review = "This place is great!" -input_ids = tokenizer.encode(review, return_tensors='tf') -pred = model(input_ids)[0][0][0].numpy() -# pred should === 1.9562385 -``` diff --git a/model_cards/squeezebert/squeezebert-mnli-headless/README.md b/model_cards/squeezebert/squeezebert-mnli-headless/README.md deleted file mode 100644 index 07c8ad18656e..000000000000 --- a/model_cards/squeezebert/squeezebert-mnli-headless/README.md +++ /dev/null @@ -1,67 +0,0 @@ -language: en -license: bsd -datasets: -- bookcorpus -- wikipedia ---- - -# SqueezeBERT pretrained model - -This model, `squeezebert-mnli-headless`, has been pretrained for the English language using a masked language modeling (MLM) and Sentence Order Prediction (SOP) objective and finetuned on the [Multi-Genre Natural Language Inference (MNLI)](https://cims.nyu.edu/~sbowman/multinli/) dataset. This is a "headless" model with the final classification layer removed, and this will allow Transformers to automatically reinitialize the final classification layer before you begin finetuning on your data. -SqueezeBERT was introduced in [this paper](https://arxiv.org/abs/2006.11316). This model is case-insensitive. The model architecture is similar to BERT-base, but with the pointwise fully-connected layers replaced with [grouped convolutions](https://blog.yani.io/filter-group-tutorial/). -The authors found that SqueezeBERT is 4.3x faster than `bert-base-uncased` on a Google Pixel 3 smartphone. - - -## Pretraining - -### Pretraining data -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of thousands of unpublished books -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) - -### Pretraining procedure -The model is pretrained using the Masked Language Model (MLM) and Sentence Order Prediction (SOP) tasks. -(Author's note: If you decide to pretrain your own model, and you prefer to train with MLM only, that should work too.) - -From the SqueezeBERT paper: -> We pretrain SqueezeBERT from scratch (without distillation) using the [LAMB](https://arxiv.org/abs/1904.00962) optimizer, and we employ the hyperparameters recommended by the LAMB authors: a global batch size of 8192, a learning rate of 2.5e-3, and a warmup proportion of 0.28. Following the LAMB paper's recommendations, we pretrain for 56k steps with a maximum sequence length of 128 and then for 6k steps with a maximum sequence length of 512. - -## Finetuning - -The SqueezeBERT paper presents 2 approaches to finetuning the model: -- "finetuning without bells and whistles" -- after pretraining the SqueezeBERT model, finetune it on each GLUE task -- "finetuning with bells and whistles" -- after pretraining the SqueezeBERT model, finetune it on a MNLI with distillation from a teacher model. Then, use the MNLI-finetuned SqueezeBERT model as a student model to finetune on each of the other GLUE tasks (e.g. RTE, MRPC, …) with distillation from a task-specific teacher model. - -A detailed discussion of the hyperparameters used for finetuning is provided in the appendix of the [SqueezeBERT paper](https://arxiv.org/abs/2006.11316). -Note that finetuning SqueezeBERT with distillation is not yet implemented in this repo. If the author (Forrest Iandola - forrest.dnn@gmail.com) gets enough encouragement from the user community, he will add example code to Transformers for finetuning SqueezeBERT with distillation. - -This model, `squeezebert/squeezebert-mnli-headless`, is the "finetuned with bells and whistles" MNLI-finetuned SqueezeBERT model. In this particular model, we have removed the final classification layer -- in other words, it is "headless." We recommend using this model if you intend to finetune the model on your own data. Using this model means that your final layer will automatically be reinitialized when you start finetuning on your data. - -### How to finetune -To try finetuning SqueezeBERT on the [MRPC](https://www.microsoft.com/en-us/download/details.aspx?id=52398) text classification task, you can run the following command: -``` -./utils/download_glue_data.py - -python examples/text-classification/run_glue.py \ - --model_name_or_path squeezebert-base-headless \ - --task_name mrpc \ - --data_dir ./glue_data/MRPC \ - --output_dir ./models/squeezebert_mrpc \ - --overwrite_output_dir \ - --do_train \ - --do_eval \ - --num_train_epochs 10 \ - --learning_rate 3e-05 \ - --per_device_train_batch_size 16 \ - --save_steps 20000 - -``` - -## BibTeX entry and citation info -``` -@article{2020_SqueezeBERT, - author = {Forrest N. Iandola and Albert E. Shaw and Ravi Krishna and Kurt W. Keutzer}, - title = {{SqueezeBERT}: What can computer vision teach NLP about efficient neural networks?}, - journal = {arXiv:2006.11316}, - year = {2020} -} -``` diff --git a/model_cards/squeezebert/squeezebert-mnli/README.md b/model_cards/squeezebert/squeezebert-mnli/README.md deleted file mode 100644 index 28d910dc442f..000000000000 --- a/model_cards/squeezebert/squeezebert-mnli/README.md +++ /dev/null @@ -1,67 +0,0 @@ -language: en -license: bsd -datasets: -- bookcorpus -- wikipedia ---- - -# SqueezeBERT pretrained model - -This model, `squeezebert-mnli`, has been pretrained for the English language using a masked language modeling (MLM) and Sentence Order Prediction (SOP) objective and finetuned on the [Multi-Genre Natural Language Inference (MNLI)](https://cims.nyu.edu/~sbowman/multinli/) dataset. -SqueezeBERT was introduced in [this paper](https://arxiv.org/abs/2006.11316). This model is case-insensitive. The model architecture is similar to BERT-base, but with the pointwise fully-connected layers replaced with [grouped convolutions](https://blog.yani.io/filter-group-tutorial/). -The authors found that SqueezeBERT is 4.3x faster than `bert-base-uncased` on a Google Pixel 3 smartphone. - - -## Pretraining - -### Pretraining data -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of thousands of unpublished books -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) - -### Pretraining procedure -The model is pretrained using the Masked Language Model (MLM) and Sentence Order Prediction (SOP) tasks. -(Author's note: If you decide to pretrain your own model, and you prefer to train with MLM only, that should work too.) - -From the SqueezeBERT paper: -> We pretrain SqueezeBERT from scratch (without distillation) using the [LAMB](https://arxiv.org/abs/1904.00962) optimizer, and we employ the hyperparameters recommended by the LAMB authors: a global batch size of 8192, a learning rate of 2.5e-3, and a warmup proportion of 0.28. Following the LAMB paper's recommendations, we pretrain for 56k steps with a maximum sequence length of 128 and then for 6k steps with a maximum sequence length of 512. - -## Finetuning - -The SqueezeBERT paper presents 2 approaches to finetuning the model: -- "finetuning without bells and whistles" -- after pretraining the SqueezeBERT model, finetune it on each GLUE task -- "finetuning with bells and whistles" -- after pretraining the SqueezeBERT model, finetune it on a MNLI with distillation from a teacher model. Then, use the MNLI-finetuned SqueezeBERT model as a student model to finetune on each of the other GLUE tasks (e.g. RTE, MRPC, …) with distillation from a task-specific teacher model. - -A detailed discussion of the hyperparameters used for finetuning is provided in the appendix of the [SqueezeBERT paper](https://arxiv.org/abs/2006.11316). -Note that finetuning SqueezeBERT with distillation is not yet implemented in this repo. If the author (Forrest Iandola - forrest.dnn@gmail.com) gets enough encouragement from the user community, he will add example code to Transformers for finetuning SqueezeBERT with distillation. - -This model, `squeezebert/squeezebert-mnli`, is the "trained with bells and whistles" MNLI-finetuned SqueezeBERT model. - -### How to finetune -To try finetuning SqueezeBERT on the [MRPC](https://www.microsoft.com/en-us/download/details.aspx?id=52398) text classification task, you can run the following command: -``` -./utils/download_glue_data.py - -python examples/text-classification/run_glue.py \ - --model_name_or_path squeezebert-base-headless \ - --task_name mrpc \ - --data_dir ./glue_data/MRPC \ - --output_dir ./models/squeezebert_mrpc \ - --overwrite_output_dir \ - --do_train \ - --do_eval \ - --num_train_epochs 10 \ - --learning_rate 3e-05 \ - --per_device_train_batch_size 16 \ - --save_steps 20000 - -``` - -## BibTeX entry and citation info -``` -@article{2020_SqueezeBERT, - author = {Forrest N. Iandola and Albert E. Shaw and Ravi Krishna and Kurt W. Keutzer}, - title = {{SqueezeBERT}: What can computer vision teach NLP about efficient neural networks?}, - journal = {arXiv:2006.11316}, - year = {2020} -} -``` diff --git a/model_cards/squeezebert/squeezebert-uncased/README.md b/model_cards/squeezebert/squeezebert-uncased/README.md deleted file mode 100644 index c1a38cab0b8d..000000000000 --- a/model_cards/squeezebert/squeezebert-uncased/README.md +++ /dev/null @@ -1,67 +0,0 @@ -language: en -license: bsd -datasets: -- bookcorpus -- wikipedia ---- - -# SqueezeBERT pretrained model - -This model, `squeezebert-uncased`, is a pretrained model for the English language using a masked language modeling (MLM) and Sentence Order Prediction (SOP) objective. -SqueezeBERT was introduced in [this paper](https://arxiv.org/abs/2006.11316). This model is case-insensitive. The model architecture is similar to BERT-base, but with the pointwise fully-connected layers replaced with [grouped convolutions](https://blog.yani.io/filter-group-tutorial/). -The authors found that SqueezeBERT is 4.3x faster than `bert-base-uncased` on a Google Pixel 3 smartphone. - - -## Pretraining - -### Pretraining data -- [BookCorpus](https://yknzhu.wixsite.com/mbweb), a dataset consisting of thousands of unpublished books -- [English Wikipedia](https://en.wikipedia.org/wiki/English_Wikipedia) - -### Pretraining procedure -The model is pretrained using the Masked Language Model (MLM) and Sentence Order Prediction (SOP) tasks. -(Author's note: If you decide to pretrain your own model, and you prefer to train with MLM only, that should work too.) - -From the SqueezeBERT paper: -> We pretrain SqueezeBERT from scratch (without distillation) using the [LAMB](https://arxiv.org/abs/1904.00962) optimizer, and we employ the hyperparameters recommended by the LAMB authors: a global batch size of 8192, a learning rate of 2.5e-3, and a warmup proportion of 0.28. Following the LAMB paper's recommendations, we pretrain for 56k steps with a maximum sequence length of 128 and then for 6k steps with a maximum sequence length of 512. - -## Finetuning - -The SqueezeBERT paper results from 2 approaches to finetuning the model: -- "finetuning without bells and whistles" -- after pretraining the SqueezeBERT model, finetune it on each GLUE task -- "finetuning with bells and whistles" -- after pretraining the SqueezeBERT model, finetune it on a MNLI with distillation from a teacher model. Then, use the MNLI-finetuned SqueezeBERT model as a student model to finetune on each of the other GLUE tasks (e.g. RTE, MRPC, …) with distillation from a task-specific teacher model. - -A detailed discussion of the hyperparameters used for finetuning is provided in the appendix of the [SqueezeBERT paper](https://arxiv.org/abs/2006.11316). -Note that finetuning SqueezeBERT with distillation is not yet implemented in this repo. If the author (Forrest Iandola - forrest.dnn@gmail.com) gets enough encouragement from the user community, he will add example code to Transformers for finetuning SqueezeBERT with distillation. - -This model, `squeezebert/squeezebert-uncased`, has been pretrained but not finetuned. For most text classification tasks, we recommend using squeezebert-mnli-headless as a starting point. - -### How to finetune -To try finetuning SqueezeBERT on the [MRPC](https://www.microsoft.com/en-us/download/details.aspx?id=52398) text classification task, you can run the following command: -``` -./utils/download_glue_data.py - -python examples/text-classification/run_glue.py \ - --model_name_or_path squeezebert-base-headless \ - --task_name mrpc \ - --data_dir ./glue_data/MRPC \ - --output_dir ./models/squeezebert_mrpc \ - --overwrite_output_dir \ - --do_train \ - --do_eval \ - --num_train_epochs 10 \ - --learning_rate 3e-05 \ - --per_device_train_batch_size 16 \ - --save_steps 20000 - -``` - -## BibTeX entry and citation info -``` -@article{2020_SqueezeBERT, - author = {Forrest N. Iandola and Albert E. Shaw and Ravi Krishna and Kurt W. Keutzer}, - title = {{SqueezeBERT}: What can computer vision teach NLP about efficient neural networks?}, - journal = {arXiv:2006.11316}, - year = {2020} -} -``` diff --git a/model_cards/stas/tiny-wmt19-en-de/README.md b/model_cards/stas/tiny-wmt19-en-de/README.md deleted file mode 100644 index e60d2bc01f1a..000000000000 --- a/model_cards/stas/tiny-wmt19-en-de/README.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -language: -- en -- de -thumbnail: -tags: -- wmt19 -- testing -license: apache-2.0 -datasets: -- wmt19 -metrics: -- bleu ---- - -# Tiny FSMT - -This is a tiny model that is used in the `transformers` test suite. It doesn't do anything useful, other than testing that `FSMT` works. diff --git a/model_cards/stevhliu/astroGPT/README.md b/model_cards/stevhliu/astroGPT/README.md deleted file mode 100644 index f2f2a9e5109c..000000000000 --- a/model_cards/stevhliu/astroGPT/README.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -language: "en" -thumbnail: "https://raw.githubusercontent.com/stevhliu/satsuma/master/images/astroGPT-thumbnail.png" -widget: -- text: "Jan 18, 2020" -- text: "Feb 14, 2020" -- text: "Jul 04, 2020" ---- - -# astroGPT 🪐 - -## Model description - -This is a GPT-2 model fine-tuned on Western zodiac signs. For more information about GPT-2, take a look at 🤗 Hugging Face's GPT-2 [model card](https://huggingface.co/gpt2). You can use astroGPT to generate a daily horoscope by entering the current date. - -## How to use - -To use this model, simply enter the current date like so `Mon DD, YEAR`: - -```python -from transformers import AutoTokenizer, AutoModelWithLMHead - -tokenizer = AutoTokenizer.from_pretrained("stevhliu/astroGPT") -model = AutoModelWithLMHead.from_pretrained("stevhliu/astroGPT") - -input_ids = tokenizer.encode('Sep 03, 2020', return_tensors='pt').to('cuda') - -sample_output = model.generate(input_ids, - do_sample=True, - max_length=75, - top_k=20, - top_p=0.97) - -print(sample_output) -``` - -## Limitations and bias - -astroGPT inherits the same biases that affect GPT-2 as a result of training on a lot of non-neutral content on the internet. The model does not currently support zodiac sign-specific generation and only returns a general horoscope. While the generated text may occasionally mention a specific zodiac sign, this is due to how the horoscopes were originally written by it's human authors. - -## Data - -The data was scraped from [Horoscope.com](https://www.horoscope.com/us/index.aspx) and trained on 4.7MB of text. The text was collected from four categories (daily, love, wellness, career) and span from 09/01/19 to 08/01/2020. The archives only store horoscopes dating a year back from the current date. - -## Training and results - -The text was tokenized using the fast GPT-2 BPE [tokenizer](https://huggingface.co/transformers/model_doc/gpt2.html#gpt2tokenizerfast). It has a vocabulary size of 50,257 and sequence length of 1024 tokens. The model was trained with on one of Google Colaboratory's GPU's for approximately 2.5 hrs with [fastai's](https://docs.fast.ai/) learning rate finder, discriminative learning rates and 1cycle policy. See table below for a quick summary of the training procedure and results. - -| dataset size | epochs | lr | training time | train_loss | valid_loss | perplexity | -|:-------------:|:------:|:-----------------:|:-------------:|:----------:|:----------:|:----------:| -| 5.9MB |32 | slice(1e-7,1e-5) | 2.5 hrs | 2.657170 | 2.642387 | 14.046692 | diff --git a/model_cards/surajp/RoBERTa-hindi-guj-san/README.md b/model_cards/surajp/RoBERTa-hindi-guj-san/README.md deleted file mode 100644 index 4d437d6e7f8b..000000000000 --- a/model_cards/surajp/RoBERTa-hindi-guj-san/README.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -language: -- hi -- sa -- gu -tags: -- Indic -license: mit -datasets: -- Wikipedia (Hindi, Sanskrit, Gujarati) -metrics: -- perplexity ---- - -# RoBERTa-hindi-guj-san - -## Model description - -Multillingual RoBERTa like model trained on Wikipedia articles of Hindi, Sanskrit, Gujarati languages. The tokenizer was trained on combined text. -However, Hindi text was used to pre-train the model and then it was fine-tuned on Sanskrit and Gujarati Text combined hoping that pre-training with Hindi -will help the model learn similar languages. - -### Configuration - -| Parameter | Value | -|---|---| -| `hidden_size` | 768 | -| `num_attention_heads` | 12 | -| `num_hidden_layers` | 6 | -| `vocab_size` | 30522 | -|`model_type`|`roberta`| - -## Intended uses & limitations - -#### How to use - -```python -# Example usage -from transformers import AutoTokenizer, AutoModelWithLMHead, pipeline - -tokenizer = AutoTokenizer.from_pretrained("surajp/RoBERTa-hindi-guj-san") -model = AutoModelWithLMHead.from_pretrained("surajp/RoBERTa-hindi-guj-san") - -fill_mask = pipeline( - "fill-mask", - model=model, - tokenizer=tokenizer -) - -# Sanskrit: इयं भाषा न केवलं भारतस्य अपि तु विश्वस्य प्राचीनतमा भाषा इति मन्यते। -# Hindi: अगर आप अब अभ्यास नहीं करते हो तो आप अपने परीक्षा में मूर्खतापूर्ण गलतियाँ करोगे। -# Gujarati: ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો હતો. -fill_mask("ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો હતો.") - -''' -Output: --------- -[ -{'score': 0.07849744707345963, 'sequence': ' ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો જ હતો.', 'token': 390}, -{'score': 0.06273336708545685, 'sequence': ' ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો ન હતો.', 'token': 478}, -{'score': 0.05160355195403099, 'sequence': ' ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો થઇ હતો.', 'token': 2075}, -{'score': 0.04751499369740486, 'sequence': ' ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો એક હતો.', 'token': 600}, -{'score': 0.03788900747895241, 'sequence': ' ગુજરાતમાં ૧૯મી માર્ચ સુધી કોઈ સકારાત્મક (પોઝીટીવ) રીપોર્ટ આવ્યો પણ હતો.', 'token': 840} -] - -``` - -## Training data - -Cleaned wikipedia articles in Hindi, Sanskrit and Gujarati on Kaggle. It contains training as well as evaluation text. -Used in [iNLTK](https://github.com/goru001/inltk) - -- [Hindi](https://www.kaggle.com/disisbig/hindi-wikipedia-articles-172k) -- [Gujarati](https://www.kaggle.com/disisbig/gujarati-wikipedia-articles) -- [Sanskrit](https://www.kaggle.com/disisbig/sanskrit-wikipedia-articles) - -## Training procedure - -- On TPU (using `xla_spawn.py`) -- For language modelling -- Iteratively increasing `--block_size` from 128 to 256 over epochs -- Tokenizer trained on combined text -- Pre-training with Hindi and fine-tuning on Sanskrit and Gujarati texts - -``` ---model_type distillroberta-base \ ---model_name_or_path "/content/SanHiGujBERTa" \ ---mlm_probability 0.20 \ ---line_by_line \ ---save_total_limit 2 \ ---per_device_train_batch_size 128 \ ---per_device_eval_batch_size 128 \ ---num_train_epochs 5 \ ---block_size 256 \ ---seed 108 \ ---overwrite_output_dir \ -``` - -## Eval results - -perplexity = 2.920005983224673 - - - -> Created by [Suraj Parmar/@parmarsuraj99](https://twitter.com/parmarsuraj99) | [LinkedIn](https://www.linkedin.com/in/parmarsuraj99/) - -> Made with ♥ in India diff --git a/model_cards/surajp/SanBERTa/README.md b/model_cards/surajp/SanBERTa/README.md deleted file mode 100644 index 89d4b138b22c..000000000000 --- a/model_cards/surajp/SanBERTa/README.md +++ /dev/null @@ -1,99 +0,0 @@ ---- -language: sa ---- - -# RoBERTa trained on Sanskrit (SanBERTa) - -**Mode size** (after training): **340MB** - -### Dataset: - -[Wikipedia articles](https://www.kaggle.com/disisbig/sanskrit-wikipedia-articles) (used in [iNLTK](https://github.com/goru001/nlp-for-sanskrit)). -It contains evaluation set. - -[Sanskrit scraps from CLTK](http://cltk.org/) - -### Configuration - -| Parameter | Value | -|---|---| -| `num_attention_heads` | 12 | -| `num_hidden_layers` | 6 | -| `hidden_size` | 768 | -| `vocab_size` | 29407 | - -### Training : -- On TPU -- For language modelling -- Iteratively increasing `--block_size` from 128 to 256 over epochs - -### Evaluation - -|Metric| # Value | -|---|---| -|Perplexity (`block_size=256`)|4.04| - -## Example of usage: - -### For Embeddings - -``` - -tokenizer = AutoTokenizer.from_pretrained("surajp/SanBERTa") -model = RobertaModel.from_pretrained("surajp/SanBERTa") - -op = tokenizer.encode("इयं भाषा न केवलं भारतस्य अपि तु विश्वस्य प्राचीनतमा भाषा इति मन्यते।", return_tensors="pt") -ps = model(op) -ps[0].shape - -``` -``` -''' -Output: --------- -torch.Size([1, 47, 768]) - -``` - - -### For \ Prediction - -``` -from transformers import pipeline - -fill_mask = pipeline( - "fill-mask", - model="surajp/SanBERTa", - tokenizer="surajp/SanBERTa" -) - -## इयं भाषा न केवलं भारतस्य अपि तु विश्वस्य प्राचीनतमा भाषा इति मन्यते। -fill_mask("इयं भाषा न केवल भारतस्य अपि तु विश्वस्य प्राचीनतमा भाषा इति मन्यते।") - -ps = model(torch.tensor(enc).unsqueeze(1)) -print(ps[0].shape) -``` -``` -''' -Output: --------- -[{'score': 0.7516744136810303, - 'sequence': ' इयं भाषा न केवलं भारतस्य अपि तु विश्वस्य प्राचीनतमा भाषा इति मन्यते।', - 'token': 280, - 'token_str': 'à¤Ĥ'}, - {'score': 0.06230105459690094, - 'sequence': ' इयं भाषा न केवली भारतस्य अपि तु विश्वस्य प्राचीनतमा भाषा इति मन्यते।', - 'token': 289, - 'token_str': 'à¥Ģ'}, - {'score': 0.055410224944353104, - 'sequence': ' इयं भाषा न केवला भारतस्य अपि तु विश्वस्य प्राचीनतमा भाषा इति मन्यते।', - 'token': 265, - 'token_str': 'ा'}, - ...] -``` - -### It works!! 🎉 🎉 🎉 - -> Created by [Suraj Parmar/@parmarsuraj99](https://twitter.com/parmarsuraj99) | [LinkedIn](https://www.linkedin.com/in/parmarsuraj99/) - -> Made with ♥ in India diff --git a/model_cards/surajp/albert-base-sanskrit/README.md b/model_cards/surajp/albert-base-sanskrit/README.md deleted file mode 100644 index 0a24bfa03330..000000000000 --- a/model_cards/surajp/albert-base-sanskrit/README.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -language: sa ---- - - -# ALBERT-base-Sanskrit - - -Explaination Notebook Colab: [SanskritALBERT.ipynb](https://colab.research.google.com/github/parmarsuraj99/suraj-parmar/blob/master/_notebooks/2020-05-02-SanskritALBERT.ipynb) - -Size of the model is **46MB** - -Example of usage: - -``` -tokenizer = AutoTokenizer.from_pretrained("surajp/albert-base-sanskrit") -model = AutoModel.from_pretrained("surajp/albert-base-sanskrit") - -enc=tokenizer.encode("ॐ सर्वे भवन्तु सुखिनः सर्वे सन्तु निरामयाः । सर्वे भद्राणि पश्यन्तु मा कश्चिद्दुःखभाग्भवेत् । ॐ शान्तिः शान्तिः शान्तिः ॥") -print(tokenizer.decode(enc)) - -ps = model(torch.tensor(enc).unsqueeze(1)) -print(ps[0].shape) -``` -``` -''' -Output: --------- -[CLS] ॐ सर्वे भवन्तु सुखिनः सर्वे सन्तु निरामयाः । सर्वे भद्राणि पश्यन्तु मा कश्चिद्दुःखभाग्भवेत् । ॐ शान्तिः शान्तिः शान्तिः ॥[SEP] -torch.Size([28, 1, 768]) -``` - - -> Created by [Suraj Parmar/@parmarsuraj99](https://twitter.com/parmarsuraj99) - -> Made with ♥ in India diff --git a/model_cards/t5-11b-README.md b/model_cards/t5-11b-README.md deleted file mode 100644 index 131667018caa..000000000000 --- a/model_cards/t5-11b-README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -language: en -datasets: -- c4 -tags: -- summarization -- translation - -license: apache-2.0 -inference: false ---- - -## Disclaimer - -**Before `transformers` v3.5.0**, due do its immense size, `t5-11b` required some special treatment. -If you're using transformers `<= v3.4.0`, `t5-11b` should be loaded with flag `use_cdn` set to `False` as follows: - -```python -t5 = transformers.T5ForConditionalGeneration.from_pretrained('t5-11b', use_cdn = False) -``` - -Secondly, a single GPU will most likely not have enough memory to even load the model into memory as the weights alone amount to over 40 GB. -Model parallelism has to be used here to overcome this problem as is explained in this [PR](https://github.com/huggingface/transformers/pull/3578). - -## [Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) - -Pretraining Dataset: [C4](https://huggingface.co/datasets/c4) - -Other Community Checkpoints: [here](https://huggingface.co/models?search=t5) - -Paper: [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) - -Authors: *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* - -## Abstract - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67) - diff --git a/model_cards/t5-3b-README.md b/model_cards/t5-3b-README.md deleted file mode 100644 index ff90f7705636..000000000000 --- a/model_cards/t5-3b-README.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -language: en -datasets: -- c4 -tags: -- summarization -- translation - -license: apache-2.0 ---- - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) - -Pretraining Dataset: [C4](https://huggingface.co/datasets/c4) - -Other Community Checkpoints: [here](https://huggingface.co/models?search=t5) - -Paper: [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) - -Authors: *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* - - -## Abstract - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67) - diff --git a/model_cards/t5-base-README.md b/model_cards/t5-base-README.md deleted file mode 100644 index ff90f7705636..000000000000 --- a/model_cards/t5-base-README.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -language: en -datasets: -- c4 -tags: -- summarization -- translation - -license: apache-2.0 ---- - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) - -Pretraining Dataset: [C4](https://huggingface.co/datasets/c4) - -Other Community Checkpoints: [here](https://huggingface.co/models?search=t5) - -Paper: [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) - -Authors: *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* - - -## Abstract - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67) - diff --git a/model_cards/t5-large-README.md b/model_cards/t5-large-README.md deleted file mode 100644 index ff90f7705636..000000000000 --- a/model_cards/t5-large-README.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -language: en -datasets: -- c4 -tags: -- summarization -- translation - -license: apache-2.0 ---- - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) - -Pretraining Dataset: [C4](https://huggingface.co/datasets/c4) - -Other Community Checkpoints: [here](https://huggingface.co/models?search=t5) - -Paper: [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) - -Authors: *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* - - -## Abstract - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67) - diff --git a/model_cards/t5-small-README.md b/model_cards/t5-small-README.md deleted file mode 100644 index ff90f7705636..000000000000 --- a/model_cards/t5-small-README.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -language: en -datasets: -- c4 -tags: -- summarization -- translation - -license: apache-2.0 ---- - -[Google's T5](https://ai.googleblog.com/2020/02/exploring-transfer-learning-with-t5.html) - -Pretraining Dataset: [C4](https://huggingface.co/datasets/c4) - -Other Community Checkpoints: [here](https://huggingface.co/models?search=t5) - -Paper: [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/pdf/1910.10683.pdf) - -Authors: *Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu* - - -## Abstract - -Transfer learning, where a model is first pre-trained on a data-rich task before being fine-tuned on a downstream task, has emerged as a powerful technique in natural language processing (NLP). The effectiveness of transfer learning has given rise to a diversity of approaches, methodology, and practice. In this paper, we explore the landscape of transfer learning techniques for NLP by introducing a unified framework that converts every language problem into a text-to-text format. Our systematic study compares pre-training objectives, architectures, unlabeled datasets, transfer approaches, and other factors on dozens of language understanding tasks. By combining the insights from our exploration with scale and our new “Colossal Clean Crawled Corpus”, we achieve state-of-the-art results on many benchmarks covering summarization, question answering, text classification, and more. To facilitate future work on transfer learning for NLP, we release our dataset, pre-trained models, and code. - -![model image](https://camo.githubusercontent.com/623b4dea0b653f2ad3f36c71ebfe749a677ac0a1/68747470733a2f2f6d69726f2e6d656469756d2e636f6d2f6d61782f343030362f312a44304a31674e51663876727255704b657944387750412e706e67) - diff --git a/model_cards/tartuNLP/EstBERT/README.md b/model_cards/tartuNLP/EstBERT/README.md deleted file mode 100644 index ab042f7cac02..000000000000 --- a/model_cards/tartuNLP/EstBERT/README.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -language: et ---- -# EstBERT - - -### What's this? -The EstBERT model is a pretrained BERTBase model exclusively trained on Estonian cased corpus on both 128 and 512 sequence length of data. - -### How to use? -You can use the model transformer library both in tensorflow and pytorch version. -``` -from transformers import AutoTokenizer, AutoModelForMaskedLM -tokenizer = AutoTokenizer.from_pretrained("tartuNLP/EstBERT") -model = AutoModelForMaskedLM.from_pretrained("tartuNLP/EstBERT") -``` -You can also download the pretrained model from here, [EstBERT_128]() [EstBERT_512]() -#### Dataset used to train the model -The EstBERT model is trained both on 128 and 512 sequence length of data. For training the EstBERT we used the [Estonian National Corpus 2017](https://metashare.ut.ee/repository/browse/estonian-national-corpus-2017/b616ceda30ce11e8a6e4005056b40024880158b577154c01bd3d3fcfc9b762b3/), which was the largest Estonian language corpus available at the time. It consists of four sub-corpora: Estonian Reference Corpus 1990-2008, Estonian Web Corpus 2013, Estonian Web Corpus 2017 and Estonian Wikipedia Corpus 2017. - -### Why would I use? -Overall EstBERT performs better in parts of speech (POS), name entity recognition (NER), rubric, and sentiment classification tasks compared to mBERT and XLM-RoBERTa. The comparative results can be found below; - -|Model |UPOS |XPOS |Morph |bf UPOS |bf XPOS |Morph | -|--------------|----------------------------|-------------|-------------|-------------|----------------------------|----------------------------| -| EstBERT | **_97.89_** | **98.40** | **96.93** | **97.84** | **_98.43_** | **_96.80_** | -| mBERT | 97.42 | 98.06 | 96.24 | 97.43 | 98.13 | 96.13 | -| XLM-RoBERTa | 97.78 | 98.36 | 96.53 | 97.80 | 98.40 | 96.69 | - - -|Model|Rubric128 |Sentiment128 | Rubric128 |Sentiment512 | -|-------------------|----------------------------|--------------------|-----------------------------------------------|----------------------------| -| EstBERT | **_81.70_** | 74.36 | **80.96** | 74.50 | -| mBERT | 75.67 | 70.23 | 74.94 | 69.52 | -| XLM\-RoBERTa | 80.34 | **74.50** | 78.62 | **_76.07_**| - -|Model |Precicion128 |Recall128 |F1-Score128 |Precision512 |Recall512 |F1-Score512 | -|--------------|----------------|----------------------------|----------------------------|----------------------------|-------------|----------------| -| EstBERT | **88.42** | 90.38 |**_89.39_** | 88.35 | 89.74 | 89.04 | -| mBERT | 85.88 | 87.09 | 86.51 |**_88.47_** | 88.28 | 88.37 | -| XLM\-RoBERTa | 87.55 |**_91.19_** | 89.34 | 87.50 | **90.76** | **89.10** | diff --git a/model_cards/tblard/tf-allocine/README.md b/model_cards/tblard/tf-allocine/README.md deleted file mode 100644 index bb5cd6b2824d..000000000000 --- a/model_cards/tblard/tf-allocine/README.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -language: fr ---- - -# tf-allociné - -A french sentiment analysis model, based on [CamemBERT](https://camembert-model.fr/), and finetuned on a large-scale dataset scraped from [Allociné.fr](http://www.allocine.fr/) user reviews. - -## Results - -| Validation Accuracy | Validation F1-Score | Test Accuracy | Test F1-Score | -|--------------------:| -------------------:| -------------:|--------------:| -| 97.39 | 97.36 | 97.44 | 97.34 | - -The dataset and the evaluation code are available on [this repo](https://github.com/TheophileBlard/french-sentiment-analysis-with-bert). - -## Usage - -```python -from transformers import AutoTokenizer, TFAutoModelForSequenceClassification -from transformers import pipeline - -tokenizer = AutoTokenizer.from_pretrained("tblard/tf-allocine") -model = TFAutoModelForSequenceClassification.from_pretrained("tblard/tf-allocine") - -nlp = pipeline('sentiment-analysis', model=model, tokenizer=tokenizer) - -print(nlp("Alad'2 est clairement le meilleur film de l'année 2018.")) # POSITIVE -print(nlp("Juste whoaaahouuu !")) # POSITIVE -print(nlp("NUL...A...CHIER ! FIN DE TRANSMISSION.")) # NEGATIVE -print(nlp("Je m'attendais à mieux de la part de Franck Dubosc !")) # NEGATIVE -``` - -## Author - -Théophile Blard – :email: theophile.blard@gmail.com - -If you use this work (code, model or dataset), please cite as: - -> Théophile Blard, French sentiment analysis with BERT, (2020), GitHub repository, diff --git a/model_cards/tuner007/pegasus_paraphrase/README.md b/model_cards/tuner007/pegasus_paraphrase/README.md deleted file mode 100644 index 311d8de61c33..000000000000 --- a/model_cards/tuner007/pegasus_paraphrase/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Pegasus for Paraphrasing -Pegasus model fine-tuned for paraphrasing - -## Model in Action 🚀 -``` -import torch -from transformers import PegasusForConditionalGeneration, PegasusTokenizer -model_name = 'tuner007/pegasus_paraphrase' -torch_device = 'cuda' if torch.cuda.is_available() else 'cpu' -tokenizer = PegasusTokenizer.from_pretrained(model_name) -model = PegasusForConditionalGeneration.from_pretrained(model_name).to(torch_device) - -def get_response(input_text,num_return_sequences): - batch = tokenizer.prepare_seq2seq_batch([input_text],truncation=True,padding='longest',max_length=60, return_tensors="pt").to(torch_device) - translated = model.generate(**batch,max_length=60,num_beams=10, num_return_sequences=num_return_sequences, temperature=1.5) - tgt_text = tokenizer.batch_decode(translated, skip_special_tokens=True) - return tgt_text -``` -#### Example 1: -``` -context = "The ultimate test of your knowledge is your capacity to convey it to another." -get_response(context,10) -# output: -['The test of your knowledge is your ability to convey it.', - 'The ability to convey your knowledge is the ultimate test of your knowledge.', - 'The ability to convey your knowledge is the most important test of your knowledge.', - 'Your capacity to convey your knowledge is the ultimate test of it.', - 'The test of your knowledge is your ability to communicate it.', - 'Your capacity to convey your knowledge is the ultimate test of your knowledge.', - 'Your capacity to convey your knowledge to another is the ultimate test of your knowledge.', - 'Your capacity to convey your knowledge is the most important test of your knowledge.', - 'The test of your knowledge is how well you can convey it.', - 'Your capacity to convey your knowledge is the ultimate test.'] -``` -#### Example 2: Question paraphrasing (was not trained on quora dataset) -``` -context = "Which course should I take to get started in data science?" -get_response(context,10) -# output: -['Which data science course should I take?', - 'Which data science course should I take first?', - 'Should I take a data science course?', - 'Which data science class should I take?', - 'Which data science course should I attend?', - 'I want to get started in data science.', - 'Which data science course should I enroll in?', - 'Which data science course is right for me?', - 'Which data science course is best for me?', - 'Which course should I take to get started?'] -``` - -> Created by Arpit Rajauria -[![Twitter icon](https://cdn0.iconfinder.com/data/icons/shift-logotypes/32/Twitter-32.png)](https://twitter.com/arpit_rajauria) diff --git a/model_cards/tuner007/pegasus_qa/README.md b/model_cards/tuner007/pegasus_qa/README.md deleted file mode 100644 index bc9397225d7b..000000000000 --- a/model_cards/tuner007/pegasus_qa/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Pegasus for question-answering -Pegasus model fine-tuned for QA using text-to-text approach - -## Model in Action 🚀 -``` -import torch -from transformers import PegasusForConditionalGeneration, PegasusTokenizer -model_name = 'tuner007/pegasus_qa' -torch_device = 'cuda' if torch.cuda.is_available() else 'cpu' -tokenizer = PegasusTokenizer.from_pretrained(model_name) -model = PegasusForConditionalGeneration.from_pretrained(model_name).to(torch_device) - -def get_answer(question, context): - input_text = "question: %s text: %s" % (question,context) - batch = tokenizer.prepare_seq2seq_batch([input_text], truncation=True, padding='longest', return_tensors="pt").to(torch_device) - translated = model.generate(**batch) - tgt_text = tokenizer.batch_decode(translated, skip_special_tokens=True) - return tgt_text[0] -``` -#### Example: -``` -context = "PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow." -question = "How many customers were affected by the shutoffs?" -get_answer(question, context) -# output: '800 thousand' -``` - - -> Created by Arpit Rajauria -[![Twitter icon](https://cdn0.iconfinder.com/data/icons/shift-logotypes/32/Twitter-32.png)](https://twitter.com/arpit_rajauria) diff --git a/model_cards/tuner007/t5_abs_qa/README.md b/model_cards/tuner007/t5_abs_qa/README.md deleted file mode 100644 index efdbb5c938df..000000000000 --- a/model_cards/tuner007/t5_abs_qa/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# T5 for abstractive question-answering -This is T5-base model fine-tuned for abstractive QA using text-to-text approach - -## Model training -This model was trained on colab TPU with 35GB RAM for 2 epochs - -## Model in Action 🚀 -``` -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("tuner007/t5_abs_qa") -model = AutoModelWithLMHead.from_pretrained("tuner007/t5_abs_qa") -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = model.to(device) - -def get_answer(question, context): -input_text = "context: %s " % (context,question) -features = tokenizer([input_text], return_tensors='pt') -out = model.generate(input_ids=features['input_ids'].to(device), attention_mask=features['attention_mask'].to(device)) -return tokenizer.decode(out[0]) -``` -#### Example 1: Answer available -``` -context = "In Norse mythology, Valhalla is a majestic, enormous hall located in Asgard, ruled over by the god Odin." -question = "What is Valhalla?" -get_answer(question, context) -# output: 'It is a hall of worship ruled by Odin.' -``` -#### Example 2: Answer not available -``` -context = "In Norse mythology, Valhalla is a majestic, enormous hall located in Asgard, ruled over by the god Odin." -question = "What is Asgard?" -get_answer(question, context) -# output: 'No answer available in context.' -``` - - -> Created by Arpit Rajauria -[![Twitter icon](https://cdn0.iconfinder.com/data/icons/shift-logotypes/32/Twitter-32.png)](https://twitter.com/arpit_rajauria) diff --git a/model_cards/twmkn9/albert-base-v2-squad2/README.md b/model_cards/twmkn9/albert-base-v2-squad2/README.md deleted file mode 100644 index 5b615c470494..000000000000 --- a/model_cards/twmkn9/albert-base-v2-squad2/README.md +++ /dev/null @@ -1,45 +0,0 @@ -This model is [ALBERT base v2](https://huggingface.co/albert-base-v2) trained on SQuAD v2 as: - -``` -export SQUAD_DIR=../../squad2 -python3 run_squad.py - --model_type albert - --model_name_or_path albert-base-v2 - --do_train - --do_eval - --overwrite_cache - --do_lower_case - --version_2_with_negative - --save_steps 100000 - --train_file $SQUAD_DIR/train-v2.0.json - --predict_file $SQUAD_DIR/dev-v2.0.json - --per_gpu_train_batch_size 8 - --num_train_epochs 3 - --learning_rate 3e-5 - --max_seq_length 384 - --doc_stride 128 - --output_dir ./tmp/albert_fine/ -``` - -Performance on a dev subset is close to the original paper: - -``` -Results: -{ - 'exact': 78.71010200723923, - 'f1': 81.89228117126069, - 'total': 6078, - 'HasAns_exact': 75.39518900343643, - 'HasAns_f1': 82.04167868004215, - 'HasAns_total': 2910, - 'NoAns_exact': 81.7550505050505, - 'NoAns_f1': 81.7550505050505, - 'NoAns_total': 3168, - 'best_exact': 78.72655478775913, - 'best_exact_thresh': 0.0, - 'best_f1': 81.90873395178066, - 'best_f1_thresh': 0.0 -} -``` - -We are hopeful this might save you time, energy, and compute. Cheers! \ No newline at end of file diff --git a/model_cards/twmkn9/bert-base-uncased-squad2/README.md b/model_cards/twmkn9/bert-base-uncased-squad2/README.md deleted file mode 100644 index 20bdf07512e0..000000000000 --- a/model_cards/twmkn9/bert-base-uncased-squad2/README.md +++ /dev/null @@ -1,45 +0,0 @@ -This model is [BERT base uncased](https://huggingface.co/bert-base-uncased) trained on SQuAD v2 as: - -``` -export SQUAD_DIR=../../squad2 -python3 run_squad.py - --model_type bert - --model_name_or_path bert-base-uncased - --do_train - --do_eval - --overwrite_cache - --do_lower_case - --version_2_with_negative - --save_steps 100000 - --train_file $SQUAD_DIR/train-v2.0.json - --predict_file $SQUAD_DIR/dev-v2.0.json - --per_gpu_train_batch_size 8 - --num_train_epochs 3 - --learning_rate 3e-5 - --max_seq_length 384 - --doc_stride 128 - --output_dir ./tmp/bert_fine_tuned/ -``` - -Performance on a dev subset is close to the original paper: - -``` -Results: -{ - 'exact': 72.35932872655479, - 'f1': 75.75355132564763, - 'total': 6078, - 'HasAns_exact': 74.29553264604812, - 'HasAns_f1': 81.38490892002987, - 'HasAns_total': 2910, - 'NoAns_exact': 70.58080808080808, - 'NoAns_f1': 70.58080808080808, - 'NoAns_total': 3168, - 'best_exact': 72.35932872655479, - 'best_exact_thresh': 0.0, - 'best_f1': 75.75355132564766, - 'best_f1_thresh': 0.0 -} -``` - -We are hopeful this might save you time, energy, and compute. Cheers! \ No newline at end of file diff --git a/model_cards/twmkn9/distilbert-base-uncased-squad2/README.md b/model_cards/twmkn9/distilbert-base-uncased-squad2/README.md deleted file mode 100644 index cb8542fb51d0..000000000000 --- a/model_cards/twmkn9/distilbert-base-uncased-squad2/README.md +++ /dev/null @@ -1,45 +0,0 @@ -This model is [Distilbert base uncased](https://huggingface.co/distilbert-base-uncased) trained on SQuAD v2 as: - -``` -export SQUAD_DIR=../../squad2 -python3 run_squad.py - --model_type distilbert - --model_name_or_path distilbert-base-uncased - --do_train - --do_eval - --overwrite_cache - --do_lower_case - --version_2_with_negative - --save_steps 100000 - --train_file $SQUAD_DIR/train-v2.0.json - --predict_file $SQUAD_DIR/dev-v2.0.json - --per_gpu_train_batch_size 8 - --num_train_epochs 3 - --learning_rate 3e-5 - --max_seq_length 384 - --doc_stride 128 - --output_dir ./tmp/distilbert_fine_tuned/ -``` - -Performance on a dev subset is close to the original paper: - -``` -Results: -{ - 'exact': 64.88976637051661, - 'f1': 68.1776176526635, - 'total': 6078, - 'HasAns_exact': 69.7594501718213, - 'HasAns_f1': 76.62665295288285, - 'HasAns_total': 2910, - 'NoAns_exact': 60.416666666666664, - 'NoAns_f1': 60.416666666666664, - 'NoAns_total': 3168, - 'best_exact': 64.88976637051661, - 'best_exact_thresh': 0.0, - 'best_f1': 68.17761765266337, - 'best_f1_thresh': 0.0 -} -``` - -We are hopeful this might save you time, energy, and compute. Cheers! \ No newline at end of file diff --git a/model_cards/twmkn9/distilroberta-base-squad2/README.md b/model_cards/twmkn9/distilroberta-base-squad2/README.md deleted file mode 100644 index c6b6569f6585..000000000000 --- a/model_cards/twmkn9/distilroberta-base-squad2/README.md +++ /dev/null @@ -1,44 +0,0 @@ -This model is [Distilroberta base](https://huggingface.co/distilroberta-base) trained on SQuAD v2 as: - -``` -export SQUAD_DIR=../../squad2 -python3 run_squad.py - --model_type robberta - --model_name_or_path distilroberta-base - --do_train - --do_eval - --overwrite_cache - --do_lower_case - --version_2_with_negative - --save_steps 100000 - --train_file $SQUAD_DIR/train-v2.0.json - --predict_file $SQUAD_DIR/dev-v2.0.json - --per_gpu_train_batch_size 8 - --num_train_epochs 3 - --learning_rate 3e-5 - --max_seq_length 384 - --doc_stride 128 - --output_dir ./tmp/distilroberta_fine_tuned/ -``` - -Performance on a dev subset is close to the original paper: - -``` -Results: -{ - 'exact': 70.9279368213228, - 'f1': 74.60439802429168, - 'total': 6078, - 'HasAns_exact': 67.62886597938144, - 'HasAns_f1': 75.30774267754136, - 'HasAns_total': 2910, - 'NoAns_exact': 73.95833333333333, - 'NoAns_f1': 73.95833333333333, 'NoAns_total': 3168, - 'best_exact': 70.94438960184272, - 'best_exact_thresh': 0.0, - 'best_f1': 74.62085080481161, - 'best_f1_thresh': 0.0 -} -``` - -We are hopeful this might save you time, energy, and compute. Cheers! \ No newline at end of file diff --git a/model_cards/uer/chinese_roberta_L-2_H-128/README.md b/model_cards/uer/chinese_roberta_L-2_H-128/README.md deleted file mode 100644 index ca646e266fae..000000000000 --- a/model_cards/uer/chinese_roberta_L-2_H-128/README.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -language: zh -datasets: -- CLUECorpus ---- - -# Chinese RoBERTa Miniatures - -## Model description - -This is the set of 24 Chinese RoBERTa models pre-trained by [UER-py](https://www.aclweb.org/anthology/D19-3041.pdf). - -You can download the 24 Chinese RoBERTa miniatures either from the [UER-py Github page](https://github.com/dbiir/UER-py/), or via HuggingFace from the links below: - -| |H=128|H=256|H=512|H=768| -|---|:---:|:---:|:---:|:---:| -| **L=2** |[**2/128 (BERT-Tiny)**][2_128]|[2/256]|[2/512]|[2/768]| -| **L=4** |[4/128]|[**4/256 (BERT-Mini)**]|[**4/512 (BERT-Small)**]|[4/768]| -| **L=6** |[6/128]|[6/256]|[6/512]|[6/768]| -| **L=8** |[8/128]|[8/256]|[**8/512 (BERT-Medium)**]|[8/768]| -| **L=10** |[10/128]|[10/256]|[10/512]|[10/768]| -| **L=12** |[12/128]|[12/256]|[12/512]|[**12/768 (BERT-Base)**]| - -## Training data - -CLUECorpus2020 and CLUECorpusSmall are used as training corpus. - -## Training procedure - -Training details can be found in [UER-py](https://github.com/dbiir/UER-py/). - -### BibTeX entry and citation info - -``` -@article{zhao2019uer, - title={UER: An Open-Source Toolkit for Pre-training Models}, - author={Zhao, Zhe and Chen, Hui and Zhang, Jinbin and Zhao, Xin and Liu, Tao and Lu, Wei and Chen, Xi and Deng, Haotang and Ju, Qi and Du, Xiaoyong}, - journal={EMNLP-IJCNLP 2019}, - pages={241}, - year={2019} -} -``` - -[2_128]: https://huggingface.co/uer/chinese_roberta_L-2_H-128 diff --git a/model_cards/uer/gpt2-chinese-couplet/README.md b/model_cards/uer/gpt2-chinese-couplet/README.md deleted file mode 100644 index 891d9e3b2c15..000000000000 --- a/model_cards/uer/gpt2-chinese-couplet/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -language: zh -widget: -- text: "[CLS]国 色 天 香 ， 姹 紫 嫣 红 ， 碧 水 青 云 欣 共 赏 -" - - ---- - -# Chinese Couplet GPT2 Model - -## Model description - -The model is used to generate Chinese couplets. You can download the model either from the [GPT2-Chinese Github page](https://github.com/Morizeyao/GPT2-Chinese), or via HuggingFace from the link [gpt2-chinese-couplet][couplet]. - -Since the parameter skip_special_tokens is used in the pipelines.py, special tokens such as [SEP], [UNK] will be deleted, and the output results may not be neat. - -## How to use - -You can use the model directly with a pipeline for text generation: - -When the parameter skip_special_tokens is True: - -```python ->>> from transformers import BertTokenizer, GPT2LMHeadModel, TextGenerationPipeline ->>> from transformers import TextGenerationPipeline, ->>> tokenizer = BertTokenizer.from_pretrained("uer/gpt2-chinese-couplet") ->>> model = GPT2LMHeadModel.from_pretrained("uer/gpt2-chinese-couplet") ->>> text_generator = TextGenerationPipeline(model, tokenizer) ->>> text_generator("[CLS]丹 枫 江 冷 人 初 去 -", max_length=25, do_sample=True) - [{'generated_text': '[CLS]丹 枫 江 冷 人 初 去 - 黄 叶 声 从 天 外 来 阅 旗'}] -``` - -When the parameter skip_special_tokens is False: - -```python ->>> from transformers import BertTokenizer, GPT2LMHeadModel, TextGenerationPipeline ->>> from transformers import TextGenerationPipeline, ->>> tokenizer = BertTokenizer.from_pretrained("uer/gpt2-chinese-poem") ->>> model = GPT2LMHeadModel.from_pretrained("uer/gpt2-chinese-poem") ->>> text_generator = TextGenerationPipeline(model, tokenizer) ->>> text_generator("[CLS]丹 枫 江 冷 人 初 去 -", max_length=25, do_sample=True) - [{'generated_text': '[CLS]丹 枫 江 冷 人 初 去 - 黄 叶 声 我 酒 不 辞 [SEP] [SEP] [SEP] [SEP] [SEP] [SEP] [SEP] [SEP] [SEP]'}] -``` - -## Training data - -Contains 700,000 Chinese couplets collected by [couplet-clean-dataset](https://github.com/v-zich/couplet-clean-dataset). - -## Training procedure - -Models are pre-trained by [UER-py](https://github.com/dbiir/UER-py/) on [Tencent Cloud TI-ONE](https://cloud.tencent.com/product/tione/). We pre-train 25,000 steps with a sequence length of 64. - -``` -python3 preprocess.py --corpus_path corpora/couplet.txt \ - --vocab_path models/google_zh_vocab.txt \ - --dataset_path couplet.pt --processes_num 16 \ - --seq_length 64 --target lm -``` - -``` -python3 pretrain.py --dataset_path couplet.pt \ - --vocab_path models/google_zh_vocab.txt \ - --output_model_path models/couplet_gpt_base_model.bin \ - --config_path models/bert_base_config.json --learning_rate 5e-4 \ - --tie_weight --world_size 8 --gpu_ranks 0 1 2 3 4 5 6 7 \ - --batch_size 64 --report_steps 1000 \ - --save_checkpoint_steps 5000 --total_steps 25000 \ - --embedding gpt --encoder gpt2 --target lm - -``` - -### BibTeX entry and citation info - -``` -@article{zhao2019uer, - title={UER: An Open-Source Toolkit for Pre-training Models}, - author={Zhao, Zhe and Chen, Hui and Zhang, Jinbin and Zhao, Xin and Liu, Tao and Lu, Wei and Chen, Xi and Deng, Haotang and Ju, Qi and Du, Xiaoyong}, - journal={EMNLP-IJCNLP 2019}, - pages={241}, - year={2019} -} -``` - -[couplet]: https://huggingface.co/uer/gpt2-chinese-couplet - diff --git a/model_cards/uer/gpt2-chinese-poem/README.md b/model_cards/uer/gpt2-chinese-poem/README.md deleted file mode 100644 index bb068eac7ff3..000000000000 --- a/model_cards/uer/gpt2-chinese-poem/README.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -language: zh -widget: -- text: "[CLS] 万 叠 春 山 积 雨 晴 ，" -- text: "[CLS] 青 山 削 芙 蓉 ，" - - ---- - -# Chinese Poem GPT2 Model - -## Model description - -The model is used to generate Chinese ancient poems. You can download the model either from the [GPT2-Chinese Github page](https://github.com/Morizeyao/GPT2-Chinese), or via HuggingFace from the link [gpt2-chinese-poem][poem]. - -Since the parameter skip_special_tokens is used in the pipelines.py, special tokens such as [SEP], [UNK] will be deleted, and the output results may not be neat. - -## How to use - -You can use the model directly with a pipeline for text generation: - -When the parameter skip_special_tokens is True: - -```python ->>> from transformers import BertTokenizer, GPT2LMHeadModel, TextGenerationPipeline ->>> from transformers import TextGenerationPipeline, ->>> tokenizer = BertTokenizer.from_pretrained("uer/gpt2-chinese-poem") ->>> model = GPT2LMHeadModel.from_pretrained("uer/gpt2-chinese-poem") ->>> text_generator = TextGenerationPipeline(model, tokenizer) ->>> text_generator("[CLS]梅 山 如 积 翠 ，", max_length=50, do_sample=True) - [{'generated_text': '[CLS]梅 山 如 积 翠 ， 的 手 堪 捧 。 遥 遥 仙 人 尉 ， 盘 盘 故 时 陇 。 丹 泉 清 可 鉴 ， 石 乳 甘 于 。 行 将 解 尘 缨 ， 于 焉 蹈 高 踵 。 我'}] -``` - -When the parameter skip_special_tokens is False: - -```python ->>> from transformers import BertTokenizer, GPT2LMHeadModel, TextGenerationPipeline ->>> from transformers import TextGenerationPipeline, ->>> tokenizer = BertTokenizer.from_pretrained("uer/gpt2-chinese-poem") ->>> model = GPT2LMHeadModel.from_pretrained("uer/gpt2-chinese-poem") ->>> text_generator = TextGenerationPipeline(model, tokenizer) ->>> text_generator("[CLS]梅 山 如 积 翠 ，", max_length=50, do_sample=True) - [{'generated_text': '[CLS]梅 山 如 积 翠 ， 的 [UNK] 手 堪 捧 。 遥 遥 仙 人 尉 ， 盘 盘 故 时 陇 。 丹 泉 清 可 鉴 ， 石 乳 甘 可 捧 。 银 汉 迟 不 来 ， 槎 头 欲 谁 揽 。 何'}] -``` - -## Training data - -Contains 800,000 Chinese ancient poems collected by [chinese-poetry](https://github.com/chinese-poetry/chinese-poetry) and [Poetry](https://github.com/Werneror/Poetry) projects. - -## Training procedure - -The model is pre-trained by [UER-py](https://github.com/dbiir/UER-py/) on [Tencent Cloud TI-ONE](https://cloud.tencent.com/product/tione/). We pre-train 200,000 steps with a sequence length of 128. - -``` -python3 preprocess.py --corpus_path corpora/poem.txt \ - --vocab_path models/google_zh_vocab.txt \ - --dataset_path poem.pt --processes_num 16 \ - --seq_length 128 --target lm -``` - -``` -python3 pretrain.py --dataset_path poem.pt \ - --vocab_path models/google_zh_vocab.txt \ - --output_model_path models/poem_gpt_base_model.bin \ - --config_path models/bert_base_config.json --learning_rate 5e-4 \ - --tie_weight --world_size 8 --gpu_ranks 0 1 2 3 4 5 6 7 \ - --batch_size 64 --report_steps 1000 \ - --save_checkpoint_steps 50000 --total_steps 200000 \ - --embedding gpt --encoder gpt2 --target lm - -``` - -### BibTeX entry and citation info - -``` -@article{zhao2019uer, - title={UER: An Open-Source Toolkit for Pre-training Models}, - author={Zhao, Zhe and Chen, Hui and Zhang, Jinbin and Zhao, Xin and Liu, Tao and Lu, Wei and Chen, Xi and Deng, Haotang and Ju, Qi and Du, Xiaoyong}, - journal={EMNLP-IJCNLP 2019}, - pages={241}, - year={2019} -} -``` - -[poem]: https://huggingface.co/uer/gpt2-chinese-poem diff --git a/model_cards/uncnlp/lxmert-base-uncased/LICENSE b/model_cards/uncnlp/lxmert-base-uncased/LICENSE deleted file mode 100644 index 52df82d35661..000000000000 --- a/model_cards/uncnlp/lxmert-base-uncased/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2019 Hao Tan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/model_cards/uncnlp/lxmert-base-uncased/README.md b/model_cards/uncnlp/lxmert-base-uncased/README.md deleted file mode 100644 index 1cb7d36f5b83..000000000000 --- a/model_cards/uncnlp/lxmert-base-uncased/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# LXMERT - -## Model Description - -[LXMERT](https://arxiv.org/abs/1908.07490) is a pre-trained multimodal transformer. The model takes an image and a sentence as input and compute cross-modal representions. The model is converted from [LXMERT github](https://github.com/airsplay/lxmert) by [Antonio Mendoza](https://avmendoza.info/) and is authored by [Hao Tan](https://www.cs.unc.edu/~airsplay/). - -![](./lxmert_model-1.jpg?raw=True) - -## Usage - - -## Training Data and Prodcedure -The model is jointly trained on multiple vision-and-language datasets. -We included two image captioning datsets (i.e., [MS COCO](http://cocodataset.org/#home), [Visual Genome](https://visualgenome.org/)) and three image-question answering datasets (i.e., [VQA](https://visualqa.org/), [GQA](https://cs.stanford.edu/people/dorarad/gqa/), [VG QA](https://github.com/yukezhu/visual7w-toolkit)). The model is pre-trained on the above datasets for 20 epochs (roughly 670K iterations with batch size 256), which takes around 8 days on 4 Titan V cards. The details of training could be found in the [LXMERT paper](https://arxiv.org/pdf/1908.07490.pdf). - -## Eval Results -| Split | [VQA](https://visualqa.org/) | [GQA](https://cs.stanford.edu/people/dorarad/gqa/) | [NLVR2](http://lil.nlp.cornell.edu/nlvr/) | -|----------- |:----: |:---: |:------:| -| Local Validation | 69.90% | 59.80% | 74.95% | -| Test-Dev | 72.42% | 60.00% | 74.45% (Test-P) | -| Test-Standard | 72.54% | 60.33% | 76.18% (Test-U) | - - -## Reference -```bibtex -@inproceedings{tan2019lxmert, - title={LXMERT: Learning Cross-Modality Encoder Representations from Transformers}, - author={Tan, Hao and Bansal, Mohit}, - booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing}, - year={2019} -} -``` - - diff --git a/model_cards/uncnlp/lxmert-base-uncased/lxmert_model-1.jpg b/model_cards/uncnlp/lxmert-base-uncased/lxmert_model-1.jpg deleted file mode 100644 index 20e3b7cce3c1..000000000000 Binary files a/model_cards/uncnlp/lxmert-base-uncased/lxmert_model-1.jpg and /dev/null differ diff --git a/model_cards/unideeplearning/polibert_sa/README.md b/model_cards/unideeplearning/polibert_sa/README.md deleted file mode 100644 index 7a71dc93614f..000000000000 --- a/model_cards/unideeplearning/polibert_sa/README.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -language: it -tags: -- sentiment -- Italian -license: MIT -widget: -- text: 'Giuseppe Rossi è un ottimo politico' ---- - -# 🤗 + polibert_SA - POLItic BERT based Sentiment Analysis - -## Model description - -This model performs sentiment analysis on Italian political twitter sentences. It was trained starting from an instance of "bert-base-italian-uncased-xxl" and fine-tuned on an Italian dataset of tweets. You can try it out at https://www.unideeplearning.com/twitter_sa/ (in italian!) - -#### Hands-on - -```python -import torch -from torch import nn -from transformers import AutoTokenizer, AutoModelForSequenceClassification - -tokenizer = AutoTokenizer.from_pretrained("unideeplearning/polibert_sa") -model = AutoModelForSequenceClassification.from_pretrained("unideeplearning/polibert_sa") - - - - -text = "Giuseppe Rossi è un pessimo politico" -input_ids = tokenizer.encode(text, add_special_tokens=True, return_tensors= 'pt') - -logits, = model(input_ids) -logits = logits.squeeze(0) - -prob = nn.functional.softmax(logits, dim=0) - -# 0 Negative, 1 Neutral, 2 Positive -print(prob.argmax().tolist()) -``` - -#### Hyperparameters - -- Optimizer: **AdamW** with learning rate of **2e-5**, epsilon of **1e-8** -- Max epochs: **2** -- Batch size: **16** - -## Acknowledgments - -Thanks to the support from: -the [Hugging Face](https://huggingface.co/), https://www.unioneprofessionisti.com - -https://www.unideeplearning.com/ diff --git a/model_cards/urduhack/roberta-urdu-small/README.md b/model_cards/urduhack/roberta-urdu-small/README.md deleted file mode 100644 index b0d6cfbf403a..000000000000 --- a/model_cards/urduhack/roberta-urdu-small/README.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -language: ur -thumbnail: https://raw.githubusercontent.com/urduhack/urduhack/master/docs/_static/urduhack.png -tags: -- roberta-urdu-small -- urdu -- transformers -license: mit ---- -## roberta-urdu-small - -[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/urduhack/urduhack/blob/master/LICENSE) -### Overview -**Language model:** roberta-urdu-small -**Model size:** 125M -**Language:** Urdu -**Training data:** News data from urdu news resources in Pakistan -### About roberta-urdu-small -roberta-urdu-small is a language model for urdu language. -``` -from transformers import pipeline -fill_mask = pipeline("fill-mask", model="urduhack/roberta-urdu-small", tokenizer="urduhack/roberta-urdu-small") -``` -## Training procedure -roberta-urdu-small was trained on urdu news corpus. Training data was normalized using normalization module from -urduhack to eliminate characters from other languages like arabic. - -### About Urduhack -Urduhack is a Natural Language Processing (NLP) library for urdu language. -Github: https://github.com/urduhack/urduhack diff --git a/model_cards/valhalla/bart-large-finetuned-squadv1/README.md b/model_cards/valhalla/bart-large-finetuned-squadv1/README.md deleted file mode 100644 index e53087c1ba97..000000000000 --- a/model_cards/valhalla/bart-large-finetuned-squadv1/README.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -datasets: -- squad ---- - -# BART-LARGE finetuned on SQuADv1 - -This is bart-large model finetuned on SQuADv1 dataset for question answering task - -## Model details -BART was propsed in the [paper](https://arxiv.org/abs/1910.13461) **BART: Denoising Sequence-to-Sequence Pre-training for Natural Language Generation, Translation, and Comprehension**. -BART is a seq2seq model intended for both NLG and NLU tasks. - -To use BART for question answering tasks, we feed the complete document into the encoder and decoder, and use the top -hidden state of the decoder as a representation for each -word. This representation is used to classify the token. As given in the paper bart-large achives comparable to ROBERTa on SQuAD. -Another notable thing about BART is that it can handle sequences with upto 1024 tokens. - -| Param | #Value | -|---------------------|--------| -| encoder layers | 12 | -| decoder layers | 12 | -| hidden size | 4096 | -| num attetion heads | 16 | -| on disk size | 1.63GB | - - -## Model training -This model was trained on google colab v100 GPU. -You can find the fine-tuning colab here -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1I5cK1M_0dLaf5xoewh6swcm5nAInfwHy?usp=sharing). - - -## Results -The results are actually slightly worse than given in the paper. -In the paper the authors mentioned that bart-large achieves 88.8 EM and 94.6 F1 - -| Metric | #Value | -|--------|--------| -| EM | 86.8022| -| F1 | 92.7342| - - -## Model in Action 🚀 -```python3 -from transformers import BartTokenizer, BartForQuestionAnswering -import torch - -tokenizer = BartTokenizer.from_pretrained('valhalla/bart-large-finetuned-squadv1') -model = BartForQuestionAnswering.from_pretrained('valhalla/bart-large-finetuned-squadv1') - -question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" -encoding = tokenizer(question, text, return_tensors='pt') -input_ids = encoding['input_ids'] -attention_mask = encoding['attention_mask'] - -start_scores, end_scores = model(input_ids, attention_mask=attention_mask, output_attentions=False)[:2] - -all_tokens = tokenizer.convert_ids_to_tokens(input_ids[0]) -answer = ' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1]) -answer = tokenizer.convert_tokens_to_ids(answer.split()) -answer = tokenizer.decode(answer) -#answer => 'a nice puppet' -``` - -> Created with ❤️ by Suraj Patil [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/patil-suraj/) -[![Twitter icon](https://cdn0.iconfinder.com/data/icons/shift-logotypes/32/Twitter-32.png)](https://twitter.com/psuraj28) diff --git a/model_cards/valhalla/distilbart-mnli-12-1/README.md b/model_cards/valhalla/distilbart-mnli-12-1/README.md deleted file mode 100644 index fd41fa7191d1..000000000000 --- a/model_cards/valhalla/distilbart-mnli-12-1/README.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -datasets: -- mnli -tags: -- distilbart -- distilbart-mnli -pipeline_tag: zero-shot-classification ---- - -# DistilBart-MNLI - -distilbart-mnli is the distilled version of bart-large-mnli created using the **No Teacher Distillation** technique proposed for BART summarisation by Huggingface, [here](https://github.com/huggingface/transformers/tree/master/examples/seq2seq#distilbart). - -We just copy alternating layers from `bart-large-mnli` and finetune more on the same data. - - -| | matched acc | mismatched acc | -| ------------------------------------------------------------------------------------ | ----------- | -------------- | -| [bart-large-mnli](https://huggingface.co/facebook/bart-large-mnli) (baseline, 12-12) | 89.9 | 90.01 | -| [distilbart-mnli-12-1](https://huggingface.co/valhalla/distilbart-mnli-12-1) | 87.08 | 87.5 | -| [distilbart-mnli-12-3](https://huggingface.co/valhalla/distilbart-mnli-12-3) | 88.1 | 88.19 | -| [distilbart-mnli-12-6](https://huggingface.co/valhalla/distilbart-mnli-12-6) | 89.19 | 89.01 | -| [distilbart-mnli-12-9](https://huggingface.co/valhalla/distilbart-mnli-12-9) | 89.56 | 89.52 | - - -This is a very simple and effective technique, as we can see the performance drop is very little. - -Detailed performace trade-offs will be posted in this [sheet](https://docs.google.com/spreadsheets/d/1dQeUvAKpScLuhDV1afaPJRRAE55s2LpIzDVA5xfqxvk/edit?usp=sharing). - - -## Fine-tuning -If you want to train these models yourself, clone the [distillbart-mnli repo](https://github.com/patil-suraj/distillbart-mnli) and follow the steps below - -Clone and install transformers from source -```bash -git clone https://github.com/huggingface/transformers.git -pip install -qqq -U ./transformers -``` - -Download MNLI data -```bash -python transformers/utils/download_glue_data.py --data_dir glue_data --tasks MNLI -``` - -Create student model -```bash -python create_student.py \ - --teacher_model_name_or_path facebook/bart-large-mnli \ - --student_encoder_layers 12 \ - --student_decoder_layers 6 \ - --save_path student-bart-mnli-12-6 \ -``` - -Start fine-tuning -```bash -python run_glue.py args.json -``` - -You can find the logs of these trained models in this [wandb project](https://wandb.ai/psuraj/distilbart-mnli). \ No newline at end of file diff --git a/model_cards/valhalla/distilbart-mnli-12-3/README.md b/model_cards/valhalla/distilbart-mnli-12-3/README.md deleted file mode 100644 index fd41fa7191d1..000000000000 --- a/model_cards/valhalla/distilbart-mnli-12-3/README.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -datasets: -- mnli -tags: -- distilbart -- distilbart-mnli -pipeline_tag: zero-shot-classification ---- - -# DistilBart-MNLI - -distilbart-mnli is the distilled version of bart-large-mnli created using the **No Teacher Distillation** technique proposed for BART summarisation by Huggingface, [here](https://github.com/huggingface/transformers/tree/master/examples/seq2seq#distilbart). - -We just copy alternating layers from `bart-large-mnli` and finetune more on the same data. - - -| | matched acc | mismatched acc | -| ------------------------------------------------------------------------------------ | ----------- | -------------- | -| [bart-large-mnli](https://huggingface.co/facebook/bart-large-mnli) (baseline, 12-12) | 89.9 | 90.01 | -| [distilbart-mnli-12-1](https://huggingface.co/valhalla/distilbart-mnli-12-1) | 87.08 | 87.5 | -| [distilbart-mnli-12-3](https://huggingface.co/valhalla/distilbart-mnli-12-3) | 88.1 | 88.19 | -| [distilbart-mnli-12-6](https://huggingface.co/valhalla/distilbart-mnli-12-6) | 89.19 | 89.01 | -| [distilbart-mnli-12-9](https://huggingface.co/valhalla/distilbart-mnli-12-9) | 89.56 | 89.52 | - - -This is a very simple and effective technique, as we can see the performance drop is very little. - -Detailed performace trade-offs will be posted in this [sheet](https://docs.google.com/spreadsheets/d/1dQeUvAKpScLuhDV1afaPJRRAE55s2LpIzDVA5xfqxvk/edit?usp=sharing). - - -## Fine-tuning -If you want to train these models yourself, clone the [distillbart-mnli repo](https://github.com/patil-suraj/distillbart-mnli) and follow the steps below - -Clone and install transformers from source -```bash -git clone https://github.com/huggingface/transformers.git -pip install -qqq -U ./transformers -``` - -Download MNLI data -```bash -python transformers/utils/download_glue_data.py --data_dir glue_data --tasks MNLI -``` - -Create student model -```bash -python create_student.py \ - --teacher_model_name_or_path facebook/bart-large-mnli \ - --student_encoder_layers 12 \ - --student_decoder_layers 6 \ - --save_path student-bart-mnli-12-6 \ -``` - -Start fine-tuning -```bash -python run_glue.py args.json -``` - -You can find the logs of these trained models in this [wandb project](https://wandb.ai/psuraj/distilbart-mnli). \ No newline at end of file diff --git a/model_cards/valhalla/distilbart-mnli-12-6/README.md b/model_cards/valhalla/distilbart-mnli-12-6/README.md deleted file mode 100644 index fd41fa7191d1..000000000000 --- a/model_cards/valhalla/distilbart-mnli-12-6/README.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -datasets: -- mnli -tags: -- distilbart -- distilbart-mnli -pipeline_tag: zero-shot-classification ---- - -# DistilBart-MNLI - -distilbart-mnli is the distilled version of bart-large-mnli created using the **No Teacher Distillation** technique proposed for BART summarisation by Huggingface, [here](https://github.com/huggingface/transformers/tree/master/examples/seq2seq#distilbart). - -We just copy alternating layers from `bart-large-mnli` and finetune more on the same data. - - -| | matched acc | mismatched acc | -| ------------------------------------------------------------------------------------ | ----------- | -------------- | -| [bart-large-mnli](https://huggingface.co/facebook/bart-large-mnli) (baseline, 12-12) | 89.9 | 90.01 | -| [distilbart-mnli-12-1](https://huggingface.co/valhalla/distilbart-mnli-12-1) | 87.08 | 87.5 | -| [distilbart-mnli-12-3](https://huggingface.co/valhalla/distilbart-mnli-12-3) | 88.1 | 88.19 | -| [distilbart-mnli-12-6](https://huggingface.co/valhalla/distilbart-mnli-12-6) | 89.19 | 89.01 | -| [distilbart-mnli-12-9](https://huggingface.co/valhalla/distilbart-mnli-12-9) | 89.56 | 89.52 | - - -This is a very simple and effective technique, as we can see the performance drop is very little. - -Detailed performace trade-offs will be posted in this [sheet](https://docs.google.com/spreadsheets/d/1dQeUvAKpScLuhDV1afaPJRRAE55s2LpIzDVA5xfqxvk/edit?usp=sharing). - - -## Fine-tuning -If you want to train these models yourself, clone the [distillbart-mnli repo](https://github.com/patil-suraj/distillbart-mnli) and follow the steps below - -Clone and install transformers from source -```bash -git clone https://github.com/huggingface/transformers.git -pip install -qqq -U ./transformers -``` - -Download MNLI data -```bash -python transformers/utils/download_glue_data.py --data_dir glue_data --tasks MNLI -``` - -Create student model -```bash -python create_student.py \ - --teacher_model_name_or_path facebook/bart-large-mnli \ - --student_encoder_layers 12 \ - --student_decoder_layers 6 \ - --save_path student-bart-mnli-12-6 \ -``` - -Start fine-tuning -```bash -python run_glue.py args.json -``` - -You can find the logs of these trained models in this [wandb project](https://wandb.ai/psuraj/distilbart-mnli). \ No newline at end of file diff --git a/model_cards/valhalla/distilbart-mnli-12-9/README.md b/model_cards/valhalla/distilbart-mnli-12-9/README.md deleted file mode 100644 index fd41fa7191d1..000000000000 --- a/model_cards/valhalla/distilbart-mnli-12-9/README.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -datasets: -- mnli -tags: -- distilbart -- distilbart-mnli -pipeline_tag: zero-shot-classification ---- - -# DistilBart-MNLI - -distilbart-mnli is the distilled version of bart-large-mnli created using the **No Teacher Distillation** technique proposed for BART summarisation by Huggingface, [here](https://github.com/huggingface/transformers/tree/master/examples/seq2seq#distilbart). - -We just copy alternating layers from `bart-large-mnli` and finetune more on the same data. - - -| | matched acc | mismatched acc | -| ------------------------------------------------------------------------------------ | ----------- | -------------- | -| [bart-large-mnli](https://huggingface.co/facebook/bart-large-mnli) (baseline, 12-12) | 89.9 | 90.01 | -| [distilbart-mnli-12-1](https://huggingface.co/valhalla/distilbart-mnli-12-1) | 87.08 | 87.5 | -| [distilbart-mnli-12-3](https://huggingface.co/valhalla/distilbart-mnli-12-3) | 88.1 | 88.19 | -| [distilbart-mnli-12-6](https://huggingface.co/valhalla/distilbart-mnli-12-6) | 89.19 | 89.01 | -| [distilbart-mnli-12-9](https://huggingface.co/valhalla/distilbart-mnli-12-9) | 89.56 | 89.52 | - - -This is a very simple and effective technique, as we can see the performance drop is very little. - -Detailed performace trade-offs will be posted in this [sheet](https://docs.google.com/spreadsheets/d/1dQeUvAKpScLuhDV1afaPJRRAE55s2LpIzDVA5xfqxvk/edit?usp=sharing). - - -## Fine-tuning -If you want to train these models yourself, clone the [distillbart-mnli repo](https://github.com/patil-suraj/distillbart-mnli) and follow the steps below - -Clone and install transformers from source -```bash -git clone https://github.com/huggingface/transformers.git -pip install -qqq -U ./transformers -``` - -Download MNLI data -```bash -python transformers/utils/download_glue_data.py --data_dir glue_data --tasks MNLI -``` - -Create student model -```bash -python create_student.py \ - --teacher_model_name_or_path facebook/bart-large-mnli \ - --student_encoder_layers 12 \ - --student_decoder_layers 6 \ - --save_path student-bart-mnli-12-6 \ -``` - -Start fine-tuning -```bash -python run_glue.py args.json -``` - -You can find the logs of these trained models in this [wandb project](https://wandb.ai/psuraj/distilbart-mnli). \ No newline at end of file diff --git a/model_cards/valhalla/electra-base-discriminator-finetuned_squadv1/README.md b/model_cards/valhalla/electra-base-discriminator-finetuned_squadv1/README.md deleted file mode 100644 index e51eebe8ae89..000000000000 --- a/model_cards/valhalla/electra-base-discriminator-finetuned_squadv1/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# ELECTRA-BASE-DISCRIMINATOR finetuned on SQuADv1 - -This is electra-base-discriminator model finetuned on SQuADv1 dataset for for question answering task. - -## Model details -As mentioned in the original paper: ELECTRA is a new method for self-supervised language representation learning. -It can be used to pre-train transformer networks using relatively little compute. -ELECTRA models are trained to distinguish "real" input tokens vs "fake" input tokens generated by another neural network, -similar to the discriminator of a GAN. At small scale, ELECTRA achieves strong results even when trained on a single GPU. -At large scale, ELECTRA achieves state-of-the-art results on the SQuAD 2.0 dataset. - -| Param | #Value | -|---------------------|--------| -| layers | 12 | -| hidden size | 768 | -| num attetion heads | 12 | -| on disk size | 436MB | - -## Model training -This model was trained on google colab v100 GPU. -You can find the fine-tuning colab here -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/11yo-LaFsgggwmDSy2P8zD3tzf5cCb-DU?usp=sharing). - -## Results -The results are actually slightly better than given in the paper. -In the paper the authors mentioned that electra-base achieves 84.5 EM and 90.8 F1 - -| Metric | #Value | -|--------|--------| -| EM | 85.0520| -| F1 | 91.6050| - - -## Model in Action 🚀 -```python3 -from transformers import pipeline - -nlp = pipeline('question-answering', model='valhalla/electra-base-discriminator-finetuned_squadv1') -nlp({ - 'question': 'What is the answer to everything ?', - 'context': '42 is the answer to life the universe and everything' -}) -=> {'answer': '42', 'end': 2, 'score': 0.981274963050339, 'start': 0} -``` - -> Created with ❤️ by Suraj Patil [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/patil-suraj/) -[![Twitter icon](https://cdn0.iconfinder.com/data/icons/shift-logotypes/32/Twitter-32.png)](https://twitter.com/psuraj28) diff --git a/model_cards/valhalla/longformer-base-4096-finetuned-squadv1/README.md b/model_cards/valhalla/longformer-base-4096-finetuned-squadv1/README.md deleted file mode 100644 index b6b096bf47cb..000000000000 --- a/model_cards/valhalla/longformer-base-4096-finetuned-squadv1/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# LONGFORMER-BASE-4096 fine-tuned on SQuAD v1 -This is longformer-base-4096 model fine-tuned on SQuAD v1 dataset for question answering task. - -[Longformer](https://arxiv.org/abs/2004.05150) model created by Iz Beltagy, Matthew E. Peters, Arman Coha from AllenAI. As the paper explains it - -> `Longformer` is a BERT-like model for long documents. - -The pre-trained model can handle sequences with upto 4096 tokens. - - -## Model Training -This model was trained on google colab v100 GPU. You can find the fine-tuning colab here [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1zEl5D-DdkBKva-DdreVOmN0hrAfzKG1o?usp=sharing). - -Few things to keep in mind while training longformer for QA task, -by default longformer uses sliding-window local attention on all tokens. But For QA, all question tokens should have global attention. For more details on this please refer the paper. The `LongformerForQuestionAnswering` model automatically does that for you. To allow it to do that -1. The input sequence must have three sep tokens, i.e the sequence should be encoded like this - ` question context`. If you encode the question and answer as a input pair, then the tokenizer already takes care of that, you shouldn't worry about it. -2. `input_ids` should always be a batch of examples. - -## Results -|Metric | # Value | -|-------------|---------| -| Exact Match | 85.1466 | -| F1 | 91.5415 | - -## Model in Action 🚀 -```python -import torch -from transformers import AutoTokenizer, AutoModelForQuestionAnswering, - -tokenizer = AutoTokenizer.from_pretrained("valhalla/longformer-base-4096-finetuned-squadv1") -model = AutoModelForQuestionAnswering.from_pretrained("valhalla/longformer-base-4096-finetuned-squadv1") - -text = "Huggingface has democratized NLP. Huge thanks to Huggingface for this." -question = "What has Huggingface done ?" -encoding = tokenizer(question, text, return_tensors="pt") -input_ids = encoding["input_ids"] - -# default is local attention everywhere -# the forward method will automatically set global attention on question tokens -attention_mask = encoding["attention_mask"] - -start_scores, end_scores = model(input_ids, attention_mask=attention_mask) -all_tokens = tokenizer.convert_ids_to_tokens(input_ids[0].tolist()) - -answer_tokens = all_tokens[torch.argmax(start_scores) :torch.argmax(end_scores)+1] -answer = tokenizer.decode(tokenizer.convert_tokens_to_ids(answer_tokens)) -# output => democratized NLP -``` - -The `LongformerForQuestionAnswering` isn't yet supported in `pipeline` . I'll update this card once the support has been added. - -> Created with ❤️ by Suraj Patil [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/patil-suraj/) -[![Twitter icon](https://cdn0.iconfinder.com/data/icons/shift-logotypes/32/Twitter-32.png)](https://twitter.com/psuraj28) diff --git a/model_cards/valhalla/t5-base-e2e-qg/README.md b/model_cards/valhalla/t5-base-e2e-qg/README.md deleted file mode 100644 index f9536523a74d..000000000000 --- a/model_cards/valhalla/t5-base-e2e-qg/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -datasets: -- squad -tags: -- question-generation -widget: -- text: "Python is a programming language. It is developed by Guido Van Rossum and released in 1991. " -license: mit ---- - -## T5 for question-generation -This is [t5-base](https://arxiv.org/abs/1910.10683) model trained for end-to-end question generation task. Simply input the text and the model will generate multile questions. - -You can play with the model using the inference API, just put the text and see the results! - -For more deatils see [this](https://github.com/patil-suraj/question_generation) repo. - -### Model in action 🚀 - -You'll need to clone the [repo](https://github.com/patil-suraj/question_generation). - -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb) - -```python3 -from pipelines import pipeline - -text = "Python is an interpreted, high-level, general-purpose programming language. Created by Guido van Rossum \ -and first released in 1991, Python's design philosophy emphasizes code \ -readability with its notable use of significant whitespace." - -nlp = pipeline("e2e-qg", model="valhalla/t5-base-e2e-qg") -nlp(text) -=> [ - 'Who created Python?', - 'When was Python first released?', - "What is Python's design philosophy?" -] -``` \ No newline at end of file diff --git a/model_cards/valhalla/t5-base-qa-qg-hl/README.md b/model_cards/valhalla/t5-base-qa-qg-hl/README.md deleted file mode 100644 index 5e2ef575bc2c..000000000000 --- a/model_cards/valhalla/t5-base-qa-qg-hl/README.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -datasets: -- squad -tags: -- question-generation -widget: -- text: "generate question: 42 is the answer to life, the universe and everything. " -- text: "question: What is 42 context: 42 is the answer to life, the universe and everything. " -license: mit ---- - -## T5 for multi-task QA and QG -This is multi-task [t5-base](https://arxiv.org/abs/1910.10683) model trained for question answering and answer aware question generation tasks. - -For question generation the answer spans are highlighted within the text with special highlight tokens (``) and prefixed with 'generate question: '. For QA the input is processed like this `question: question_text context: context_text ` - -You can play with the model using the inference API. Here's how you can use it - -For QG - -`generate question: 42 is the answer to life, the universe and everything. ` - -For QA - -`question: What is 42 context: 42 is the answer to life, the universe and everything. ` - -For more deatils see [this](https://github.com/patil-suraj/question_generation) repo. - - -### Model in action 🚀 - -You'll need to clone the [repo](https://github.com/patil-suraj/question_generation). - -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb) - -```python3 -from pipelines import pipeline -nlp = pipeline("multitask-qa-qg", model="valhalla/t5-base-qa-qg-hl") - -# to generate questions simply pass the text -nlp("42 is the answer to life, the universe and everything.") -=> [{'answer': '42', 'question': 'What is the answer to life, the universe and everything?'}] - -# for qa pass a dict with "question" and "context" -nlp({ - "question": "What is 42 ?", - "context": "42 is the answer to life, the universe and everything." -}) -=> 'the answer to life, the universe and everything' -``` \ No newline at end of file diff --git a/model_cards/valhalla/t5-base-qg-hl/README.md b/model_cards/valhalla/t5-base-qg-hl/README.md deleted file mode 100644 index 40d603dda5e7..000000000000 --- a/model_cards/valhalla/t5-base-qg-hl/README.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -datasets: -- squad -tags: -- question-generation -widget: -- text: " 42 is the answer to life, the universe and everything. " -- text: "Python is a programming language. It is developed by Guido Van Rossum . " -- text: "Although practicality beats purity " -license: mit ---- - -## T5 for question-generation -This is [t5-base](https://arxiv.org/abs/1910.10683) model trained for answer aware question generation task. The answer spans are highlighted within the text with special highlight tokens. - -You can play with the model using the inference API, just highlight the answer spans with `` tokens and end the text with ``. For example - -` 42 is the answer to life, the universe and everything. ` - -For more deatils see [this](https://github.com/patil-suraj/question_generation) repo. - -### Model in action 🚀 - -You'll need to clone the [repo](https://github.com/patil-suraj/question_generation). - -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb) - -```python3 -from pipelines import pipeline -nlp = pipeline("question-generation", model="valhalla/t5-base-qg-hl") -nlp("42 is the answer to life, universe and everything.") -=> [{'answer': '42', 'question': 'What is the answer to life, universe and everything?'}] -``` \ No newline at end of file diff --git a/model_cards/valhalla/t5-base-squad/README.md b/model_cards/valhalla/t5-base-squad/README.md deleted file mode 100644 index 18f314759a44..000000000000 --- a/model_cards/valhalla/t5-base-squad/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# T5 for question-answering -This is T5-base model fine-tuned on SQuAD1.1 for QA using text-to-text approach - -## Model training -This model was trained on colab TPU with 35GB RAM for 4 epochs - -## Results: -| Metric | #Value | -|-------------|---------| -| Exact Match | 81.5610 | -| F1 | 89.9601 | - -## Model in Action 🚀 -``` -from transformers import AutoModelWithLMHead, AutoTokenizer - -tokenizer = AutoTokenizer.from_pretrained("valhalla/t5-base-squad") -model = AutoModelWithLMHead.from_pretrained("valhalla/t5-base-squad") - -def get_answer(question, context): - input_text = "question: %s context: %s " % (question, context) - features = tokenizer([input_text], return_tensors='pt') - - out = model.generate(input_ids=features['input_ids'], - attention_mask=features['attention_mask']) - - return tokenizer.decode(out[0]) - -context = "In Norse mythology, Valhalla is a majestic, enormous hall located in Asgard, ruled over by the god Odin." -question = "What is Valhalla ?" - -get_answer(question, context) -# output: 'a majestic, enormous hall located in Asgard, ruled over by the god Odin' -``` -Play with this model [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1a5xpJiUjZybfU9Mi-aDkOp116PZ9-wni?usp=sharing) - -> Created by Suraj Patil [![Github icon](https://cdn0.iconfinder.com/data/icons/octicons/1024/mark-github-32.png)](https://github.com/patil-suraj/) -[![Twitter icon](https://cdn0.iconfinder.com/data/icons/shift-logotypes/32/Twitter-32.png)](https://twitter.com/psuraj28) diff --git a/model_cards/valhalla/t5-samll-qg-prepend/README.md b/model_cards/valhalla/t5-samll-qg-prepend/README.md deleted file mode 100644 index c4f35c3da8fe..000000000000 --- a/model_cards/valhalla/t5-samll-qg-prepend/README.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -datasets: -- squad -tags: -- question-generation -widget: -- text: "answer: 42 context: 42 is the answer to life, the universe and everything. " -- text: "answer: Guido Van Rossum context: Python is a programming language. It is developed by Guido Van Rossum. " -- text: "answer: Explicit context: Explicit is better than implicit " -license: mit ---- - -## T5 for question-generation -This is [t5-small](https://arxiv.org/abs/1910.10683) model trained for answer aware question generation task. The answer text is prepended before the context text. - -You can play with the model using the inference API, just get the input text in this format and see the results! -`answer: answer_text context: context_text ` - -For example - -`answer: 42 context: 42 is the answer to life, the universe and everything. ` - -For more deatils see [this](https://github.com/patil-suraj/question_generation) repo. - -### Model in action 🚀 - -You'll need to clone the [repo](https://github.com/patil-suraj/question_generation). - -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb) - -```python3 -from pipelines import pipeline -nlp = pipeline("question-generation", qg_format="prepend") -nlp("42 is the answer to life, universe and everything.") -=> [{'answer': '42', 'question': 'What is the answer to life, universe and everything?'}] -``` \ No newline at end of file diff --git a/model_cards/valhalla/t5-small-e2e-qg/README.md b/model_cards/valhalla/t5-small-e2e-qg/README.md deleted file mode 100644 index 743b0334e9d2..000000000000 --- a/model_cards/valhalla/t5-small-e2e-qg/README.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -datasets: -- squad -tags: -- question-generation -widget: -- text: "Python is developed by Guido Van Rossum and released in 1991. " -license: mit ---- - -## T5 for question-generation -This is [t5-small](https://arxiv.org/abs/1910.10683) model trained for end-to-end question generation task. Simply input the text and the model will generate multile questions. - -You can play with the model using the inference API, just put the text and see the results! - -For more deatils see [this](https://github.com/patil-suraj/question_generation) repo. - -### Model in action 🚀 - -You'll need to clone the [repo](https://github.com/patil-suraj/question_generation). - -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb) - -```python3 -from pipelines import pipeline - -text = "Python is an interpreted, high-level, general-purpose programming language. Created by Guido van Rossum \ -and first released in 1991, Python's design philosophy emphasizes code \ -readability with its notable use of significant whitespace." - -nlp = pipeline("e2e-qg") -nlp(text) -=> [ - 'Who created Python?', - 'When was Python first released?', - "What is Python's design philosophy?" -] -``` \ No newline at end of file diff --git a/model_cards/valhalla/t5-small-qa-qg-hl/README.md b/model_cards/valhalla/t5-small-qa-qg-hl/README.md deleted file mode 100644 index 7f3a24c346bc..000000000000 --- a/model_cards/valhalla/t5-small-qa-qg-hl/README.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -datasets: -- squad -tags: -- question-generation -widget: -- text: "generate question: 42 is the answer to life, the universe and everything. " -- text: "question: What is 42 context: 42 is the answer to life, the universe and everything. " -license: mit ---- - -## T5 for multi-task QA and QG -This is multi-task [t5-small](https://arxiv.org/abs/1910.10683) model trained for question answering and answer aware question generation tasks. - -For question generation the answer spans are highlighted within the text with special highlight tokens (``) and prefixed with 'generate question: '. For QA the input is processed like this `question: question_text context: context_text ` - -You can play with the model using the inference API. Here's how you can use it - -For QG - -`generate question: 42 is the answer to life, the universe and everything. ` - -For QA - -`question: What is 42 context: 42 is the answer to life, the universe and everything. ` - -For more deatils see [this](https://github.com/patil-suraj/question_generation) repo. - -### Model in action 🚀 - -You'll need to clone the [repo](https://github.com/patil-suraj/question_generation). - -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb) - -```python3 -from pipelines import pipeline -nlp = pipeline("multitask-qa-qg") - -# to generate questions simply pass the text -nlp("42 is the answer to life, the universe and everything.") -=> [{'answer': '42', 'question': 'What is the answer to life, the universe and everything?'}] - -# for qa pass a dict with "question" and "context" -nlp({ - "question": "What is 42 ?", - "context": "42 is the answer to life, the universe and everything." -}) -=> 'the answer to life, the universe and everything' -``` \ No newline at end of file diff --git a/model_cards/valhalla/t5-small-qg-hl/README.md b/model_cards/valhalla/t5-small-qg-hl/README.md deleted file mode 100644 index ca295f1dfa6c..000000000000 --- a/model_cards/valhalla/t5-small-qg-hl/README.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -datasets: -- squad -tags: -- question-generation -widget: -- text: " 42 is the answer to life, the universe and everything. " -- text: "Python is a programming language. It is developed by Guido Van Rossum . " -- text: "Simple is better than complex . " -license: mit ---- - -## T5 for question-generation -This is [t5-small](https://arxiv.org/abs/1910.10683) model trained for answer aware question generation task. The answer spans are highlighted within the text with special highlight tokens. - -You can play with the model using the inference API, just highlight the answer spans with `` tokens and end the text with ``. For example - -` 42 is the answer to life, the universe and everything. ` - -For more deatils see [this](https://github.com/patil-suraj/question_generation) repo. - -### Model in action 🚀 - -You'll need to clone the [repo](https://github.com/patil-suraj/question_generation). - -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/question_generation/blob/master/question_generation.ipynb) - -```python3 -from pipelines import pipeline -nlp = pipeline("question-generation") -nlp("42 is the answer to life, universe and everything.") -=> [{'answer': '42', 'question': 'What is the answer to life, universe and everything?'}] -``` \ No newline at end of file diff --git a/model_cards/vinai/bertweet-base/README.md b/model_cards/vinai/bertweet-base/README.md deleted file mode 100644 index 4d6b041f5d43..000000000000 --- a/model_cards/vinai/bertweet-base/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# BERTweet: A pre-trained language model for English Tweets - - - BERTweet is the first public large-scale language model pre-trained for English Tweets. BERTweet is trained based on the [RoBERTa](https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.md) pre-training procedure, using the same model configuration as [BERT-base](https://github.com/google-research/bert). - - The corpus used to pre-train BERTweet consists of 850M English Tweets (16B word tokens ~ 80GB), containing 845M Tweets streamed from 01/2012 to 08/2019 and 5M Tweets related to the **COVID-19** pandemic. - - BERTweet does better than its competitors RoBERTa-base and [XLM-R-base](https://arxiv.org/abs/1911.02116) and outperforms previous state-of-the-art models on three downstream Tweet NLP tasks of Part-of-speech tagging, Named entity recognition and text classification. - -The general architecture and experimental results of BERTweet can be found in our [paper](https://arxiv.org/abs/2005.10200): - - @inproceedings{bertweet, - title = {{BERTweet: A pre-trained language model for English Tweets}}, - author = {Dat Quoc Nguyen and Thanh Vu and Anh Tuan Nguyen}, - booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations}, - year = {2020} - } - -**Please CITE** our paper when BERTweet is used to help produce published results or is incorporated into other software. - -For further information or requests, please go to [BERTweet's homepage](https://github.com/VinAIResearch/BERTweet)! - -### Installation - - - Python 3.6+, and PyTorch 1.1.0+ (or TensorFlow 2.0+) - - Install `transformers`: - - `git clone https://github.com/huggingface/transformers.git` - - `cd transformers` - - `pip3 install --upgrade .` - - Install `emoji`: `pip3 install emoji` - -### Pre-trained models - - -Model | #params | Arch. | Pre-training data ----|---|---|--- -`vinai/bertweet-base` | 135M | base | 845M English Tweets (cased) -`vinai/bertweet-covid19-base-cased` | 135M | base | 23M COVID-19 English Tweets (cased) -`vinai/bertweet-covid19-base-uncased` | 135M | base | 23M COVID-19 English Tweets (uncased) - -Two pre-trained models `vinai/bertweet-covid19-base-cased` and `vinai/bertweet-covid19-base-uncased` are resulted by further pre-training the pre-trained model `vinai/bertweet-base` on a corpus of 23M COVID-19 English Tweets for 40 epochs. - -### Example usage - - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -bertweet = AutoModel.from_pretrained("vinai/bertweet-base") -tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-base") - -# INPUT TWEET IS ALREADY NORMALIZED! -line = "SC has first two presumptive cases of coronavirus , DHEC confirms HTTPURL via @USER :cry:" - -input_ids = torch.tensor([tokenizer.encode(line)]) - -with torch.no_grad(): - features = bertweet(input_ids) # Models outputs are now tuples - -## With TensorFlow 2.0+: -# from transformers import TFAutoModel -# bertweet = TFAutoModel.from_pretrained("vinai/bertweet-base") -``` - -### Normalize raw input Tweets - -Before applying `fastBPE` to the pre-training corpus of 850M English Tweets, we tokenized these Tweets using `TweetTokenizer` from the NLTK toolkit and used the `emoji` package to translate emotion icons into text strings (here, each icon is referred to as a word token). We also normalized the Tweets by converting user mentions and web/url links into special tokens `@USER` and `HTTPURL`, respectively. Thus it is recommended to also apply the same pre-processing step for BERTweet-based downstream applications w.r.t. the raw input Tweets. BERTweet provides this pre-processing step by enabling the `normalization` argument. - -```python -import torch -from transformers import AutoTokenizer - -# Load the AutoTokenizer with a normalization mode if the input Tweet is raw -tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-base", normalization=True) - -# from transformers import BertweetTokenizer -# tokenizer = BertweetTokenizer.from_pretrained("vinai/bertweet-base", normalization=True) - -line = "SC has first two presumptive cases of coronavirus, DHEC confirms https://postandcourier.com/health/covid19/sc-has-first-two-presumptive-cases-of-coronavirus-dhec-confirms/article_bddfe4ae-5fd3-11ea-9ce4-5f495366cee6.html?utm_medium=social&utm_source=twitter&utm_campaign=user-share… via @postandcourier" - -input_ids = torch.tensor([tokenizer.encode(line)]) -``` diff --git a/model_cards/vinai/bertweet-covid19-base-cased/README.md b/model_cards/vinai/bertweet-covid19-base-cased/README.md deleted file mode 100644 index e09c71e4b71b..000000000000 --- a/model_cards/vinai/bertweet-covid19-base-cased/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# BERTweet: A pre-trained language model for English Tweets - - - BERTweet is the first public large-scale language model pre-trained for English Tweets. BERTweet is trained based on the [RoBERTa](https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.md) pre-training procedure, using the same model configuration as [BERT-base](https://github.com/google-research/bert). - - The corpus used to pre-train BERTweet consists of 850M English Tweets (16B word tokens ~ 80GB), containing 845M Tweets streamed from 01/2012 to 08/2019 and 5M Tweets related to the **COVID-19** pandemic. - - BERTweet does better than its competitors RoBERTa-base and [XLM-R-base](https://arxiv.org/abs/1911.02116) and outperforms previous state-of-the-art models on three downstream Tweet NLP tasks of Part-of-speech tagging, Named entity recognition and text classification. - -The general architecture and experimental results of BERTweet can be found in our [paper](https://arxiv.org/abs/2005.10200): - - @inproceedings{bertweet, - title = {{BERTweet: A pre-trained language model for English Tweets}}, - author = {Dat Quoc Nguyen and Thanh Vu and Anh Tuan Nguyen}, - booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations}, - year = {2020} - } - -**Please CITE** our paper when BERTweet is used to help produce published results or is incorporated into other software. - -For further information or requests, please go to [BERTweet's homepage](https://github.com/VinAIResearch/BERTweet)! - -### Installation - - - Python 3.6+, and PyTorch 1.1.0+ (or TensorFlow 2.0+) - - Install `transformers`: - - `git clone https://github.com/huggingface/transformers.git` - - `cd transformers` - - `pip3 install --upgrade .` - - Install `emoji`: `pip3 install emoji` - -### Pre-trained models - - -Model | #params | Arch. | Pre-training data ----|---|---|--- -`vinai/bertweet-base` | 135M | base | 845M English Tweets (cased) -`vinai/bertweet-covid19-base-cased` | 135M | base | 23M COVID-19 English Tweets (cased) -`vinai/bertweet-covid19-base-uncased` | 135M | base | 23M COVID-19 English Tweets (uncased) - -Two pre-trained models `vinai/bertweet-covid19-base-cased` and `vinai/bertweet-covid19-base-uncased` are resulted by further pre-training the pre-trained model `vinai/bertweet-base` on a corpus of 23M COVID-19 English Tweets for 40 epochs. - -### Example usage - - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -bertweet = AutoModel.from_pretrained("vinai/bertweet-covid19-base-cased") -tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-covid19-base-cased") - -# INPUT TWEET IS ALREADY NORMALIZED! -line = "SC has first two presumptive cases of coronavirus , DHEC confirms HTTPURL via @USER :cry:" - -input_ids = torch.tensor([tokenizer.encode(line)]) - -with torch.no_grad(): - features = bertweet(input_ids) # Models outputs are now tuples - -## With TensorFlow 2.0+: -# from transformers import TFAutoModel -# bertweet = TFAutoModel.from_pretrained("vinai/bertweet-covid19-base-cased") -``` - -### Normalize raw input Tweets - -Before applying `fastBPE` to the pre-training corpus of 850M English Tweets, we tokenized these Tweets using `TweetTokenizer` from the NLTK toolkit and used the `emoji` package to translate emotion icons into text strings (here, each icon is referred to as a word token). We also normalized the Tweets by converting user mentions and web/url links into special tokens `@USER` and `HTTPURL`, respectively. Thus it is recommended to also apply the same pre-processing step for BERTweet-based downstream applications w.r.t. the raw input Tweets. BERTweet provides this pre-processing step by enabling the `normalization` argument. - -```python -import torch -from transformers import AutoTokenizer - -# Load the AutoTokenizer with a normalization mode if the input Tweet is raw -tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-covid19-base-cased", normalization=True) - -# from transformers import BertweetTokenizer -# tokenizer = BertweetTokenizer.from_pretrained("vinai/bertweet-covid19-base-cased", normalization=True) - -line = "SC has first two presumptive cases of coronavirus, DHEC confirms https://postandcourier.com/health/covid19/sc-has-first-two-presumptive-cases-of-coronavirus-dhec-confirms/article_bddfe4ae-5fd3-11ea-9ce4-5f495366cee6.html?utm_medium=social&utm_source=twitter&utm_campaign=user-share… via @postandcourier" - -input_ids = torch.tensor([tokenizer.encode(line)]) -``` diff --git a/model_cards/vinai/bertweet-covid19-base-uncased/README.md b/model_cards/vinai/bertweet-covid19-base-uncased/README.md deleted file mode 100644 index 4f807de06aa5..000000000000 --- a/model_cards/vinai/bertweet-covid19-base-uncased/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# BERTweet: A pre-trained language model for English Tweets - - - BERTweet is the first public large-scale language model pre-trained for English Tweets. BERTweet is trained based on the [RoBERTa](https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.md) pre-training procedure, using the same model configuration as [BERT-base](https://github.com/google-research/bert). - - The corpus used to pre-train BERTweet consists of 850M English Tweets (16B word tokens ~ 80GB), containing 845M Tweets streamed from 01/2012 to 08/2019 and 5M Tweets related to the **COVID-19** pandemic. - - BERTweet does better than its competitors RoBERTa-base and [XLM-R-base](https://arxiv.org/abs/1911.02116) and outperforms previous state-of-the-art models on three downstream Tweet NLP tasks of Part-of-speech tagging, Named entity recognition and text classification. - -The general architecture and experimental results of BERTweet can be found in our [paper](https://arxiv.org/abs/2005.10200): - - @inproceedings{bertweet, - title = {{BERTweet: A pre-trained language model for English Tweets}}, - author = {Dat Quoc Nguyen and Thanh Vu and Anh Tuan Nguyen}, - booktitle = {Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations}, - year = {2020} - } - -**Please CITE** our paper when BERTweet is used to help produce published results or is incorporated into other software. - -For further information or requests, please go to [BERTweet's homepage](https://github.com/VinAIResearch/BERTweet)! - -### Installation - - - Python 3.6+, and PyTorch 1.1.0+ (or TensorFlow 2.0+) - - Install `transformers`: - - `git clone https://github.com/huggingface/transformers.git` - - `cd transformers` - - `pip3 install --upgrade .` - - Install `emoji`: `pip3 install emoji` - -### Pre-trained models - - -Model | #params | Arch. | Pre-training data ----|---|---|--- -`vinai/bertweet-base` | 135M | base | 845M English Tweets (cased) -`vinai/bertweet-covid19-base-cased` | 135M | base | 23M COVID-19 English Tweets (cased) -`vinai/bertweet-covid19-base-uncased` | 135M | base | 23M COVID-19 English Tweets (uncased) - -Two pre-trained models `vinai/bertweet-covid19-base-cased` and `vinai/bertweet-covid19-base-uncased` are resulted by further pre-training the pre-trained model `vinai/bertweet-base` on a corpus of 23M COVID-19 English Tweets for 40 epochs. - -### Example usage - - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -bertweet = AutoModel.from_pretrained("vinai/bertweet-covid19-base-uncased") -tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-covid19-base-uncased") - -# INPUT TWEET IS ALREADY NORMALIZED! -line = "SC has first two presumptive cases of coronavirus , DHEC confirms HTTPURL via @USER :cry:" - -input_ids = torch.tensor([tokenizer.encode(line)]) - -with torch.no_grad(): - features = bertweet(input_ids) # Models outputs are now tuples - -## With TensorFlow 2.0+: -# from transformers import TFAutoModel -# bertweet = TFAutoModel.from_pretrained("vinai/bertweet-covid19-base-uncased") -``` - -### Normalize raw input Tweets - -Before applying `fastBPE` to the pre-training corpus of 850M English Tweets, we tokenized these Tweets using `TweetTokenizer` from the NLTK toolkit and used the `emoji` package to translate emotion icons into text strings (here, each icon is referred to as a word token). We also normalized the Tweets by converting user mentions and web/url links into special tokens `@USER` and `HTTPURL`, respectively. Thus it is recommended to also apply the same pre-processing step for BERTweet-based downstream applications w.r.t. the raw input Tweets. BERTweet provides this pre-processing step by enabling the `normalization` argument. - -```python -import torch -from transformers import AutoTokenizer - -# Load the AutoTokenizer with a normalization mode if the input Tweet is raw -tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-covid19-base-uncased", normalization=True) - -# from transformers import BertweetTokenizer -# tokenizer = BertweetTokenizer.from_pretrained("vinai/bertweet-covid19-base-uncased", normalization=True) - -line = "SC has first two presumptive cases of coronavirus, DHEC confirms https://postandcourier.com/health/covid19/sc-has-first-two-presumptive-cases-of-coronavirus-dhec-confirms/article_bddfe4ae-5fd3-11ea-9ce4-5f495366cee6.html?utm_medium=social&utm_source=twitter&utm_campaign=user-share… via @postandcourier" - -input_ids = torch.tensor([tokenizer.encode(line)]) -``` diff --git a/model_cards/vinai/phobert-base/README.md b/model_cards/vinai/phobert-base/README.md deleted file mode 100644 index afae1177c722..000000000000 --- a/model_cards/vinai/phobert-base/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# PhoBERT: Pre-trained language models for Vietnamese - -Pre-trained PhoBERT models are the state-of-the-art language models for Vietnamese ([Pho](https://en.wikipedia.org/wiki/Pho), i.e. "Phở", is a popular food in Vietnam): - - - Two PhoBERT versions of "base" and "large" are the first public large-scale monolingual language models pre-trained for Vietnamese. PhoBERT pre-training approach is based on [RoBERTa](https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.md) which optimizes the [BERT](https://github.com/google-research/bert) pre-training procedure for more robust performance. - - PhoBERT outperforms previous monolingual and multilingual approaches, obtaining new state-of-the-art performances on four downstream Vietnamese NLP tasks of Part-of-speech tagging, Dependency parsing, Named-entity recognition and Natural language inference. - -The general architecture and experimental results of PhoBERT can be found in our EMNLP-2020 Findings [paper](https://arxiv.org/abs/2003.00744): - - @article{phobert, - title = {{PhoBERT: Pre-trained language models for Vietnamese}}, - author = {Dat Quoc Nguyen and Anh Tuan Nguyen}, - journal = {Findings of EMNLP}, - year = {2020} - } - -**Please CITE** our paper when PhoBERT is used to help produce published results or is incorporated into other software. - -For further information or requests, please go to [PhoBERT's homepage](https://github.com/VinAIResearch/PhoBERT)! - -### Installation - - Python 3.6+, and PyTorch 1.1.0+ (or TensorFlow 2.0+) - - Install `transformers`: - - `git clone https://github.com/huggingface/transformers.git` - - `cd transformers` - - `pip3 install --upgrade .` - -### Pre-trained models - -Model | #params | Arch. | Pre-training data ----|---|---|--- -`vinai/phobert-base` | 135M | base | 20GB of texts -`vinai/phobert-large` | 370M | large | 20GB of texts - -### Example usage - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -phobert = AutoModel.from_pretrained("vinai/phobert-base") -tokenizer = AutoTokenizer.from_pretrained("vinai/phobert-base") - -# INPUT TEXT MUST BE ALREADY WORD-SEGMENTED! -line = "Tôi là sinh_viên trường đại_học Công_nghệ ." - -input_ids = torch.tensor([tokenizer.encode(line)]) - -with torch.no_grad(): - features = phobert(input_ids) # Models outputs are now tuples - -## With TensorFlow 2.0+: -# from transformers import TFAutoModel -# phobert = TFAutoModel.from_pretrained("vinai/phobert-base") -``` diff --git a/model_cards/vinai/phobert-large/README.md b/model_cards/vinai/phobert-large/README.md deleted file mode 100644 index 7bbf4521ef92..000000000000 --- a/model_cards/vinai/phobert-large/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# PhoBERT: Pre-trained language models for Vietnamese - -Pre-trained PhoBERT models are the state-of-the-art language models for Vietnamese ([Pho](https://en.wikipedia.org/wiki/Pho), i.e. "Phở", is a popular food in Vietnam): - - - Two PhoBERT versions of "base" and "large" are the first public large-scale monolingual language models pre-trained for Vietnamese. PhoBERT pre-training approach is based on [RoBERTa](https://github.com/pytorch/fairseq/blob/master/examples/roberta/README.md) which optimizes the [BERT](https://github.com/google-research/bert) pre-training procedure for more robust performance. - - PhoBERT outperforms previous monolingual and multilingual approaches, obtaining new state-of-the-art performances on four downstream Vietnamese NLP tasks of Part-of-speech tagging, Dependency parsing, Named-entity recognition and Natural language inference. - -The general architecture and experimental results of PhoBERT can be found in our EMNLP-2020 Findings [paper](https://arxiv.org/abs/2003.00744): - - @article{phobert, - title = {{PhoBERT: Pre-trained language models for Vietnamese}}, - author = {Dat Quoc Nguyen and Anh Tuan Nguyen}, - journal = {Findings of EMNLP}, - year = {2020} - } - -**Please CITE** our paper when PhoBERT is used to help produce published results or is incorporated into other software. - -For further information or requests, please go to [PhoBERT's homepage](https://github.com/VinAIResearch/PhoBERT)! - -### Installation - - Python 3.6+, and PyTorch 1.1.0+ (or TensorFlow 2.0+) - - Install `transformers`: - - `git clone https://github.com/huggingface/transformers.git` - - `cd transformers` - - `pip3 install --upgrade .` - -### Pre-trained models - -Model | #params | Arch. | Pre-training data ----|---|---|--- -`vinai/phobert-base` | 135M | base | 20GB of texts -`vinai/phobert-large` | 370M | large | 20GB of texts - -### Example usage - -```python -import torch -from transformers import AutoModel, AutoTokenizer - -phobert = AutoModel.from_pretrained("vinai/phobert-large") -tokenizer = AutoTokenizer.from_pretrained("vinai/phobert-large") - -# INPUT TEXT MUST BE ALREADY WORD-SEGMENTED! -line = "Tôi là sinh_viên trường đại_học Công_nghệ ." - -input_ids = torch.tensor([tokenizer.encode(line)]) - -with torch.no_grad(): - features = phobert(input_ids) # Models outputs are now tuples - -## With TensorFlow 2.0+: -# from transformers import TFAutoModel -# phobert = TFAutoModel.from_pretrained("vinai/phobert-large") -``` diff --git a/model_cards/voidful/albert_chinese_base/README.md b/model_cards/voidful/albert_chinese_base/README.md deleted file mode 100644 index d86eec8871f3..000000000000 --- a/model_cards/voidful/albert_chinese_base/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: zh ---- - -# albert_chinese_base - -This a albert_chinese_base model from [Google's github](https://github.com/google-research/ALBERT) -converted by huggingface's [script](https://github.com/huggingface/transformers/blob/master/src/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py) - -## Attention (注意) - -Since sentencepiece is not used in albert_chinese_base model -you have to call BertTokenizer instead of AlbertTokenizer !!! -we can eval it using an example on MaskedLM - -由於 albert_chinese_base 模型沒有用 sentencepiece -用AlbertTokenizer會載不進詞表，因此需要改用BertTokenizer !!! -我們可以跑MaskedLM預測來驗證這個做法是否正確 - -## Justify (驗證有效性) -[colab trial](https://colab.research.google.com/drive/1Wjz48Uws6-VuSHv_-DcWLilv77-AaYgj) -```python -from transformers import * -import torch -from torch.nn.functional import softmax - -pretrained = 'voidful/albert_chinese_base' -tokenizer = BertTokenizer.from_pretrained(pretrained) -model = AlbertForMaskedLM.from_pretrained(pretrained) - -inputtext = "今天[MASK]情很好" - -maskpos = tokenizer.encode(inputtext, add_special_tokens=True).index(103) - -input_ids = torch.tensor(tokenizer.encode(inputtext, add_special_tokens=True)).unsqueeze(0) # Batch size 1 -outputs = model(input_ids, masked_lm_labels=input_ids) -loss, prediction_scores = outputs[:2] -logit_prob = softmax(prediction_scores[0, maskpos]).data.tolist() -predicted_index = torch.argmax(prediction_scores[0, maskpos]).item() -predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] -print(predicted_token,logit_prob[predicted_index]) -``` -Result: `感 0.36333346366882324` diff --git a/model_cards/voidful/albert_chinese_large/README.md b/model_cards/voidful/albert_chinese_large/README.md deleted file mode 100644 index 4311563c043d..000000000000 --- a/model_cards/voidful/albert_chinese_large/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: zh ---- - -# albert_chinese_large - -This a albert_chinese_large model from [Google's github](https://github.com/google-research/ALBERT) -converted by huggingface's [script](https://github.com/huggingface/transformers/blob/master/src/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py) - -## Attention (注意) - -Since sentencepiece is not used in albert_chinese_large model -you have to call BertTokenizer instead of AlbertTokenizer !!! -we can eval it using an example on MaskedLM - -由於 albert_chinese_large 模型沒有用 sentencepiece -用AlbertTokenizer會載不進詞表，因此需要改用BertTokenizer !!! -我們可以跑MaskedLM預測來驗證這個做法是否正確 - -## Justify (驗證有效性) -[colab trial](https://colab.research.google.com/drive/1Wjz48Uws6-VuSHv_-DcWLilv77-AaYgj) -```python -from transformers import * -import torch -from torch.nn.functional import softmax - -pretrained = 'voidful/albert_chinese_large' -tokenizer = BertTokenizer.from_pretrained(pretrained) -model = AlbertForMaskedLM.from_pretrained(pretrained) - -inputtext = "今天[MASK]情很好" - -maskpos = tokenizer.encode(inputtext, add_special_tokens=True).index(103) - -input_ids = torch.tensor(tokenizer.encode(inputtext, add_special_tokens=True)).unsqueeze(0) # Batch size 1 -outputs = model(input_ids, masked_lm_labels=input_ids) -loss, prediction_scores = outputs[:2] -logit_prob = softmax(prediction_scores[0, maskpos]).data.tolist() -predicted_index = torch.argmax(prediction_scores[0, maskpos]).item() -predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] -print(predicted_token,logit_prob[predicted_index]) -``` -Result: `心 0.9422469735145569` diff --git a/model_cards/voidful/albert_chinese_small/README.md b/model_cards/voidful/albert_chinese_small/README.md deleted file mode 100644 index 00a212752fe7..000000000000 --- a/model_cards/voidful/albert_chinese_small/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: zh ---- - -# albert_chinese_small - -This a albert_chinese_small model from [brightmart/albert_zh project](https://github.com/brightmart/albert_zh), albert_small_google_zh model -converted by huggingface's [script](https://github.com/huggingface/transformers/blob/master/src/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py) - -## Attention (注意) - -Since sentencepiece is not used in albert_chinese_small model -you have to call BertTokenizer instead of AlbertTokenizer !!! -we can eval it using an example on MaskedLM - -由於 albert_chinese_small 模型沒有用 sentencepiece -用AlbertTokenizer會載不進詞表，因此需要改用BertTokenizer !!! -我們可以跑MaskedLM預測來驗證這個做法是否正確 - -## Justify (驗證有效性) -[colab trial](https://colab.research.google.com/drive/1Wjz48Uws6-VuSHv_-DcWLilv77-AaYgj) -```python -from transformers import * -import torch -from torch.nn.functional import softmax - -pretrained = 'voidful/albert_chinese_small' -tokenizer = BertTokenizer.from_pretrained(pretrained) -model = AlbertForMaskedLM.from_pretrained(pretrained) - -inputtext = "今天[MASK]情很好" - -maskpos = tokenizer.encode(inputtext, add_special_tokens=True).index(103) - -input_ids = torch.tensor(tokenizer.encode(inputtext, add_special_tokens=True)).unsqueeze(0) # Batch size 1 -outputs = model(input_ids, masked_lm_labels=input_ids) -loss, prediction_scores = outputs[:2] -logit_prob = softmax(prediction_scores[0, maskpos]).data.tolist() -predicted_index = torch.argmax(prediction_scores[0, maskpos]).item() -predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] -print(predicted_token,logit_prob[predicted_index]) -``` -Result: `感 0.6390823125839233` diff --git a/model_cards/voidful/albert_chinese_tiny/README.md b/model_cards/voidful/albert_chinese_tiny/README.md deleted file mode 100644 index 6d495bb49cda..000000000000 --- a/model_cards/voidful/albert_chinese_tiny/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: zh ---- - -# albert_chinese_tiny - -This a albert_chinese_tiny model from [brightmart/albert_zh project](https://github.com/brightmart/albert_zh), albert_tiny_google_zh model -converted by huggingface's [script](https://github.com/huggingface/transformers/blob/master/src/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py) - -## Attention (注意) - -Since sentencepiece is not used in albert_chinese_tiny model -you have to call BertTokenizer instead of AlbertTokenizer !!! -we can eval it using an example on MaskedLM - -由於 albert_chinese_tiny 模型沒有用 sentencepiece -用AlbertTokenizer會載不進詞表，因此需要改用BertTokenizer !!! -我們可以跑MaskedLM預測來驗證這個做法是否正確 - -## Justify (驗證有效性) -[colab trial](https://colab.research.google.com/drive/1Wjz48Uws6-VuSHv_-DcWLilv77-AaYgj) -```python -from transformers import * -import torch -from torch.nn.functional import softmax - -pretrained = 'voidful/albert_chinese_tiny' -tokenizer = BertTokenizer.from_pretrained(pretrained) -model = AlbertForMaskedLM.from_pretrained(pretrained) - -inputtext = "今天[MASK]情很好" - -maskpos = tokenizer.encode(inputtext, add_special_tokens=True).index(103) - -input_ids = torch.tensor(tokenizer.encode(inputtext, add_special_tokens=True)).unsqueeze(0) # Batch size 1 -outputs = model(input_ids, masked_lm_labels=input_ids) -loss, prediction_scores = outputs[:2] -logit_prob = softmax(prediction_scores[0, maskpos]).data.tolist() -predicted_index = torch.argmax(prediction_scores[0, maskpos]).item() -predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] -print(predicted_token,logit_prob[predicted_index]) -``` -Result: `感 0.40312355756759644` diff --git a/model_cards/voidful/albert_chinese_xlarge/README.md b/model_cards/voidful/albert_chinese_xlarge/README.md deleted file mode 100644 index f5ddba938de1..000000000000 --- a/model_cards/voidful/albert_chinese_xlarge/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: zh ---- - -# albert_chinese_xlarge - -This a albert_chinese_xlarge model from [Google's github](https://github.com/google-research/ALBERT) -converted by huggingface's [script](https://github.com/huggingface/transformers/blob/master/src/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py) - -## Attention (注意) - -Since sentencepiece is not used in albert_chinese_xlarge model -you have to call BertTokenizer instead of AlbertTokenizer !!! -we can eval it using an example on MaskedLM - -由於 albert_chinese_xlarge 模型沒有用 sentencepiece -用AlbertTokenizer會載不進詞表，因此需要改用BertTokenizer !!! -我們可以跑MaskedLM預測來驗證這個做法是否正確 - -## Justify (驗證有效性) -[colab trial](https://colab.research.google.com/drive/1Wjz48Uws6-VuSHv_-DcWLilv77-AaYgj) -```python -from transformers import * -import torch -from torch.nn.functional import softmax - -pretrained = 'voidful/albert_chinese_xlarge' -tokenizer = BertTokenizer.from_pretrained(pretrained) -model = AlbertForMaskedLM.from_pretrained(pretrained) - -inputtext = "今天[MASK]情很好" - -maskpos = tokenizer.encode(inputtext, add_special_tokens=True).index(103) - -input_ids = torch.tensor(tokenizer.encode(inputtext, add_special_tokens=True)).unsqueeze(0) # Batch size 1 -outputs = model(input_ids, masked_lm_labels=input_ids) -loss, prediction_scores = outputs[:2] -logit_prob = softmax(prediction_scores[0, maskpos]).data.tolist() -predicted_index = torch.argmax(prediction_scores[0, maskpos]).item() -predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] -print(predicted_token,logit_prob[predicted_index]) -``` -Result: `心 0.9942440390586853` diff --git a/model_cards/voidful/albert_chinese_xxlarge/README.md b/model_cards/voidful/albert_chinese_xxlarge/README.md deleted file mode 100644 index 800cfe12a33f..000000000000 --- a/model_cards/voidful/albert_chinese_xxlarge/README.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -language: zh ---- - -# albert_chinese_xxlarge - -This a albert_chinese_xxlarge model from [Google's github](https://github.com/google-research/ALBERT) -converted by huggingface's [script](https://github.com/huggingface/transformers/blob/master/src/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py) - -## Attention (注意) - -Since sentencepiece is not used in albert_chinese_xxlarge model -you have to call BertTokenizer instead of AlbertTokenizer !!! -we can eval it using an example on MaskedLM - -由於 albert_chinese_xxlarge 模型沒有用 sentencepiece -用AlbertTokenizer會載不進詞表，因此需要改用BertTokenizer !!! -我們可以跑MaskedLM預測來驗證這個做法是否正確 - -## Justify (驗證有效性) -[colab trial](https://colab.research.google.com/drive/1Wjz48Uws6-VuSHv_-DcWLilv77-AaYgj) -```python -from transformers import * -import torch -from torch.nn.functional import softmax - -pretrained = 'voidful/albert_chinese_xxlarge' -tokenizer = BertTokenizer.from_pretrained(pretrained) -model = AlbertForMaskedLM.from_pretrained(pretrained) - -inputtext = "今天[MASK]情很好" - -maskpos = tokenizer.encode(inputtext, add_special_tokens=True).index(103) - -input_ids = torch.tensor(tokenizer.encode(inputtext, add_special_tokens=True)).unsqueeze(0) # Batch size 1 -outputs = model(input_ids, masked_lm_labels=input_ids) -loss, prediction_scores = outputs[:2] -logit_prob = softmax(prediction_scores[0, maskpos]).data.tolist() -predicted_index = torch.argmax(prediction_scores[0, maskpos]).item() -predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] -print(predicted_token,logit_prob[predicted_index]) -``` -Result: `心 0.995713472366333` diff --git a/model_cards/wietsedv/bert-base-dutch-cased/README.md b/model_cards/wietsedv/bert-base-dutch-cased/README.md deleted file mode 100644 index ec4c18dab9a2..000000000000 --- a/model_cards/wietsedv/bert-base-dutch-cased/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# BERTje: A Dutch BERT model - -BERTje is a Dutch pre-trained BERT model developed at the University of Groningen. - -⚠️ **The new home of this model is the [GroNLP](https://huggingface.co/GroNLP) organization.** - -BERTje now lives at: [`GroNLP/bert-base-dutch-cased`](https://huggingface.co/GroNLP/bert-base-dutch-cased) - -The model weights of the versions at `wietsedv/` and `GroNLP/` are the same, so do not worry if you use(d) `wietsedv/bert-base-dutch-cased`. - - - diff --git a/model_cards/wptoux/albert-chinese-large-qa/README.md b/model_cards/wptoux/albert-chinese-large-qa/README.md deleted file mode 100644 index cf43f31714da..000000000000 --- a/model_cards/wptoux/albert-chinese-large-qa/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# albert-chinese-large-qa -Albert large QA model pretrained from baidu webqa and baidu dureader datasets. - -## Data source -+ baidu webqa 1.0 -+ baidu dureader - -## Traing Method -We combined the two datasets together and created a new dataset in squad format, including 705139 samples for training and 69638 samples for validation. -We finetune the model based on the albert chinese large model. - -## Hyperparams -+ learning_rate 1e-5 -+ max_seq_length 512 -+ max_query_length 50 -+ max_answer_length 300 -+ doc_stride 256 -+ num_train_epochs 2 -+ warmup_steps 1000 -+ per_gpu_train_batch_size 8 -+ gradient_accumulation_steps 3 -+ n_gpu 2 (Nvidia Tesla P100) - -## Usage -``` -from transformers import AutoModelForQuestionAnswering, BertTokenizer - -model = AutoModelForQuestionAnswering.from_pretrained('wptoux/albert-chinese-large-qa') -tokenizer = BertTokenizer.from_pretrained('wptoux/albert-chinese-large-qa') -``` -***Important: use BertTokenizer*** - -## MoreInfo -Please visit https://github.com/wptoux/albert-chinese-large-webqa for details. diff --git a/model_cards/xlm-mlm-en-2048-README.md b/model_cards/xlm-mlm-en-2048-README.md deleted file mode 100644 index a3a2b5e0021a..000000000000 --- a/model_cards/xlm-mlm-en-2048-README.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -tags: -- exbert - -license: cc-by-nc-4.0 ---- - - - - diff --git a/model_cards/xlm-roberta-base-README.md b/model_cards/xlm-roberta-base-README.md deleted file mode 100644 index 8230171a7dd7..000000000000 --- a/model_cards/xlm-roberta-base-README.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -tags: -- exbert - -license: mit ---- - - - - diff --git a/model_cards/xlm-roberta-large-finetuned-conll03-german-README.md b/model_cards/xlm-roberta-large-finetuned-conll03-german-README.md deleted file mode 100644 index e05bb660a38a..000000000000 --- a/model_cards/xlm-roberta-large-finetuned-conll03-german-README.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -language: de ---- - -## xlm-roberta-large-finetuned-conll03-german diff --git a/model_cards/yjernite/bart_eli5/README.md b/model_cards/yjernite/bart_eli5/README.md deleted file mode 100644 index 168be620f8c9..000000000000 --- a/model_cards/yjernite/bart_eli5/README.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -language: en -license: apache-2.0 -datasets: -- eli5 ---- - -## BART ELI5 - -Read the article at https://yjernite.github.io/lfqa.html and try the demo at https://huggingface.co/qa/ diff --git a/model_cards/ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli/README.md b/model_cards/ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli/README.md deleted file mode 100644 index 15679d0e5b6f..000000000000 --- a/model_cards/ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli/README.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -datasets: -- snli -- anli -- multi_nli -- multi_nli_mismatch -- fever -license: mit ---- -This is a strong pre-trained RoBERTa-Large NLI model. - -The training data is a combination of well-known NLI datasets: [`SNLI`](https://nlp.stanford.edu/projects/snli/), [`MNLI`](https://cims.nyu.edu/~sbowman/multinli/), [`FEVER-NLI`](https://github.com/easonnie/combine-FEVER-NSMN/blob/master/other_resources/nli_fever.md), [`ANLI (R1, R2, R3)`](https://github.com/facebookresearch/anli). -Other pre-trained NLI models including `RoBERTa`, `ALBert`, `BART`, `ELECTRA`, `XLNet` are also available. - -Trained by [Yixin Nie](https://easonnie.github.io), [original source](https://github.com/facebookresearch/anli). - -Try the code snippet below. -``` -from transformers import AutoTokenizer, AutoModelForSequenceClassification -import torch - -if __name__ == '__main__': - max_length = 256 - - premise = "Two women are embracing while holding to go packages." - hypothesis = "The men are fighting outside a deli." - - hg_model_hub_name = "ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli" - # hg_model_hub_name = "ynie/albert-xxlarge-v2-snli_mnli_fever_anli_R1_R2_R3-nli" - # hg_model_hub_name = "ynie/bart-large-snli_mnli_fever_anli_R1_R2_R3-nli" - # hg_model_hub_name = "ynie/electra-large-discriminator-snli_mnli_fever_anli_R1_R2_R3-nli" - # hg_model_hub_name = "ynie/xlnet-large-cased-snli_mnli_fever_anli_R1_R2_R3-nli" - - tokenizer = AutoTokenizer.from_pretrained(hg_model_hub_name) - model = AutoModelForSequenceClassification.from_pretrained(hg_model_hub_name) - - tokenized_input_seq_pair = tokenizer.encode_plus(premise, hypothesis, - max_length=max_length, - return_token_type_ids=True, truncation=True) - - input_ids = torch.Tensor(tokenized_input_seq_pair['input_ids']).long().unsqueeze(0) - # remember bart doesn't have 'token_type_ids', remove the line below if you are using bart. - token_type_ids = torch.Tensor(tokenized_input_seq_pair['token_type_ids']).long().unsqueeze(0) - attention_mask = torch.Tensor(tokenized_input_seq_pair['attention_mask']).long().unsqueeze(0) - - outputs = model(input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - labels=None) - # Note: - # "id2label": { - # "0": "entailment", - # "1": "neutral", - # "2": "contradiction" - # }, - - predicted_probability = torch.softmax(outputs[0], dim=1)[0].tolist() # batch_size only one - - print("Premise:", premise) - print("Hypothesis:", hypothesis) - print("Entailment:", predicted_probability[0]) - print("Neutral:", predicted_probability[1]) - print("Contradiction:", predicted_probability[2]) -``` - -More in [here](https://github.com/facebookresearch/anli/blob/master/src/hg_api/interactive_eval.py). - -Citation: -``` -@inproceedings{nie-etal-2020-adversarial, - title = "Adversarial {NLI}: A New Benchmark for Natural Language Understanding", - author = "Nie, Yixin and - Williams, Adina and - Dinan, Emily and - Bansal, Mohit and - Weston, Jason and - Kiela, Douwe", - booktitle = "Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics", - year = "2020", - publisher = "Association for Computational Linguistics", -} -``` diff --git a/model_cards/youscan/ukr-roberta-base/README.md b/model_cards/youscan/ukr-roberta-base/README.md deleted file mode 100644 index c698a26d989d..000000000000 --- a/model_cards/youscan/ukr-roberta-base/README.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -language: -- uk ---- - -# ukr-roberta-base - -## Pre-training corpora -Below is the list of corpora used along with the output of wc command (counting lines, words and characters). These corpora were concatenated and tokenized with HuggingFace Roberta Tokenizer. - -| Tables | Lines | Words | Characters | -| ------------- |--------------:| -----:| -----:| -| [Ukrainian Wikipedia - May 2020](https://dumps.wikimedia.org/ukwiki/latest/ukwiki-latest-pages-articles.xml.bz2) | 18 001 466| 201 207 739 | 2 647 891 947 | -| [Ukrainian OSCAR deduplicated dataset](https://oscar-public.huma-num.fr/shuffled/uk_dedup.txt.gz) | 56 560 011 | 2 250 210 650 | 29 705 050 592 | -| Sampled mentions from social networks | 11 245 710 | 128 461 796 | 1 632 567 763 | -| Total | 85 807 187 | 2 579 880 185 | 33 985 510 302 | - -## Pre-training details - -* Ukrainian Roberta was trained with code provided in [HuggingFace tutorial](https://huggingface.co/blog/how-to-train) -* Currently released model follows roberta-base-cased model architecture (12-layer, 768-hidden, 12-heads, 125M parameters) -* The model was trained on 4xV100 (85 hours) -* Training configuration you can find in the [original repository](https://github.com/youscan/language-models) - -## Author -Vitalii Radchenko - contact me on Twitter [@vitaliradchenko](https://twitter.com/vitaliradchenko) diff --git a/model_cards/yuvraj/summarizer-cnndm/README.md b/model_cards/yuvraj/summarizer-cnndm/README.md deleted file mode 100644 index f124c031a30e..000000000000 --- a/model_cards/yuvraj/summarizer-cnndm/README.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -language: "en" -tags: -- summarization ---- -​ -# Summarization -​ -## Model description -​ -BartForConditionalGeneration model fine tuned for summarization on 10000 samples from the cnn-dailymail dataset -​ -## How to use -​ -PyTorch model available -​ -```python -from transformers import AutoTokenizer, AutoModelWithLMHead, pipeline -​ -tokenizer = AutoTokenizer.from_pretrained("yuvraj/summarizer-cnndm") -AutoModelWithLMHead.from_pretrained("yuvraj/summarizer-cnndm") -​ -summarizer = pipeline('summarization', model=model, tokenizer=tokenizer) -summarizer("") -​ -## Limitations and bias -Trained on a small dataset diff --git a/model_cards/yuvraj/xSumm/README.md b/model_cards/yuvraj/xSumm/README.md deleted file mode 100644 index 12c1141984d3..000000000000 --- a/model_cards/yuvraj/xSumm/README.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -language: "en" -tags: -- summarization -- extreme summarization ---- -​ -## Model description -​ -BartForConditionalGenerationModel for extreme summarization- creates a one line abstractive summary of a given article -​ -## How to use -​ -PyTorch model available -​ -```python -from transformers import AutoTokenizer, AutoModelWithLMHead, pipeline -​ -tokenizer = AutoTokenizer.from_pretrained("yuvraj/xSumm") -model = AutoModelWithLMHead.from_pretrained("yuvraj/xSumm") -​ -xsumm = pipeline('summarization', model=model, tokenizer=tokenizer) -xsumm("") -​ -## Limitations and bias -Trained on a small fraction of the xsumm training dataset diff --git a/model_cards/zanelim/singbert-large-sg/README.md b/model_cards/zanelim/singbert-large-sg/README.md deleted file mode 100644 index e3be8882d137..000000000000 --- a/model_cards/zanelim/singbert-large-sg/README.md +++ /dev/null @@ -1,215 +0,0 @@ ---- -language: en -tags: -- singapore -- sg -- singlish -- malaysia -- ms -- manglish -- bert-large-uncased -license: mit -datasets: -- reddit singapore, malaysia -- hardwarezone -widget: -- text: "kopi c siew [MASK]" -- text: "die [MASK] must try" ---- - -# Model name - -SingBert Large - Bert for Singlish (SG) and Manglish (MY). - -## Model description - -Similar to [SingBert](https://huggingface.co/zanelim/singbert) but the large version, which was initialized from [BERT large uncased (whole word masking)](https://github.com/google-research/bert#pre-trained-models), with pre-training finetuned on -[singlish](https://en.wikipedia.org/wiki/Singlish) and [manglish](https://en.wikipedia.org/wiki/Manglish) data. - -## Intended uses & limitations - -#### How to use - -```python ->>> from transformers import pipeline ->>> nlp = pipeline('fill-mask', model='zanelim/singbert-large-sg') ->>> nlp("kopi c siew [MASK]") - -[{'sequence': '[CLS] kopi c siew dai [SEP]', - 'score': 0.9003700017929077, - 'token': 18765, - 'token_str': 'dai'}, - {'sequence': '[CLS] kopi c siew mai [SEP]', - 'score': 0.0779474675655365, - 'token': 14736, - 'token_str': 'mai'}, - {'sequence': '[CLS] kopi c siew. [SEP]', - 'score': 0.0032227332703769207, - 'token': 1012, - 'token_str': '.'}, - {'sequence': '[CLS] kopi c siew bao [SEP]', - 'score': 0.0017727474914863706, - 'token': 25945, - 'token_str': 'bao'}, - {'sequence': '[CLS] kopi c siew peng [SEP]', - 'score': 0.0012526646023616195, - 'token': 26473, - 'token_str': 'peng'}] - ->>> nlp("one teh c siew dai, and one kopi [MASK]") - -[{'sequence': '[CLS] one teh c siew dai, and one kopi. [SEP]', - 'score': 0.5249741077423096, - 'token': 1012, - 'token_str': '.'}, - {'sequence': '[CLS] one teh c siew dai, and one kopi o [SEP]', - 'score': 0.27349168062210083, - 'token': 1051, - 'token_str': 'o'}, - {'sequence': '[CLS] one teh c siew dai, and one kopi peng [SEP]', - 'score': 0.057190295308828354, - 'token': 26473, - 'token_str': 'peng'}, - {'sequence': '[CLS] one teh c siew dai, and one kopi c [SEP]', - 'score': 0.04022320732474327, - 'token': 1039, - 'token_str': 'c'}, - {'sequence': '[CLS] one teh c siew dai, and one kopi? [SEP]', - 'score': 0.01191170234233141, - 'token': 1029, - 'token_str': '?'}] - ->>> nlp("die [MASK] must try") - -[{'sequence': '[CLS] die die must try [SEP]', - 'score': 0.9921030402183533, - 'token': 3280, - 'token_str': 'die'}, - {'sequence': '[CLS] die also must try [SEP]', - 'score': 0.004993876442313194, - 'token': 2036, - 'token_str': 'also'}, - {'sequence': '[CLS] die liao must try [SEP]', - 'score': 0.000317625846946612, - 'token': 727, - 'token_str': 'liao'}, - {'sequence': '[CLS] die still must try [SEP]', - 'score': 0.0002260878391098231, - 'token': 2145, - 'token_str': 'still'}, - {'sequence': '[CLS] die i must try [SEP]', - 'score': 0.00016935862367972732, - 'token': 1045, - 'token_str': 'i'}] - ->>> nlp("dont play [MASK] leh") - -[{'sequence': '[CLS] dont play play leh [SEP]', - 'score': 0.9079819321632385, - 'token': 2377, - 'token_str': 'play'}, - {'sequence': '[CLS] dont play punk leh [SEP]', - 'score': 0.006846973206847906, - 'token': 7196, - 'token_str': 'punk'}, - {'sequence': '[CLS] dont play games leh [SEP]', - 'score': 0.004041737411171198, - 'token': 2399, - 'token_str': 'games'}, - {'sequence': '[CLS] dont play politics leh [SEP]', - 'score': 0.003728888463228941, - 'token': 4331, - 'token_str': 'politics'}, - {'sequence': '[CLS] dont play cheat leh [SEP]', - 'score': 0.0032805048394948244, - 'token': 21910, - 'token_str': 'cheat'}] - ->>> nlp("confirm plus [MASK]") - -{'sequence': '[CLS] confirm plus chop [SEP]', - 'score': 0.9749826192855835, - 'token': 24494, - 'token_str': 'chop'}, - {'sequence': '[CLS] confirm plus chopped [SEP]', - 'score': 0.017554156482219696, - 'token': 24881, - 'token_str': 'chopped'}, - {'sequence': '[CLS] confirm plus minus [SEP]', - 'score': 0.002725469646975398, - 'token': 15718, - 'token_str': 'minus'}, - {'sequence': '[CLS] confirm plus guarantee [SEP]', - 'score': 0.000900257145985961, - 'token': 11302, - 'token_str': 'guarantee'}, - {'sequence': '[CLS] confirm plus one [SEP]', - 'score': 0.0004384620988275856, - 'token': 2028, - 'token_str': 'one'}] - ->>> nlp("catch no [MASK]") - -[{'sequence': '[CLS] catch no ball [SEP]', - 'score': 0.9381157159805298, - 'token': 3608, - 'token_str': 'ball'}, - {'sequence': '[CLS] catch no balls [SEP]', - 'score': 0.060842301696538925, - 'token': 7395, - 'token_str': 'balls'}, - {'sequence': '[CLS] catch no fish [SEP]', - 'score': 0.00030917322146706283, - 'token': 3869, - 'token_str': 'fish'}, - {'sequence': '[CLS] catch no breath [SEP]', - 'score': 7.552534952992573e-05, - 'token': 3052, - 'token_str': 'breath'}, - {'sequence': '[CLS] catch no tail [SEP]', - 'score': 4.208395694149658e-05, - 'token': 5725, - 'token_str': 'tail'}] - -``` - -Here is how to use this model to get the features of a given text in PyTorch: -```python -from transformers import BertTokenizer, BertModel -tokenizer = BertTokenizer.from_pretrained('zanelim/singbert-large-sg') -model = BertModel.from_pretrained("zanelim/singbert-large-sg") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained("zanelim/singbert-large-sg") -model = TFBertModel.from_pretrained("zanelim/singbert-large-sg") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -#### Limitations and bias -This model was finetuned on colloquial Singlish and Manglish corpus, hence it is best applied on downstream tasks involving the main -constituent languages- english, mandarin, malay. Also, as the training data is mainly from forums, beware of existing inherent bias. - -## Training data -Colloquial singlish and manglish (both are a mixture of English, Mandarin, Tamil, Malay, and other local dialects like Hokkien, Cantonese or Teochew) -corpus. The corpus is collected from subreddits- `r/singapore` and `r/malaysia`, and forums such as `hardwarezone`. - -## Training procedure - -Initialized with [bert large uncased (whole word masking)](https://github.com/google-research/bert#pre-trained-models) vocab and checkpoints (pre-trained weights). -Top 1000 custom vocab tokens (non-overlapped with original bert vocab) were further extracted from training data and filled into unused tokens in original bert vocab. - -Pre-training was further finetuned on training data with the following hyperparameters -* train_batch_size: 512 -* max_seq_length: 128 -* num_train_steps: 300000 -* num_warmup_steps: 5000 -* learning_rate: 2e-5 -* hardware: TPU v3-8 diff --git a/model_cards/zanelim/singbert-lite-sg/README.md b/model_cards/zanelim/singbert-lite-sg/README.md deleted file mode 100644 index 13819e064b6f..000000000000 --- a/model_cards/zanelim/singbert-lite-sg/README.md +++ /dev/null @@ -1,168 +0,0 @@ ---- -language: en -tags: -- singapore -- sg -- singlish -- malaysia -- ms -- manglish -- albert-base-v2 -license: mit -datasets: -- reddit singapore, malaysia -- hardwarezone -widget: -- text: "dont play [MASK] leh" -- text: "die [MASK] must try" ---- - -# Model name - -SingBert Lite - Bert for Singlish (SG) and Manglish (MY). - -## Model description - -Similar to [SingBert](https://huggingface.co/zanelim/singbert) but the lite-version, which was initialized from [Albert base v2](https://github.com/google-research/albert#albert), with pre-training finetuned on -[singlish](https://en.wikipedia.org/wiki/Singlish) and [manglish](https://en.wikipedia.org/wiki/Manglish) data. - -## Intended uses & limitations - -#### How to use - -```python ->>> from transformers import pipeline ->>> nlp = pipeline('fill-mask', model='zanelim/singbert-lite-sg') ->>> nlp("die [MASK] must try") - -[{'sequence': '[CLS] die die must try[SEP]', - 'score': 0.7731555700302124, - 'token': 1327, - 'token_str': '▁die'}, - {'sequence': '[CLS] die also must try[SEP]', - 'score': 0.04763784259557724, - 'token': 67, - 'token_str': '▁also'}, - {'sequence': '[CLS] die still must try[SEP]', - 'score': 0.01859409362077713, - 'token': 174, - 'token_str': '▁still'}, - {'sequence': '[CLS] die u must try[SEP]', - 'score': 0.015824034810066223, - 'token': 287, - 'token_str': '▁u'}, - {'sequence': '[CLS] die is must try[SEP]', - 'score': 0.011271446943283081, - 'token': 25, - 'token_str': '▁is'}] - ->>> nlp("dont play [MASK] leh") - -[{'sequence': '[CLS] dont play play leh[SEP]', - 'score': 0.4365769624710083, - 'token': 418, - 'token_str': '▁play'}, - {'sequence': '[CLS] dont play punk leh[SEP]', - 'score': 0.06880936771631241, - 'token': 6769, - 'token_str': '▁punk'}, - {'sequence': '[CLS] dont play game leh[SEP]', - 'score': 0.051739856600761414, - 'token': 250, - 'token_str': '▁game'}, - {'sequence': '[CLS] dont play games leh[SEP]', - 'score': 0.045703962445259094, - 'token': 466, - 'token_str': '▁games'}, - {'sequence': '[CLS] dont play around leh[SEP]', - 'score': 0.013458190485835075, - 'token': 140, - 'token_str': '▁around'}] - ->>> nlp("catch no [MASK]") - -[{'sequence': '[CLS] catch no ball[SEP]', - 'score': 0.6197211146354675, - 'token': 1592, - 'token_str': '▁ball'}, - {'sequence': '[CLS] catch no balls[SEP]', - 'score': 0.08441998809576035, - 'token': 7152, - 'token_str': '▁balls'}, - {'sequence': '[CLS] catch no joke[SEP]', - 'score': 0.0676785409450531, - 'token': 8186, - 'token_str': '▁joke'}, - {'sequence': '[CLS] catch no?[SEP]', - 'score': 0.040638409554958344, - 'token': 60, - 'token_str': '?'}, - {'sequence': '[CLS] catch no one[SEP]', - 'score': 0.03546864539384842, - 'token': 53, - 'token_str': '▁one'}] - ->>> nlp("confirm plus [MASK]") - -[{'sequence': '[CLS] confirm plus chop[SEP]', - 'score': 0.9608421921730042, - 'token': 17144, - 'token_str': '▁chop'}, - {'sequence': '[CLS] confirm plus guarantee[SEP]', - 'score': 0.011784233152866364, - 'token': 9120, - 'token_str': '▁guarantee'}, - {'sequence': '[CLS] confirm plus confirm[SEP]', - 'score': 0.010571340098977089, - 'token': 10265, - 'token_str': '▁confirm'}, - {'sequence': '[CLS] confirm plus egg[SEP]', - 'score': 0.0033525123726576567, - 'token': 6387, - 'token_str': '▁egg'}, - {'sequence': '[CLS] confirm plus bet[SEP]', - 'score': 0.0008760977652855217, - 'token': 5676, - 'token_str': '▁bet'}] - -``` - -Here is how to use this model to get the features of a given text in PyTorch: -```python -from transformers import AlbertTokenizer, AlbertModel -tokenizer = AlbertTokenizer.from_pretrained('zanelim/singbert-lite-sg') -model = AlbertModel.from_pretrained("zanelim/singbert-lite-sg") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: -```python -from transformers import AlbertTokenizer, TFAlbertModel -tokenizer = AlbertTokenizer.from_pretrained("zanelim/singbert-lite-sg") -model = TFAlbertModel.from_pretrained("zanelim/singbert-lite-sg") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -#### Limitations and bias -This model was finetuned on colloquial Singlish and Manglish corpus, hence it is best applied on downstream tasks involving the main -constituent languages- english, mandarin, malay. Also, as the training data is mainly from forums, beware of existing inherent bias. - -## Training data -Colloquial singlish and manglish (both are a mixture of English, Mandarin, Tamil, Malay, and other local dialects like Hokkien, Cantonese or Teochew) -corpus. The corpus is collected from subreddits- `r/singapore` and `r/malaysia`, and forums such as `hardwarezone`. - -## Training procedure - -Initialized with [albert base v2](https://github.com/google-research/albert#albert) vocab and checkpoints (pre-trained weights). - -Pre-training was further finetuned on training data with the following hyperparameters -* train_batch_size: 4096 -* max_seq_length: 128 -* num_train_steps: 125000 -* num_warmup_steps: 5000 -* learning_rate: 0.00176 -* hardware: TPU v3-8 diff --git a/model_cards/zanelim/singbert/README.md b/model_cards/zanelim/singbert/README.md deleted file mode 100644 index bd5a0f96f20e..000000000000 --- a/model_cards/zanelim/singbert/README.md +++ /dev/null @@ -1,215 +0,0 @@ ---- -language: en -tags: -- singapore -- sg -- singlish -- malaysia -- ms -- manglish -- bert-base-uncased -license: mit -datasets: -- reddit singapore, malaysia -- hardwarezone -widget: -- text: "kopi c siew [MASK]" -- text: "die [MASK] must try" ---- - -# Model name - -SingBert - Bert for Singlish (SG) and Manglish (MY). - -## Model description - -[BERT base uncased](https://github.com/google-research/bert#pre-trained-models), with pre-training finetuned on -[singlish](https://en.wikipedia.org/wiki/Singlish) and [manglish](https://en.wikipedia.org/wiki/Manglish) data. - -## Intended uses & limitations - -#### How to use - -```python ->>> from transformers import pipeline ->>> nlp = pipeline('fill-mask', model='zanelim/singbert') ->>> nlp("kopi c siew [MASK]") - -[{'sequence': '[CLS] kopi c siew dai [SEP]', - 'score': 0.5092713236808777, - 'token': 18765, - 'token_str': 'dai'}, - {'sequence': '[CLS] kopi c siew mai [SEP]', - 'score': 0.3515934646129608, - 'token': 14736, - 'token_str': 'mai'}, - {'sequence': '[CLS] kopi c siew bao [SEP]', - 'score': 0.05576375499367714, - 'token': 25945, - 'token_str': 'bao'}, - {'sequence': '[CLS] kopi c siew. [SEP]', - 'score': 0.006019321270287037, - 'token': 1012, - 'token_str': '.'}, - {'sequence': '[CLS] kopi c siew sai [SEP]', - 'score': 0.0038361591286957264, - 'token': 18952, - 'token_str': 'sai'}] - ->>> nlp("one teh c siew dai, and one kopi [MASK].") - -[{'sequence': '[CLS] one teh c siew dai, and one kopi c [SEP]', - 'score': 0.6176503300666809, - 'token': 1039, - 'token_str': 'c'}, - {'sequence': '[CLS] one teh c siew dai, and one kopi o [SEP]', - 'score': 0.21094971895217896, - 'token': 1051, - 'token_str': 'o'}, - {'sequence': '[CLS] one teh c siew dai, and one kopi. [SEP]', - 'score': 0.13027705252170563, - 'token': 1012, - 'token_str': '.'}, - {'sequence': '[CLS] one teh c siew dai, and one kopi! [SEP]', - 'score': 0.004680239595472813, - 'token': 999, - 'token_str': '!'}, - {'sequence': '[CLS] one teh c siew dai, and one kopi w [SEP]', - 'score': 0.002034128177911043, - 'token': 1059, - 'token_str': 'w'}] - ->>> nlp("dont play [MASK] leh") - -[{'sequence': '[CLS] dont play play leh [SEP]', - 'score': 0.9281464219093323, - 'token': 2377, - 'token_str': 'play'}, - {'sequence': '[CLS] dont play politics leh [SEP]', - 'score': 0.010990909300744534, - 'token': 4331, - 'token_str': 'politics'}, - {'sequence': '[CLS] dont play punk leh [SEP]', - 'score': 0.005583590362221003, - 'token': 7196, - 'token_str': 'punk'}, - {'sequence': '[CLS] dont play dirty leh [SEP]', - 'score': 0.0025784350000321865, - 'token': 6530, - 'token_str': 'dirty'}, - {'sequence': '[CLS] dont play cheat leh [SEP]', - 'score': 0.0025066907983273268, - 'token': 21910, - 'token_str': 'cheat'}] - ->>> nlp("catch no [MASK]") - -[{'sequence': '[CLS] catch no ball [SEP]', - 'score': 0.7922210693359375, - 'token': 3608, - 'token_str': 'ball'}, - {'sequence': '[CLS] catch no balls [SEP]', - 'score': 0.20503675937652588, - 'token': 7395, - 'token_str': 'balls'}, - {'sequence': '[CLS] catch no tail [SEP]', - 'score': 0.0006608376861549914, - 'token': 5725, - 'token_str': 'tail'}, - {'sequence': '[CLS] catch no talent [SEP]', - 'score': 0.0002158183924620971, - 'token': 5848, - 'token_str': 'talent'}, - {'sequence': '[CLS] catch no prisoners [SEP]', - 'score': 5.3481446229852736e-05, - 'token': 5895, - 'token_str': 'prisoners'}] - ->>> nlp("confirm plus [MASK]") - -[{'sequence': '[CLS] confirm plus chop [SEP]', - 'score': 0.992355227470398, - 'token': 24494, - 'token_str': 'chop'}, - {'sequence': '[CLS] confirm plus one [SEP]', - 'score': 0.0037301010452210903, - 'token': 2028, - 'token_str': 'one'}, - {'sequence': '[CLS] confirm plus minus [SEP]', - 'score': 0.0014284878270700574, - 'token': 15718, - 'token_str': 'minus'}, - {'sequence': '[CLS] confirm plus 1 [SEP]', - 'score': 0.0011354683665558696, - 'token': 1015, - 'token_str': '1'}, - {'sequence': '[CLS] confirm plus chopped [SEP]', - 'score': 0.0003804611915256828, - 'token': 24881, - 'token_str': 'chopped'}] - ->>> nlp("die [MASK] must try") - -[{'sequence': '[CLS] die die must try [SEP]', - 'score': 0.9552758932113647, - 'token': 3280, - 'token_str': 'die'}, - {'sequence': '[CLS] die also must try [SEP]', - 'score': 0.03644804656505585, - 'token': 2036, - 'token_str': 'also'}, - {'sequence': '[CLS] die liao must try [SEP]', - 'score': 0.003282855963334441, - 'token': 727, - 'token_str': 'liao'}, - {'sequence': '[CLS] die already must try [SEP]', - 'score': 0.0004937972989864647, - 'token': 2525, - 'token_str': 'already'}, - {'sequence': '[CLS] die hard must try [SEP]', - 'score': 0.0003659659414552152, - 'token': 2524, - 'token_str': 'hard'}] - -``` - -Here is how to use this model to get the features of a given text in PyTorch: -```python -from transformers import BertTokenizer, BertModel -tokenizer = BertTokenizer.from_pretrained('zanelim/singbert') -model = BertModel.from_pretrained("zanelim/singbert") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='pt') -output = model(**encoded_input) -``` - -and in TensorFlow: -```python -from transformers import BertTokenizer, TFBertModel -tokenizer = BertTokenizer.from_pretrained("zanelim/singbert") -model = TFBertModel.from_pretrained("zanelim/singbert") -text = "Replace me by any text you'd like." -encoded_input = tokenizer(text, return_tensors='tf') -output = model(encoded_input) -``` - -#### Limitations and bias -This model was finetuned on colloquial Singlish and Manglish corpus, hence it is best applied on downstream tasks involving the main -constituent languages- english, mandarin, malay. Also, as the training data is mainly from forums, beware of existing inherent bias. - -## Training data -Colloquial singlish and manglish (both are a mixture of English, Mandarin, Tamil, Malay, and other local dialects like Hokkien, Cantonese or Teochew) -corpus. The corpus is collected from subreddits- `r/singapore` and `r/malaysia`, and forums such as `hardwarezone`. - -## Training procedure - -Initialized with [bert base uncased](https://github.com/google-research/bert#pre-trained-models) vocab and checkpoints (pre-trained weights). -Top 1000 custom vocab tokens (non-overlapped with original bert vocab) were further extracted from training data and filled into unused tokens in original bert vocab. - -Pre-training was further finetuned on training data with the following hyperparameters -* train_batch_size: 512 -* max_seq_length: 128 -* num_train_steps: 300000 -* num_warmup_steps: 5000 -* learning_rate: 2e-5 -* hardware: TPU v3-8 diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index f5ec2bae08cb..704af85ed999 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -164,6 +164,7 @@ from .models.roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, RobertaTokenizer from .models.squeezebert import SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, SqueezeBertConfig, SqueezeBertTokenizer from .models.t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config +from .models.tapas import TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP, TapasConfig, TapasTokenizer from .models.transfo_xl import ( TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig, @@ -605,6 +606,13 @@ T5PreTrainedModel, load_tf_weights_in_t5, ) + from .models.tapas import ( + TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST, + TapasForMaskedLM, + TapasForQuestionAnswering, + TapasForSequenceClassification, + TapasModel, + ) from .models.transfo_xl import ( TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST, AdaptiveEmbedding, diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index 02b7b4ec3a17..0e345bb28ab6 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -216,6 +216,29 @@ _tokenizers_available = False +try: + import pandas # noqa: F401 + + _pandas_available = True + +except ImportError: + _pandas_available = False + + +try: + import torch_scatter + + # Check we're not importing a "torch_scatter" directory somewhere + _scatter_available = hasattr(torch_scatter, "__version__") and hasattr(torch_scatter, "scatter") + if _scatter_available: + logger.debug(f"Succesfully imported torch-scatter version {torch_scatter.__version__}") + else: + logger.debug("Imported a torch_scatter object but this doesn't seem to be the torch-scatter library.") + +except ImportError: + _scatter_available = False + + old_default_cache_path = os.path.join(torch_cache_home, "transformers") # New default cache, shared with the Datasets library hf_cache_home = os.path.expanduser( @@ -325,6 +348,14 @@ def is_in_notebook(): return _in_notebook +def is_scatter_available(): + return _scatter_available + + +def is_pandas_available(): + return _pandas_available + + def torch_only_method(fn): def wrapper(*args, **kwargs): if not _torch_available: @@ -427,6 +458,13 @@ def wrapper(*args, **kwargs): """ +# docstyle-ignore +SCATTER_IMPORT_ERROR = """ +{0} requires the torch-scatter library but it was not found in your environment. You can install it with pip as +explained here: https://github.com/rusty1s/pytorch_scatter. +""" + + def requires_datasets(obj): name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__ if not is_datasets_available(): @@ -481,6 +519,12 @@ def requires_protobuf(obj): raise ImportError(PROTOBUF_IMPORT_ERROR.format(name)) +def requires_scatter(obj): + name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__ + if not is_scatter_available(): + raise ImportError(SCATTER_IMPORT_ERROR.format(name)) + + def add_start_docstrings(*docstr): def docstring_decorator(fn): fn.__doc__ = "".join(docstr) + (fn.__doc__ if fn.__doc__ is not None else "") diff --git a/src/transformers/modeling_tf_utils.py b/src/transformers/modeling_tf_utils.py index 850333ca8df8..f7e982c1caf5 100644 --- a/src/transformers/modeling_tf_utils.py +++ b/src/transformers/modeling_tf_utils.py @@ -614,13 +614,32 @@ def set_input_embeddings(self, value): def get_output_embeddings(self) -> tf.keras.layers.Layer: """ - Returns the model's output embeddings. + Returns the model's output embeddings Returns: :obj:`tf.keras.layers.Layer`: A torch module mapping hidden states to vocabulary. """ return None # Overwrite for models with output embeddings + def get_output_layer_with_bias(self) -> Union[None, tf.keras.layers.Layer]: + """ + Get the layer that handles a bias attribute in case the model has an LM head with weights tied to the + embeddings. + + Return: + :obj:`tf.keras.layers.Layer`: The layer that handles the bias, None if not an LM model. + """ + return None + + def get_prefix_bias_name(self) -> Union[None, str]: + """ + Get the concatenated prefix name of the bias from the model name to the parent layer. + + Return: + :obj:`str`: The prefix name of the bias. + """ + return None + def resize_token_embeddings(self, new_num_tokens=None) -> tf.Variable: """ Resizes input token embeddings matrix of the model if :obj:`new_num_tokens != config.vocab_size`. @@ -662,7 +681,17 @@ def _get_word_embeddings(self, embeddings): # TFSharedEmbeddings return embeddings.weight else: - raise ValueError("word embedding is not defined.") + # Here we build the word embeddings weights if not exists. + # And then we retry to get the attribute once built. + embeddings.build([]) + if hasattr(embeddings, "word_embeddings"): + # TFBertEmbeddings, TFAlbertEmbeddings, TFElectraEmbeddings + return embeddings.word_embeddings + elif hasattr(embeddings, "weight"): + # TFSharedEmbeddings + return embeddings.weight + else: + raise ValueError("word embedding is not defined.") def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None) -> tf.Variable: """ @@ -684,28 +713,87 @@ def _get_resized_embeddings(self, old_embeddings, new_num_tokens=None) -> tf.Var :obj:`new_num_tokens` is :obj:`None` """ word_embeddings = self._get_word_embeddings(old_embeddings) + bias_layer = self.get_output_layer_with_bias() + if new_num_tokens is None: return word_embeddings + old_num_tokens, old_embedding_dim = word_embeddings.shape + if old_num_tokens == new_num_tokens: return word_embeddings # initialize new embeddings # todo: initializer range is not always passed in config. init_range = getattr(self.config, "initializer_range", 0.02) + name = ( + self.name + + "/" + + self.base_model_prefix + + "/" + + old_embeddings.name + + "/" + + word_embeddings.name.split(":")[0] + ) new_embeddings = self.add_weight( - "weight", + name=name, shape=[new_num_tokens, old_embedding_dim], initializer=get_initializer(init_range), dtype=tf.float32, ) - init_weights = new_embeddings.numpy() + init_weights = tf.make_ndarray(tf.make_tensor_proto(new_embeddings.value())) # Copy token embeddings from the previous weights num_tokens_to_copy = min(old_num_tokens, new_num_tokens) - init_weights[:num_tokens_to_copy] = word_embeddings[:num_tokens_to_copy, :] + init_weights[:num_tokens_to_copy] = word_embeddings.value()[:num_tokens_to_copy, :] new_embeddings.assign(init_weights) + if bias_layer is not None: + if not hasattr(bias_layer, "bias"): + bias_layer.build([]) + + # Second check in order to be sure the attribute has been properly created + if not hasattr(bias_layer, "bias"): + raise ValueError("bias is not defined.") + + # initialize bias + init_bias = np.zeros((new_num_tokens,)) + init_bias[:num_tokens_to_copy] = bias_layer.bias.value()[ + :num_tokens_to_copy + ] # tf.make_ndarray(tf.make_tensor_proto(bias_layer.bias.value()))[:num_tokens_to_copy] + + bias_layer.bias = self.add_weight( + shape=(new_num_tokens,), + initializer="zeros", + trainable=True, + name=self.get_prefix_bias_name() + "/bias", + ) + + bias_layer.bias.assign(init_bias) + + output_embeddings = self.get_output_embeddings() + + if output_embeddings is not None: + if self.get_input_embeddings() != output_embeddings: + if not hasattr(output_embeddings, "decoder"): + output_embeddings.build([]) + + # Second check in order to be sure the attribute has been properly created + if not hasattr(output_embeddings, "decoder"): + raise ValueError("decoder is not defined.") + + # initialize decoder + init_weights = np.zeros((new_num_tokens, old_embedding_dim)) + init_weights[:num_tokens_to_copy] = output_embeddings.decoder.value()[:num_tokens_to_copy, :] + + output_embeddings.decoder = self.add_weight( + shape=(new_num_tokens, old_embedding_dim), + initializer="zeros", + trainable=True, + name=self.get_prefix_bias_name() + "/decoder/weight", + ) + output_embeddings.decoder.assign(init_weights) + return new_embeddings def prune_heads(self, heads_to_prune): diff --git a/src/transformers/models/albert/modeling_tf_albert.py b/src/transformers/models/albert/modeling_tf_albert.py index 671f19629681..222e815b80c9 100644 --- a/src/transformers/models/albert/modeling_tf_albert.py +++ b/src/transformers/models/albert/modeling_tf_albert.py @@ -467,6 +467,7 @@ def build(self, input_shape): self.decoder_bias = self.add_weight( shape=(self.vocab_size,), initializer="zeros", trainable=True, name="decoder/bias" ) + super().build(input_shape) def call(self, hidden_states): @@ -825,6 +826,32 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.albert.embeddings + def resize_token_embeddings(self, new_num_tokens): + super().resize_token_embeddings(new_num_tokens=new_num_tokens) + + # ALBERT is a special case where there are two bias to update + # even though self.bias is not used anywhere and is here + # just to make the loading weights from a PT model happy + if new_num_tokens is not None: + num_tokens_to_copy = min(self.predictions.bias.shape[0], new_num_tokens) + self.predictions.vocab_size = num_tokens_to_copy + init_bias = tf.zeros((new_num_tokens,)) + init_bias[:num_tokens_to_copy] = self.predictions.bias.value()[:num_tokens_to_copy] + name = self.name + "/" + self.predictions.name + "/bias" + self.predictions.bias = self.add_weight( + shape=(new_num_tokens,), initializer="zeros", trainable=True, name=name + ) + self.predictions.bias.assign(init_bias) + + init_decoder_bias = tf.zeros((new_num_tokens,)) + init_decoder_bias[:num_tokens_to_copy] = self.predictions.decoder_bias.value()[:num_tokens_to_copy] + name = self.name + "/" + self.predictions.name + "/decoder_bias" + self.predictions.decoder_bias = self.add_weight( + shape=(new_num_tokens,), initializer="zeros", trainable=True, name=name + ) + + self.predictions.decoder_bias.assign(init_decoder_bias) + @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @replace_return_docstrings(output_type=TFAlbertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) def call( @@ -933,6 +960,32 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.albert.embeddings + def resize_token_embeddings(self, new_num_tokens): + super().resize_token_embeddings(new_num_tokens=new_num_tokens) + + # ALBERT is a special case where there are two bias to update + # even though self.bias is not used anywhere and is here + # just to make the loading weights from a PT model happy + if new_num_tokens is not None: + num_tokens_to_copy = min(self.predictions.bias.shape[0], new_num_tokens) + self.predictions.vocab_size = num_tokens_to_copy + init_bias = tf.zeros((new_num_tokens,)) + init_bias[:num_tokens_to_copy] = self.predictions.bias.value()[:num_tokens_to_copy] + name = self.name + "/" + self.predictions.name + "/bias" + self.predictions.bias = self.add_weight( + shape=(new_num_tokens,), initializer="zeros", trainable=True, name=name + ) + self.predictions.bias.assign(init_bias) + + init_decoder_bias = tf.zeros((new_num_tokens,)) + init_decoder_bias[:num_tokens_to_copy] = self.predictions.decoder_bias.value()[:num_tokens_to_copy] + name = self.name + "/" + self.predictions.name + "/decoder_bias" + self.predictions.decoder_bias = self.add_weight( + shape=(new_num_tokens,), initializer="zeros", trainable=True, name=name + ) + + self.predictions.decoder_bias.assign(init_decoder_bias) + @add_start_docstrings_to_model_forward(ALBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index a5f3c93ce1e1..5a83846634d0 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -51,6 +51,7 @@ from ..roberta.configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig from ..squeezebert.configuration_squeezebert import SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, SqueezeBertConfig from ..t5.configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config +from ..tapas.configuration_tapas import TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP, TapasConfig from ..transfo_xl.configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig from ..xlm.configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig from ..xlm_prophetnet.configuration_xlm_prophetnet import ( @@ -95,6 +96,7 @@ XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP, + TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP, ] for key, value, in pretrained_map.items() ) @@ -141,6 +143,7 @@ ("dpr", DPRConfig), ("layoutlm", LayoutLMConfig), ("rag", RagConfig), + ("tapas", TapasConfig), ] ) @@ -185,6 +188,7 @@ ("prophetnet", "ProphetNet"), ("mt5", "mT5"), ("mpnet", "MPNet"), + ("tapas", "TAPAS"), ] ) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 3e649bcf9828..72b691a58914 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -165,6 +165,12 @@ SqueezeBertModel, ) from ..t5.modeling_t5 import T5ForConditionalGeneration, T5Model +from ..tapas.modeling_tapas import ( + TapasForMaskedLM, + TapasForQuestionAnswering, + TapasForSequenceClassification, + TapasModel, +) from ..transfo_xl.modeling_transfo_xl import TransfoXLForSequenceClassification, TransfoXLLMHeadModel, TransfoXLModel from ..xlm.modeling_xlm import ( XLMForMultipleChoice, @@ -230,6 +236,7 @@ RobertaConfig, SqueezeBertConfig, T5Config, + TapasConfig, TransfoXLConfig, XLMConfig, XLMProphetNetConfig, @@ -277,6 +284,7 @@ (XLMProphetNetConfig, XLMProphetNetModel), (ProphetNetConfig, ProphetNetModel), (MPNetConfig, MPNetModel), + (TapasConfig, TapasModel), ] ) @@ -308,6 +316,7 @@ (LxmertConfig, LxmertForPreTraining), (FunnelConfig, FunnelForPreTraining), (MPNetConfig, MPNetForMaskedLM), + (TapasConfig, TapasForMaskedLM), ] ) @@ -340,6 +349,7 @@ (ReformerConfig, ReformerModelWithLMHead), (FunnelConfig, FunnelForMaskedLM), (MPNetConfig, MPNetForMaskedLM), + (TapasConfig, TapasForMaskedLM), ] ) @@ -386,6 +396,7 @@ (ReformerConfig, ReformerForMaskedLM), (FunnelConfig, FunnelForMaskedLM), (MPNetConfig, MPNetForMaskedLM), + (TapasConfig, TapasForMaskedLM), ] ) @@ -431,6 +442,7 @@ (CTRLConfig, CTRLForSequenceClassification), (TransfoXLConfig, TransfoXLForSequenceClassification), (MPNetConfig, MPNetForSequenceClassification), + (TapasConfig, TapasForSequenceClassification), ] ) @@ -455,6 +467,7 @@ (FunnelConfig, FunnelForQuestionAnswering), (LxmertConfig, LxmertForQuestionAnswering), (MPNetConfig, MPNetForQuestionAnswering), + (TapasConfig, TapasForQuestionAnswering), ] ) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 92160fc583ca..ecbc2d9ee784 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -47,6 +47,7 @@ from ..retribert.tokenization_retribert import RetriBertTokenizer from ..roberta.tokenization_roberta import RobertaTokenizer from ..squeezebert.tokenization_squeezebert import SqueezeBertTokenizer +from ..tapas.tokenization_tapas import TapasTokenizer from ..transfo_xl.tokenization_transfo_xl import TransfoXLTokenizer from ..xlm.tokenization_xlm import XLMTokenizer from .configuration_auto import ( @@ -84,6 +85,7 @@ RobertaConfig, SqueezeBertConfig, T5Config, + TapasConfig, TransfoXLConfig, XLMConfig, XLMProphetNetConfig, @@ -223,6 +225,7 @@ (XLMProphetNetConfig, (XLMProphetNetTokenizer, None)), (ProphetNetConfig, (ProphetNetTokenizer, None)), (MPNetConfig, (MPNetTokenizer, MPNetTokenizerFast)), + (TapasConfig, (TapasTokenizer, None)), ] ) diff --git a/src/transformers/models/bart/modeling_tf_bart.py b/src/transformers/models/bart/modeling_tf_bart.py index 17d2308df926..88af7f8336ea 100644 --- a/src/transformers/models/bart/modeling_tf_bart.py +++ b/src/transformers/models/bart/modeling_tf_bart.py @@ -1049,6 +1049,24 @@ def __init__(self, config, *inputs, **kwargs): name="/final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=False ) + def resize_token_embeddings(self, new_num_tokens): + super().resize_token_embeddings(new_num_tokens=new_num_tokens) + + # BART is a special case where the bias has two dimensions + # and not named just `bias` + if new_num_tokens is not None: + num_tokens_to_copy = min(self.final_logits_bias.shape[0], new_num_tokens) + init_bias = tf.zeros((new_num_tokens,)) + init_bias[:num_tokens_to_copy] = self.final_logits_bias.value()[:num_tokens_to_copy] + name = self.name + "/final_logits_bias" + self.final_logits_bias = self.add_weight( + shape=(1, new_num_tokens), + initializer="zeros", + trainable=False, + name=name, + ) + self.final_logits_bias.assign(init_bias) + @add_start_docstrings_to_model_forward(BART_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) def call( diff --git a/src/transformers/models/bert/modeling_tf_bert.py b/src/transformers/models/bert/modeling_tf_bert.py index 61d9dad62a32..987b1d9dc065 100644 --- a/src/transformers/models/bert/modeling_tf_bert.py +++ b/src/transformers/models/bert/modeling_tf_bert.py @@ -893,6 +893,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.bert.embeddings + def get_output_layer_with_bias(self): + return self.mlm.predictions + + def get_prefix_bias_name(self): + return self.name + "/" + self.mlm.name + "/" + self.mlm.predictions.name + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @replace_return_docstrings(output_type=TFBertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) def call( @@ -1002,6 +1008,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.bert.embeddings + def get_output_layer_with_bias(self): + return self.mlm.predictions + + def get_prefix_bias_name(self): + return self.name + "/" + self.mlm.name + "/" + self.mlm.predictions.name + @add_start_docstrings_to_model_forward(BERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, @@ -1095,6 +1107,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.bert.embeddings + def get_output_layer_with_bias(self): + return self.mlm.predictions + + def get_prefix_bias_name(self): + return self.name + "/" + self.mlm.name + "/" + self.mlm.predictions.name + @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="bert-base-cased", diff --git a/src/transformers/models/ctrl/modeling_tf_ctrl.py b/src/transformers/models/ctrl/modeling_tf_ctrl.py index 6d1680bda2cc..abbb5d0a5776 100644 --- a/src/transformers/models/ctrl/modeling_tf_ctrl.py +++ b/src/transformers/models/ctrl/modeling_tf_ctrl.py @@ -629,6 +629,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.lm_head.input_embeddings + def get_output_layer_with_bias(self): + return self.lm_head + + def get_prefix_bias_name(self): + return self.name + "/" + self.lm_head.name + def prepare_inputs_for_generation(self, inputs, past, **kwargs): # only last token for inputs_ids if past is defined in kwargs if past: diff --git a/src/transformers/models/distilbert/modeling_tf_distilbert.py b/src/transformers/models/distilbert/modeling_tf_distilbert.py index 9e887b4a1cd9..1f5a1c7d8ca0 100644 --- a/src/transformers/models/distilbert/modeling_tf_distilbert.py +++ b/src/transformers/models/distilbert/modeling_tf_distilbert.py @@ -655,6 +655,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.vocab_projector.input_embeddings + def get_output_layer_with_bias(self): + return self.vocab_projector + + def get_prefix_bias_name(self): + return self.name + "/" + self.vocab_projector.name + @add_start_docstrings_to_model_forward(DISTILBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, diff --git a/src/transformers/models/electra/modeling_tf_electra.py b/src/transformers/models/electra/modeling_tf_electra.py index c97b4ed37391..3a39b0376207 100644 --- a/src/transformers/models/electra/modeling_tf_electra.py +++ b/src/transformers/models/electra/modeling_tf_electra.py @@ -882,7 +882,7 @@ def build(self, input_shape): super().build(input_shape) - def call(self, hidden_states): + def call(self, hidden_states, training=False): hidden_states = self.input_embeddings(hidden_states, mode="linear") hidden_states = hidden_states + self.bias @@ -914,8 +914,14 @@ def __init__(self, config, **kwargs): self.generator_lm_head = TFElectraMaskedLMHead(config, self.electra.embeddings, name="generator_lm_head") def get_output_embeddings(self): + return self.electra.embeddings + + def get_output_layer_with_bias(self): return self.generator_lm_head + def get_prefix_bias_name(self): + return self.name + "/" + self.generator_lm_head.name + @add_start_docstrings_to_model_forward(ELECTRA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, diff --git a/src/transformers/models/flaubert/modeling_tf_flaubert.py b/src/transformers/models/flaubert/modeling_tf_flaubert.py index 71b21780d771..c1711b7f7399 100644 --- a/src/transformers/models/flaubert/modeling_tf_flaubert.py +++ b/src/transformers/models/flaubert/modeling_tf_flaubert.py @@ -766,6 +766,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.pred_layer.input_embeddings + def get_output_layer_with_bias(self): + return self.pred_layer + + def get_prefix_bias_name(self): + return self.name + "/" + self.pred_layer.name + def prepare_inputs_for_generation(self, inputs, **kwargs): mask_token_id = self.config.mask_token_id lang_id = self.config.lang_id diff --git a/src/transformers/models/funnel/modeling_tf_funnel.py b/src/transformers/models/funnel/modeling_tf_funnel.py index 57368134ea7d..38208112bfff 100644 --- a/src/transformers/models/funnel/modeling_tf_funnel.py +++ b/src/transformers/models/funnel/modeling_tf_funnel.py @@ -1320,6 +1320,15 @@ def __init__(self, config, *inputs, **kwargs): self.funnel = TFFunnelMainLayer(config, name="funnel") self.lm_head = TFFunnelMaskedLMHead(config, self.funnel.embeddings, name="lm_head") + def get_output_embeddings(self): + return self.funnel.embeddings + + def get_output_layer_with_bias(self): + return self.lm_head + + def get_prefix_bias_name(self): + return self.name + "/" + self.lm_head.name + @add_start_docstrings_to_model_forward(FUNNEL_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, diff --git a/src/transformers/models/longformer/modeling_tf_longformer.py b/src/transformers/models/longformer/modeling_tf_longformer.py index 71595b0564ee..db30435be567 100644 --- a/src/transformers/models/longformer/modeling_tf_longformer.py +++ b/src/transformers/models/longformer/modeling_tf_longformer.py @@ -2009,6 +2009,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.lm_head.decoder + def get_output_layer_with_bias(self): + return self.lm_head + + def get_prefix_bias_name(self): + return self.name + "/" + self.lm_head.name + @add_start_docstrings_to_model_forward(LONGFORMER_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, diff --git a/src/transformers/models/lxmert/modeling_tf_lxmert.py b/src/transformers/models/lxmert/modeling_tf_lxmert.py index bfe159c1541d..43cd3b5fc40d 100644 --- a/src/transformers/models/lxmert/modeling_tf_lxmert.py +++ b/src/transformers/models/lxmert/modeling_tf_lxmert.py @@ -1257,6 +1257,15 @@ def dummy_inputs(self): **({"obj_labels": obj_labels} if self.config.task_obj_predict else {}), } + def get_output_embeddings(self): + return self.lxmert.embeddings + + def get_output_layer_with_bias(self): + return self.cls.predictions + + def get_prefix_bias_name(self): + return self.name + "/" + self.cls.name + "/" + self.cls.predictions.name + @add_start_docstrings_to_model_forward(LXMERT_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=TFLxmertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) def call( diff --git a/src/transformers/models/mobilebert/modeling_tf_mobilebert.py b/src/transformers/models/mobilebert/modeling_tf_mobilebert.py index 3b637cc4e1b1..2891223ad353 100644 --- a/src/transformers/models/mobilebert/modeling_tf_mobilebert.py +++ b/src/transformers/models/mobilebert/modeling_tf_mobilebert.py @@ -702,6 +702,10 @@ def __init__(self, config, add_pooling_layer=True, **kwargs): def get_input_embeddings(self): return self.embeddings + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + self.embeddings.vocab_size = value.shape[0] + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -1024,7 +1028,13 @@ def __init__(self, config, *inputs, **kwargs): self.seq_relationship = TFMobileBertOnlyNSPHead(2, name="seq_relationship___cls") def get_output_embeddings(self): - return self.mobilebert.embeddings + return self.predictions.predictions + + def get_output_layer_with_bias(self): + return self.predictions.predictions + + def get_prefix_bias_name(self): + return self.name + "/" + self.predictions.name + "/" + self.predictions.predictions.name @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @replace_return_docstrings(output_type=TFMobileBertForPreTrainingOutput, config_class=_CONFIG_FOR_DOC) @@ -1117,7 +1127,13 @@ def __init__(self, config, *inputs, **kwargs): self.mlm = TFMobileBertMLMHead(config, name="mlm___cls") def get_output_embeddings(self): - return self.mobilebert.embeddings + return self.mlm.predictions + + def get_output_layer_with_bias(self): + return self.mlm.predictions + + def get_prefix_bias_name(self): + return self.name + "/" + self.mlm.name + "/" + self.mlm.predictions.name @add_start_docstrings_to_model_forward(MOBILEBERT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( diff --git a/src/transformers/models/mpnet/modeling_tf_mpnet.py b/src/transformers/models/mpnet/modeling_tf_mpnet.py index b65c133f0949..02f462572d41 100644 --- a/src/transformers/models/mpnet/modeling_tf_mpnet.py +++ b/src/transformers/models/mpnet/modeling_tf_mpnet.py @@ -830,6 +830,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.mpnet.embeddings + def get_output_layer_with_bias(self): + return self.lm_head + + def get_prefix_bias_name(self): + return self.name + "/" + self.lm_head.name + @add_start_docstrings_to_model_forward(MPNET_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, diff --git a/src/transformers/models/roberta/modeling_tf_roberta.py b/src/transformers/models/roberta/modeling_tf_roberta.py index 3764efac8384..ae5f3dd22393 100644 --- a/src/transformers/models/roberta/modeling_tf_roberta.py +++ b/src/transformers/models/roberta/modeling_tf_roberta.py @@ -810,6 +810,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.lm_head.decoder + def get_output_layer_with_bias(self): + return self.lm_head + + def get_prefix_bias_name(self): + return self.name + "/" + self.lm_head.name + @add_start_docstrings_to_model_forward(ROBERTA_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, diff --git a/src/transformers/models/tapas/__init__.py b/src/transformers/models/tapas/__init__.py new file mode 100644 index 000000000000..a6dfb3145388 --- /dev/null +++ b/src/transformers/models/tapas/__init__.py @@ -0,0 +1,31 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...file_utils import is_torch_available +from .configuration_tapas import TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP, TapasConfig +from .tokenization_tapas import TapasTokenizer + + +if is_torch_available(): + from .modeling_tapas import ( + TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST, + TapasForMaskedLM, + TapasForQuestionAnswering, + TapasForSequenceClassification, + TapasModel, + ) diff --git a/src/transformers/models/tapas/configuration_tapas.py b/src/transformers/models/tapas/configuration_tapas.py new file mode 100644 index 000000000000..e818c34591b7 --- /dev/null +++ b/src/transformers/models/tapas/configuration_tapas.py @@ -0,0 +1,219 @@ +# coding=utf-8 +# Copyright 2020 Google Research and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +TAPAS configuration. Adds additional hyperparameters to the configuration of BERT. + +Hyperparameters are taken from run_task_main.py and hparam_utils.py of the original implementation. URLS: + +- https://github.com/google-research/tapas/blob/master/tapas/run_task_main.py +- https://github.com/google-research/tapas/blob/master/tapas/utils/hparam_utils.py + +""" + + +from ...configuration_utils import PretrainedConfig + + +TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "nielsr/tapas-base-finetuned-sqa": "https://huggingface.co/nielsr/tapas-base-finetuned-sqa/resolve/main/config.json", + "nielsr/tapas-base-finetuned-wtq": "https://huggingface.co/nielsr/tapas-base-finetuned-wtq/resolve/main/config.json", + "nielsr/tapas-base-finetuned-wikisql-supervised": "https://huggingface.co/nielsr/tapas-base-finetuned-wikisql-supervised/resolve/main/config.json", + "nielsr/tapas-base-finetuned-tabfact": "https://huggingface.co/nielsr/tapas-base-finetuned-tabfact/resolve/main/config.json", +} + + +class TapasConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a :class:`~transformers.TapasModel`. It is used to + instantiate a TAPAS model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the TAPAS `tapas-base-finetuned-sqa` + architecture. Configuration objects inherit from :class:`~transformers.PreTrainedConfig` and can be used to control + the model outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + Hyperparameters additional to BERT are taken from run_task_main.py and hparam_utils.py of the original + implementation. Original implementation available at https://github.com/google-research/tapas/tree/master. + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 30522): + Vocabulary size of the TAPAS model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.TapasModel`. + hidden_size (:obj:`int`, `optional`, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + num_hidden_layers (:obj:`int`, `optional`, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (:obj:`int`, `optional`, defaults to 3072): + Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder. + hidden_act (:obj:`str` or :obj:`Callable`, `optional`, defaults to :obj:`"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, + :obj:`"gelu"`, :obj:`"relu"`, :obj:`"swish"` and :obj:`"gelu_new"` are supported. + hidden_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob (:obj:`float`, `optional`, defaults to 0.1): + The dropout ratio for the attention probabilities. + max_position_embeddings (:obj:`int`, `optional`, defaults to 1024): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + type_vocab_sizes (:obj:`List[int]`, `optional`, defaults to [3, 256, 256, 2, 256, 256, 10]): + The vocabulary sizes of the :obj:`token_type_ids` passed when calling :class:`~transformers.TapasModel`. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + layer_norm_eps (:obj:`float`, `optional`, defaults to 1e-12): + The epsilon used by the layer normalization layers. + gradient_checkpointing (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to use gradient checkpointing to save memory at the expense of a slower backward pass. + positive_label_weight (:obj:`float`, `optional`, defaults to 10.0): + Weight for positive labels. + num_aggregation_labels (:obj:`int`, `optional`, defaults to 0): + The number of aggregation operators to predict. + aggregation_loss_weight (:obj:`float`, `optional`, defaults to 1.0): + Importance weight for the aggregation loss. + use_answer_as_supervision (:obj:`bool`, `optional`): + Whether to use the answer as the only supervision for aggregation examples. + answer_loss_importance (:obj:`float`, `optional`, defaults to 1.0): + Importance weight for the regression loss. + use_normalized_answer_loss (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to normalize the answer loss by the maximum of the predicted and expected value. + huber_loss_delta: (:obj:`float`, `optional`): + Delta parameter used to calculate the regression loss. + temperature: (:obj:`float`, `optional`, defaults to 1.0): + Value used to control (OR change) the skewness of cell logits probabilities. + aggregation_temperature: (:obj:`float`, `optional`, defaults to 1.0): + Scales aggregation logits to control the skewness of probabilities. + use_gumbel_for_cells: (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to apply Gumbel-Softmax to cell selection. + use_gumbel_for_aggregation: (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to apply Gumbel-Softmax to aggregation selection. + average_approximation_function: (:obj:`string`, `optional`, defaults to :obj:`"ratio"`): + Method to calculate the expected average of cells in the weak supervision case. One of :obj:`"ratio"`, + :obj:`"first_order"` or :obj:`"second_order"`. + cell_selection_preference: (:obj:`float`, `optional`): + Preference for cell selection in ambiguous cases. Only applicable in case of weak supervision for + aggregation (WTQ, WikiSQL). If the total mass of the aggregation probabilities (excluding the "NONE" + operator) is higher than this hyperparameter, then aggregation is predicted for an example. + answer_loss_cutoff: (:obj:`float`, `optional`): + Ignore examples with answer loss larger than cutoff. + max_num_rows: (:obj:`int`, `optional`, defaults to 64): + Maximum number of rows. + max_num_columns: (:obj:`int`, `optional`, defaults to 32): + Maximum number of columns. + average_logits_per_cell: (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to average logits per cell. + select_one_column: (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to constrain the model to only select cells from a single column. + allow_empty_column_selection: (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to allow not to select any column. + init_cell_selection_weights_to_zero: (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to initialize cell selection weights to 0 so that the initial probabilities are 50%. + reset_position_index_per_cell: (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to restart position indexes at every cell (i.e. use relative position embeddings). + disable_per_token_loss: (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to disable any (strong or weak) supervision on cells. + + Example:: + + >>> from transformers import TapasModel, TapasConfig + >>> # Initializing a default (SQA) Tapas configuration + >>> configuration = TapasConfig() + >>> # Initializing a model from the configuration + >>> model = TapasModel(configuration) + >>> # Accessing the model configuration + >>> configuration = model.config + """ + + model_type = "tapas" + + def __init__( + self, + vocab_size=30522, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=1024, + type_vocab_sizes=[3, 256, 256, 2, 256, 256, 10], + initializer_range=0.02, + layer_norm_eps=1e-12, + pad_token_id=0, + gradient_checkpointing=False, + positive_label_weight=10.0, + num_aggregation_labels=0, + aggregation_loss_weight=1.0, + use_answer_as_supervision=None, + answer_loss_importance=1.0, + use_normalized_answer_loss=False, + huber_loss_delta=None, + temperature=1.0, + aggregation_temperature=1.0, + use_gumbel_for_cells=False, + use_gumbel_for_aggregation=False, + average_approximation_function="ratio", + cell_selection_preference=None, + answer_loss_cutoff=None, + max_num_rows=64, + max_num_columns=32, + average_logits_per_cell=False, + select_one_column=True, + allow_empty_column_selection=False, + init_cell_selection_weights_to_zero=False, + reset_position_index_per_cell=True, + disable_per_token_loss=False, + **kwargs + ): + + super().__init__(pad_token_id=pad_token_id, **kwargs) + + # BERT hyperparameters (with updated max_position_embeddings and type_vocab_sizes) + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_sizes = type_vocab_sizes + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps + self.gradient_checkpointing = gradient_checkpointing + + # Fine-tuning task hyperparameters + self.positive_label_weight = positive_label_weight + self.num_aggregation_labels = num_aggregation_labels + self.aggregation_loss_weight = aggregation_loss_weight + self.use_answer_as_supervision = use_answer_as_supervision + self.answer_loss_importance = answer_loss_importance + self.use_normalized_answer_loss = use_normalized_answer_loss + self.huber_loss_delta = huber_loss_delta + self.temperature = temperature + self.aggregation_temperature = aggregation_temperature + self.use_gumbel_for_cells = use_gumbel_for_cells + self.use_gumbel_for_aggregation = use_gumbel_for_aggregation + self.average_approximation_function = average_approximation_function + self.cell_selection_preference = cell_selection_preference + self.answer_loss_cutoff = answer_loss_cutoff + self.max_num_rows = max_num_rows + self.max_num_columns = max_num_columns + self.average_logits_per_cell = average_logits_per_cell + self.select_one_column = select_one_column + self.allow_empty_column_selection = allow_empty_column_selection + self.init_cell_selection_weights_to_zero = init_cell_selection_weights_to_zero + self.reset_position_index_per_cell = reset_position_index_per_cell + self.disable_per_token_loss = disable_per_token_loss diff --git a/src/transformers/models/tapas/convert_tapas_original_tf_checkpoint_to_pytorch.py b/src/transformers/models/tapas/convert_tapas_original_tf_checkpoint_to_pytorch.py new file mode 100644 index 000000000000..463bd9413ab5 --- /dev/null +++ b/src/transformers/models/tapas/convert_tapas_original_tf_checkpoint_to_pytorch.py @@ -0,0 +1,141 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert TAPAS checkpoint.""" + + +import argparse + +import torch + +from transformers.models.tapas import ( + TapasConfig, + TapasForMaskedLM, + TapasForQuestionAnswering, + TapasForSequenceClassification, + TapasModel, + load_tf_weights_in_tapas, +) +from transformers.utils import logging + + +logging.set_verbosity_info() + + +def convert_tf_checkpoint_to_pytorch( + task, reset_position_index_per_cell, tf_checkpoint_path, tapas_config_file, pytorch_dump_path +): + # Initialise PyTorch model. Defaults to TapasForQuestionAnswering with default SQA config. + # If you want to convert a checkpoint that uses absolute position embeddings, make sure to set reset_position_index_per_cell of + # TapasConfig to False. + + if task == "SQA": + config = TapasConfig( + reset_position_index_per_cell=reset_position_index_per_cell, + ) + model = TapasForQuestionAnswering(config=config) + elif task == "WTQ": + # WTQ config + config = TapasConfig( + reset_position_index_per_cell=reset_position_index_per_cell, + # run_task_main.py hparams + num_aggregation_labels=4, + use_answer_as_supervision=True, + # hparam_utils.py hparams + answer_loss_cutoff=0.664694, + cell_selection_preference=0.207951, + huber_loss_delta=0.121194, + init_cell_selection_weights_to_zero=True, + select_one_column=True, + allow_empty_column_selection=False, + temperature=0.0352513, + ) + model = TapasForQuestionAnswering(config=config) + elif task == "WIKISQL_SUPERVISED": + # WikiSQL-supervised config + config = TapasConfig( + reset_position_index_per_cell=reset_position_index_per_cell, + # run_task_main.py hparams + num_aggregation_labels=4, + use_answer_as_supervision=False, + # hparam_utils.py hparams + answer_loss_cutoff=36.4519, + cell_selection_preference=0.903421, + huber_loss_delta=222.088, + init_cell_selection_weights_to_zero=True, + select_one_column=True, + allow_empty_column_selection=True, + temperature=0.763141, + ) + model = TapasForQuestionAnswering(config=config) + elif task == "TABFACT": + config = TapasConfig( + reset_position_index_per_cell=reset_position_index_per_cell, + ) + model = TapasForSequenceClassification(config=config) + elif task == "MLM": + config = TapasConfig( + reset_position_index_per_cell=reset_position_index_per_cell, + ) + model = TapasForMaskedLM(config=config) + elif task == "INTERMEDIATE_PRETRAINING": + config = TapasConfig( + reset_position_index_per_cell=reset_position_index_per_cell, + ) + model = TapasModel(config=config) + + print("Building PyTorch model from configuration: {}".format(str(config))) + + # Load weights from tf checkpoint + load_tf_weights_in_tapas(model, config, tf_checkpoint_path) + + # Save pytorch-model + print("Save PyTorch model to {}".format(pytorch_dump_path)) + torch.save(model.state_dict(), pytorch_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # Required parameters + parser.add_argument( + "--task", default="SQA", type=str, help="Model task for which to convert a checkpoint. Defaults to SQA." + ) + parser.add_argument( + "--reset_position_index_per_cell", + default=True, + type=bool, + help="Whether to use relative position embeddings or not. Defaults to True.", + ) + parser.add_argument( + "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path." + ) + parser.add_argument( + "--tapas_config_file", + default=None, + type=str, + required=True, + help="The config json file corresponding to the pre-trained TAPAS model. \n" + "This specifies the model architecture.", + ) + parser.add_argument( + "--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model." + ) + args = parser.parse_args() + convert_tf_checkpoint_to_pytorch( + args.task, + args.reset_position_index_per_cell, + args.tf_checkpoint_path, + args.tapas_config_file, + args.pytorch_dump_path, + ) diff --git a/src/transformers/models/tapas/modeling_tapas.py b/src/transformers/models/tapas/modeling_tapas.py new file mode 100644 index 000000000000..d13f16751985 --- /dev/null +++ b/src/transformers/models/tapas/modeling_tapas.py @@ -0,0 +1,2311 @@ +# coding=utf-8 +# Copyright 2020 Google Research and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch TAPAS model. """ + + +import enum +import math +import os +from dataclasses import dataclass +from typing import Optional, Tuple + +import torch +import torch.nn as nn +from torch.nn import CrossEntropyLoss, MSELoss + +from ...activations import ACT2FN +from ...file_utils import ( + ModelOutput, + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_scatter_available, + replace_return_docstrings, + requires_scatter, +) +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, MaskedLMOutput, SequenceClassifierOutput +from ...modeling_utils import ( + PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer, +) +from ...utils import logging +from .configuration_tapas import TapasConfig + + +# soft dependency +if is_scatter_available(): + from torch_scatter import scatter + +logger = logging.get_logger(__name__) + +_CONFIG_FOR_DOC = "TapasConfig" +_TOKENIZER_FOR_DOC = "TapasTokenizer" + +TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST = [ + # large models + "nielsr/tapas-large", + "nielsr/tapas-large-finetuned-sqa", + "nielsr/tapas-large-finetuned-wtq", + "nielsr/tapas-large-finetuned-wikisql-supervised", + "nielsr/tapas-large-finetuned-tabfact", + # base models + "nielsr/tapas-base", + "nielsr/tapas-base-finetuned-sqa", + "nielsr/tapas-base-finetuned-wtq", + "nielsr/tapas-base-finetuned-wikisql-supervised", + "nielsr/tapas-base-finetuned-tabfact", + # small models + "nielsr/tapas-small", + "nielsr/tapas-small-finetuned-sqa", + "nielsr/tapas-small-finetuned-wtq", + "nielsr/tapas-small-finetuned-wikisql-supervised", + "nielsr/tapas-small-finetuned-tabfact", + # mini models + "nielsr/tapas-mini", + "nielsr/tapas-mini-finetuned-sqa", + "nielsr/tapas-mini-finetuned-wtq", + "nielsr/tapas-mini-finetuned-wikisql-supervised", + "nielsr/tapas-mini-finetuned-tabfact", + # tiny models + "nielsr/tapas-tiny", + "nielsr/tapas-tiny-finetuned-sqa", + "nielsr/tapas-tiny-finetuned-wtq", + "nielsr/tapas-tiny-finetuned-wikisql-supervised", + "nielsr/tapas-tiny-finetuned-tabfact", + # See all TAPAS models at https://huggingface.co/models?filter=tapas +] + +EPSILON_ZERO_DIVISION = 1e-10 +CLOSE_ENOUGH_TO_LOG_ZERO = -10000.0 + + +@dataclass +class TableQuestionAnsweringOutput(ModelOutput): + """ + Output type of :class:`~transformers.TapasForQuestionAnswering`. + + Args: + loss (:obj:`torch.FloatTensor` of shape :obj:`(1,)`, `optional`, returned when :obj:`labels` (and possibly :obj:`answer`, :obj:`aggregation_labels`, :obj:`numeric_values` and :obj:`numeric_values_scale` are provided)): + Total loss as the sum of the hierarchical cell selection log-likelihood loss and (optionally) the + semi-supervised regression loss and (optionally) supervised loss for aggregations. + logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`): + Prediction scores of the cell selection head, for every token. + logits_aggregation (:obj:`torch.FloatTensor`, `optional`, of shape :obj:`(batch_size, num_aggregation_labels)`): + Prediction scores of the aggregation head, for every aggregation operator. + hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``): + Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) + of shape :obj:`(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of + each layer plus the initial embedding outputs. + attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``): + Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape :obj:`(batch_size, num_heads, + sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the + weighted average in the self-attention heads. + """ + + loss: Optional[torch.FloatTensor] = None + logits: torch.FloatTensor = None + logits_aggregation: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +def load_tf_weights_in_tapas(model, config, tf_checkpoint_path): + """ + Load tf checkpoints in a PyTorch model. This is an adaptation from load_tf_weights_in_bert + + - add cell selection and aggregation heads + - take into account additional token type embedding layers + """ + try: + import re + + import numpy as np + import tensorflow as tf + except ImportError: + logger.error( + "Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions." + ) + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + name = name.split("/") + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculate m and v + # which are not required for using pretrained model + if any( + n + in [ + "adam_v", + "adam_m", + "AdamWeightDecayOptimizer", + "AdamWeightDecayOptimizer_1", + "global_step", + "seq_relationship", + ] + for n in name + ): + logger.info("Skipping {}".format("/".join(name))) + continue + # in case the model is TapasForSequenceClassification, we skip output_bias and output_weights + # since these are not used for classification + if isinstance(model, TapasForSequenceClassification): + if any( + n + in [ + "output_bias", + "output_weights", + ] + for n in name + ): + logger.info("Skipping {}".format("/".join(name))) + continue + # in case the model is TapasModel, we skip output_bias, output_weights, output_bias_cls and output_weights_cls + # since this model does not have MLM and NSP heads + if isinstance(model, TapasModel): + if any( + n + in [ + "output_bias", + "output_weights", + "output_bias_cls", + "output_weights_cls", + ] + for n in name + ): + logger.info("Skipping {}".format("/".join(name))) + continue + # if first scope name starts with "bert", change it to "tapas" + if name[0] == "bert": + name[0] = "tapas" + pointer = model + for m_name in name: + if re.fullmatch(r"[A-Za-z]+_\d+", m_name): + scope_names = re.split(r"_(\d+)", m_name) + else: + scope_names = [m_name] + if scope_names[0] == "kernel" or scope_names[0] == "gamma": + pointer = getattr(pointer, "weight") + elif scope_names[0] == "beta": + pointer = getattr(pointer, "bias") + # cell selection heads + elif scope_names[0] == "output_bias": + pointer = getattr(pointer, "output_bias") + elif scope_names[0] == "output_weights": + pointer = getattr(pointer, "output_weights") + elif scope_names[0] == "column_output_bias": + pointer = getattr(pointer, "column_output_bias") + elif scope_names[0] == "column_output_weights": + pointer = getattr(pointer, "column_output_weights") + # aggregation head + elif scope_names[0] == "output_bias_agg": + pointer = getattr(pointer, "aggregation_classifier") + pointer = getattr(pointer, "bias") + elif scope_names[0] == "output_weights_agg": + pointer = getattr(pointer, "aggregation_classifier") + pointer = getattr(pointer, "weight") + # classification head + elif scope_names[0] == "output_bias_cls": + pointer = getattr(pointer, "classifier") + pointer = getattr(pointer, "bias") + elif scope_names[0] == "output_weights_cls": + pointer = getattr(pointer, "classifier") + pointer = getattr(pointer, "weight") + else: + try: + pointer = getattr(pointer, scope_names[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(scope_names) >= 2: + num = int(scope_names[1]) + pointer = pointer[num] + if m_name[-11:] == "_embeddings": + pointer = getattr(pointer, "weight") + elif m_name[-13:] in [ + "_embeddings_0", + "_embeddings_1", + "_embeddings_2", + "_embeddings_3", + "_embeddings_4", + "_embeddings_5", + "_embeddings_6", + ]: + pointer = getattr(pointer, "weight") + elif m_name == "kernel": + array = np.transpose(array) + try: + assert ( + pointer.shape == array.shape + ), f"Pointer shape {pointer.shape} and array shape {array.shape} mismatched" + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info("Initialize PyTorch weight {}".format(name)) + # added a check to see whether the array is a scalar (because bias terms in Tapas checkpoints can be scalar => should first be converted to numpy arrays) + if np.isscalar(array): + array = np.array(array) + pointer.data = torch.from_numpy(array) + return model + + +class TapasEmbeddings(nn.Module): + """ + Construct the embeddings from word, position and token_type embeddings. Same as BertEmbeddings but with a number of + additional token type embeddings to encode tabular structure. + """ + + def __init__(self, config): + super().__init__() + # we do not include config.disabled_features and config.disable_position_embeddings from the original implementation + # word embeddings + self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id) + # position embeddings + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size) + # token type embeddings + for i, type_vocab_sizes in enumerate(config.type_vocab_sizes): + name = f"token_type_embeddings_{i}" + setattr(self, name, nn.Embedding(type_vocab_sizes, config.hidden_size)) + + self.number_of_token_type_embeddings = len(config.type_vocab_sizes) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + self.config = config + + def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + device = input_ids.device if input_ids is not None else inputs_embeds.device + + if position_ids is None: + # create absolute position embeddings + position_ids = torch.arange(seq_length, dtype=torch.long, device=device) + position_ids = position_ids.unsqueeze(0).expand(input_shape) + # when self.config.reset_position_index_per_cell is set to True, create relative position embeddings + if self.config.reset_position_index_per_cell: + col_index = IndexMap( + token_type_ids[:, :, 1], self.config.type_vocab_sizes[1], batch_dims=1 + ) # shape (batch_size, seq_len) + row_index = IndexMap( + token_type_ids[:, :, 2], self.config.type_vocab_sizes[2], batch_dims=1 + ) # shape (batch_size, seq_len) + full_index = ProductIndexMap(col_index, row_index) # shape (batch_size, seq_len) + + first_position_per_segment = reduce_min(position_ids, full_index)[ + 0 + ] # shape (max_rows * max_columns,). First absolute position for every cell + first_position = gather( + first_position_per_segment, full_index + ) # ? shape (batch_size, seq_len). First absolute position of the cell for every token + position = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0) # shape (1, seq_len) + position_ids = torch.min( + torch.as_tensor(self.config.max_position_embeddings - 1, device=device), position - first_position + ) + + if token_type_ids is None: + token_type_ids = torch.zeros( + (input_shape + self.number_of_token_type_embeddings), dtype=torch.long, device=device + ) + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + position_embeddings = self.position_embeddings(position_ids) + + embeddings = inputs_embeds + position_embeddings + + for i in range(self.number_of_token_type_embeddings): + name = f"token_type_embeddings_{i}" + embeddings += getattr(self, name)(token_type_ids[:, :, i]) + + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class TapasSelfAttention(nn.Module): + def __init__(self, config): + super().__init__() + if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"): + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads) + ) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + if encoder_hidden_states is not None: + mixed_key_layer = self.key(encoder_hidden_states) + mixed_value_layer = self.value(encoder_hidden_states) + attention_mask = encoder_attention_mask + else: + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TapasModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + return outputs + + +# Copied from transformers.models.bert.modeling_bert.BertSelfOutput +class TapasSelfOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +# Copied from transformers.models.bert.modeling_bert.BertAttention with Bert->Tapas +class TapasAttention(nn.Module): + def __init__(self, config): + super().__init__() + self.self = TapasSelfAttention(config) + self.output = TapasSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads + ) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len(heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + return outputs + + +# Copied from transformers.models.bert.modeling_bert.BertIntermediate +class TapasIntermediate(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +# Copied from transformers.models.bert.modeling_bert.BertOutput +class TapasOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +# Copied from transformers.models.bert.modeling_bert.BertLayer with Bert->Tapas +class TapasLayer(nn.Module): + def __init__(self, config): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = TapasAttention(config) + self.is_decoder = config.is_decoder + self.add_cross_attention = config.add_cross_attention + if self.add_cross_attention: + assert self.is_decoder, f"{self} should be used as a decoder model if cross attention is added" + self.crossattention = TapasAttention(config) + self.intermediate = TapasIntermediate(config) + self.output = TapasOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=False, + ): + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + ) + attention_output = self_attention_outputs[0] + outputs = self_attention_outputs[1:] # add self attentions if we output attention weights + + if self.is_decoder and encoder_hidden_states is not None: + assert hasattr( + self, "crossattention" + ), f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`" + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[1:] # add cross attentions if we output attention weights + + layer_output = apply_chunking_to_forward( + self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output + ) + outputs = (layer_output,) + outputs + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class TapasEncoder(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList([TapasLayer(config) for _ in range(config.num_hidden_layers)]) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + ): + all_hidden_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + for i, layer_module in enumerate(self.layer): + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_head_mask = head_mask[i] if head_mask is not None else None + + if getattr(self.config, "gradient_checkpointing", False): + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions, + ) + hidden_states = layer_outputs[0] + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, all_hidden_states, all_attentions] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, hidden_states=all_hidden_states, attentions=all_attentions + ) + + +# Copied from transformers.models.bert.modeling_bert.BertPooler +class TapasPooler(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class TapasPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = TapasConfig + base_model_prefix = "tapas" + + # Copied from transformers.models.bert.modeling_bert.BertPreTrainedModel._init_weights + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +TAPAS_START_DOCSTRING = r""" + This model inherits from :class:`~transformers.PreTrainedModel`. Check the superclass documentation for the generic + methods the library implements for all its models (such as downloading or saving, resizing the input embeddings, + pruning heads etc.) + + This model is also a PyTorch `torch.nn.Module `__ + subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to + general usage and behavior. + + Parameters: + config (:class:`~transformers.TapasConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model + weights. +""" + +TAPAS_INPUTS_DOCSTRING = r""" + Args: + input_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`): + Indices of input sequence tokens in the vocabulary. Indices can be obtained using + :class:`~transformers.TapasTokenizer`. See :meth:`transformers.PreTrainedTokenizer.encode` and + :meth:`transformers.PreTrainedTokenizer.__call__` for details. + + `What are input IDs? <../glossary.html#input-ids>`__ + attention_mask (:obj:`torch.FloatTensor` of shape :obj:`({0})`, `optional`): + Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + `What are attention masks? <../glossary.html#attention-mask>`__ + token_type_ids (:obj:`torch.LongTensor` of shape :obj:`({0}, 7)`, `optional`): + Token indices that encode tabular structure. Indices can be obtained using + :class:`~transformers.TapasTokenizer`. See this class for more info. + + `What are token type IDs? <../glossary.html#token-type-ids>`_ + position_ids (:obj:`torch.LongTensor` of shape :obj:`({0})`, `optional`): + Indices of positions of each input sequence tokens in the position embeddings. If + ``reset_position_index_per_cell`` of :class:`~transformers.TapasConfig` is set to ``True``, relative + position embeddings will be used. Selected in the range ``[0, config.max_position_embeddings - 1]``. + + `What are position IDs? <../glossary.html#position-ids>`_ + head_mask (:obj:`torch.FloatTensor` of shape :obj:`(num_heads,)` or :obj:`(num_layers, num_heads)`, `optional`): + Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: - 1 + indicates the head is **not masked**, - 0 indicates the head is **masked**. + inputs_embeds (:obj:`torch.FloatTensor` of shape :obj:`({0}, hidden_size)`, `optional`): + Optionally, instead of passing :obj:`input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert :obj:`input_ids` indices into associated + vectors than the model's internal embedding lookup matrix. + output_attentions (:obj:`bool`, `optional`): + Whether or not to return the attentions tensors of all attention layers. See ``attentions`` under returned + tensors for more detail. + output_hidden_states (:obj:`bool`, `optional`): + Whether or not to return the hidden states of all layers. See ``hidden_states`` under returned tensors for + more detail. + return_dict (:obj:`bool`, `optional`): + Whether or not to return a :class:`~transformers.file_utils.ModelOutput` instead of a plain tuple. +""" + + +@add_start_docstrings( + "The bare Tapas Model transformer outputting raw hidden-states without any specific head on top.", + TAPAS_START_DOCSTRING, +) +class TapasModel(TapasPreTrainedModel): + """ + This class is a small change compared to :class:`~transformers.BertModel`, taking into account the additional token + type ids. + + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in `Attention is + all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + + """ + + def __init__(self, config, add_pooling_layer=True): + requires_scatter(self) + super().__init__(config) + self.config = config + + self.embeddings = TapasEmbeddings(config) + self.encoder = TapasEncoder(config) + + self.pooler = TapasPooler(config) if add_pooling_layer else None + + self.init_weights() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + @add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + Returns: + + Examples:: + + >>> from transformers import TapasTokenizer, TapasModel + >>> import pandas as pd + + >>> tokenizer = TapasTokenizer.from_pretrained('google/tapas-base') + >>> model = TapasModel.from_pretrained('google/tapas-base') + + >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], + ... 'Age': ["56", "45", "59"], + ... 'Number of movies': ["87", "53", "69"] + ... } + >>> table = pd.DataFrame.from_dict(data) + >>> queries = ["How many movies has George Clooney played in?", "How old is Brad Pitt?"] + + >>> inputs = tokenizer(table=table, queries=queries, padding="max_length", return_tensors="pt") + >>> outputs = model(**inputs) + + >>> last_hidden_states = outputs.last_hidden_state + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + if attention_mask is None: + attention_mask = torch.ones(input_shape, device=device) + if token_type_ids is None: + token_type_ids = torch.zeros( + (*input_shape, len(self.config.type_vocab_sizes)), dtype=torch.long, device=device + ) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, device) + + # If a 2D ou 3D attention mask is provided for the cross-attention + # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) + + embedding_output = self.embeddings( + input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds + ) + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +@add_start_docstrings("""Tapas Model with a `language modeling` head on top. """, TAPAS_START_DOCSTRING) +class TapasForMaskedLM(TapasPreTrainedModel): + config_class = TapasConfig + base_model_prefix = "tapas" + + def __init__(self, config): + super().__init__(config) + + self.tapas = TapasModel(config, add_pooling_layer=False) + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size) + + self.init_weights() + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, word_embeddings): + self.lm_head = word_embeddings + + @add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=MaskedLMOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + **kwargs + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the masked language modeling loss. Indices should be in ``[-100, 0, ..., + config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored + (masked), the loss is only computed for the tokens with labels in ``[0, ..., config.vocab_size]`` + + Returns: + + Examples:: + + >>> from transformers import TapasTokenizer, TapasForMaskedLM + >>> import pandas as pd + + >>> tokenizer = TapasTokenizer.from_pretrained('google/tapas-base') + >>> model = TapasForMaskedLM.from_pretrained('google/tapas-base') + + >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], + ... 'Age': ["56", "45", "59"], + ... 'Number of movies': ["87", "53", "69"] + ... } + >>> table = pd.DataFrame.from_dict(data) + + >>> inputs = tokenizer(table=table, queries="How many [MASK] has George [MASK] played in?", return_tensors="pt") + >>> labels = tokenizer(table=table, queries="How many movies has George Clooney played in?", return_tensors="pt")["input_ids"] + + >>> outputs = model(**inputs, labels=labels) + >>> last_hidden_states = outputs.last_hidden_state + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.tapas( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + prediction_scores = self.lm_head(sequence_output) + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() # -100 index = padding token + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) + + if not return_dict: + output = (prediction_scores,) + outputs[2:] + return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output + + return MaskedLMOutput( + loss=masked_lm_loss, + logits=prediction_scores, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + Tapas Model with a cell selection head and optional aggregation head on top for question-answering tasks on tables + (linear layers on top of the hidden-states output to compute `logits` and optional `logits_aggregation`), e.g. for + SQA, WTQ or WikiSQL-supervised tasks. + """, + TAPAS_START_DOCSTRING, +) +class TapasForQuestionAnswering(TapasPreTrainedModel): + def __init__(self, config: TapasConfig): + super().__init__(config) + + # base model + self.tapas = TapasModel(config) + + # dropout (only used when training) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # cell selection heads + if config.init_cell_selection_weights_to_zero: + # init_cell_selection_weights_to_zero: Whether the initial weights should be + # set to 0. This ensures that all tokens have the same prior probability. + self.output_weights = nn.Parameter(torch.zeros(config.hidden_size)) + self.column_output_weights = nn.Parameter(torch.zeros(config.hidden_size)) + else: + self.output_weights = nn.Parameter(torch.empty(config.hidden_size)) + nn.init.normal_( + self.output_weights, std=config.initializer_range + ) # here, a truncated normal is used in the original implementation + self.column_output_weights = nn.Parameter(torch.empty(config.hidden_size)) + nn.init.normal_( + self.column_output_weights, std=config.initializer_range + ) # here, a truncated normal is used in the original implementation + self.output_bias = nn.Parameter(torch.zeros([])) + self.column_output_bias = nn.Parameter(torch.zeros([])) + + # aggregation head + if config.num_aggregation_labels > 0: + self.aggregation_classifier = nn.Linear(config.hidden_size, config.num_aggregation_labels) + + self.init_weights() + + @add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=TableQuestionAnsweringOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + table_mask=None, + labels=None, + aggregation_labels=None, + float_answer=None, + numeric_values=None, + numeric_values_scale=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + table_mask (:obj:`torch.LongTensor` of shape :obj:`(batch_size, seq_length)`, `optional`): + Mask for the table. Indicates which tokens belong to the table (1). Question tokens, table headers and + padding are 0. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, seq_length)`, `optional`): + Labels per token for computing the hierarchical cell selection loss. This encodes the positions of the + answer appearing in the table. Can be obtained using :class:`~transformers.TapasTokenizer`. + + - 1 for tokens that are **part of the answer**, + - 0 for tokens that are **not part of the answer**. + + aggregation_labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, )`, `optional`): + Aggregation function index for every example in the batch for computing the aggregation loss. Indices + should be in :obj:`[0, ..., config.num_aggregation_labels - 1]`. Only required in case of strong + supervision for aggregation (WikiSQL-supervised). + float_answer (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, )`, `optional`): + Float answer for every example in the batch. Set to `float('nan')` for cell selection questions. Only + required in case of weak supervision (WTQ) to calculate the aggregate mask and regression loss. + numeric_values (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`, `optional`): + Numeric values of every token, NaN for tokens which are not numeric values. Can be obtained using + :class:`~transformers.TapasTokenizer`. Only required in case of weak supervision for aggregation (WTQ) to + calculate the regression loss. + numeric_values_scale (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`, `optional`): + Scale of the numeric values of every token. Can be obtained using :class:`~transformers.TapasTokenizer`. + Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss. + + Returns: + + Examples:: + + >>> from transformers import TapasTokenizer, TapasForQuestionAnswering + >>> import pandas as pd + + >>> tokenizer = TapasTokenizer.from_pretrained('google/tapas-base-finetuned-wtq') + >>> model = TapasForQuestionAnswering.from_pretrained('google/tapas-base-finetuned-wtq') + + >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], + ... 'Age': ["56", "45", "59"], + ... 'Number of movies': ["87", "53", "69"] + ... } + >>> table = pd.DataFrame.from_dict(data) + >>> queries = ["How many movies has George Clooney played in?", "How old is Brad Pitt?"] + + >>> inputs = tokenizer(table=table, queries=queries, padding="max_length", return_tensors="pt") + >>> outputs = model(**inputs) + + >>> logits = outputs.logits + >>> logits_aggregation = outputs.logits_aggregation + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.tapas( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + pooled_output = outputs[1] + + sequence_output = self.dropout(sequence_output) + + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + device = input_ids.device if input_ids is not None else inputs_embeds.device + + # Construct indices for the table. + if token_type_ids is None: + token_type_ids = torch.zeros( + (*input_shape, len(self.config.type_vocab_sizes)), dtype=torch.long, device=device + ) + + token_types = [ + "segment_ids", + "column_ids", + "row_ids", + "prev_labels", + "column_ranks", + "inv_column_ranks", + "numeric_relations", + ] + + row_ids = token_type_ids[:, :, token_types.index("row_ids")] + column_ids = token_type_ids[:, :, token_types.index("column_ids")] + + row_index = IndexMap( + indices=torch.min(row_ids, torch.as_tensor(self.config.max_num_rows - 1, device=row_ids.device)), + num_segments=self.config.max_num_rows, + batch_dims=1, + ) + col_index = IndexMap( + indices=torch.min(column_ids, torch.as_tensor(self.config.max_num_columns - 1, device=column_ids.device)), + num_segments=self.config.max_num_columns, + batch_dims=1, + ) + cell_index = ProductIndexMap(row_index, col_index) + + # Masks. + input_shape = input_ids.size() if input_ids is not None else inputs_embeds.size()[:-1] + device = input_ids.device if input_ids is not None else inputs_embeds.device + if attention_mask is None: + attention_mask = torch.ones(input_shape, device=device) + # Table cells only, without question tokens and table headers. + if table_mask is None: + table_mask = torch.where(row_ids > 0, torch.ones_like(row_ids), torch.zeros_like(row_ids)) + # torch.FloatTensor[batch_size, seq_length] + input_mask_float = attention_mask.float().to(device) + table_mask_float = table_mask.float().to(device) + # Mask for cells that exist in the table (i.e. that are not padding). + cell_mask, _ = reduce_mean(input_mask_float, cell_index) + + # Compute logits per token. These are used to select individual cells. + logits = compute_token_logits(sequence_output, self.config.temperature, self.output_weights, self.output_bias) + + # Compute logits per column. These are used to select a column. + column_logits = None + if self.config.select_one_column: + column_logits = compute_column_logits( + sequence_output, + self.column_output_weights, + self.column_output_bias, + cell_index, + cell_mask, + self.config.allow_empty_column_selection, + ) + + # Aggregation logits + logits_aggregation = None + if self.config.num_aggregation_labels > 0: + logits_aggregation = self.aggregation_classifier(pooled_output) + + # Total loss calculation + total_loss = 0.0 + calculate_loss = False + if labels is not None: + calculate_loss = True + is_supervised = not self.config.num_aggregation_labels > 0 or not self.config.use_answer_as_supervision + + # Semi-supervised cell selection in case of no aggregation: + # If the answer (the denotation) appears directly in the table we might + # select the answer without applying any aggregation function. There are + # some ambiguous cases, see utils._calculate_aggregate_mask for more info. + # `aggregate_mask` is 1 for examples where we chose to aggregate and 0 + # for examples where we chose to select the answer directly. + # `labels` encodes the positions of the answer appearing in the table. + if is_supervised: + aggregate_mask = None + else: + if float_answer is not None: + assert ( + labels.shape[0] == float_answer.shape[0] + ), "Make sure the answers are a FloatTensor of shape (batch_size,)" + # [batch_size] + aggregate_mask = _calculate_aggregate_mask( + float_answer, + pooled_output, + self.config.cell_selection_preference, + labels, + self.aggregation_classifier, + ) + else: + raise ValueError("You have to specify float answers in order to calculate the aggregate mask") + + # Cell selection log-likelihood + if self.config.average_logits_per_cell: + logits_per_cell, _ = reduce_mean(logits, cell_index) + logits = gather(logits_per_cell, cell_index) + dist_per_token = torch.distributions.Bernoulli(logits=logits) + + # Compute cell selection loss per example. + selection_loss_per_example = None + if not self.config.select_one_column: + weight = torch.where( + labels == 0, + torch.ones_like(labels, dtype=torch.float32), + self.config.positive_label_weight * torch.ones_like(labels, dtype=torch.float32), + ) + selection_loss_per_token = -dist_per_token.log_prob(labels) * weight + selection_loss_per_example = torch.sum(selection_loss_per_token * input_mask_float, dim=1) / ( + torch.sum(input_mask_float, dim=1) + EPSILON_ZERO_DIVISION + ) + else: + selection_loss_per_example, logits = _single_column_cell_selection_loss( + logits, column_logits, labels, cell_index, col_index, cell_mask + ) + dist_per_token = torch.distributions.Bernoulli(logits=logits) + + # Supervised cell selection + if self.config.disable_per_token_loss: + pass + elif is_supervised: + total_loss += torch.mean(selection_loss_per_example) + else: + # For the not supervised case, do not assign loss for cell selection + total_loss += torch.mean(selection_loss_per_example * (1.0 - aggregate_mask)) + + # Semi-supervised regression loss and supervised loss for aggregations + if self.config.num_aggregation_labels > 0: + if is_supervised: + # Note that `aggregate_mask` is None if the setting is supervised. + if aggregation_labels is not None: + assert ( + labels.shape[0] == aggregation_labels.shape[0] + ), "Make sure the aggregation labels are a LongTensor of shape (batch_size,)" + per_example_additional_loss = _calculate_aggregation_loss( + logits_aggregation, + aggregate_mask, + aggregation_labels, + self.config.use_answer_as_supervision, + self.config.num_aggregation_labels, + self.config.aggregation_loss_weight, + ) + else: + raise ValueError( + "You have to specify aggregation labels in order to calculate the aggregation loss" + ) + else: + # Set aggregation labels to zeros + aggregation_labels = torch.zeros(labels.shape[0], dtype=torch.long, device=labels.device) + per_example_additional_loss = _calculate_aggregation_loss( + logits_aggregation, + aggregate_mask, + aggregation_labels, + self.config.use_answer_as_supervision, + self.config.num_aggregation_labels, + self.config.aggregation_loss_weight, + ) + + if self.config.use_answer_as_supervision: + if numeric_values is not None and numeric_values_scale is not None: + assert numeric_values.shape == numeric_values_scale.shape + # Add regression loss for numeric answers which require aggregation. + answer_loss, large_answer_loss_mask = _calculate_regression_loss( + float_answer, + aggregate_mask, + dist_per_token, + numeric_values, + numeric_values_scale, + table_mask_float, + logits_aggregation, + self.config, + ) + per_example_additional_loss += answer_loss + # Zero loss for examples with answer_loss > cutoff. + per_example_additional_loss *= large_answer_loss_mask + else: + raise ValueError( + "You have to specify numeric values and numeric values scale in order to calculate the regression loss" + ) + + total_loss += torch.mean(per_example_additional_loss) + + else: + # if no label ids are provided, set them to zeros in order to properly compute logits + labels = torch.zeros_like(logits) + _, logits = _single_column_cell_selection_loss( + logits, column_logits, labels, cell_index, col_index, cell_mask + ) + if not return_dict: + output = (logits, logits_aggregation) + outputs[2:] + return ((total_loss,) + output) if calculate_loss else output + + return TableQuestionAnsweringOutput( + loss=total_loss if calculate_loss else None, + logits=logits, + logits_aggregation=logits_aggregation, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + Tapas Model with a sequence classification head on top (a linear layer on top of the pooled output), e.g. for table + entailment tasks, such as TabFact (Chen et al., 2020). + """, + TAPAS_START_DOCSTRING, +) +class TapasForSequenceClassification(TapasPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + + self.tapas = TapasModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + @add_start_docstrings_to_model_forward(TAPAS_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @replace_return_docstrings(output_type=SequenceClassifierOutput, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids=None, + attention_mask=None, + token_type_ids=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + labels=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`): + Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ..., + config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss), + If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy). Note: this is called + "classification_class_index" in the original implementation. + + Returns: + + Examples:: + + >>> from transformers import TapasTokenizer, TapasForSequenceClassification + >>> import torch + >>> import pandas as pd + + >>> tokenizer = TapasTokenizer.from_pretrained('google/tapas-base-finetuned-tabfact') + >>> model = TapasForSequenceClassification.from_pretrained('google/tapas-base-finetuned-tabfact') + + >>> data = {'Actors': ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], + ... 'Age': ["56", "45", "59"], + ... 'Number of movies': ["87", "53", "69"] + ... } + >>> table = pd.DataFrame.from_dict(data) + >>> queries = ["There is only one actor who is 45 years old", "There are 3 actors which played in more than 60 movies"] + + >>> inputs = tokenizer(table=table, queries=queries, padding="max_length", return_tensors="pt") + >>> labels = torch.tensor([1, 0]) # 1 means entailed, 0 means refuted + + >>> outputs = model(**inputs, labels=labels) + >>> loss = outputs.loss + >>> logits = outputs.logits + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.tapas( + input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + loss = None + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +""" TAPAS utilities.""" + + +class AverageApproximationFunction(str, enum.Enum): + RATIO = "ratio" + FIRST_ORDER = "first_order" + SECOND_ORDER = "second_order" + + +# Beginning of everything related to segmented tensors + + +class IndexMap(object): + """Index grouping entries within a tensor.""" + + def __init__(self, indices, num_segments, batch_dims=0): + """ + Creates an index + + Args: + indices (:obj:`torch.LongTensor`, same shape as a `values` Tensor to which the indices refer): + Tensor containing the indices. + num_segments (:obj:`torch.LongTensor`): + Scalar tensor, the number of segments. All elements in a batched segmented tensor must have the same + number of segments (although many segments can be empty). + batch_dims (:obj:`int`, `optional`, defaults to 0): + The number of batch dimensions. The first `batch_dims` dimensions of a SegmentedTensor are treated as + batch dimensions. Segments in different batch elements are always distinct even if they have the same + index. + """ + self.indices = torch.as_tensor(indices) + self.num_segments = torch.as_tensor(num_segments, device=indices.device) + self.batch_dims = batch_dims + + def batch_shape(self): + return self.indices.size()[: self.batch_dims] # returns a torch.Size object + + +class ProductIndexMap(IndexMap): + """The product of two indices.""" + + def __init__(self, outer_index, inner_index): + """ + Combines indices i and j into pairs (i, j). The result is an index where each segment (i, j) is the + intersection of segments i and j. For example if the inputs represent table cells indexed by respectively rows + and columns the output will be a table indexed by (row, column) pairs, i.e. by cell. The implementation + combines indices {0, .., n - 1} and {0, .., m - 1} into {0, .., nm - 1}. The output has `num_segments` equal to + `outer_index.num_segments` * `inner_index.num_segments` + + Args: + outer_index (:obj:`IndexMap`): + IndexMap. + inner_index (:obj:`IndexMap`): + IndexMap, must have the same shape as `outer_index`. + """ + if outer_index.batch_dims != inner_index.batch_dims: + raise ValueError("outer_index.batch_dims and inner_index.batch_dims must be the same.") + + super().__init__( + indices=(inner_index.indices + outer_index.indices * inner_index.num_segments), + num_segments=inner_index.num_segments * outer_index.num_segments, + batch_dims=inner_index.batch_dims, + ) + self.outer_index = outer_index + self.inner_index = inner_index + + def project_outer(self, index): + """Projects an index with the same index set onto the outer components.""" + return IndexMap( + indices=(index.indices // self.inner_index.num_segments).type(torch.float).floor().type(torch.long), + num_segments=self.outer_index.num_segments, + batch_dims=index.batch_dims, + ) + + def project_inner(self, index): + """Projects an index with the same index set onto the inner components.""" + return IndexMap( + indices=torch.fmod(index.indices, self.inner_index.num_segments) + .type(torch.float) + .floor() + .type(torch.long), + num_segments=self.inner_index.num_segments, + batch_dims=index.batch_dims, + ) + + +def gather(values, index, name="segmented_gather"): + """ + Gathers from `values` using the index map. For each element in the domain of the index map this operation looks up + a value for that index in `values`. Two elements from the same segment always get assigned the same value. + + Args: + values (:obj:`torch.Tensor` of shape (B1, ..., Bn, num_segments, V1, ...)): + Tensor with segment values. + index (:obj:`IndexMap` of shape (B1, ..., Bn, I1, ..., Ik)): + IndexMap. + name (:obj:`str`, `optional`, defaults to 'segmented_gather'): + Name for the operation. Currently not used + + Returns: + :obj:`tuple(torch.Tensor)`: Tensor of shape (B1, ..., Bn, I1, ..., Ik, V1, ...) with the gathered values. + """ + indices = index.indices + # first, check whether the indices of the index represent scalar values (i.e. not vectorized) + if len(values.shape[index.batch_dims :]) < 2: + return torch.gather( + values, + index.batch_dims, + indices.view( + values.size()[0], -1 + ), # torch.gather expects index to have the same number of dimensions as values + ).view(indices.size()) + else: + # this means we have a vectorized version + # we have to adjust the index + indices = indices.unsqueeze(-1).expand(values.shape) + return torch.gather(values, index.batch_dims, indices) + + +def flatten(index, name="segmented_flatten"): + """ + Flattens a batched index map (which is typically of shape batch_size, seq_length) to a 1d index map. This operation + relabels the segments to keep batch elements distinct. The k-th batch element will have indices shifted by + `num_segments` * (k - 1). The result is a tensor with `num_segments` multiplied by the number of elements in the + batch. + + Args: + index (:obj:`IndexMap`): + IndexMap to flatten. + name (:obj:`str`, `optional`, defaults to 'segmented_flatten'): + Name for the operation. Currently not used + + Returns: + (:obj:`IndexMap`): The flattened IndexMap. + """ + # first, get batch_size as scalar tensor + batch_size = torch.prod(torch.tensor(list(index.batch_shape()))) + # next, create offset as 1-D tensor of length batch_size, + # and multiply element-wise by num segments (to offset different elements in the batch) e.g. if batch size is 2: [0, 64] + offset = torch.arange(start=0, end=batch_size, device=index.num_segments.device) * index.num_segments + offset = offset.view(index.batch_shape()) + for _ in range(index.batch_dims, len(index.indices.size())): # typically range(1,2) + offset = offset.unsqueeze(-1) + + indices = offset + index.indices + return IndexMap(indices=indices.view(-1), num_segments=index.num_segments * batch_size, batch_dims=0) + + +def range_index_map(batch_shape, num_segments, name="range_index_map"): + """ + Constructs an index map equal to range(num_segments). + + Args: + batch_shape (:obj:`torch.Size`): + Batch shape + num_segments (:obj:`int`): + Number of segments + name (:obj:`str`, `optional`, defaults to 'range_index_map'): + Name for the operation. Currently not used + + Returns: + (:obj:`IndexMap`): IndexMap of shape batch_shape with elements equal to range(num_segments). + """ + batch_shape = torch.as_tensor( + batch_shape, dtype=torch.long + ) # create a rank 1 tensor vector containing batch_shape (e.g. [2]) + assert len(batch_shape.size()) == 1 + num_segments = torch.as_tensor(num_segments) # create a rank 0 tensor (scalar) containing num_segments (e.g. 64) + assert len(num_segments.size()) == 0 + + indices = torch.arange( + start=0, end=num_segments, device=num_segments.device + ) # create a rank 1 vector with num_segments elements + new_tensor = torch.cat( + [torch.ones_like(batch_shape, dtype=torch.long, device=num_segments.device), num_segments.unsqueeze(dim=0)], + dim=0, + ) + # new_tensor is just a vector of [1 64] for example (assuming only 1 batch dimension) + new_shape = [int(x) for x in new_tensor.tolist()] + indices = indices.view(new_shape) + + multiples = torch.cat([batch_shape, torch.as_tensor([1])], dim=0) + indices = indices.repeat(multiples.tolist()) + # equivalent (in Numpy:) + # indices = torch.as_tensor(np.tile(indices.numpy(), multiples.tolist())) + + return IndexMap(indices=indices, num_segments=num_segments, batch_dims=list(batch_shape.size())[0]) + + +def _segment_reduce(values, index, segment_reduce_fn, name): + """ + Applies a segment reduction segment-wise. + + Args: + values (:obj:`torch.Tensor`): + Tensor with segment values. + index (:obj:`IndexMap`): + IndexMap. + segment_reduce_fn (:obj:`str`): + Name for the reduce operation. One of "sum", "mean", "max" or "min". + name (:obj:`str`): + Name for the operation. Currently not used + + Returns: + (:obj:`IndexMap`): IndexMap of shape batch_shape with elements equal to range(num_segments). + """ + # Flatten the batch dimensions, as segments ops (scatter) do not support batching. + # However if `values` has extra dimensions to the right keep them + # unflattened. Segmented ops support vector-valued operations. + flat_index = flatten(index) + vector_shape = values.size()[len(index.indices.size()) :] # torch.Size object + flattened_shape = torch.cat( + [torch.as_tensor([-1], dtype=torch.long), torch.as_tensor(vector_shape, dtype=torch.long)], dim=0 + ) + # changed "view" by "reshape" in the following line + flat_values = values.reshape(flattened_shape.tolist()) + + segment_means = scatter( + src=flat_values, + index=flat_index.indices.type(torch.long), + dim=0, + dim_size=flat_index.num_segments, + reduce=segment_reduce_fn, + ) + + # Unflatten the values. + new_shape = torch.cat( + [ + torch.as_tensor(index.batch_shape(), dtype=torch.long), + torch.as_tensor([index.num_segments], dtype=torch.long), + torch.as_tensor(vector_shape, dtype=torch.long), + ], + dim=0, + ) + + output_values = segment_means.view(new_shape.tolist()) + output_index = range_index_map(index.batch_shape(), index.num_segments) + return output_values, output_index + + +def reduce_sum(values, index, name="segmented_reduce_sum"): + """ + Sums a tensor over its segments. + + Outputs 0 for empty segments. + + This operations computes the sum over segments, with support for: + + - Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices. + - Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be a sum of + vectors rather than scalars. Only the middle dimensions [I1, ..., Ik] are reduced by the operation. + + Args: + values (:obj:`torch.Tensor` of shape [B1, B2, ..., Bn, I1, .., Ik, V1, V2, ..]): + Tensor containing the values of which the sum must be taken segment-wise. + index (:obj:`IndexMap`, indices are of shape [B1, B2, ..., Bn, I1, .., Ik].): + Index defining the segments. + name (:obj:`str`, `optional`, defaults to 'segmented_reduce_sum'): + Name for the operation. Currently not used + + Returns: + output_values (:obj:`torch.Tensor`of shape [B1, B2, ..., Bn, num_segments, V1, V2, ..]): Tensor containing the + output values. output_index (:obj:`IndexMap`): IndexMap with shape [B1, B2, ..., Bn, num_segments]. . + """ + return _segment_reduce(values, index, "sum", name) + + +def reduce_mean(values, index, name="segmented_reduce_mean"): + """ + Averages a tensor over its segments. + + Outputs 0 for empty segments. + + This operations computes the mean over segments, with support for: + + - Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices. + - Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be a mean of + vectors rather than scalars. + + Only the middle dimensions [I1, ..., Ik] are reduced by the operation. + + Args: + values (:obj:`torch.Tensor` of shape [B1, B2, ..., Bn, I1, .., Ik, V1, V2, ..]): + Tensor containing the values of which the mean must be taken segment-wise. + index (:obj:`IndexMap`, indices are of shape [B1, B2, ..., Bn, I1, .., Ik].): + Index defining the segments. + name (:obj:`str`, `optional`, defaults to 'segmented_reduce_sum'): + Name for the operation. Currently not used + + Returns: + output_values (:obj:`torch.Tensor`of shape [B1, B2, ..., Bn, num_segments, V1, V2, ..]): Tensor containing the + output values. output_index (:obj:`IndexMap`): IndexMap with shape [B1, B2, ..., Bn, num_segments]. + """ + return _segment_reduce(values, index, "mean", name) + + +def reduce_max(values, index, name="segmented_reduce_max"): + """ + Computes the maximum over segments. + + This operation computes the maximum over segments, with support for: + + - Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices. + - Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be an element-wise + maximum of vectors rather than scalars. + + Only the middle dimensions [I1, ..., Ik] are reduced by the operation. + + Args: + values (:obj:`torch.Tensor` of shape [B1, B2, ..., Bn, I1, .., Ik, V1, V2, ..]): + Tensor containing the values of which the max must be taken segment-wise. + index (:obj:`IndexMap`, indices are of shape [B1, B2, ..., Bn, I1, .., Ik].): + Index defining the segments. + name (:obj:`str`, `optional`, defaults to 'segmented_reduce_sum'): + Name for the operation. Currently not used + + Returns: + output_values (:obj:`torch.Tensor`of shape [B1, B2, ..., Bn, num_segments, V1, V2, ..]): Tensor containing the + output values. output_index (:obj:`IndexMap`): IndexMap with shape [B1, B2, ..., Bn, num_segments]. + """ + return _segment_reduce(values, index, "max", name) + + +def reduce_min(values, index, name="segmented_reduce_min"): + """ + Computes the minimum over segments. + + This operations computes the minimum over segments, with support for: + + - Batching using the first dimensions [B1, B2, ..., Bn]. Each element in a batch can have different indices. + - Vectorization using the last dimension [V1, V2, ...]. If they are present, the output will be an element-wise + minimum of vectors rather than scalars. + + Only the middle dimensions [I1, ..., Ik] are reduced by the operation. + + Args: + values (:obj:`torch.Tensor` of shape [B1, B2, ..., Bn, I1, .., Ik, V1, V2, ..]): + Tensor containing the values of which the min must be taken segment-wise. + index (:obj:`IndexMap`, indices are of shape [B1, B2, ..., Bn, I1, .., Ik].): + Index defining the segments. + name (:obj:`str`, `optional`, defaults to 'segmented_reduce_sum'): + Name for the operation. Currently not used + + Returns: + output_values (:obj:`torch.Tensor`of shape [B1, B2, ..., Bn, num_segments, V1, V2, ..]): Tensor containing the + output values. output_index (:obj:`IndexMap`): IndexMap with shape [B1, B2, ..., Bn, num_segments]. + """ + return _segment_reduce(values, index, "min", name) + + +# End of everything related to segmented tensors + + +def compute_column_logits( + sequence_output, column_output_weights, column_output_bias, cell_index, cell_mask, allow_empty_column_selection +): + """ + Computes the column logits. + + Args: + sequence_output (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`): + Also known as last_hidden_state. Sequence of hidden-states at the output of the last layer of the model. + column_output_weights (:obj:`torch.FloatTensor` of shape :obj:`(hidden_size)`): + Weights of the linear layer for column selection. + column_output_bias (:obj:`torch.FloatTensor` of shape :obj:`()`): + Bias of the linear layer for column selection. + cell_index (:obj:`ProductIndexMap`): + Index that groups tokens into cells. + cell_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, max_num_rows * max_num_cols)`): + Mask for cells that exist in the table (i.e. that are not padding). + allow_empty_column_selection (:obj:`bool`): + Whether to allow not to select any column + + Returns: + column_logits (:obj:`torch.FloatTensor`of shape :obj:`(batch_size, max_num_cols)`): Tensor containing the + column logits for every example in the batch. + """ + + # First, compute the token logits (batch_size, seq_len) - without temperature + token_logits = torch.einsum("bsj,j->bs", sequence_output, column_output_weights) + column_output_bias + + # Next, average the logits per cell (batch_size, max_num_cols*max_num_rows) + cell_logits, cell_logits_index = reduce_mean(token_logits, cell_index) + + # Finally, average the logits per column (batch_size, max_num_cols) + column_index = cell_index.project_inner(cell_logits_index) + column_logits, out_index = reduce_sum(cell_logits * cell_mask, column_index) + + cell_count, _ = reduce_sum(cell_mask, column_index) + column_logits /= cell_count + EPSILON_ZERO_DIVISION + + # Mask columns that do not appear in the example. + is_padding = torch.logical_and(cell_count < 0.5, ~torch.eq(out_index.indices, 0)) + column_logits += CLOSE_ENOUGH_TO_LOG_ZERO * torch.as_tensor( + is_padding, dtype=torch.float32, device=is_padding.device + ) + + if not allow_empty_column_selection: + column_logits += CLOSE_ENOUGH_TO_LOG_ZERO * torch.as_tensor( + torch.eq(out_index.indices, 0), dtype=torch.float32, device=out_index.indices.device + ) + + return column_logits + + +def _single_column_cell_selection_loss(token_logits, column_logits, labels, cell_index, col_index, cell_mask): + """ + Computes the loss for cell selection constrained to a single column. The loss is a hierarchical log-likelihood. The + model first predicts a column and then selects cells within that column (conditioned on the column). Cells outside + the selected column are never selected. + + Args: + token_logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`): + Tensor containing the logits per token. + column_logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, max_num_cols)`): + Tensor containing the logits per column. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Labels per token. + cell_index (:obj:`ProductIndexMap`): + Index that groups tokens into cells. + col_index (:obj:`IndexMap`): + Index that groups tokens into columns. + cell_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, max_num_rows * max_num_cols)`): + Mask for cells that exist in the table (i.e. that are not padding). + + Returns: + selection_loss_per_example (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): Loss for each example. + logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`): New logits which are only + allowed to select cells in a single column. Logits outside of the most likely column according to + `column_logits` will be set to a very low value (such that the probabilities are 0). + """ + # Part 1: column loss + + # First find the column we should select. We use the column with maximum number of selected cells. + labels_per_column, _ = reduce_sum(torch.as_tensor(labels, dtype=torch.float32, device=labels.device), col_index) + # shape of labels_per_column is (batch_size, max_num_cols). It contains the number of label ids for every column, for every example + column_label = torch.argmax(labels_per_column, dim=-1) # shape (batch_size,) + # Check if there are no selected cells in the column. In that case the model + # should predict the special column id 0, which means "select nothing". + no_cell_selected = torch.eq( + torch.max(labels_per_column, dim=-1)[0], 0 + ) # no_cell_selected is of shape (batch_size,) and equals True + # if an example of the batch has no cells selected (i.e. if there are no labels set to 1 for that example) + column_label = torch.where( + no_cell_selected.view(column_label.size()), torch.zeros_like(column_label), column_label + ) + + column_dist = torch.distributions.Categorical(logits=column_logits) # shape (batch_size, max_num_cols) + column_loss_per_example = -column_dist.log_prob(column_label) + + # Part 2: cell loss + + # Reduce the labels and logits to per-cell from per-token. + # logits_per_cell: shape (batch_size, max_num_rows*max_num_cols) i.e. (batch_size, 64*32) + logits_per_cell, _ = reduce_mean(token_logits, cell_index) + # labels_per_cell: shape (batch_size, 64*32), indicating whether each cell should be selected (1) or not (0) + labels_per_cell, labels_index = reduce_max( + torch.as_tensor(labels, dtype=torch.long, device=labels.device), cell_index + ) + + # Mask for the selected column. + # column_id_for_cells: shape (batch_size, 64*32), indicating to which column each cell belongs + column_id_for_cells = cell_index.project_inner(labels_index).indices + # column_mask: shape (batch_size, 64*32), equal to 1 if cell belongs to column to be selected + column_mask = torch.as_tensor( + torch.eq(column_id_for_cells, torch.unsqueeze(column_label, dim=-1)), + dtype=torch.float32, + device=cell_mask.device, + ) + + # Compute the log-likelihood for cells, but only for the selected column. + cell_dist = torch.distributions.Bernoulli(logits=logits_per_cell) # shape (batch_size, 64*32) + cell_log_prob = cell_dist.log_prob(labels_per_cell.type(torch.float32)) # shape(batch_size, 64*32) + + cell_loss = -torch.sum(cell_log_prob * column_mask * cell_mask, dim=1) + + # We need to normalize the loss by the number of cells in the column. + cell_loss /= torch.sum(column_mask * cell_mask, dim=1) + EPSILON_ZERO_DIVISION + + selection_loss_per_example = column_loss_per_example + selection_loss_per_example += torch.where( + no_cell_selected.view(selection_loss_per_example.size()), + torch.zeros_like(selection_loss_per_example), + cell_loss, + ) + + # Set the probs outside the selected column (selected by the *model*) + # to 0. This ensures backwards compatibility with models that select + # cells from multiple columns. + selected_column_id = torch.as_tensor( + torch.argmax(column_logits, dim=-1), dtype=torch.long, device=column_logits.device + ) # shape (batch_size,) + + # selected_column_mask: shape (batch_size, 64*32), equal to 1 if cell belongs to column selected by the model + selected_column_mask = torch.as_tensor( + torch.eq(column_id_for_cells, torch.unsqueeze(selected_column_id, dim=-1)), + dtype=torch.float32, + device=selected_column_id.device, + ) + + # Never select cells with the special column id 0. + selected_column_mask = torch.where( + torch.eq(column_id_for_cells, 0).view(selected_column_mask.size()), + torch.zeros_like(selected_column_mask), + selected_column_mask, + ) + new_logits_per_cell = logits_per_cell + CLOSE_ENOUGH_TO_LOG_ZERO * (1.0 - cell_mask * selected_column_mask) + logits = gather(new_logits_per_cell, cell_index) + + return selection_loss_per_example, logits + + +def compute_token_logits(sequence_output, temperature, output_weights, output_bias): + """ + Computes logits per token + + Args: + sequence_output (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`): + Also known as last_hidden_state. Sequence of hidden-states at the output of the last layer of the model. + temperature (:obj:`float`): + Temperature for the Bernoulli distribution. + output_weights (:obj:`torch.FloatTensor` of shape :obj:`(hidden_size,)`): + Weights of the linear layer for cell selection. + output_bias (:obj:`torch.FloatTensor` of shape :obj:`()`): + Bias of the linear layer for cell selection + + Returns: + logits (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`): Logits per token. + """ + logits = (torch.einsum("bsj,j->bs", sequence_output, output_weights) + output_bias) / temperature + + return logits + + +def _calculate_aggregate_mask(answer, pooled_output, cell_selection_preference, labels, aggregation_classifier): + """ + Finds examples where the model should select cells with no aggregation. + + Returns a mask that determines for which examples should the model select answers directly from the table, without + any aggregation function. If the answer is a piece of text the case is unambiguous as aggregation functions only + apply to numbers. If the answer is a number but does not appear in the table then we must use some aggregation + case. The ambiguous case is when the answer is a number that also appears in the table. In this case we use the + aggregation function probabilities predicted by the model to decide whether to select or aggregate. The threshold + for this is a hyperparameter `cell_selection_preference + + Args: + answer (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, )`): + Answer for every example in the batch. Nan if there is no scalar answer. + pooled_output (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, hidden_size)`): + Output of the pooler (BertPooler) on top of the encoder layer. + cell_selection_preference (:obj:`float`): + Preference for cell selection in ambiguous cases. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`): + Labels per token. aggregation_classifier (:obj:`torch.nn.Linear`): Aggregation head + + Returns: + aggregate_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): A mask set to 1 for examples that + should use aggregation functions. + """ + # torch.FloatTensor(batch_size,) + aggregate_mask_init = torch.logical_not(torch.isnan(answer)).type(torch.FloatTensor).to(answer.device) + logits_aggregation = aggregation_classifier(pooled_output) + dist_aggregation = torch.distributions.categorical.Categorical(logits=logits_aggregation) + # Index 0 correponds to "no aggregation". + aggregation_ops_total_mass = torch.sum(dist_aggregation.probs[:, 1:], dim=1) + + # Cell selection examples according to current model. + is_pred_cell_selection = aggregation_ops_total_mass <= cell_selection_preference + + # Examples with non-empty cell selection supervision. + is_cell_supervision_available = torch.sum(labels, dim=1) > 0 + + # torch.where is not equivalent to tf.where (in tensorflow 1) + # hence the added .view on the condition to match the shape of the first tensor + aggregate_mask = torch.where( + torch.logical_and(is_pred_cell_selection, is_cell_supervision_available).view(aggregate_mask_init.size()), + torch.zeros_like(aggregate_mask_init, dtype=torch.float32), + aggregate_mask_init, + ) + + aggregate_mask = aggregate_mask.detach() + + return aggregate_mask + + +def _calculate_aggregation_loss_known( + logits_aggregation, aggregate_mask, aggregation_labels, use_answer_as_supervision, num_aggregation_labels +): + """ + Calculates aggregation loss when its type is known during training. + + In the weakly supervised setting, the only known information is that for cell selection examples, "no aggregation" + should be predicted. For other examples (those that require aggregation), no loss is accumulated. In the setting + where aggregation type is always known, standard cross entropy loss is accumulated for all examples + + Args: + logits_aggregation (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, num_aggregation_labels)`): + Logits per aggregation operation. + aggregate_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, )`): + A mask set to 1 for examples that should use aggregation functions. + aggregation_labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, )`): + Aggregation function id for every example in the batch. + use_answer_as_supervision (:obj:`bool`, `optional`): + Whether to use the answer as the only supervision for aggregation examples. + num_aggregation_labels (:obj:`int`, `optional`, defaults to 0): + The number of aggregation operators to predict. + + Returns: + aggregation_loss_known (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): Aggregation loss (when its + type is known during training) per example. + """ + if use_answer_as_supervision: + # Prepare "no aggregation" targets for cell selection examples. + target_aggregation = torch.zeros_like(aggregate_mask, dtype=torch.long) + else: + # Use aggregation supervision as the target. + target_aggregation = aggregation_labels + + one_hot_labels = torch.nn.functional.one_hot(target_aggregation, num_classes=num_aggregation_labels).type( + torch.float32 + ) + log_probs = torch.nn.functional.log_softmax(logits_aggregation, dim=-1) + + # torch.FloatTensor[batch_size] + per_example_aggregation_intermediate = -torch.sum(one_hot_labels * log_probs, dim=-1) + if use_answer_as_supervision: + # Accumulate loss only for examples requiring cell selection + # (no aggregation). + return per_example_aggregation_intermediate * (1 - aggregate_mask) + else: + return per_example_aggregation_intermediate + + +def _calculate_aggregation_loss_unknown(logits_aggregation, aggregate_mask): + """ + Calculates aggregation loss in the case of answer supervision. + + Args: + logits_aggregation (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, num_aggregation_labels)`): + Logits per aggregation operation. + aggregate_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, )`): + A mask set to 1 for examples that should use aggregation functions + + Returns: + aggregation_loss_unknown (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): Aggregation loss (in case of + answer supervision) per example. + """ + dist_aggregation = torch.distributions.categorical.Categorical(logits=logits_aggregation) + # Index 0 correponds to "no aggregation". + aggregation_ops_total_mass = torch.sum(dist_aggregation.probs[:, 1:], dim=1) + # Predict some aggregation in case of an answer that needs aggregation. + # This increases the probability of all aggregation functions, in a way + # similar to MML, but without considering whether the function gives the + # correct answer. + return -torch.log(aggregation_ops_total_mass) * aggregate_mask + + +def _calculate_aggregation_loss( + logits_aggregation, + aggregate_mask, + aggregation_labels, + use_answer_as_supervision, + num_aggregation_labels, + aggregation_loss_weight, +): + """ + Calculates the aggregation loss per example. + + Args: + logits_aggregation (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, num_aggregation_labels)`): + Logits per aggregation operation. + aggregate_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, )`): + A mask set to 1 for examples that should use aggregation functions. + aggregation_labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, )`): + Aggregation function id for every example in the batch. + use_answer_as_supervision (:obj:`bool`, `optional`): + Whether to use the answer as the only supervision for aggregation examples. + num_aggregation_labels (:obj:`int`, `optional`, defaults to 0): + The number of aggregation operators to predict. + aggregation_loss_weight (:obj:`float`, `optional`, defaults to 1.0): + Importance weight for the aggregation loss. + + Returns: + aggregation_loss (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): Aggregation loss per example. + """ + per_example_aggregation_loss = _calculate_aggregation_loss_known( + logits_aggregation, aggregate_mask, aggregation_labels, use_answer_as_supervision, num_aggregation_labels + ) + + if use_answer_as_supervision: + # Add aggregation loss for numeric answers that need aggregation. + per_example_aggregation_loss += _calculate_aggregation_loss_unknown(logits_aggregation, aggregate_mask) + return aggregation_loss_weight * per_example_aggregation_loss + + +def _calculate_expected_result( + dist_per_cell, numeric_values, numeric_values_scale, input_mask_float, logits_aggregation, config +): + """ + Calculates the expected result given cell and aggregation probabilities. + + Args: + dist_per_cell (:obj:`torch.distributions.Bernoulli`): + Cell selection distribution for each cell. + numeric_values (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`): + Numeric values of every token. Nan for tokens which are not numeric values. + numeric_values_scale (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`): + Scale of the numeric values of every token. + input_mask_float (:obj: `torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`): + Mask for the table, without question tokens and table headers. + logits_aggregation (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, num_aggregation_labels)`): + Logits per aggregation operation. + config (:class:`~transformers.TapasConfig`): + Model configuration class with all the hyperparameters of the model + + Returns: + expected_result (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): The expected result per example. + """ + if config.use_gumbel_for_cells: + gumbel_dist = torch.distributions.RelaxedBernoulli( + # The token logits where already divided by the temperature and used for + # computing cell selection errors so we need to multiply it again here + temperature=config.temperature, + logits=dist_per_cell.logits * config.temperature, + ) + scaled_probability_per_cell = gumbel_dist.sample() + else: + scaled_probability_per_cell = dist_per_cell.probs + + # [batch_size, seq_length] + scaled_probability_per_cell = (scaled_probability_per_cell / numeric_values_scale) * input_mask_float + count_result = torch.sum(scaled_probability_per_cell, dim=1) + numeric_values_masked = torch.where( + torch.isnan(numeric_values), torch.zeros_like(numeric_values), numeric_values + ) # Mask non-numeric table values to zero. + sum_result = torch.sum(scaled_probability_per_cell * numeric_values_masked, dim=1) + avg_approximation = config.average_approximation_function + if avg_approximation == AverageApproximationFunction.RATIO: + average_result = sum_result / (count_result + EPSILON_ZERO_DIVISION) + elif avg_approximation == AverageApproximationFunction.FIRST_ORDER: + # The sum of all probabilities except that correspond to other cells + # Ex here stands for expectation, more explicitly the expectation of the sum of N-1 Bernoulli random variables plus + # the constant 1, which is computed as adding all N expected values and subtracting the extra one. It corresponds to X_c + # in Appendix D of the original TAPAS paper which is trying to approximate the average of a random set. + ex = torch.sum(scaled_probability_per_cell, dim=1, keepdim=True) - scaled_probability_per_cell + 1 + average_result = torch.sum(numeric_values_masked * scaled_probability_per_cell / ex, dim=1) + elif avg_approximation == AverageApproximationFunction.SECOND_ORDER: + # The sum of all probabilities except that correspond to other cells + ex = torch.sum(scaled_probability_per_cell, dim=1, keepdim=True) - scaled_probability_per_cell + 1 + pointwise_var = scaled_probability_per_cell * (1 - scaled_probability_per_cell) + var = torch.sum(pointwise_var, dim=1, keepdim=True) - pointwise_var + + multiplier = (var / torch.square(ex) + 1) / ex + average_result = torch.sum(numeric_values_masked * scaled_probability_per_cell * multiplier, dim=1) + else: + raise ValueError(f"Invalid average_approximation_function: {config.average_approximation_function}") + + if config.use_gumbel_for_aggregation: + gumbel_dist = torch.distributions.RelaxedOneHotCategorical( + config.aggregation_temperature, logits=logits_aggregation[:, 1:] + ) + # [batch_size, num_aggregation_labels - 1] + aggregation_op_only_probs = gumbel_dist.sample() + else: + # [batch_size, num_aggregation_labels - 1] + aggregation_op_only_probs = torch.nn.functional.softmax( + logits_aggregation[:, 1:] / config.aggregation_temperature, dim=-1 + ) + + all_results = torch.cat( + [ + torch.unsqueeze(sum_result, dim=1), + torch.unsqueeze(average_result, dim=1), + torch.unsqueeze(count_result, dim=1), + ], + dim=1, + ) + + expected_result = torch.sum(all_results * aggregation_op_only_probs, dim=1) + return expected_result + + +# PyTorch does not currently support Huber loss with custom delta so we define it ourself +def huber_loss(input, target, delta: float = 1.0): + errors = torch.abs(input - target) # shape (batch_size,) + return torch.where(errors < delta, 0.5 * errors ** 2, errors * delta - (0.5 * delta ** 2)) + + +def _calculate_regression_loss( + answer, + aggregate_mask, + dist_per_cell, + numeric_values, + numeric_values_scale, + input_mask_float, + logits_aggregation, + config, +): + """ + Calculates the regression loss per example. + + Args: + answer (:obj: `torch.FloatTensor` of shape :obj:`(batch_size,)`): + Answer for every example in the batch. Nan if there is no scalar answer. + aggregate_mask (:obj: `torch.FloatTensor` of shape :obj:`(batch_size,)`): + A mask set to 1 for examples that should use aggregation functions. + dist_per_cell (:obj:`torch.distributions.Bernoulli`): + Cell selection distribution for each cell. + numeric_values (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`): + Numeric values of every token. Nan for tokens which are not numeric values. + numeric_values_scale (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`): + Scale of the numeric values of every token. + input_mask_float (:obj: `torch.FloatTensor` of shape :obj:`(batch_size, seq_length)`): + Mask for the table, without question tokens and table headers. + logits_aggregation (:obj: `torch.FloatTensor` of shape :obj:`(batch_size, num_aggregation_labels)`): + Logits per aggregation operation. + config (:class:`~transformers.TapasConfig`): + Model configuration class with all the parameters of the model + + Returns: + per_example_answer_loss_scaled (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): Scales answer loss for + each example in the batch. large_answer_loss_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size,)`): A + mask which is 1 for examples for which their answer loss is larger than the answer_loss_cutoff. + """ + # float32 (batch_size,) + expected_result = _calculate_expected_result( + dist_per_cell, numeric_values, numeric_values_scale, input_mask_float, logits_aggregation, config + ) + + # float32 (batch_size,) + answer_masked = torch.where(torch.isnan(answer), torch.zeros_like(answer), answer) + + if config.use_normalized_answer_loss: + normalizer = (torch.max(torch.abs(expected_result), torch.abs(answer_masked)) + EPSILON_ZERO_DIVISION).detach() + + normalized_answer_masked = answer_masked / normalizer + normalized_expected_result = expected_result / normalizer + per_example_answer_loss = huber_loss( + normalized_expected_result * aggregate_mask, normalized_answer_masked * aggregate_mask + ) + else: + per_example_answer_loss = huber_loss( + expected_result * aggregate_mask, answer_masked * aggregate_mask, delta=config.huber_loss_delta + ) + + if config.answer_loss_cutoff is None: + large_answer_loss_mask = torch.ones_like(per_example_answer_loss, dtype=torch.float32) + + else: + large_answer_loss_mask = torch.where( + per_example_answer_loss > config.answer_loss_cutoff, + torch.zeros_like(per_example_answer_loss, dtype=torch.float32), + torch.ones_like(per_example_answer_loss, dtype=torch.float32), + ) + per_example_answer_loss_scaled = config.answer_loss_importance * (per_example_answer_loss * aggregate_mask) + + return per_example_answer_loss_scaled, large_answer_loss_mask diff --git a/src/transformers/models/tapas/tokenization_tapas.py b/src/transformers/models/tapas/tokenization_tapas.py new file mode 100644 index 000000000000..f31109ae77fa --- /dev/null +++ b/src/transformers/models/tapas/tokenization_tapas.py @@ -0,0 +1,2782 @@ +# coding=utf-8 +# Copyright 2020 Google Research and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization class for TAPAS model.""" + + +import collections +import datetime +import enum +import itertools +import math +import os +import re +import unicodedata +from dataclasses import dataclass +from typing import Callable, Dict, Generator, List, Optional, Text, Tuple, Union + +import numpy as np + +from transformers import add_end_docstrings + +from ...file_utils import is_pandas_available +from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace +from ...tokenization_utils_base import ( + ENCODE_KWARGS_DOCSTRING, + BatchEncoding, + EncodedInput, + ExplicitEnum, + PaddingStrategy, + PreTokenizedInput, + TensorType, + TextInput, +) +from ...utils import logging + + +if is_pandas_available(): + import pandas as pd + +logger = logging.get_logger(__name__) + + +VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "nielsr/tapas-base-finetuned-sqa": "https://huggingface.co/nielsr/tapas-base-finetuned-sqa/resolve/main/vocab.txt", + "nielsr/tapas-base-finetuned-wtq": "https://huggingface.co/nielsr/tapas-base-finetuned-wtq/resolve/main/vocab.txt", + "nielsr/tapas-base-finetuned-wikisql-supervised": "https://huggingface.co/nielsr/tapas-base-finetuned-wikisql-supervised/resolve/main/vocab.txt", + "nielsr/tapas-base-finetuned-tabfact": "https://huggingface.co/nielsr/tapas-base-finetuned-tabfact/resolve/main/vocab.txt", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + # large models + "nielsr/tapas-large-finetuned-sqa": 512, + "nielsr/tapas-large-finetuned-wtq": 512, + "nielsr/tapas-large-finetuned-wikisql-supervised": 512, + "nielsr/tapas-large-finetuned-tabfact": 512, + # base models + "nielsr/tapas-base-finetuned-sqa": 512, + "nielsr/tapas-base-finetuned-wtq": 512, + "nielsr/tapas-base-finetuned-wikisql-supervised": 512, + "nielsr/tapas-base-finetuned-tabfact": 512, + # medium models + "nielsr/tapas-medium-finetuned-sqa": 512, + "nielsr/tapas-medium-finetuned-wtq": 512, + "nielsr/tapas-medium-finetuned-wikisql-supervised": 512, + "nielsr/tapas-medium-finetuned-tabfact": 512, + # small models + "nielsr/tapas-base-finetuned-sqa": 512, + "nielsr/tapas-base-finetuned-wtq": 512, + "nielsr/tapas-base-finetuned-wikisql-supervised": 512, + "nielsr/tapas-base-finetuned-tabfact": 512, + # tiny models + "nielsr/tapas-tiny-finetuned-sqa": 512, + "nielsr/tapas-tiny-finetuned-wtq": 512, + "nielsr/tapas-tiny-finetuned-wikisql-supervised": 512, + "nielsr/tapas-tiny-finetuned-tabfact": 512, + # mini models + "nielsr/tapas-mini-finetuned-sqa": 512, + "nielsr/tapas-mini-finetuned-wtq": 512, + "nielsr/tapas-mini-finetuned-wikisql-supervised": 512, + "nielsr/tapas-mini-finetuned-tabfact": 512, +} + +PRETRAINED_INIT_CONFIGURATION = { + "nielsr/tapas-base-finetuned-sqa": {"do_lower_case": True}, + "nielsr/tapas-base-finetuned-wtq": {"do_lower_case": True}, + "nielsr/tapas-base-finetuned-wikisql-supervised": {"do_lower_case": True}, + "nielsr/tapas-base-finetuned-tabfact": {"do_lower_case": True}, +} + + +class TapasTruncationStrategy(ExplicitEnum): + """ + Possible values for the ``truncation`` argument in :meth:`~transformers.TapasTokenizer.__call__`. Useful for + tab-completion in an IDE. + """ + + DROP_ROWS_TO_FIT = "drop_rows_to_fit" + DO_NOT_TRUNCATE = "do_not_truncate" + + +TableValue = collections.namedtuple("TokenValue", ["token", "column_id", "row_id"]) + + +@dataclass(frozen=True) +class TokenCoordinates: + column_index: int + row_index: int + token_index: int + + +@dataclass +class TokenizedTable: + rows: List[List[List[Text]]] + selected_tokens: List[TokenCoordinates] + + +@dataclass(frozen=True) +class SerializedExample: + tokens: List[Text] + column_ids: List[int] + row_ids: List[int] + segment_ids: List[int] + + +def _is_inner_wordpiece(token: Text): + return token.startswith("##") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + with open(vocab_file, "r", encoding="utf-8") as reader: + tokens = reader.readlines() + for index, token in enumerate(tokens): + token = token.rstrip("\n") + vocab[token] = index + return vocab + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r""" + add_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to encode the sequences with the special tokens relative to their model. + padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`False`): + Activates and controls padding. Accepts the following values: + + * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a + single sequence if provided). + * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the + maximum acceptable input length for the model if that argument is not provided. + * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of + different lengths). + truncation (:obj:`bool`, :obj:`str` or :class:`~transformers.TapasTruncationStrategy`, `optional`, defaults to :obj:`False`): + Activates and controls truncation. Accepts the following values: + + * :obj:`True` or :obj:`'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument + :obj:`max_length` or to the maximum acceptable input length for the model if that argument is not + provided. This will truncate row by row, removing rows from the table. + * :obj:`False` or :obj:`'do_not_truncate'` (default): No truncation (i.e., can output batch with + sequence lengths greater than the model maximum admissible input size). + max_length (:obj:`int`, `optional`): + Controls the maximum length to use by one of the truncation/padding parameters. + + If left unset or set to :obj:`None`, this will use the predefined model maximum length if a maximum + length is required by one of the truncation/padding parameters. If the model has no specific maximum + input length (like XLNet) truncation/padding to a maximum length will be deactivated. + is_split_into_words (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether or not the input is already pre-tokenized (e.g., split into words), in which case the tokenizer + will skip the pre-tokenization step. This is useful for NER or token classification. + pad_to_multiple_of (:obj:`int`, `optional`): + If set will pad the sequence to a multiple of the provided value. This is especially useful to enable + the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta). + return_tensors (:obj:`str` or :class:`~transformers.tokenization_utils_base.TensorType`, `optional`): + If set, will return tensors instead of list of python integers. Acceptable values are: + + * :obj:`'tf'`: Return TensorFlow :obj:`tf.constant` objects. + * :obj:`'pt'`: Return PyTorch :obj:`torch.Tensor` objects. + * :obj:`'np'`: Return Numpy :obj:`np.ndarray` objects. +""" + + +class TapasTokenizer(PreTrainedTokenizer): + r""" + Construct a TAPAS tokenizer. Based on WordPiece. Flattens a table and one or more related sentences to be used by + TAPAS models. + + This tokenizer inherits from :class:`~transformers.PreTrainedTokenizer` which contains most of the main methods. + Users should refer to this superclass for more information regarding those methods. + :class:`~transformers.TapasTokenizer` creates several token type ids to encode tabular structure. To be more + precise, it adds 7 token type ids, in the following order: :obj:`segment_ids`, :obj:`column_ids`, :obj:`row_ids`, + :obj:`prev_labels`, :obj:`column_ranks`, :obj:`inv_column_ranks` and :obj:`numeric_relations`: + + - segment_ids: indicate whether a token belongs to the question (0) or the table (1). 0 for special tokens and + padding. + - column_ids: indicate to which column of the table a token belongs (starting from 1). Is 0 for all question + tokens, special tokens and padding. + - row_ids: indicate to which row of the table a token belongs (starting from 1). Is 0 for all question tokens, + special tokens and padding. Tokens of column headers are also 0. + - prev_labels: indicate whether a token was (part of) an answer to the previous question (1) or not (0). Useful in + a conversational setup (such as SQA). + - column_ranks: indicate the rank of a table token relative to a column, if applicable. For example, if you have a + column "number of movies" with values 87, 53 and 69, then the column ranks of these tokens are 3, 1 and 2 + respectively. 0 for all question tokens, special tokens and padding. + - inv_column_ranks: indicate the inverse rank of a table token relative to a column, if applicable. For example, if + you have a column "number of movies" with values 87, 53 and 69, then the inverse column ranks of these tokens are + 1, 3 and 2 respectively. 0 for all question tokens, special tokens and padding. + - numeric_relations: indicate numeric relations between the question and the tokens of the table. 0 for all + question tokens, special tokens and padding. + + :class:`~transformers.TapasTokenizer` runs end-to-end tokenization on a table and associated sentences: punctuation + splitting and wordpiece. + + Args: + vocab_file (:obj:`str`): + File containing the vocabulary. + do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to lowercase the input when tokenizing. + do_basic_tokenize (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to do basic tokenization before WordPiece. + never_split (:obj:`Iterable`, `optional`): + Collection of tokens which will never be split during tokenization. Only has an effect when + :obj:`do_basic_tokenize=True` + unk_token (:obj:`str`, `optional`, defaults to :obj:`"[UNK]"`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + sep_token (:obj:`str`, `optional`, defaults to :obj:`"[SEP]"`): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for + sequence classification or for a text and a question for question answering. It is also used as the last + token of a sequence built with special tokens. + pad_token (:obj:`str`, `optional`, defaults to :obj:`"[PAD]"`): + The token used for padding, for example when batching sequences of different lengths. + cls_token (:obj:`str`, `optional`, defaults to :obj:`"[CLS]"`): + The classifier token which is used when doing sequence classification (classification of the whole sequence + instead of per-token classification). It is the first token of the sequence when built with special tokens. + mask_token (:obj:`str`, `optional`, defaults to :obj:`"[MASK]"`): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + empty_token (:obj:`str`, `optional`, defaults to :obj:`"[EMPTY]"`): + The token used for empty cell values in a table. Empty cell values include "", "n/a", "nan" and "?". + tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see this + `issue `__). + strip_accents: (:obj:`bool`, `optional`): + Whether or not to strip all accents. If this option is not specified, then it will be determined by the + value for :obj:`lowercase` (as in the original BERT). + cell_trim_length (:obj:`int`, `optional`, defaults to -1): + If > 0: Trim cells so that the length is <= this value. Also disables further cell trimming, should thus be + used with 'drop_rows_to_fit' below. + max_column_id (:obj:`int`, `optional`): + Max column id to extract. + max_row_id (:obj:`int`, `optional`): + Max row id to extract. + strip_column_names (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to add empty strings instead of column names. + update_answer_coordinates (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to recompute the answer coordinates from the answer text. + drop_rows_to_fit (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to drop the last rows if a table doesn't fit within max sequence length. + + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__( + self, + vocab_file, + do_lower_case=True, + do_basic_tokenize=True, + never_split=None, + unk_token="[UNK]", + sep_token="[SEP]", + pad_token="[PAD]", + cls_token="[CLS]", + mask_token="[MASK]", + empty_token="[EMPTY]", + tokenize_chinese_chars=True, + strip_accents=None, + cell_trim_length: int = -1, + max_column_id: int = None, + max_row_id: int = None, + strip_column_names: bool = False, + update_answer_coordinates: bool = False, + drop_rows_to_fit: bool = False, + model_max_length: int = 512, + additional_special_tokens: Optional[List[str]] = None, + **kwargs + ): + if not is_pandas_available(): + raise ImportError("Pandas is required for the TAPAS tokenizer.") + + if additional_special_tokens is not None: + if empty_token not in additional_special_tokens: + additional_special_tokens.append(empty_token) + else: + additional_special_tokens = [empty_token] + + super().__init__( + do_lower_case=do_lower_case, + do_basic_tokenize=do_basic_tokenize, + never_split=never_split, + unk_token=unk_token, + sep_token=sep_token, + pad_token=pad_token, + cls_token=cls_token, + mask_token=mask_token, + empty_token=empty_token, + tokenize_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + cell_trim_length=cell_trim_length, + max_column_id=max_column_id, + max_row_id=max_row_id, + strip_column_names=strip_column_names, + update_answer_coordinates=update_answer_coordinates, + drop_rows_to_fit=drop_rows_to_fit, + model_max_length=model_max_length, + additional_special_tokens=additional_special_tokens, + **kwargs, + ) + + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file) + ) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()]) + self.do_basic_tokenize = do_basic_tokenize + if do_basic_tokenize: + self.basic_tokenizer = BasicTokenizer( + do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=tokenize_chinese_chars, + strip_accents=strip_accents, + ) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=self.unk_token) + + # Additional properties + self.cell_trim_length = cell_trim_length + self.max_column_id = max_column_id if max_column_id is not None else self.model_max_length + self.max_row_id = max_row_id if max_row_id is not None else self.model_max_length + self.strip_column_names = strip_column_names + self.update_answer_coordinates = update_answer_coordinates + self.drop_rows_to_fit = drop_rows_to_fit + + @property + def do_lower_case(self): + return self.basic_tokenizer.do_lower_case + + @property + def vocab_size(self): + return len(self.vocab) + + def get_vocab(self): + return dict(self.vocab, **self.added_tokens_encoder) + + def _tokenize(self, text): + if format_text(text) == EMPTY_TEXT: + return [self.additional_special_tokens[0]] + split_tokens = [] + if self.do_basic_tokenize: + for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens): + + # If the token is part of the never_split set + if token in self.basic_tokenizer.never_split: + split_tokens.append(token) + else: + split_tokens += self.wordpiece_tokenizer.tokenize(token) + else: + split_tokens = self.wordpiece_tokenizer.tokenize(text) + return split_tokens + + def _convert_token_to_id(self, token): + """ Converts a token (str) in an id using the vocab. """ + return self.vocab.get(token, self.vocab.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.ids_to_tokens.get(index, self.unk_token) + + def convert_tokens_to_string(self, tokens): + """ Converts a sequence of tokens (string) in a single string. """ + out_string = " ".join(tokens).replace(" ##", "").strip() + return out_string + + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + index = 0 + if os.path.isdir(save_directory): + vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + else: + vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory + with open(vocab_file, "w", encoding="utf-8") as writer: + for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive." + " Please check that the vocabulary is not corrupted!" + ) + index = token_index + writer.write(token + "\n") + index += 1 + return (vocab_file,) + + def create_attention_mask_from_sequences(self, query_ids: List[int], table_values: List[TableValue]) -> List[int]: + """ + Creates the attention mask according to the query token IDs and a list of table values. + + Args: + query_ids (:obj:`List[int]`): list of token IDs corresponding to the ID. + table_values (:obj:`List[TableValue]`): lift of table values, which are named tuples containing the + token value, the column ID and the row ID of said token. + + Returns: + :obj:`List[int]`: List of ints containing the attention mask values. + """ + return [1] * (1 + len(query_ids) + 1 + len(table_values)) + + def create_segment_token_type_ids_from_sequences( + self, query_ids: List[int], table_values: List[TableValue] + ) -> List[int]: + """ + Creates the segment token type IDs according to the query token IDs and a list of table values. + + Args: + query_ids (:obj:`List[int]`): list of token IDs corresponding to the ID. + table_values (:obj:`List[TableValue]`): lift of table values, which are named tuples containing the + token value, the column ID and the row ID of said token. + + Returns: + :obj:`List[int]`: List of ints containing the segment token type IDs values. + """ + table_ids = list(zip(*table_values))[0] if table_values else [] + return [0] * (1 + len(query_ids) + 1) + [1] * len(table_ids) + + def create_column_token_type_ids_from_sequences( + self, query_ids: List[int], table_values: List[TableValue] + ) -> List[int]: + """ + Creates the column token type IDs according to the query token IDs and a list of table values. + + Args: + query_ids (:obj:`List[int]`): list of token IDs corresponding to the ID. + table_values (:obj:`List[TableValue]`): lift of table values, which are named tuples containing the + token value, the column ID and the row ID of said token. + + Returns: + :obj:`List[int]`: List of ints containing the column token type IDs values. + """ + table_column_ids = list(zip(*table_values))[1] if table_values else [] + return [0] * (1 + len(query_ids) + 1) + list(table_column_ids) + + def create_row_token_type_ids_from_sequences( + self, query_ids: List[int], table_values: List[TableValue] + ) -> List[int]: + """ + Creates the row token type IDs according to the query token IDs and a list of table values. + + Args: + query_ids (:obj:`List[int]`): list of token IDs corresponding to the ID. + table_values (:obj:`List[TableValue]`): lift of table values, which are named tuples containing the + token value, the column ID and the row ID of said token. + + Returns: + :obj:`List[int]`: List of ints containing the row token type IDs values. + """ + table_row_ids = list(zip(*table_values))[2] if table_values else [] + return [0] * (1 + len(query_ids) + 1) + list(table_row_ids) + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Build model inputs from a question and flattened table for question answering or sequence classification tasks + by concatenating and adding special tokens. + + Args: + token_ids_0 (:obj:`List[int]`): The ids of the question. + token_ids_1 (:obj:`List[int]`, `optional`): The ids of the flattened table. + + Returns: + :obj:`List[int]`: The model input with special tokens. + """ + if token_ids_1 is None: + raise ValueError("With TAPAS, you must provide both question IDs and table IDs.") + + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + token_ids_1 + + def get_special_tokens_mask( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` method. + + Args: + token_ids_0 (:obj:`List[int]`): + List of question IDs. + token_ids_1 (:obj:`List[int]`, `optional`): + List of flattened table IDs. + already_has_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + :obj:`List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError( + "You should not supply a second sequence if the provided sequence of " + "ids is already formatted with special tokens for the model." + ) + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + return [1] + ([0] * len(token_ids_0)) + [1] + + @add_end_docstrings(TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING) + def __call__( + self, + table: "pd.DataFrame", + queries: Optional[ + Union[ + TextInput, + PreTokenizedInput, + EncodedInput, + List[TextInput], + List[PreTokenizedInput], + List[EncodedInput], + ] + ] = None, + answer_coordinates: Optional[Union[List[Tuple], List[List[Tuple]]]] = None, + answer_text: Optional[Union[List[TextInput], List[List[TextInput]]]] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TapasTruncationStrategy] = False, + max_length: Optional[int] = None, + pad_to_multiple_of: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_token_type_ids: Optional[bool] = None, + return_attention_mask: Optional[bool] = None, + return_overflowing_tokens: bool = False, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_length: bool = False, + verbose: bool = True, + **kwargs + ) -> BatchEncoding: + """ + Main method to tokenize and prepare for the model one or several sequence(s) related to a table. + + Args: + table (:obj:`pd.DataFrame`): + Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas + dataframe to convert it to string. + queries (:obj:`str` or :obj:`List[str]`): + Question or batch of questions related to a table to be encoded. Note that in case of a batch, all + questions must refer to the **same** table. + answer_coordinates (:obj:`List[Tuple]` or :obj:`List[List[Tuple]]`, `optional`): + Answer coordinates of each table-question pair in the batch. In case only a single table-question pair + is provided, then the answer_coordinates must be a single list of one or more tuples. Each tuple must + be a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The + first column has index 0. In case a batch of table-question pairs is provided, then the + answer_coordinates must be a list of lists of tuples (each list corresponding to a single + table-question pair). + answer_text (:obj:`List[str]` or :obj:`List[List[str]]`, `optional`): + Answer text of each table-question pair in the batch. In case only a single table-question pair is + provided, then the answer_text must be a single list of one or more strings. Each string must be the + answer text of a corresponding answer coordinate. In case a batch of table-question pairs is provided, + then the answer_coordinates must be a list of lists of strings (each list corresponding to a single + table-question pair). + """ + assert isinstance(table, pd.DataFrame), "Table must be of type pd.DataFrame" + + # Input type checking for clearer error + assert ( + queries is None + or isinstance(queries, str) + or ( + isinstance(queries, (list, tuple)) + and ( + len(queries) == 0 + or ( + isinstance(queries[0], str) + or ( + isinstance(queries[0], (list, tuple)) + and (len(queries[0]) == 0 or isinstance(queries[0][0], str)) + ) + ) + ) + ) + ), ( + "queries input must of type `str` (single example), `List[str]` (batch or single pretokenized example) " + "or `List[List[str]]` (batch of pretokenized examples)." + ) + + is_batched = isinstance(queries, (list, tuple)) + + if is_batched: + return self.batch_encode_plus( + table=table, + queries=queries, + answer_coordinates=answer_coordinates, + answer_text=answer_text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + pad_to_multiple_of=pad_to_multiple_of, + return_tensors=return_tensors, + return_token_type_ids=return_token_type_ids, + return_attention_mask=return_attention_mask, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_length=return_length, + verbose=verbose, + **kwargs, + ) + else: + return self.encode_plus( + table=table, + query=queries, + answer_coordinates=answer_coordinates, + answer_text=answer_text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + pad_to_multiple_of=pad_to_multiple_of, + return_tensors=return_tensors, + return_token_type_ids=return_token_type_ids, + return_attention_mask=return_attention_mask, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_length=return_length, + verbose=verbose, + **kwargs, + ) + + @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING) + def batch_encode_plus( + self, + table: "pd.DataFrame", + queries: Optional[ + Union[ + List[TextInput], + List[PreTokenizedInput], + List[EncodedInput], + ] + ] = None, + answer_coordinates: Optional[List[List[Tuple]]] = None, + answer_text: Optional[List[List[TextInput]]] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TapasTruncationStrategy] = False, + max_length: Optional[int] = None, + pad_to_multiple_of: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_token_type_ids: Optional[bool] = None, + return_attention_mask: Optional[bool] = None, + return_overflowing_tokens: bool = False, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_length: bool = False, + verbose: bool = True, + **kwargs + ) -> BatchEncoding: + """ + Prepare a table and a list of strings for the model. + + .. warning:: + This method is deprecated, ``__call__`` should be used instead. + + Args: + table (:obj:`pd.DataFrame`): + Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas + dataframe to convert it to string. + queries (:obj:`List[str]`): + Batch of questions related to a table to be encoded. Note that all questions must refer to the **same** + table. + answer_coordinates (:obj:`List[Tuple]` or :obj:`List[List[Tuple]]`, `optional`): + Answer coordinates of each table-question pair in the batch. Each tuple must be a (row_index, + column_index) pair. The first data row (not the column header row) has index 0. The first column has + index 0. The answer_coordinates must be a list of lists of tuples (each list corresponding to a single + table-question pair). + answer_text (:obj:`List[str]` or :obj:`List[List[str]]`, `optional`): + Answer text of each table-question pair in the batch. In case a batch of table-question pairs is + provided, then the answer_coordinates must be a list of lists of strings (each list corresponding to a + single table-question pair). Each string must be the answer text of a corresponding answer coordinate. + """ + if return_token_type_ids is not None and not add_special_tokens: + raise ValueError( + "Asking to return token_type_ids while setting add_special_tokens to False " + "results in an undefined behavior. Please set add_special_tokens to True or " + "set return_token_type_ids to None." + ) + + if (answer_coordinates and not answer_text) or (not answer_coordinates and answer_text): + raise ValueError("In case you provide answers, both answer_coordinates and answer_text should be provided") + elif answer_coordinates is None and answer_text is None: + answer_coordinates = answer_text = [None] * len(queries) + + if "is_split_into_words" in kwargs: + raise NotImplementedError("Currently TapasTokenizer only supports questions as strings.") + + if return_offsets_mapping: + raise NotImplementedError( + "return_offset_mapping is not available when using Python tokenizers." + "To use this feature, change your tokenizer to one deriving from " + "transformers.PreTrainedTokenizerFast." + ) + + return self._batch_encode_plus( + table=table, + queries=queries, + answer_coordinates=answer_coordinates, + answer_text=answer_text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + pad_to_multiple_of=pad_to_multiple_of, + return_tensors=return_tensors, + return_token_type_ids=return_token_type_ids, + return_attention_mask=return_attention_mask, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_length=return_length, + verbose=verbose, + **kwargs, + ) + + def _batch_encode_plus( + self, + table, + queries: Union[ + List[TextInput], + List[PreTokenizedInput], + List[EncodedInput], + ], + answer_coordinates: Optional[List[List[Tuple]]] = None, + answer_text: Optional[List[List[TextInput]]] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TapasTruncationStrategy] = False, + max_length: Optional[int] = None, + pad_to_multiple_of: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_token_type_ids: Optional[bool] = True, + return_attention_mask: Optional[bool] = None, + return_overflowing_tokens: bool = False, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_length: bool = False, + verbose: bool = True, + **kwargs + ) -> BatchEncoding: + table_tokens = self._tokenize_table(table) + + queries_tokens = [] + for query in queries: + query_tokens = self.tokenize(query) + queries_tokens.append(query_tokens) + + batch_outputs = self._batch_prepare_for_model( + table, + queries, + tokenized_table=table_tokens, + queries_tokens=queries_tokens, + answer_coordinates=answer_coordinates, + padding=padding, + truncation=truncation, + answer_text=answer_text, + add_special_tokens=add_special_tokens, + max_length=max_length, + pad_to_multiple_of=pad_to_multiple_of, + return_tensors=return_tensors, + prepend_batch_axis=True, + return_attention_mask=return_attention_mask, + return_token_type_ids=return_token_type_ids, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_length=return_length, + verbose=verbose, + ) + + return BatchEncoding(batch_outputs) + + def _batch_prepare_for_model( + self, + raw_table: "pd.DataFrame", + raw_queries: Union[ + List[TextInput], + List[PreTokenizedInput], + List[EncodedInput], + ], + tokenized_table: Optional[TokenizedTable] = None, + queries_tokens: Optional[List[List[str]]] = None, + answer_coordinates: Optional[List[List[Tuple]]] = None, + answer_text: Optional[List[List[TextInput]]] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TapasTruncationStrategy] = False, + max_length: Optional[int] = None, + pad_to_multiple_of: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_token_type_ids: Optional[bool] = True, + return_attention_mask: Optional[bool] = True, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_length: bool = False, + verbose: bool = True, + prepend_batch_axis: bool = False, + **kwargs + ) -> BatchEncoding: + batch_outputs = {} + + for index, example in enumerate(zip(raw_queries, queries_tokens, answer_coordinates, answer_text)): + raw_query, query_tokens, answer_coords, answer_txt = example + outputs = self.prepare_for_model( + raw_table, + raw_query, + tokenized_table=tokenized_table, + query_tokens=query_tokens, + answer_coordinates=answer_coords, + answer_text=answer_txt, + add_special_tokens=add_special_tokens, + padding=PaddingStrategy.DO_NOT_PAD.value, # we pad in batch afterwards + truncation=truncation, + max_length=max_length, + pad_to_multiple_of=None, # we pad in batch afterwards + return_attention_mask=False, # we pad in batch afterwards + return_token_type_ids=return_token_type_ids, + return_special_tokens_mask=return_special_tokens_mask, + return_length=return_length, + return_tensors=None, # We convert the whole batch to tensors at the end + prepend_batch_axis=False, + verbose=verbose, + prev_answer_coordinates=answer_coordinates[index - 1] if index != 0 else None, + prev_answer_text=answer_text[index - 1] if index != 0 else None, + ) + + for key, value in outputs.items(): + if key not in batch_outputs: + batch_outputs[key] = [] + batch_outputs[key].append(value) + + batch_outputs = self.pad( + batch_outputs, + padding=padding, + max_length=max_length, + pad_to_multiple_of=pad_to_multiple_of, + return_attention_mask=return_attention_mask, + ) + + batch_outputs = BatchEncoding(batch_outputs, tensor_type=return_tensors) + + return batch_outputs + + @add_end_docstrings(ENCODE_KWARGS_DOCSTRING) + def encode( + self, + table: "pd.DataFrame", + query: Optional[ + Union[ + TextInput, + PreTokenizedInput, + EncodedInput, + ] + ] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TapasTruncationStrategy] = False, + max_length: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + **kwargs + ) -> List[int]: + """ + Prepare a table and a string for the model. This method does not return token type IDs, attention masks, etc. + which are necessary for the model to work correctly. Use that method if you want to build your processing on + your own, otherwise refer to ``__call__``. + + Args: + table (:obj:`pd.DataFrame`): + Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas + dataframe to convert it to string. + query (:obj:`str` or :obj:`List[str]`): + Question related to a table to be encoded. + """ + encoded_inputs = self.encode_plus( + table, + query=query, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + **kwargs, + ) + + return encoded_inputs["input_ids"] + + @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING) + def encode_plus( + self, + table: "pd.DataFrame", + query: Optional[ + Union[ + TextInput, + PreTokenizedInput, + EncodedInput, + ] + ] = None, + answer_coordinates: Optional[List[Tuple]] = None, + answer_text: Optional[List[TextInput]] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TapasTruncationStrategy] = False, + max_length: Optional[int] = None, + pad_to_multiple_of: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_token_type_ids: Optional[bool] = None, + return_attention_mask: Optional[bool] = None, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_length: bool = False, + verbose: bool = True, + **kwargs + ) -> BatchEncoding: + """ + Prepare a table and a string for the model. + + Args: + table (:obj:`pd.DataFrame`): + Table containing tabular data. Note that all cell values must be text. Use `.astype(str)` on a Pandas + dataframe to convert it to string. + query (:obj:`str` or :obj:`List[str]`): + Question related to a table to be encoded. + answer_coordinates (:obj:`List[Tuple]` or :obj:`List[List[Tuple]]`, `optional`): + Answer coordinates of each table-question pair in the batch. The answer_coordinates must be a single + list of one or more tuples. Each tuple must be a (row_index, column_index) pair. The first data row + (not the column header row) has index 0. The first column has index 0. + answer_text (:obj:`List[str]` or :obj:`List[List[str]]`, `optional`): + Answer text of each table-question pair in the batch. The answer_text must be a single list of one or + more strings. Each string must be the answer text of a corresponding answer coordinate. + """ + if return_token_type_ids is not None and not add_special_tokens: + raise ValueError( + "Asking to return token_type_ids while setting add_special_tokens to False " + "results in an undefined behavior. Please set add_special_tokens to True or " + "set return_token_type_ids to None." + ) + + if (answer_coordinates and not answer_text) or (not answer_coordinates and answer_text): + raise ValueError("In case you provide answers, both answer_coordinates and answer_text should be provided") + + if "is_split_into_words" in kwargs: + raise NotImplementedError("Currently TapasTokenizer only supports questions as strings.") + + if return_offsets_mapping: + raise NotImplementedError( + "return_offset_mapping is not available when using Python tokenizers." + "To use this feature, change your tokenizer to one deriving from " + "transformers.PreTrainedTokenizerFast." + ) + + return self._encode_plus( + table=table, + query=query, + answer_coordinates=answer_coordinates, + answer_text=answer_text, + add_special_tokens=add_special_tokens, + truncation=truncation, + padding=padding, + max_length=max_length, + pad_to_multiple_of=pad_to_multiple_of, + return_tensors=return_tensors, + return_token_type_ids=return_token_type_ids, + return_attention_mask=return_attention_mask, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_length=return_length, + verbose=verbose, + **kwargs, + ) + + def _encode_plus( + self, + table: "pd.DataFrame", + query: Union[ + TextInput, + PreTokenizedInput, + EncodedInput, + ], + answer_coordinates: Optional[List[Tuple]] = None, + answer_text: Optional[List[TextInput]] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TapasTruncationStrategy] = False, + max_length: Optional[int] = None, + pad_to_multiple_of: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_token_type_ids: Optional[bool] = True, + return_attention_mask: Optional[bool] = True, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_length: bool = False, + verbose: bool = True, + **kwargs + ): + if query is None: + query = "" + logger.warning( + "TAPAS is a question answering model but you have not passed a query. Please be aware that the " + "model will probably not behave correctly." + ) + + table_tokens = self._tokenize_table(table) + query_tokens = self.tokenize(query) + + return self.prepare_for_model( + table, + query, + tokenized_table=table_tokens, + query_tokens=query_tokens, + answer_coordinates=answer_coordinates, + answer_text=answer_text, + add_special_tokens=add_special_tokens, + truncation=truncation, + padding=padding, + max_length=max_length, + pad_to_multiple_of=pad_to_multiple_of, + return_tensors=return_tensors, + prepend_batch_axis=True, + return_attention_mask=return_attention_mask, + return_token_type_ids=return_token_type_ids, + return_special_tokens_mask=return_special_tokens_mask, + return_length=return_length, + verbose=verbose, + ) + + @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING) + def prepare_for_model( + self, + raw_table: "pd.DataFrame", + raw_query: Union[ + TextInput, + PreTokenizedInput, + EncodedInput, + ], + tokenized_table: Optional[TokenizedTable] = None, + query_tokens: Optional[TokenizedTable] = None, + answer_coordinates: Optional[List[Tuple]] = None, + answer_text: Optional[List[TextInput]] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TapasTruncationStrategy] = False, + max_length: Optional[int] = None, + pad_to_multiple_of: Optional[int] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + return_token_type_ids: Optional[bool] = True, + return_attention_mask: Optional[bool] = True, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_length: bool = False, + verbose: bool = True, + prepend_batch_axis: bool = False, + **kwargs + ) -> BatchEncoding: + """ + Prepares a sequence of input id so that it can be used by the model. It adds special tokens, truncates + sequences if overflowing while taking into account the special tokens. + + Args: + raw_table (:obj:`pd.DataFrame`): + The original table before any transformation (like tokenization) was applied to it. + raw_query (:obj:`TextInput` or :obj:`PreTokenizedInput` or :obj:`EncodedInput`): + The original query before any transformation (like tokenization) was applied to it. + tokenized_table (:obj:`TokenizedTable`): + The table after tokenization. + query_tokens (:obj:`List[str]`): + The query after tokenization. + answer_coordinates (:obj:`List[Tuple]` or :obj:`List[List[Tuple]]`, `optional`): + Answer coordinates of each table-question pair in the batch. The answer_coordinates must be a single + list of one or more tuples. Each tuple must be a (row_index, column_index) pair. The first data row + (not the column header row) has index 0. The first column has index 0. + answer_text (:obj:`List[str]` or :obj:`List[List[str]]`, `optional`): + Answer text of each table-question pair in the batch. The answer_text must be a single list of one or + more strings. Each string must be the answer text of a corresponding answer coordinate. + """ + if isinstance(padding, bool): + if padding and (max_length is not None or pad_to_multiple_of is not None): + padding = PaddingStrategy.MAX_LENGTH + else: + padding = PaddingStrategy.DO_NOT_PAD + elif not isinstance(padding, PaddingStrategy): + padding = PaddingStrategy(padding) + + if isinstance(truncation, bool): + if truncation: + truncation = TapasTruncationStrategy.DROP_ROWS_TO_FIT + else: + truncation = TapasTruncationStrategy.DO_NOT_TRUNCATE + elif not isinstance(truncation, TapasTruncationStrategy): + truncation = TapasTruncationStrategy(truncation) + + encoded_inputs = {} + + is_part_of_batch = False + prev_answer_coordinates, prev_answer_text = None, None + if "prev_answer_coordinates" in kwargs and "prev_answer_text" in kwargs: + is_part_of_batch = True + prev_answer_coordinates = kwargs["prev_answer_coordinates"] + prev_answer_text = kwargs["prev_answer_text"] + + num_rows = self._get_num_rows(raw_table, self.drop_rows_to_fit) + num_columns = self._get_num_columns(raw_table) + _, _, num_tokens = self._get_table_boundaries(tokenized_table) + + if truncation != TapasTruncationStrategy.DO_NOT_TRUNCATE: + num_rows, num_tokens = self._get_truncated_table_rows( + query_tokens, tokenized_table, num_rows, num_columns, max_length, truncation_strategy=truncation + ) + table_data = list(self._get_table_values(tokenized_table, num_columns, num_rows, num_tokens)) + + query_ids = self.convert_tokens_to_ids(query_tokens) + table_ids = list(zip(*table_data))[0] if len(table_data) > 0 else list(zip(*table_data)) + table_ids = self.convert_tokens_to_ids(list(table_ids)) + + if "return_overflowing_tokens" in kwargs and kwargs["return_overflowing_tokens"]: + raise ValueError("TAPAS does not return overflowing tokens as it works on tables.") + + if add_special_tokens: + input_ids = self.build_inputs_with_special_tokens(query_ids, table_ids) + else: + input_ids = query_ids + table_ids + + if max_length is not None and len(input_ids) > max_length: + raise ValueError( + "Could not encode the query and table header given the maximum length. Encoding the query and table" + f"header results in a length of {len(input_ids)} which is higher than the max_length of {max_length}" + ) + + encoded_inputs["input_ids"] = input_ids + + segment_ids = self.create_segment_token_type_ids_from_sequences(query_ids, table_data) + column_ids = self.create_column_token_type_ids_from_sequences(query_ids, table_data) + row_ids = self.create_row_token_type_ids_from_sequences(query_ids, table_data) + if not is_part_of_batch or (prev_answer_coordinates is None and prev_answer_text is None): + # simply set the prev_labels to zeros + prev_labels = [0] * len(row_ids) + else: + prev_labels = self.get_answer_ids( + column_ids, row_ids, table_data, prev_answer_text, prev_answer_coordinates + ) + + # FIRST: parse both the table and question in terms of numeric values + + raw_table = add_numeric_table_values(raw_table) + raw_query = add_numeric_values_to_question(raw_query) + + # SECOND: add numeric-related features (and not parse them in these functions): + + column_ranks, inv_column_ranks = self._get_numeric_column_ranks(column_ids, row_ids, raw_table) + numeric_relations = self._get_numeric_relations(raw_query, column_ids, row_ids, raw_table) + + # Load from model defaults + if return_token_type_ids is None: + return_token_type_ids = "token_type_ids" in self.model_input_names + if return_attention_mask is None: + return_attention_mask = "attention_mask" in self.model_input_names + + if return_attention_mask: + attention_mask = self.create_attention_mask_from_sequences(query_ids, table_data) + encoded_inputs["attention_mask"] = attention_mask + + if answer_coordinates is not None and answer_text is not None: + labels = self.get_answer_ids(column_ids, row_ids, table_data, answer_text, answer_coordinates) + numeric_values = self._get_numeric_values(raw_table, column_ids, row_ids) + numeric_values_scale = self._get_numeric_values_scale(raw_table, column_ids, row_ids) + + encoded_inputs["labels"] = labels + encoded_inputs["numeric_values"] = numeric_values + encoded_inputs["numeric_values_scale"] = numeric_values_scale + + if return_token_type_ids: + token_type_ids = [ + segment_ids, + column_ids, + row_ids, + prev_labels, + column_ranks, + inv_column_ranks, + numeric_relations, + ] + + token_type_ids = [list(ids) for ids in list(zip(*token_type_ids))] + encoded_inputs["token_type_ids"] = token_type_ids + + if return_special_tokens_mask: + if add_special_tokens: + encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(query_ids, table_ids) + else: + encoded_inputs["special_tokens_mask"] = [0] * len(input_ids) + + # Check lengths + if max_length is None and len(encoded_inputs["input_ids"]) > self.model_max_length and verbose: + if not self.deprecation_warnings.get("sequence-length-is-longer-than-the-specified-maximum", False): + logger.warning( + "Token indices sequence length is longer than the specified maximum sequence length " + "for this model ({} > {}). Running this sequence through the model will result in " + "indexing errors".format(len(encoded_inputs["input_ids"]), self.model_max_length) + ) + self.deprecation_warnings["sequence-length-is-longer-than-the-specified-maximum"] = True + + # Padding + if padding != PaddingStrategy.DO_NOT_PAD or return_attention_mask: + encoded_inputs = self.pad( + encoded_inputs, + max_length=max_length, + padding=padding.value, + pad_to_multiple_of=pad_to_multiple_of, + return_attention_mask=return_attention_mask, + ) + + if return_length: + encoded_inputs["length"] = len(encoded_inputs["input_ids"]) + + batch_outputs = BatchEncoding( + encoded_inputs, tensor_type=return_tensors, prepend_batch_axis=prepend_batch_axis + ) + + return batch_outputs + + def _get_truncated_table_rows( + self, + query_tokens: List[str], + tokenized_table: TokenizedTable, + num_rows: int, + num_columns: int, + max_length: int, + truncation_strategy: Union[str, TapasTruncationStrategy], + ) -> Tuple[int, int]: + """ + Truncates a sequence pair in-place following the strategy. + + Args: + query_tokens (:obj:`List[str]`): + List of strings corresponding to the tokenized query. + tokenized_table (:obj:`TokenizedTable`): + Tokenized table + num_rows (:obj:`int`): + Total number of table rows + num_columns (:obj:`int`): + Total number of table columns + max_length (:obj:`int`): + Total maximum length. + truncation_strategy (:obj:`str` or :obj:`~transformers.TapasTruncationStrategy`): + Truncation strategy to use. Seeing as this method should only be called when truncating, the only + available strategy is the :obj:`"drop_rows_to_fit"` strategy. + + Returns: + :obj:`Tuple(int, int)`: tuple containing the number of rows after truncation, and the number of tokens + available for each table element. + """ + if not isinstance(truncation_strategy, TapasTruncationStrategy): + truncation_strategy = TapasTruncationStrategy(truncation_strategy) + + if max_length is None: + max_length = self.model_max_length + + if truncation_strategy == TapasTruncationStrategy.DROP_ROWS_TO_FIT: + while True: + num_tokens = self._get_max_num_tokens( + query_tokens, tokenized_table, num_rows=num_rows, num_columns=num_columns, max_length=max_length + ) + + if num_tokens is not None: + # We could fit the table. + break + + # Try to drop a row to fit the table. + num_rows -= 1 + + if num_rows < 1: + break + elif truncation_strategy != TapasTruncationStrategy.DO_NOT_TRUNCATE: + raise ValueError(f"Unknown truncation strategy {truncation_strategy}.") + + return num_rows, num_tokens or 1 + + def _tokenize_table( + self, + table=None, + ): + """ + Tokenizes column headers and cell texts of a table. + + Args: + table (:obj:`pd.Dataframe`): + Table. Returns: :obj:`TokenizedTable`: TokenizedTable object. + """ + tokenized_rows = [] + tokenized_row = [] + # tokenize column headers + for column in table: + if self.strip_column_names: + tokenized_row.append(self.tokenize("")) + else: + tokenized_row.append(self.tokenize(column)) + tokenized_rows.append(tokenized_row) + + # tokenize cell values + for idx, row in table.iterrows(): + tokenized_row = [] + for cell in row: + tokenized_row.append(self.tokenize(cell)) + tokenized_rows.append(tokenized_row) + + token_coordinates = [] + for row_index, row in enumerate(tokenized_rows): + for column_index, cell in enumerate(row): + for token_index, _ in enumerate(cell): + token_coordinates.append( + TokenCoordinates( + row_index=row_index, + column_index=column_index, + token_index=token_index, + ) + ) + + return TokenizedTable( + rows=tokenized_rows, + selected_tokens=token_coordinates, + ) + + def _question_encoding_cost(self, question_tokens): + # Two extra spots of SEP and CLS. + return len(question_tokens) + 2 + + def _get_token_budget(self, question_tokens, max_length=None): + """ + Computes the number of tokens left for the table after tokenizing a question, taking into account the max + sequence length of the model. + + Args: + question_tokens (:obj:`List[String]`): + List of question tokens. Returns: :obj:`int`: the number of tokens left for the table, given the model + max length. + """ + return (max_length if max_length is not None else self.model_max_length) - self._question_encoding_cost( + question_tokens + ) + + def _get_table_values(self, table, num_columns, num_rows, num_tokens) -> Generator[TableValue, None, None]: + """Iterates over partial table and returns token, column and row indexes.""" + for tc in table.selected_tokens: + # First row is header row. + if tc.row_index >= num_rows + 1: + continue + if tc.column_index >= num_columns: + continue + cell = table.rows[tc.row_index][tc.column_index] + token = cell[tc.token_index] + word_begin_index = tc.token_index + # Don't add partial words. Find the starting word piece and check if it + # fits in the token budget. + while word_begin_index >= 0 and _is_inner_wordpiece(cell[word_begin_index]): + word_begin_index -= 1 + if word_begin_index >= num_tokens: + continue + yield TableValue(token, tc.column_index + 1, tc.row_index) + + def _get_table_boundaries(self, table): + """Return maximal number of rows, columns and tokens.""" + max_num_tokens = 0 + max_num_columns = 0 + max_num_rows = 0 + for tc in table.selected_tokens: + max_num_columns = max(max_num_columns, tc.column_index + 1) + max_num_rows = max(max_num_rows, tc.row_index + 1) + max_num_tokens = max(max_num_tokens, tc.token_index + 1) + max_num_columns = min(self.max_column_id, max_num_columns) + max_num_rows = min(self.max_row_id, max_num_rows) + return max_num_rows, max_num_columns, max_num_tokens + + def _get_table_cost(self, table, num_columns, num_rows, num_tokens): + return sum(1 for _ in self._get_table_values(table, num_columns, num_rows, num_tokens)) + + def _get_max_num_tokens(self, question_tokens, tokenized_table, num_columns, num_rows, max_length): + """Computes max number of tokens that can be squeezed into the budget.""" + token_budget = self._get_token_budget(question_tokens, max_length) + _, _, max_num_tokens = self._get_table_boundaries(tokenized_table) + if self.cell_trim_length >= 0 and max_num_tokens > self.cell_trim_length: + max_num_tokens = self.cell_trim_length + num_tokens = 0 + for num_tokens in range(max_num_tokens + 1): + cost = self._get_table_cost(tokenized_table, num_columns, num_rows, num_tokens + 1) + if cost > token_budget: + break + if num_tokens < max_num_tokens: + if self.cell_trim_length >= 0: + # We don't allow dynamic trimming if a cell_trim_length is set. + return None + if num_tokens == 0: + return None + return num_tokens + + def _get_num_columns(self, table): + num_columns = table.shape[1] + if num_columns >= self.max_column_id: + raise ValueError("Too many columns") + return num_columns + + def _get_num_rows(self, table, drop_rows_to_fit): + num_rows = table.shape[0] + if num_rows >= self.max_row_id: + if drop_rows_to_fit: + num_rows = self.max_row_id - 1 + else: + raise ValueError("Too many rows") + return num_rows + + def _serialize_text(self, question_tokens): + """Serializes texts in index arrays.""" + tokens = [] + segment_ids = [] + column_ids = [] + row_ids = [] + + # add [CLS] token at the beginning + tokens.append(self.cls_token) + segment_ids.append(0) + column_ids.append(0) + row_ids.append(0) + + for token in question_tokens: + tokens.append(token) + segment_ids.append(0) + column_ids.append(0) + row_ids.append(0) + + return tokens, segment_ids, column_ids, row_ids + + def _serialize( + self, + question_tokens, + table, + num_columns, + num_rows, + num_tokens, + ): + """Serializes table and text.""" + tokens, segment_ids, column_ids, row_ids = self._serialize_text(question_tokens) + + # add [SEP] token between question and table tokens + tokens.append(self.sep_token) + segment_ids.append(0) + column_ids.append(0) + row_ids.append(0) + + for token, column_id, row_id in self._get_table_values(table, num_columns, num_rows, num_tokens): + tokens.append(token) + segment_ids.append(1) + column_ids.append(column_id) + row_ids.append(row_id) + + return SerializedExample( + tokens=tokens, + segment_ids=segment_ids, + column_ids=column_ids, + row_ids=row_ids, + ) + + def _get_column_values(self, table, col_index): + table_numeric_values = {} + for row_index, row in table.iterrows(): + cell = row[col_index] + if cell.numeric_value is not None: + table_numeric_values[row_index] = cell.numeric_value + return table_numeric_values + + def _get_cell_token_indexes(self, column_ids, row_ids, column_id, row_id): + for index in range(len(column_ids)): + if column_ids[index] - 1 == column_id and row_ids[index] - 1 == row_id: + yield index + + def _get_numeric_column_ranks(self, column_ids, row_ids, table): + """Returns column ranks for all numeric columns.""" + + ranks = [0] * len(column_ids) + inv_ranks = [0] * len(column_ids) + + # original code from tf_example_utils.py of the original implementation + if table is not None: + for col_index in range(len(table.columns)): + table_numeric_values = self._get_column_values(table, col_index) + + if not table_numeric_values: + continue + + try: + key_fn = get_numeric_sort_key_fn(table_numeric_values.values()) + except ValueError: + continue + + table_numeric_values = {row_index: key_fn(value) for row_index, value in table_numeric_values.items()} + + table_numeric_values_inv = collections.defaultdict(list) + for row_index, value in table_numeric_values.items(): + table_numeric_values_inv[value].append(row_index) + + unique_values = sorted(table_numeric_values_inv.keys()) + + for rank, value in enumerate(unique_values): + for row_index in table_numeric_values_inv[value]: + for index in self._get_cell_token_indexes(column_ids, row_ids, col_index, row_index): + ranks[index] = rank + 1 + inv_ranks[index] = len(unique_values) - rank + + return ranks, inv_ranks + + def _get_numeric_sort_key_fn(self, table_numeric_values, value): + """ + Returns the sort key function for comparing value to table values. The function returned will be a suitable + input for the key param of the sort(). See number_annotation_utils._get_numeric_sort_key_fn for details + + Args: + table_numeric_values: Numeric values of a column + value: Numeric value in the question + + Returns: + A function key function to compare column and question values. + """ + if not table_numeric_values: + return None + all_values = list(table_numeric_values.values()) + all_values.append(value) + try: + return get_numeric_sort_key_fn(all_values) + except ValueError: + return None + + def _get_numeric_relations(self, question, column_ids, row_ids, table): + """ + Returns numeric relations embeddings + + Args: + question: Question object. + column_ids: Maps word piece position to column id. + row_ids: Maps word piece position to row id. + table: The table containing the numeric cell values. + """ + + numeric_relations = [0] * len(column_ids) + + # first, we add any numeric value spans to the question: + # Create a dictionary that maps a table cell to the set of all relations + # this cell has with any value in the question. + cell_indices_to_relations = collections.defaultdict(set) + if question is not None and table is not None: + for numeric_value_span in question.numeric_spans: + for value in numeric_value_span.values: + for column_index in range(len(table.columns)): + table_numeric_values = self._get_column_values(table, column_index) + sort_key_fn = self._get_numeric_sort_key_fn(table_numeric_values, value) + if sort_key_fn is None: + continue + for row_index, cell_value in table_numeric_values.items(): + relation = get_numeric_relation(value, cell_value, sort_key_fn) + if relation is not None: + cell_indices_to_relations[column_index, row_index].add(relation) + + # For each cell add a special feature for all its word pieces. + for (column_index, row_index), relations in cell_indices_to_relations.items(): + relation_set_index = 0 + for relation in relations: + assert relation.value >= Relation.EQ.value + relation_set_index += 2 ** (relation.value - Relation.EQ.value) + for cell_token_index in self._get_cell_token_indexes(column_ids, row_ids, column_index, row_index): + numeric_relations[cell_token_index] = relation_set_index + + return numeric_relations + + def _get_numeric_values(self, table, column_ids, row_ids): + """Returns numeric values for computation of answer loss.""" + + numeric_values = [float("nan")] * len(column_ids) + + if table is not None: + num_rows = table.shape[0] + num_columns = table.shape[1] + + for col_index in range(num_columns): + for row_index in range(num_rows): + numeric_value = table.iloc[row_index, col_index].numeric_value + if numeric_value is not None: + if numeric_value.float_value is None: + continue + float_value = numeric_value.float_value + if float_value == float("inf"): + continue + for index in self._get_cell_token_indexes(column_ids, row_ids, col_index, row_index): + numeric_values[index] = float_value + + return numeric_values + + def _get_numeric_values_scale(self, table, column_ids, row_ids): + """Returns a scale to each token to down weigh the value of long words.""" + + numeric_values_scale = [1.0] * len(column_ids) + + if table is None: + return numeric_values_scale + + num_rows = table.shape[0] + num_columns = table.shape[1] + + for col_index in range(num_columns): + for row_index in range(num_rows): + indices = [index for index in self._get_cell_token_indexes(column_ids, row_ids, col_index, row_index)] + num_indices = len(indices) + if num_indices > 1: + for index in indices: + numeric_values_scale[index] = float(num_indices) + + return numeric_values_scale + + def _pad_to_seq_length(self, inputs): + while len(inputs) > self.model_max_length: + inputs.pop() + while len(inputs) < self.model_max_length: + inputs.append(0) + + def _get_all_answer_ids_from_coordinates( + self, + column_ids, + row_ids, + answers_list, + ): + """Maps lists of answer coordinates to token indexes.""" + answer_ids = [0] * len(column_ids) + found_answers = set() + all_answers = set() + for answers in answers_list: + column_index, row_index = answers + all_answers.add((column_index, row_index)) + for index in self._get_cell_token_indexes(column_ids, row_ids, column_index, row_index): + found_answers.add((column_index, row_index)) + answer_ids[index] = 1 + + missing_count = len(all_answers) - len(found_answers) + return answer_ids, missing_count + + def _get_all_answer_ids(self, column_ids, row_ids, answer_coordinates): + """ + Maps answer coordinates of a question to token indexes. + + In the SQA format (TSV), the coordinates are given as (row, column) tuples. Here, we first swap them to + (column, row) format before calling _get_all_answer_ids_from_coordinates. + """ + + def _to_coordinates(answer_coordinates_question): + return [(coords[1], coords[0]) for coords in answer_coordinates_question] + + return self._get_all_answer_ids_from_coordinates( + column_ids, row_ids, answers_list=(_to_coordinates(answer_coordinates)) + ) + + def _find_tokens(self, text, segment): + """Return start index of segment in text or None.""" + logging.info("text: %s %s", text, segment) + for index in range(1 + len(text) - len(segment)): + for seg_index, seg_token in enumerate(segment): + if text[index + seg_index].piece != seg_token.piece: + break + else: + return index + return None + + def _find_answer_coordinates_from_answer_text( + self, + tokenized_table, + answer_text, + ): + """Returns all occurrences of answer_text in the table.""" + logging.info("answer text: %s", answer_text) + for row_index, row in enumerate(tokenized_table.rows): + if row_index == 0: + # We don't search for answers in the header. + continue + for col_index, cell in enumerate(row): + token_index = self._find_tokens(cell, answer_text) + if token_index is not None: + yield TokenCoordinates( + row_index=row_index, + column_index=col_index, + token_index=token_index, + ) + + def _find_answer_ids_from_answer_texts( + self, + column_ids, + row_ids, + tokenized_table, + answer_texts, + ): + """Maps question with answer texts to the first matching token indexes.""" + answer_ids = [0] * len(column_ids) + for answer_text in answer_texts: + for coordinates in self._find_answer_coordinates_from_answer_text( + tokenized_table, + answer_text, + ): + # Maps answer coordinates to indexes this can fail if tokens / rows have + # been pruned. + indexes = list( + self._get_cell_token_indexes( + column_ids, + row_ids, + column_id=coordinates.column_index, + row_id=coordinates.row_index - 1, + ) + ) + indexes.sort() + coordinate_answer_ids = [] + if indexes: + begin_index = coordinates.token_index + indexes[0] + end_index = begin_index + len(answer_text) + for index in indexes: + if index >= begin_index and index < end_index: + coordinate_answer_ids.append(index) + if len(coordinate_answer_ids) == len(answer_text): + for index in coordinate_answer_ids: + answer_ids[index] = 1 + break + return answer_ids + + def _get_answer_ids(self, column_ids, row_ids, answer_coordinates): + """Maps answer coordinates of a question to token indexes.""" + answer_ids, missing_count = self._get_all_answer_ids(column_ids, row_ids, answer_coordinates) + + if missing_count: + raise ValueError("Couldn't find all answers") + return answer_ids + + def get_answer_ids(self, column_ids, row_ids, tokenized_table, answer_texts_question, answer_coordinates_question): + if self.update_answer_coordinates: + return self._find_answer_ids_from_answer_texts( + column_ids, + row_ids, + tokenized_table, + answer_texts=[self.tokenize(at) for at in answer_texts_question], + ) + return self._get_answer_ids(column_ids, row_ids, answer_coordinates_question) + + def _pad( + self, + encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], + max_length: Optional[int] = None, + padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, + pad_to_multiple_of: Optional[int] = None, + return_attention_mask: Optional[bool] = None, + ) -> dict: + """ + Pad encoded inputs (on left/right and up to predefined length or max length in the batch) + + Args: + encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`). + max_length: maximum length of the returned list and optionally padding length (see below). + Will truncate by taking into account the special tokens. + padding_strategy: PaddingStrategy to use for padding. + + - PaddingStrategy.LONGEST Pad to the longest sequence in the batch + - PaddingStrategy.MAX_LENGTH: Pad to the max length (default) + - PaddingStrategy.DO_NOT_PAD: Do not pad + The tokenizer padding sides are defined in self.padding_side: + + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability + >= 7.5 (Volta). + return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics) + """ + # Load from model defaults + if return_attention_mask is None: + return_attention_mask = "attention_mask" in self.model_input_names + + if padding_strategy == PaddingStrategy.LONGEST: + max_length = len(encoded_inputs["input_ids"]) + + if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0): + max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of + + needs_to_be_padded = ( + padding_strategy != PaddingStrategy.DO_NOT_PAD and len(encoded_inputs["input_ids"]) != max_length + ) + + if needs_to_be_padded: + difference = max_length - len(encoded_inputs["input_ids"]) + if self.padding_side == "right": + if return_attention_mask: + encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + [0] * difference + if "token_type_ids" in encoded_inputs: + encoded_inputs["token_type_ids"] = ( + encoded_inputs["token_type_ids"] + [[self.pad_token_type_id] * 7] * difference + ) + if "labels" in encoded_inputs: + encoded_inputs["labels"] = encoded_inputs["labels"] + [0] * difference + if "numeric_values" in encoded_inputs: + encoded_inputs["numeric_values"] = encoded_inputs["numeric_values"] + [float("nan")] * difference + if "numeric_values_scale" in encoded_inputs: + encoded_inputs["numeric_values_scale"] = ( + encoded_inputs["numeric_values_scale"] + [1.0] * difference + ) + if "special_tokens_mask" in encoded_inputs: + encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference + encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference + elif self.padding_side == "left": + if return_attention_mask: + encoded_inputs["attention_mask"] = [0] * difference + [1] * len(encoded_inputs["input_ids"]) + if "token_type_ids" in encoded_inputs: + encoded_inputs["token_type_ids"] = [[self.pad_token_type_id] * 7] * difference + encoded_inputs[ + "token_type_ids" + ] + if "labels" in encoded_inputs: + encoded_inputs["labels"] = [0] * difference + encoded_inputs["labels"] + if "numeric_values" in encoded_inputs: + encoded_inputs["numeric_values"] = [float("nan")] * difference + encoded_inputs["numeric_values"] + if "numeric_values_scale" in encoded_inputs: + encoded_inputs["numeric_values_scale"] = [1.0] * difference + encoded_inputs[ + "numeric_values_scale" + ] + if "special_tokens_mask" in encoded_inputs: + encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"] + encoded_inputs["input_ids"] = [self.pad_token_id] * difference + encoded_inputs["input_ids"] + else: + raise ValueError("Invalid padding strategy:" + str(self.padding_side)) + else: + if return_attention_mask: + encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + + return encoded_inputs + + # Everything related to converting logits to predictions + + def _get_cell_token_probs(self, probabilities, segment_ids, row_ids, column_ids): + for i, p in enumerate(probabilities): + segment_id = segment_ids[i] + col = column_ids[i] - 1 + row = row_ids[i] - 1 + if col >= 0 and row >= 0 and segment_id == 1: + yield i, p + + def _get_mean_cell_probs(self, probabilities, segment_ids, row_ids, column_ids): + """Computes average probability per cell, aggregating over tokens.""" + coords_to_probs = collections.defaultdict(list) + for i, prob in self._get_cell_token_probs(probabilities, segment_ids, row_ids, column_ids): + col = column_ids[i] - 1 + row = row_ids[i] - 1 + coords_to_probs[(col, row)].append(prob) + return {coords: np.array(cell_probs).mean() for coords, cell_probs in coords_to_probs.items()} + + def convert_logits_to_predictions(self, data, logits, logits_agg=None, cell_classification_threshold=0.5): + """ + Converts logits of :class:`~transformers.TapasForQuestionAnswering` to actual predicted answer coordinates and + optional aggregation indices. + + The original implementation, on which this function is based, can be found `here + `__. + + Args: + data (:obj:`dict`): + Dictionary mapping features to actual values. Should be created using + :class:`~transformers.TapasTokenizer`. + logits (:obj:`np.ndarray` of shape ``(batch_size, sequence_length)``): + Tensor containing the logits at the token level. + logits_agg (:obj:`np.ndarray` of shape ``(batch_size, num_aggregation_labels)``, `optional`): + Tensor containing the aggregation logits. + cell_classification_threshold (:obj:`float`, `optional`, defaults to 0.5): + Threshold to be used for cell selection. All table cells for which their probability is larger than + this threshold will be selected. + + Returns: + :obj:`tuple` comprising various elements depending on the inputs: predicted_answer_coordinates + (``List[List[[tuple]]`` of length ``batch_size``): Predicted answer coordinates as a list of lists of + tuples. Each element in the list contains the predicted answer coordinates of a single example in the + batch, as a list of tuples. Each tuple is a cell, i.e. (row index, column index). + predicted_aggregation_indices (`optional`, returned when ``logits_aggregation`` is provided) ``List[int]`` + of length ``batch_size``: Predicted aggregation operator indices of the aggregation head. + """ + # input data is of type float32 + # np.log(np.finfo(np.float32).max) = 88.72284 + # Any value over 88.72284 will overflow when passed through the exponential, sending a warning + # We disable this warning by truncating the logits. + logits[logits < -88.7] = -88.7 + + # Compute probabilities from token logits + probabilities = 1 / (1 + np.exp(-logits)) * data["attention_mask"] + token_types = [ + "segment_ids", + "column_ids", + "row_ids", + "prev_labels", + "column_ranks", + "inv_column_ranks", + "numeric_relations", + ] + + # collect input_ids, segment ids, row ids and column ids of batch. Shape (batch_size, seq_len) + input_ids = data["input_ids"] + segment_ids = data["token_type_ids"][:, :, token_types.index("segment_ids")] + row_ids = data["token_type_ids"][:, :, token_types.index("row_ids")] + column_ids = data["token_type_ids"][:, :, token_types.index("column_ids")] + + # next, get answer coordinates for every example in the batch + num_batch = input_ids.shape[0] + predicted_answer_coordinates = [] + for i in range(num_batch): + probabilities_example = probabilities[i].tolist() + segment_ids_example = segment_ids[i] + row_ids_example = row_ids[i] + column_ids_example = column_ids[i] + + max_width = column_ids_example.max() + max_height = row_ids_example.max() + + if max_width == 0 and max_height == 0: + continue + + cell_coords_to_prob = self._get_mean_cell_probs( + probabilities_example, + segment_ids_example.tolist(), + row_ids_example.tolist(), + column_ids_example.tolist(), + ) + + # Select the answers above the classification threshold. + answer_coordinates = [] + for col in range(max_width): + for row in range(max_height): + cell_prob = cell_coords_to_prob.get((col, row), None) + if cell_prob is not None: + if cell_prob > cell_classification_threshold: + answer_coordinates.append((row, col)) + answer_coordinates = sorted(answer_coordinates) + predicted_answer_coordinates.append(answer_coordinates) + + output = predicted_answer_coordinates + + if logits_agg is not None: + predicted_aggregation_indices = logits_agg.argmax(dim=-1) + output = (output, predicted_aggregation_indices.tolist()) + + return output + + # End of everything related to converting logits to predictions + + +# Copied from transformers.models.bert.tokenization_bert.BasicTokenizer +class BasicTokenizer(object): + """ + Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.). + + Args: + do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to lowercase the input when tokenizing. + never_split (:obj:`Iterable`, `optional`): + Collection of tokens which will never be split during tokenization. Only has an effect when + :obj:`do_basic_tokenize=True` + tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not to tokenize Chinese characters. + + This should likely be deactivated for Japanese (see this `issue + `__). + strip_accents: (:obj:`bool`, `optional`): + Whether or not to strip all accents. If this option is not specified, then it will be determined by the + value for :obj:`lowercase` (as in the original BERT). + """ + + def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True, strip_accents=None): + if never_split is None: + never_split = [] + self.do_lower_case = do_lower_case + self.never_split = set(never_split) + self.tokenize_chinese_chars = tokenize_chinese_chars + self.strip_accents = strip_accents + + def tokenize(self, text, never_split=None): + """ + Basic Tokenization of a piece of text. Split on "white spaces" only, for sub-word tokenization, see + WordPieceTokenizer. + + Args: + **never_split**: (`optional`) list of str + Kept for backward compatibility purposes. Now implemented directly at the base class level (see + :func:`PreTrainedTokenizer.tokenize`) List of token not to split. + """ + # union() returns a new set by concatenating the two sets. + never_split = self.never_split.union(set(never_split)) if never_split else self.never_split + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + if self.tokenize_chinese_chars: + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if token not in never_split: + if self.do_lower_case: + token = token.lower() + if self.strip_accents is not False: + token = self._run_strip_accents(token) + elif self.strip_accents: + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token, never_split)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text, never_split=None): + """Splits punctuation on a piece of text.""" + if never_split is not None and text in never_split: + return [text] + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xFFFD or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +# Copied from transformers.models.bert.tokenization_bert.WordpieceTokenizer +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token, max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """ + Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform + tokenization using the given vocabulary. + + For example, :obj:`input = "unaffable"` wil return as output :obj:`["un", "##aff", "##able"]`. + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer`. + + Returns: + A list of wordpiece tokens. + """ + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +# Below: utilities for TAPAS tokenizer (independent from PyTorch/Tensorflow). +# This includes functions to parse numeric values (dates and numbers) from both the table and questions in order +# to create the column_ranks, inv_column_ranks, numeric_values, numeric values_scale and numeric_relations in +# prepare_for_model of TapasTokenizer. +# These are meant to be used in an academic setup, for production use cases Gold mine or Aqua should be used. + + +# taken from constants.py of the original implementation +# URL: https://github.com/google-research/tapas/blob/master/tapas/utils/constants.py +class Relation(enum.Enum): + HEADER_TO_CELL = 1 # Connects header to cell. + CELL_TO_HEADER = 2 # Connects cell to header. + QUERY_TO_HEADER = 3 # Connects query to headers. + QUERY_TO_CELL = 4 # Connects query to cells. + ROW_TO_CELL = 5 # Connects row to cells. + CELL_TO_ROW = 6 # Connects cells to row. + EQ = 7 # Annotation value is same as cell value + LT = 8 # Annotation value is less than cell value + GT = 9 # Annotation value is greater than cell value + + +@dataclass +class Date: + year: Optional[int] = None + month: Optional[int] = None + day: Optional[int] = None + + +@dataclass +class NumericValue: + float_value: Optional[float] = None + date: Optional[Date] = None + + +@dataclass +class NumericValueSpan: + begin_index: int = None + end_index: int = None + values: List[NumericValue] = None + + +@dataclass +class Cell: + text: Text + numeric_value: Optional[NumericValue] = None + + +@dataclass +class Question: + original_text: Text # The original raw question string. + text: Text # The question string after normalization. + numeric_spans: Optional[List[NumericValueSpan]] = None + + +# Below: all functions from number_utils.py as well as 2 functions (namely get_all_spans and normalize_for_match) +# from text_utils.py of the original implementation. URL's: +# - https://github.com/google-research/tapas/blob/master/tapas/utils/number_utils.py +# - https://github.com/google-research/tapas/blob/master/tapas/utils/text_utils.py + + +# Constants for parsing date expressions. +# Masks that specify (by a bool) which of (year, month, day) will be populated. +_DateMask = collections.namedtuple("_DateMask", ["year", "month", "day"]) + +_YEAR = _DateMask(True, False, False) +_YEAR_MONTH = _DateMask(True, True, False) +_YEAR_MONTH_DAY = _DateMask(True, True, True) +_MONTH = _DateMask(False, True, False) +_MONTH_DAY = _DateMask(False, True, True) + +# Pairs of patterns to pass to 'datetime.strptime' and masks specifying which +# fields will be set by the corresponding pattern. +_DATE_PATTERNS = ( + ("%B", _MONTH), + ("%Y", _YEAR), + ("%Ys", _YEAR), + ("%b %Y", _YEAR_MONTH), + ("%B %Y", _YEAR_MONTH), + ("%B %d", _MONTH_DAY), + ("%b %d", _MONTH_DAY), + ("%d %b", _MONTH_DAY), + ("%d %B", _MONTH_DAY), + ("%B %d, %Y", _YEAR_MONTH_DAY), + ("%d %B %Y", _YEAR_MONTH_DAY), + ("%m-%d-%Y", _YEAR_MONTH_DAY), + ("%Y-%m-%d", _YEAR_MONTH_DAY), + ("%Y-%m", _YEAR_MONTH), + ("%B %Y", _YEAR_MONTH), + ("%d %b %Y", _YEAR_MONTH_DAY), + ("%Y-%m-%d", _YEAR_MONTH_DAY), + ("%b %d, %Y", _YEAR_MONTH_DAY), + ("%d.%m.%Y", _YEAR_MONTH_DAY), + ("%A, %b %d", _MONTH_DAY), + ("%A, %B %d", _MONTH_DAY), +) + +# This mapping is used to convert date patterns to regex patterns. +_FIELD_TO_REGEX = ( + ("%A", r"\w+"), # Weekday as locale’s full name. + ("%B", r"\w+"), # Month as locale’s full name. + ("%Y", r"\d{4}"), # Year with century as a decimal number. + ("%b", r"\w{3}"), # Month as locale’s abbreviated name. + ("%d", r"\d{1,2}"), # Day of the month as a zero-padded decimal number. + ("%m", r"\d{1,2}"), # Month as a zero-padded decimal number. +) + + +def _process_date_pattern(dp): + """Compute a regex for each date pattern to use as a prefilter.""" + pattern, mask = dp + regex = pattern + regex = regex.replace(".", re.escape(".")) + regex = regex.replace("-", re.escape("-")) + regex = regex.replace(" ", r"\s+") + for field, field_regex in _FIELD_TO_REGEX: + regex = regex.replace(field, field_regex) + # Make sure we didn't miss any of the fields. + assert "%" not in regex, regex + return pattern, mask, re.compile("^" + regex + "$") + + +def _process_date_patterns(): + return tuple(_process_date_pattern(dp) for dp in _DATE_PATTERNS) + + +_PROCESSED_DATE_PATTERNS = _process_date_patterns() + +_MAX_DATE_NGRAM_SIZE = 5 + +# Following DynSp: +# https://github.com/Microsoft/DynSP/blob/master/util.py#L414. +_NUMBER_WORDS = [ + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", +] + +_ORDINAL_WORDS = [ + "zeroth", + "first", + "second", + "third", + "fourth", + "fith", + "sixth", + "seventh", + "eighth", + "ninth", + "tenth", + "eleventh", + "twelfth", +] + +_ORDINAL_SUFFIXES = ["st", "nd", "rd", "th"] + +_NUMBER_PATTERN = re.compile(r"((^|\s)[+-])?((\.\d+)|(\d+(,\d\d\d)*(\.\d*)?))") + +# Following DynSp: +# https://github.com/Microsoft/DynSP/blob/master/util.py#L293. +_MIN_YEAR = 1700 +_MAX_YEAR = 2016 + +_INF = float("INF") + + +def _get_numeric_value_from_date(date, mask): + """Converts date (datetime Python object) to a NumericValue object with a Date object value.""" + if date.year < _MIN_YEAR or date.year > _MAX_YEAR: + raise ValueError("Invalid year: %d" % date.year) + + new_date = Date() + if mask.year: + new_date.year = date.year + if mask.month: + new_date.month = date.month + if mask.day: + new_date.day = date.day + return NumericValue(date=new_date) + + +def _get_span_length_key(span): + """Sorts span by decreasing length first and incresing first index second.""" + return span[1] - span[0], -span[0] + + +def _get_numeric_value_from_float(value): + """Converts float (Python) to a NumericValue object with a float value.""" + return NumericValue(float_value=value) + + +# Doesn't parse ordinal expressions such as '18th of february 1655'. +def _parse_date(text): + """Attempts to format a text as a standard date string (yyyy-mm-dd).""" + text = re.sub(r"Sept\b", "Sep", text) + for in_pattern, mask, regex in _PROCESSED_DATE_PATTERNS: + if not regex.match(text): + continue + try: + date = datetime.datetime.strptime(text, in_pattern).date() + except ValueError: + continue + try: + return _get_numeric_value_from_date(date, mask) + except ValueError: + continue + return None + + +def _parse_number(text): + """Parses simple cardinal and ordinals numbers.""" + for suffix in _ORDINAL_SUFFIXES: + if text.endswith(suffix): + text = text[: -len(suffix)] + break + text = text.replace(",", "") + try: + value = float(text) + except ValueError: + return None + if math.isnan(value): + return None + if value == _INF: + return None + return value + + +def get_all_spans(text, max_ngram_length): + """ + Split a text into all possible ngrams up to 'max_ngram_length'. Split points are white space and punctuation. + + Args: + text: Text to split. + max_ngram_length: maximal ngram length. + Yields: + Spans, tuples of begin-end index. + """ + start_indexes = [] + for index, char in enumerate(text): + if not char.isalnum(): + continue + if index == 0 or not text[index - 1].isalnum(): + start_indexes.append(index) + if index + 1 == len(text) or not text[index + 1].isalnum(): + for start_index in start_indexes[-max_ngram_length:]: + yield start_index, index + 1 + + +def normalize_for_match(text): + return " ".join(text.lower().split()) + + +def format_text(text): + """Lowercases and strips punctuation.""" + text = text.lower().strip() + if text == "n/a" or text == "?" or text == "nan": + text = EMPTY_TEXT + + text = re.sub(r"[^\w\d]+", " ", text).replace("_", " ") + text = " ".join(text.split()) + text = text.strip() + if text: + return text + return EMPTY_TEXT + + +def parse_text(text): + """ + Extracts longest number and date spans. + + Args: + text: text to annotate + + Returns: + List of longest numeric value spans. + """ + span_dict = collections.defaultdict(list) + for match in _NUMBER_PATTERN.finditer(text): + span_text = text[match.start() : match.end()] + number = _parse_number(span_text) + if number is not None: + span_dict[match.span()].append(_get_numeric_value_from_float(number)) + + for begin_index, end_index in get_all_spans(text, max_ngram_length=1): + if (begin_index, end_index) in span_dict: + continue + span_text = text[begin_index:end_index] + + number = _parse_number(span_text) + if number is not None: + span_dict[begin_index, end_index].append(_get_numeric_value_from_float(number)) + for number, word in enumerate(_NUMBER_WORDS): + if span_text == word: + span_dict[begin_index, end_index].append(_get_numeric_value_from_float(float(number))) + break + for number, word in enumerate(_ORDINAL_WORDS): + if span_text == word: + span_dict[begin_index, end_index].append(_get_numeric_value_from_float(float(number))) + break + + for begin_index, end_index in get_all_spans(text, max_ngram_length=_MAX_DATE_NGRAM_SIZE): + span_text = text[begin_index:end_index] + date = _parse_date(span_text) + if date is not None: + span_dict[begin_index, end_index].append(date) + + spans = sorted(span_dict.items(), key=lambda span_value: _get_span_length_key(span_value[0]), reverse=True) + selected_spans = [] + for span, value in spans: + for selected_span, _ in selected_spans: + if selected_span[0] <= span[0] and span[1] <= selected_span[1]: + break + else: + selected_spans.append((span, value)) + + selected_spans.sort(key=lambda span_value: span_value[0][0]) + + numeric_value_spans = [] + for span, values in selected_spans: + numeric_value_spans.append(NumericValueSpan(begin_index=span[0], end_index=span[1], values=values)) + return numeric_value_spans + + +# Below: all functions from number_annotation_utils.py and 2 functions (namely filter_invalid_unicode +# and filter_invalid_unicode_from_table) from text_utils.py of the original implementation. URL's: +# - https://github.com/google-research/tapas/blob/master/tapas/utils/number_annotation_utils.py +# - https://github.com/google-research/tapas/blob/master/tapas/utils/text_utils.py + + +_PrimitiveNumericValue = Union[float, Tuple[Optional[float], Optional[float], Optional[float]]] +_SortKeyFn = Callable[[NumericValue], Tuple[float, Ellipsis]] + +_DATE_TUPLE_SIZE = 3 + +EMPTY_TEXT = "EMPTY" + +NUMBER_TYPE = "number" +DATE_TYPE = "date" + + +def _get_value_type(numeric_value): + if numeric_value.float_value is not None: + return NUMBER_TYPE + elif numeric_value.date is not None: + return DATE_TYPE + raise ValueError("Unknown type: %s" % numeric_value) + + +def _get_value_as_primitive_value(numeric_value): + """Maps a NumericValue proto to a float or tuple of float.""" + if numeric_value.float_value is not None: + return numeric_value.float_value + if numeric_value.date is not None: + date = numeric_value.date + value_tuple = [None, None, None] + # All dates fields are cased to float to produce a simple primitive value. + if date.year is not None: + value_tuple[0] = float(date.year) + if date.month is not None: + value_tuple[1] = float(date.month) + if date.day is not None: + value_tuple[2] = float(date.day) + return tuple(value_tuple) + raise ValueError("Unknown type: %s" % numeric_value) + + +def _get_all_types(numeric_values): + return {_get_value_type(value) for value in numeric_values} + + +def get_numeric_sort_key_fn(numeric_values): + """ + Creates a function that can be used as a sort key or to compare the values. Maps to primitive types and finds the + biggest common subset. Consider the values "05/05/2010" and "August 2007". With the corresponding primitive values + (2010.,5.,5.) and (2007.,8., None). These values can be compared by year and date so we map to the sequence (2010., + 5.), (2007., 8.). If we added a third value "2006" with primitive value (2006., None, None), we could only compare + by the year so we would map to (2010.,), (2007.,) and (2006.,). + + Args: + numeric_values: Values to compare + + Returns: + A function that can be used as a sort key function (mapping numeric values to a comparable tuple) + + Raises: + ValueError if values don't have a common type or are not comparable. + """ + value_types = _get_all_types(numeric_values) + if len(value_types) != 1: + raise ValueError("No common value type in %s" % numeric_values) + + value_type = next(iter(value_types)) + if value_type == NUMBER_TYPE: + # Primitive values are simple floats, nothing to do here. + return _get_value_as_primitive_value + + # The type can only be Date at this point which means the primitive type + # is a float triple. + valid_indexes = set(range(_DATE_TUPLE_SIZE)) + + for numeric_value in numeric_values: + value = _get_value_as_primitive_value(numeric_value) + assert isinstance(value, tuple) + for tuple_index, inner_value in enumerate(value): + if inner_value is None: + valid_indexes.discard(tuple_index) + + if not valid_indexes: + raise ValueError("No common value in %s" % numeric_values) + + def _sort_key_fn(numeric_value): + value = _get_value_as_primitive_value(numeric_value) + return tuple(value[index] for index in valid_indexes) + + return _sort_key_fn + + +def _consolidate_numeric_values(row_index_to_values, min_consolidation_fraction, debug_info): + """ + Finds the most common numeric values in a column and returns them + + Args: + row_index_to_values: + For each row index all the values in that cell. + min_consolidation_fraction: + Fraction of cells that need to have consolidated value. + debug_info: + Additional information only used for logging + + Returns: + For each row index the first value that matches the most common value. Rows that don't have a matching value + are dropped. Empty list if values can't be consolidated. + """ + type_counts = collections.Counter() + for numeric_values in row_index_to_values.values(): + type_counts.update(_get_all_types(numeric_values)) + if not type_counts: + return {} + max_count = max(type_counts.values()) + if max_count < len(row_index_to_values) * min_consolidation_fraction: + # logging.log_every_n(logging.INFO, 'Can\'t consolidate types: %s %s %d', 100, + # debug_info, row_index_to_values, max_count) + return {} + + valid_types = set() + for value_type, count in type_counts.items(): + if count == max_count: + valid_types.add(value_type) + if len(valid_types) > 1: + assert DATE_TYPE in valid_types + max_type = DATE_TYPE + else: + max_type = next(iter(valid_types)) + + new_row_index_to_value = {} + for index, values in row_index_to_values.items(): + # Extract the first matching value. + for value in values: + if _get_value_type(value) == max_type: + new_row_index_to_value[index] = value + break + + return new_row_index_to_value + + +def _get_numeric_values(text): + """Parses text and returns numeric values.""" + numeric_spans = parse_text(text) + return itertools.chain(*(span.values for span in numeric_spans)) + + +def _get_column_values(table, col_index): + """ + Parses text in column and returns a dict mapping row_index to values. This is the _get_column_values function from + number_annotation_utils.py of the original implementation + + Args: + table: Pandas dataframe + col_index: integer, indicating the index of the column to get the numeric values of + """ + index_to_values = {} + for row_index, row in table.iterrows(): + text = normalize_for_match(row[col_index].text) + index_to_values[row_index] = list(_get_numeric_values(text)) + return index_to_values + + +def get_numeric_relation(value, other_value, sort_key_fn): + """Compares two values and returns their relation or None.""" + value = sort_key_fn(value) + other_value = sort_key_fn(other_value) + if value == other_value: + return Relation.EQ + if value < other_value: + return Relation.LT + if value > other_value: + return Relation.GT + return None + + +def add_numeric_values_to_question(question): + """Adds numeric value spans to a question.""" + original_text = question + question = normalize_for_match(question) + numeric_spans = parse_text(question) + return Question(original_text=original_text, text=question, numeric_spans=numeric_spans) + + +def filter_invalid_unicode(text): + """Return an empty string and True if 'text' is in invalid unicode.""" + return ("", True) if isinstance(text, bytes) else (text, False) + + +def filter_invalid_unicode_from_table(table): + """ + Removes invalid unicode from table. Checks whether a table cell text contains an invalid unicode encoding. If yes, + reset the table cell text to an empty str and log a warning for each invalid cell + + Args: + table: table to clean. + """ + # to do: add table id support + if not hasattr(table, "table_id"): + table.table_id = 0 + + for row_index, row in table.iterrows(): + for col_index, cell in enumerate(row): + cell, is_invalid = filter_invalid_unicode(cell) + if is_invalid: + logging.warning( + "Scrub an invalid table body @ table_id: %s, row_index: %d, " "col_index: %d", + table.table_id, + row_index, + col_index, + ) + for col_index, column in enumerate(table.columns): + column, is_invalid = filter_invalid_unicode(column) + if is_invalid: + logging.warning("Scrub an invalid table header @ table_id: %s, col_index: %d", table.table_id, col_index) + + +def add_numeric_table_values(table, min_consolidation_fraction=0.7, debug_info=None): + """ + Parses text in table column-wise and adds the consolidated values. Consolidation refers to finding values with a + common types (date or number) + + Args: + table: + Table to annotate. + min_consolidation_fraction: + Fraction of cells in a column that need to have consolidated value. + debug_info: + Additional information used for logging. + """ + table = table.copy() + # First, filter table on invalid unicode + filter_invalid_unicode_from_table(table) + + # Second, replace cell values by Cell objects + for row_index, row in table.iterrows(): + for col_index, cell in enumerate(row): + table.iloc[row_index, col_index] = Cell(text=cell) + + # Third, add numeric_value attributes to these Cell objects + for col_index, column in enumerate(table.columns): + column_values = _consolidate_numeric_values( + _get_column_values(table, col_index), + min_consolidation_fraction=min_consolidation_fraction, + debug_info=(debug_info, column), + ) + + for row_index, numeric_value in column_values.items(): + table.iloc[row_index, col_index].numeric_value = numeric_value + + return table diff --git a/src/transformers/models/xlm/modeling_tf_xlm.py b/src/transformers/models/xlm/modeling_tf_xlm.py index fc20415a2f90..c03022e14102 100644 --- a/src/transformers/models/xlm/modeling_tf_xlm.py +++ b/src/transformers/models/xlm/modeling_tf_xlm.py @@ -803,6 +803,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.pred_layer.input_embeddings + def get_output_layer_with_bias(self): + return self.pred_layer + + def get_prefix_bias_name(self): + return self.name + "/" + self.pred_layer.name + def prepare_inputs_for_generation(self, inputs, **kwargs): mask_token_id = self.config.mask_token_id lang_id = self.config.lang_id diff --git a/src/transformers/models/xlnet/modeling_tf_xlnet.py b/src/transformers/models/xlnet/modeling_tf_xlnet.py index e678944d6c24..7e72df370a62 100644 --- a/src/transformers/models/xlnet/modeling_tf_xlnet.py +++ b/src/transformers/models/xlnet/modeling_tf_xlnet.py @@ -1221,6 +1221,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.lm_loss.input_embeddings + def get_output_layer_with_bias(self): + return self.lm_loss + + def get_prefix_bias_name(self): + return self.name + "/" + self.lm_loss.name + def prepare_inputs_for_generation(self, inputs, past, use_mems=None, **kwargs): # Add dummy token at the end (no attention on this one) diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index 58aa26148ba7..c664664f60d8 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -28,6 +28,8 @@ _datasets_available, _faiss_available, _flax_available, + _pandas_available, + _scatter_available, _sentencepiece_available, _tf_available, _tokenizers_available, @@ -221,6 +223,30 @@ def require_tokenizers(test_case): return test_case +def require_pandas(test_case): + """ + Decorator marking a test that requires pandas. + + These tests are skipped when pandas isn't installed. + + """ + if not _pandas_available: + return unittest.skip("test requires pandas")(test_case) + else: + return test_case + + +def require_scatter(test_case): + """ + Decorator marking a test that requires PyTorch Scatter. These tests are skipped when PyTorch Scatter isn't + installed. + """ + if not _scatter_available: + return unittest.skip("test requires PyTorch Scatter")(test_case) + else: + return test_case + + def require_torch_multi_gpu(test_case): """ Decorator marking a test that requires a multi-GPU setup (in PyTorch). diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index b684d21ebe94..0669a4e09584 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -1867,6 +1867,45 @@ def load_tf_weights_in_t5(*args, **kwargs): requires_pytorch(load_tf_weights_in_t5) +TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class TapasForMaskedLM: + def __init__(self, *args, **kwargs): + requires_pytorch(self) + + @classmethod + def from_pretrained(self, *args, **kwargs): + requires_pytorch(self) + + +class TapasForQuestionAnswering: + def __init__(self, *args, **kwargs): + requires_pytorch(self) + + @classmethod + def from_pretrained(self, *args, **kwargs): + requires_pytorch(self) + + +class TapasForSequenceClassification: + def __init__(self, *args, **kwargs): + requires_pytorch(self) + + @classmethod + def from_pretrained(self, *args, **kwargs): + requires_pytorch(self) + + +class TapasModel: + def __init__(self, *args, **kwargs): + requires_pytorch(self) + + @classmethod + def from_pretrained(self, *args, **kwargs): + requires_pytorch(self) + + TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py b/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py index 9cd82c00d1e2..109b9f310b4e 100644 --- a/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py +++ b/templates/adding_a_new_model/cookiecutter-template-{{cookiecutter.modelname}}/modeling_tf_{{cookiecutter.lowercase_modelname}}.py @@ -772,10 +772,16 @@ def __init__(self, config, *inputs, **kwargs): self.{{cookiecutter.lowercase_modelname}} = TF{{cookiecutter.camelcase_modelname}}MainLayer(config, name="{{cookiecutter.lowercase_modelname}}") self.mlm = TF{{cookiecutter.camelcase_modelname}}MLMHead(config, self.{{cookiecutter.lowercase_modelname}}.embeddings, name="mlm___cls") - + def get_output_embeddings(self): return self.{{cookiecutter.lowercase_modelname}}.embeddings + def get_output_layer_with_bias(self): + return self.mlm.predictions + + def get_prefix_bias_name(self): + return self.name + "/" + self.mlm.name + "/" + self.mlm.predictions.name + @add_start_docstrings_to_model_forward({{cookiecutter.uppercase_modelname}}_INPUTS_DOCSTRING.format("batch_size, sequence_length")) @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, @@ -866,6 +872,12 @@ def __init__(self, config, *inputs, **kwargs): def get_output_embeddings(self): return self.{{cookiecutter.lowercase_modelname}}.embeddings + def get_output_layer_with_bias(self): + return self.mlm.predictions + + def get_prefix_bias_name(self): + return self.name + "/" + self.mlm.name + "/" + self.mlm.predictions.name + @add_code_sample_docstrings( tokenizer_class=_TOKENIZER_FOR_DOC, checkpoint="{{cookiecutter.checkpoint_identifier}}", diff --git a/tests/test_modeling_tapas.py b/tests/test_modeling_tapas.py new file mode 100644 index 000000000000..b751bac6ab7c --- /dev/null +++ b/tests/test_modeling_tapas.py @@ -0,0 +1,1035 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import copy +import unittest + +import numpy as np +import pandas as pd + +from transformers import ( + MODEL_FOR_CAUSAL_LM_MAPPING, + MODEL_FOR_MASKED_LM_MAPPING, + MODEL_FOR_MULTIPLE_CHOICE_MAPPING, + MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING, + MODEL_FOR_QUESTION_ANSWERING_MAPPING, + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING, + MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING, + MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING, + is_torch_available, +) +from transformers.file_utils import cached_property +from transformers.testing_utils import require_scatter, require_torch, slow, torch_device + +from .test_configuration_common import ConfigTester +from .test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask + + +if is_torch_available(): + import torch + + from transformers import ( + TapasConfig, + TapasForMaskedLM, + TapasForQuestionAnswering, + TapasForSequenceClassification, + TapasModel, + TapasTokenizer, + ) + from transformers.models.tapas.modeling_tapas import ( + IndexMap, + ProductIndexMap, + flatten, + gather, + range_index_map, + reduce_max, + reduce_mean, + reduce_sum, + ) + + +class TapasModelTester: + """You can also import this e.g from .test_modeling_tapas import TapasModelTester """ + + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + initializer_range=0.02, + max_position_embeddings=512, + type_vocab_sizes=[3, 256, 256, 2, 256, 256, 10], + type_sequence_label_size=2, + positive_weight=10.0, + num_aggregation_labels=4, + num_labels=2, + aggregation_loss_importance=0.8, + use_answer_as_supervision=True, + answer_loss_importance=0.001, + use_normalized_answer_loss=False, + huber_loss_delta=25.0, + temperature=1.0, + agg_temperature=1.0, + use_gumbel_for_cells=False, + use_gumbel_for_agg=False, + average_approximation_function="ratio", + cell_selection_preference=0.5, + answer_loss_cutoff=100, + max_num_rows=64, + max_num_columns=32, + average_logits_per_cell=True, + select_one_column=True, + allow_empty_column_selection=False, + init_cell_selection_weights_to_zero=False, + reset_position_index_per_cell=True, + disable_per_token_loss=False, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.initializer_range = initializer_range + self.max_position_embeddings = max_position_embeddings + self.type_vocab_sizes = type_vocab_sizes + self.type_sequence_label_size = type_sequence_label_size + self.positive_weight = positive_weight + self.num_aggregation_labels = num_aggregation_labels + self.num_labels = num_labels + self.aggregation_loss_importance = aggregation_loss_importance + self.use_answer_as_supervision = use_answer_as_supervision + self.answer_loss_importance = answer_loss_importance + self.use_normalized_answer_loss = use_normalized_answer_loss + self.huber_loss_delta = huber_loss_delta + self.temperature = temperature + self.agg_temperature = agg_temperature + self.use_gumbel_for_cells = use_gumbel_for_cells + self.use_gumbel_for_agg = use_gumbel_for_agg + self.average_approximation_function = average_approximation_function + self.cell_selection_preference = cell_selection_preference + self.answer_loss_cutoff = answer_loss_cutoff + self.max_num_rows = max_num_rows + self.max_num_columns = max_num_columns + self.average_logits_per_cell = average_logits_per_cell + self.select_one_column = select_one_column + self.allow_empty_column_selection = allow_empty_column_selection + self.init_cell_selection_weights_to_zero = init_cell_selection_weights_to_zero + self.reset_position_index_per_cell = reset_position_index_per_cell + self.disable_per_token_loss = disable_per_token_loss + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size).to(torch_device) + + input_mask = None + if self.use_input_mask: + input_mask = random_attention_mask([self.batch_size, self.seq_length]).to(torch_device) + + token_type_ids = [] + for type_vocab_size in self.type_vocab_sizes: + token_type_ids.append(ids_tensor(shape=[self.batch_size, self.seq_length], vocab_size=type_vocab_size)) + token_type_ids = torch.stack(token_type_ids, dim=2).to(torch_device) + + sequence_labels = None + token_labels = None + labels = None + numeric_values = None + numeric_values_scale = None + float_answer = None + aggregation_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size).to(torch_device) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels).to(torch_device) + labels = ids_tensor([self.batch_size, self.seq_length], vocab_size=2).to(torch_device) + numeric_values = floats_tensor([self.batch_size, self.seq_length]).to(torch_device) + numeric_values_scale = floats_tensor([self.batch_size, self.seq_length]).to(torch_device) + float_answer = floats_tensor([self.batch_size]).to(torch_device) + aggregation_labels = ids_tensor([self.batch_size], self.num_aggregation_labels).to(torch_device) + + config = TapasConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_sizes=self.type_vocab_sizes, + initializer_range=self.initializer_range, + positive_weight=self.positive_weight, + num_aggregation_labels=self.num_aggregation_labels, + num_labels=self.num_labels, + aggregation_loss_importance=self.aggregation_loss_importance, + use_answer_as_supervision=self.use_answer_as_supervision, + answer_loss_importance=self.answer_loss_importance, + use_normalized_answer_loss=self.use_normalized_answer_loss, + huber_loss_delta=self.huber_loss_delta, + temperature=self.temperature, + agg_temperature=self.agg_temperature, + use_gumbel_for_cells=self.use_gumbel_for_cells, + use_gumbel_for_agg=self.use_gumbel_for_agg, + average_approximation_function=self.average_approximation_function, + cell_selection_preference=self.cell_selection_preference, + answer_loss_cutoff=self.answer_loss_cutoff, + max_num_rows=self.max_num_rows, + max_num_columns=self.max_num_columns, + average_logits_per_cell=self.average_logits_per_cell, + select_one_column=self.select_one_column, + allow_empty_column_selection=self.allow_empty_column_selection, + init_cell_selection_weights_to_zero=self.init_cell_selection_weights_to_zero, + reset_position_index_per_cell=self.reset_position_index_per_cell, + disable_per_token_loss=self.disable_per_token_loss, + ) + + return ( + config, + input_ids, + input_mask, + token_type_ids, + sequence_labels, + token_labels, + labels, + numeric_values, + numeric_values_scale, + float_answer, + aggregation_labels, + ) + + def create_and_check_model( + self, + config, + input_ids, + input_mask, + token_type_ids, + sequence_labels, + token_labels, + labels, + numeric_values, + numeric_values_scale, + float_answer, + aggregation_labels, + ): + model = TapasModel(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + result = model(input_ids, token_type_ids=token_type_ids) + result = model(input_ids) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) + + def create_and_check_for_masked_lm( + self, + config, + input_ids, + input_mask, + token_type_ids, + sequence_labels, + token_labels, + labels, + numeric_values, + numeric_values_scale, + float_answer, + aggregation_labels, + ): + model = TapasForMaskedLM(config=config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size)) + + def create_and_check_for_question_answering( + self, + config, + input_ids, + input_mask, + token_type_ids, + sequence_labels, + token_labels, + labels, + numeric_values, + numeric_values_scale, + float_answer, + aggregation_labels, + ): + # inference: without aggregation head (SQA). Model only returns logits + sqa_config = copy.copy(config) + sqa_config.num_aggregation_labels = 0 + sqa_config.use_answer_as_supervision = False + model = TapasForQuestionAnswering(config=sqa_config) + model.to(torch_device) + model.eval() + result = model( + input_ids=input_ids, + attention_mask=input_mask, + token_type_ids=token_type_ids, + ) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length)) + + # inference: with aggregation head (WTQ, WikiSQL-supervised). Model returns logits and aggregation logits + model = TapasForQuestionAnswering(config=config) + model.to(torch_device) + model.eval() + result = model( + input_ids=input_ids, + attention_mask=input_mask, + token_type_ids=token_type_ids, + ) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length)) + self.parent.assertEqual(result.logits_aggregation.shape, (self.batch_size, self.num_aggregation_labels)) + + # training: can happen in 3 main ways + # case 1: conversational (SQA) + model = TapasForQuestionAnswering(config=sqa_config) + model.to(torch_device) + model.eval() + result = model( + input_ids, + attention_mask=input_mask, + token_type_ids=token_type_ids, + labels=labels, + ) + self.parent.assertEqual(result.loss.shape, ()) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length)) + + # case 2: weak supervision for aggregation (WTQ) + model = TapasForQuestionAnswering(config=config) + model.to(torch_device) + model.eval() + result = model( + input_ids=input_ids, + attention_mask=input_mask, + token_type_ids=token_type_ids, + labels=labels, + numeric_values=numeric_values, + numeric_values_scale=numeric_values_scale, + float_answer=float_answer, + ) + self.parent.assertEqual(result.loss.shape, ()) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length)) + self.parent.assertEqual(result.logits_aggregation.shape, (self.batch_size, self.num_aggregation_labels)) + + # case 3: strong supervision for aggregation (WikiSQL-supervised) + wikisql_config = copy.copy(config) + wikisql_config.use_answer_as_supervision = False + model = TapasForQuestionAnswering(config=wikisql_config) + model.to(torch_device) + model.eval() + result = model( + input_ids, + attention_mask=input_mask, + token_type_ids=token_type_ids, + labels=labels, + aggregation_labels=aggregation_labels, + ) + self.parent.assertEqual(result.loss.shape, ()) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length)) + self.parent.assertEqual(result.logits_aggregation.shape, (self.batch_size, self.num_aggregation_labels)) + + def create_and_check_for_sequence_classification( + self, + config, + input_ids, + input_mask, + token_type_ids, + sequence_labels, + token_labels, + labels, + numeric_values, + numeric_values_scale, + float_answer, + aggregation_labels, + ): + config.num_labels = self.num_labels + model = TapasForSequenceClassification(config) + model.to(torch_device) + model.eval() + result = model(input_ids, attention_mask=input_mask, labels=sequence_labels) + self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + ( + config, + input_ids, + input_mask, + token_type_ids, + sequence_labels, + token_labels, + labels, + numeric_values, + numeric_values_scale, + float_answer, + aggregation_labels, + ) = config_and_inputs + inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask} + return config, inputs_dict + + +@require_torch +@require_scatter +class TapasModelTest(ModelTesterMixin, unittest.TestCase): + + all_model_classes = ( + ( + TapasModel, + TapasForMaskedLM, + TapasForQuestionAnswering, + TapasForSequenceClassification, + ) + if is_torch_available() + else None + ) + test_pruning = False + test_torchscript = False + test_resize_embeddings = True + test_head_masking = False + + def _prepare_for_class(self, inputs_dict, model_class, return_labels=False): + inputs_dict = copy.deepcopy(inputs_dict) + if model_class in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.values(): + inputs_dict = { + k: v.unsqueeze(1).expand(-1, self.model_tester.num_choices, -1).contiguous() + if isinstance(v, torch.Tensor) and v.ndim > 1 + else v + for k, v in inputs_dict.items() + } + + if return_labels: + if model_class in MODEL_FOR_MULTIPLE_CHOICE_MAPPING.values(): + inputs_dict["labels"] = torch.ones(self.model_tester.batch_size, dtype=torch.long, device=torch_device) + elif model_class in MODEL_FOR_QUESTION_ANSWERING_MAPPING.values(): + inputs_dict["labels"] = torch.zeros( + (self.model_tester.batch_size, self.model_tester.seq_length), dtype=torch.long, device=torch_device + ) + inputs_dict["aggregation_labels"] = torch.zeros( + self.model_tester.batch_size, dtype=torch.long, device=torch_device + ) + inputs_dict["numeric_values"] = torch.zeros( + (self.model_tester.batch_size, self.model_tester.seq_length), + dtype=torch.float, + device=torch_device, + ) + inputs_dict["numeric_values_scale"] = torch.zeros( + (self.model_tester.batch_size, self.model_tester.seq_length), + dtype=torch.float, + device=torch_device, + ) + inputs_dict["float_answer"] = torch.zeros( + self.model_tester.batch_size, dtype=torch.float, device=torch_device + ) + elif model_class in [ + *MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING.values(), + *MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING.values(), + ]: + inputs_dict["labels"] = torch.zeros( + self.model_tester.batch_size, dtype=torch.long, device=torch_device + ) + elif model_class in [ + *MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING.values(), + *MODEL_FOR_CAUSAL_LM_MAPPING.values(), + *MODEL_FOR_MASKED_LM_MAPPING.values(), + *MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.values(), + ]: + inputs_dict["labels"] = torch.zeros( + (self.model_tester.batch_size, self.model_tester.seq_length), dtype=torch.long, device=torch_device + ) + return inputs_dict + + def setUp(self): + self.model_tester = TapasModelTester(self) + self.config_tester = ConfigTester(self, config_class=TapasConfig, dim=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + # def test_for_masked_lm(self): + # config_and_inputs = self.model_tester.prepare_config_and_inputs() + # self.model_tester.create_and_check_for_masked_lm(*config_and_inputs) + + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs) + + +def prepare_tapas_single_inputs_for_inference(): + # Here we prepare a single table-question pair to test TAPAS inference on: + data = { + "Footballer": ["Lionel Messi", "Cristiano Ronaldo"], + "Age": ["33", "35"], + } + queries = "Which footballer is 33 years old?" + table = pd.DataFrame.from_dict(data) + + return table, queries + + +def prepare_tapas_batch_inputs_for_inference(): + # Here we prepare a batch of 2 table-question pairs to test TAPAS inference on: + data = { + "Footballer": ["Lionel Messi", "Cristiano Ronaldo"], + "Age": ["33", "35"], + "Number of goals": ["712", "750"], + } + queries = ["Which footballer is 33 years old?", "How many goals does Ronaldo have?"] + table = pd.DataFrame.from_dict(data) + + return table, queries + + +def prepare_tapas_batch_inputs_for_training(): + # Here we prepare a DIFFERENT batch of 2 table-question pairs to test TAPAS training on: + data = { + "Footballer": ["Lionel Messi", "Cristiano Ronaldo"], + "Age": ["33", "35"], + "Number of goals": ["712", "750"], + } + queries = ["Which footballer is 33 years old?", "What's the total number of goals?"] + table = pd.DataFrame.from_dict(data) + + answer_coordinates = [[(0, 0)], [(0, 2), (1, 2)]] + answer_text = [["Lionel Messi"], ["1462"]] + float_answer = [float("NaN"), float("1462")] + + return table, queries, answer_coordinates, answer_text, float_answer + + +TOLERANCE = 1 + + +@require_torch +@require_scatter +class TapasModelIntegrationTest(unittest.TestCase): + @cached_property + def default_tokenizer(self): + return TapasTokenizer.from_pretrained("nielsr/tapas-base-finetuned-wtq") + + @slow + def test_inference_no_head(self): + # ideally we want to test this with the weights of tapas_inter_masklm_base_reset, + # but since it's not straightforward to do this with the TF 1 implementation, we test it with + # the weights of the WTQ base model (i.e. tapas_wtq_wikisql_sqa_inter_masklm_base_reset) + model = TapasModel.from_pretrained("nielsr/tapas-base-finetuned-wtq").to(torch_device) + + tokenizer = self.default_tokenizer + table, queries = prepare_tapas_single_inputs_for_inference() + inputs = tokenizer(table=table, queries=queries, return_tensors="pt") + inputs = {k: v.to(torch_device) for k, v in inputs.items()} + outputs = model(**inputs) + # test the sequence output + expected_slice = torch.tensor( + [ + [ + [-0.141581565, -0.599805772, 0.747186482], + [-0.143664181, -0.602008104, 0.749218345], + [-0.15169853, -0.603363097, 0.741370678], + ] + ], + device=torch_device, + ) + + self.assertTrue(torch.allclose(outputs.last_hidden_state[:, :3, :3], expected_slice, atol=TOLERANCE)) + + # test the pooled output + expected_slice = torch.tensor([[0.987518311, -0.970520139, -0.994303405]], device=torch_device) + + self.assertTrue(torch.allclose(outputs.pooler_output[:, :3], expected_slice, atol=TOLERANCE)) + + @unittest.skip(reason="Model not available yet") + def test_inference_masked_lm(self): + pass + + # TapasForQuestionAnswering has 3 possible ways of being fine-tuned: + # - conversational set-up (SQA) + # - weak supervision for aggregation (WTQ, WikiSQL) + # - strong supervision for aggregation (WikiSQL-supervised) + # We test all of them: + @slow + def test_inference_question_answering_head_conversational(self): + # note that nielsr/tapas-base-finetuned-sqa should correspond to tapas_sqa_inter_masklm_base_reset + model = TapasForQuestionAnswering.from_pretrained("nielsr/tapas-base-finetuned-sqa").to(torch_device) + + tokenizer = self.default_tokenizer + table, queries = prepare_tapas_single_inputs_for_inference() + inputs = tokenizer(table=table, queries=queries, return_tensors="pt") + inputs = {k: v.to(torch_device) for k, v in inputs.items()} + outputs = model(**inputs) + # test the logits + logits = outputs.logits + expected_shape = torch.Size((1, 21)) + self.assertEqual(logits.shape, expected_shape) + + expected_tensor = torch.tensor( + [ + [ + -9997.22461, + -9997.22461, + -9997.22461, + -9997.22461, + -9997.22461, + -9997.22461, + -9997.22461, + -9997.22461, + -9997.22461, + -16.2628059, + -10004.082, + 15.4330549, + 15.4330549, + 15.4330549, + -9990.42, + -16.3270779, + -16.3270779, + -16.3270779, + -16.3270779, + -16.3270779, + -10004.8506, + ] + ], + device=torch_device, + ) # ok + + self.assertTrue(torch.allclose(logits, expected_tensor, atol=TOLERANCE)) + + @slow + def test_inference_question_answering_head_weak_supervision(self): + # note that nielsr/tapas-base-finetuned-wtq should correspond to tapas_wtq_wikisql_sqa_inter_masklm_base_reset + model = TapasForQuestionAnswering.from_pretrained("nielsr/tapas-base-finetuned-wtq").to(torch_device) + + tokenizer = self.default_tokenizer + # let's test on a batch + table, queries = prepare_tapas_batch_inputs_for_inference() + inputs = tokenizer(table=table, queries=queries, padding="longest", return_tensors="pt") + inputs_on_device = {k: v.to(torch_device) for k, v in inputs.items()} + + outputs = model(**inputs_on_device) + # test the logits + logits = outputs.logits + expected_shape = torch.Size((2, 28)) + self.assertEqual(logits.shape, expected_shape) + + expected_slice = torch.tensor( + [ + [-160.375504, -160.375504, -160.375504, -10072.3965, -10070.9414, -10094.9736], + [-9861.6123, -9861.6123, -9861.6123, -9861.6123, -9891.01172, 146.600677], + ], + device=torch_device, + ) # ok (batch size = 2) + + self.assertTrue(torch.allclose(logits[:, -6:], expected_slice, atol=TOLERANCE)) + + # test the aggregation logits + logits_aggregation = outputs.logits_aggregation + expected_shape = torch.Size((2, 4)) + self.assertEqual(logits_aggregation.shape, expected_shape) + expected_tensor = torch.tensor( + [[18.8545208, -9.76614857, -6.3128891, -2.93525243], [-4.05782509, 40.0351, -5.35329962, 23.3978653]], + device=torch_device, + ) # ok (batch size = 2) + + self.assertTrue(torch.allclose(logits_aggregation, expected_tensor, atol=TOLERANCE)) + + # test the predicted answer coordinates and aggregation indices + EXPECTED_PREDICTED_ANSWER_COORDINATES = [[(0, 0)], [(1, 2)]] + EXPECTED_PREDICTED_AGGREGATION_INDICES = [0, 1] + + predicted_answer_coordinates, predicted_aggregation_indices = tokenizer.convert_logits_to_predictions( + inputs, outputs.logits.detach().cpu(), outputs.logits_aggregation.detach().cpu() + ) + + self.assertEqual(EXPECTED_PREDICTED_ANSWER_COORDINATES, predicted_answer_coordinates) + self.assertEqual(EXPECTED_PREDICTED_AGGREGATION_INDICES, predicted_aggregation_indices) + + @slow + def test_training_question_answering_head_weak_supervision(self): + # note that nielsr/tapas-base-finetuned-wtq should correspond to tapas_wtq_wikisql_sqa_inter_masklm_base_reset + model = TapasForQuestionAnswering.from_pretrained("nielsr/tapas-base-finetuned-wtq").to(torch_device) + model.to(torch_device) + # normally we should put the model in training mode but it's a pain to do this with the TF 1 implementation + + tokenizer = self.default_tokenizer + # let's test on a batch + table, queries, answer_coordinates, answer_text, float_answer = prepare_tapas_batch_inputs_for_training() + inputs = tokenizer( + table=table, + queries=queries, + answer_coordinates=answer_coordinates, + answer_text=answer_text, + padding="longest", + return_tensors="pt", + ) + + # prepare data (created by the tokenizer) and move to torch_device + input_ids = inputs["input_ids"].to(torch_device) + attention_mask = inputs["attention_mask"].to(torch_device) + token_type_ids = inputs["token_type_ids"].to(torch_device) + labels = inputs["labels"].to(torch_device) + numeric_values = inputs["numeric_values"].to(torch_device) + numeric_values_scale = inputs["numeric_values_scale"].to(torch_device) + + # the answer should be prepared by the user + float_answer = torch.FloatTensor(float_answer).to(torch_device) + + # forward pass to get loss + logits: + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + labels=labels, + numeric_values=numeric_values, + numeric_values_scale=numeric_values_scale, + float_answer=float_answer, + ) + + # test the loss + loss = outputs.loss + expected_loss = torch.tensor(3.3527612686157227e-08, device=torch_device) # ok + self.assertTrue(torch.allclose(loss, expected_loss, atol=TOLERANCE)) + + # test the logits on the first example + logits = outputs.logits + expected_shape = torch.Size((2, 29)) + self.assertEqual(logits.shape, expected_shape) + expected_slice = torch.tensor( + [ + -160.0156, + -160.0156, + -160.0156, + -160.0156, + -160.0156, + -10072.2266, + -10070.8896, + -10092.6006, + -10092.6006, + ], + device=torch_device, + ) # ok + + self.assertTrue(torch.allclose(logits[0, -9:], expected_slice, atol=TOLERANCE)) + + # test the aggregation logits on the second example + logits_aggregation = outputs.logits_aggregation + expected_shape = torch.Size((2, 4)) + self.assertEqual(logits_aggregation.shape, expected_shape) + expected_slice = torch.tensor([-4.0538, 40.0304, -5.3554, 23.3965], device=torch_device) # ok + + self.assertTrue(torch.allclose(logits_aggregation[1, -4:], expected_slice, atol=TOLERANCE)) + + @slow + def test_inference_question_answering_head_strong_supervision(self): + # note that nielsr/tapas-base-finetuned-wikisql-supervised should correspond to tapas_wikisql_sqa_inter_masklm_base_reset + model = TapasForQuestionAnswering.from_pretrained("nielsr/tapas-base-finetuned-wikisql-supervised").to( + torch_device + ) + + tokenizer = self.default_tokenizer + table, queries = prepare_tapas_single_inputs_for_inference() + inputs = tokenizer(table=table, queries=queries, return_tensors="pt") + inputs = {k: v.to(torch_device) for k, v in inputs.items()} + outputs = model(**inputs) + # test the logits + logits = outputs.logits + expected_shape = torch.Size((1, 21)) + self.assertEqual(logits.shape, expected_shape) + expected_tensor = torch.tensor( + [ + [ + -10011.1084, + -10011.1084, + -10011.1084, + -10011.1084, + -10011.1084, + -10011.1084, + -10011.1084, + -10011.1084, + -10011.1084, + -18.6185989, + -10008.7969, + 17.6355762, + 17.6355762, + 17.6355762, + -10002.4404, + -18.7111301, + -18.7111301, + -18.7111301, + -18.7111301, + -18.7111301, + -10007.0977, + ] + ], + device=torch_device, + ) # ok + + self.assertTrue(torch.allclose(logits, expected_tensor, atol=TOLERANCE)) + + # test the aggregation logits + logits_aggregation = outputs.logits_aggregation + expected_shape = torch.Size((1, 4)) + self.assertEqual(logits_aggregation.shape, expected_shape) + expected_tensor = torch.tensor( + [[16.5659733, -3.06624889, -2.34152961, -0.970244825]], device=torch_device + ) # ok, PyTorch model outputs [[16.5679, -3.0668, -2.3442, -0.9674]] + + self.assertTrue(torch.allclose(logits_aggregation, expected_tensor, atol=TOLERANCE)) + + @slow + def test_inference_classification_head(self): + # note that nielsr/tapas-base-finetuned-tabfact should correspond to tapas_tabfact_inter_masklm_base_reset + model = TapasForSequenceClassification.from_pretrained("nielsr/tapas-base-finetuned-tabfact").to(torch_device) + + tokenizer = self.default_tokenizer + table, queries = prepare_tapas_single_inputs_for_inference() + inputs = tokenizer(table=table, queries=queries, padding="longest", return_tensors="pt") + inputs = {k: v.to(torch_device) for k, v in inputs.items()} + outputs = model(**inputs) + + # test the classification logits + logits = outputs.logits + expected_shape = torch.Size((1, 2)) + self.assertEqual(logits.shape, expected_shape) + expected_tensor = torch.tensor( + [[0.795137286, 9.5572]], device=torch_device + ) # ok. Note that the PyTorch model outputs [[0.8057, 9.5281]] + + self.assertTrue(torch.allclose(outputs.logits, expected_tensor, atol=TOLERANCE)) + + +# Below: tests for Tapas utilities which are defined in modeling_tapas.py. +# These are based on segmented_tensor_test.py of the original implementation. +# URL: https://github.com/google-research/tapas/blob/master/tapas/models/segmented_tensor_test.py +@require_scatter +class TapasUtilitiesTest(unittest.TestCase): + def _prepare_tables(self): + """Prepares two tables, both with three distinct rows. + The first table has two columns: + 1.0, 2.0 | 3.0 + 2.0, 0.0 | 1.0 + 1.0, 3.0 | 4.0 + The second table has three columns: + 1.0 | 2.0 | 3.0 + 2.0 | 0.0 | 1.0 + 1.0 | 3.0 | 4.0 + Returns: + SegmentedTensors with the tables. + """ + values = torch.tensor( + [ + [[1.0, 2.0, 3.0], [2.0, 0.0, 1.0], [1.0, 3.0, 4.0]], + [[1.0, 2.0, 3.0], [2.0, 0.0, 1.0], [1.0, 3.0, 4.0]], + ] + ) + row_index = IndexMap( + indices=torch.tensor( + [ + [[0, 0, 0], [1, 1, 1], [2, 2, 2]], + [[0, 0, 0], [1, 1, 1], [2, 2, 2]], + ] + ), + num_segments=3, + batch_dims=1, + ) + col_index = IndexMap( + indices=torch.tensor( + [ + [[0, 0, 1], [0, 0, 1], [0, 0, 1]], + [[0, 1, 2], [0, 1, 2], [0, 1, 2]], + ] + ), + num_segments=3, + batch_dims=1, + ) + return values, row_index, col_index + + def test_product_index(self): + _, row_index, col_index = self._prepare_tables() + cell_index = ProductIndexMap(row_index, col_index) + row_index_proj = cell_index.project_outer(cell_index) + col_index_proj = cell_index.project_inner(cell_index) + + ind = cell_index.indices + self.assertEqual(cell_index.num_segments, 9) + + # Projections should give back the original indices. + # we use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_array_equal(row_index.indices.numpy(), row_index_proj.indices.numpy()) + self.assertEqual(row_index.num_segments, row_index_proj.num_segments) + self.assertEqual(row_index.batch_dims, row_index_proj.batch_dims) + # We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_array_equal(col_index.indices.numpy(), col_index_proj.indices.numpy()) + self.assertEqual(col_index.batch_dims, col_index_proj.batch_dims) + + # The first and second "column" are identified in the first table. + for i in range(3): + self.assertEqual(ind[0, i, 0], ind[0, i, 1]) + self.assertNotEqual(ind[0, i, 0], ind[0, i, 2]) + + # All rows are distinct in the first table. + for i, i_2 in zip(range(3), range(3)): + for j, j_2 in zip(range(3), range(3)): + if i != i_2 and j != j_2: + self.assertNotEqual(ind[0, i, j], ind[0, i_2, j_2]) + + # All cells are distinct in the second table. + for i, i_2 in zip(range(3), range(3)): + for j, j_2 in zip(range(3), range(3)): + if i != i_2 or j != j_2: + self.assertNotEqual(ind[1, i, j], ind[1, i_2, j_2]) + + def test_flatten(self): + _, row_index, col_index = self._prepare_tables() + row_index_flat = flatten(row_index) + col_index_flat = flatten(col_index) + + shape = [3, 4, 5] + batched_index = IndexMap(indices=torch.zeros(shape).type(torch.LongTensor), num_segments=1, batch_dims=3) + batched_index_flat = flatten(batched_index) + + # We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_array_equal( + row_index_flat.indices.numpy(), [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5] + ) + np.testing.assert_array_equal( + col_index_flat.indices.numpy(), [0, 0, 1, 0, 0, 1, 0, 0, 1, 3, 4, 5, 3, 4, 5, 3, 4, 5] + ) + self.assertEqual(batched_index_flat.num_segments.numpy(), np.prod(shape)) + np.testing.assert_array_equal(batched_index_flat.indices.numpy(), range(np.prod(shape))) + + def test_range_index_map(self): + batch_shape = [3, 4] + num_segments = 5 + index = range_index_map(batch_shape, num_segments) + + self.assertEqual(num_segments, index.num_segments) + self.assertEqual(2, index.batch_dims) + indices = index.indices + # We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_array_equal(list(indices.size()), [3, 4, 5]) + for i in range(batch_shape[0]): + for j in range(batch_shape[1]): + # We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_array_equal(indices[i, j, :].numpy(), range(num_segments)) + + def test_reduce_sum(self): + values, row_index, col_index = self._prepare_tables() + cell_index = ProductIndexMap(row_index, col_index) + row_sum, _ = reduce_sum(values, row_index) + col_sum, _ = reduce_sum(values, col_index) + cell_sum, _ = reduce_sum(values, cell_index) + + # We use np.testing.assert_allclose rather than Tensorflow's assertAllClose + np.testing.assert_allclose(row_sum.numpy(), [[6.0, 3.0, 8.0], [6.0, 3.0, 8.0]]) + np.testing.assert_allclose(col_sum.numpy(), [[9.0, 8.0, 0.0], [4.0, 5.0, 8.0]]) + np.testing.assert_allclose( + cell_sum.numpy(), + [[3.0, 3.0, 0.0, 2.0, 1.0, 0.0, 4.0, 4.0, 0.0], [1.0, 2.0, 3.0, 2.0, 0.0, 1.0, 1.0, 3.0, 4.0]], + ) + + def test_reduce_mean(self): + values, row_index, col_index = self._prepare_tables() + cell_index = ProductIndexMap(row_index, col_index) + row_mean, _ = reduce_mean(values, row_index) + col_mean, _ = reduce_mean(values, col_index) + cell_mean, _ = reduce_mean(values, cell_index) + + # We use np.testing.assert_allclose rather than Tensorflow's assertAllClose + np.testing.assert_allclose( + row_mean.numpy(), [[6.0 / 3.0, 3.0 / 3.0, 8.0 / 3.0], [6.0 / 3.0, 3.0 / 3.0, 8.0 / 3.0]] + ) + np.testing.assert_allclose(col_mean.numpy(), [[9.0 / 6.0, 8.0 / 3.0, 0.0], [4.0 / 3.0, 5.0 / 3.0, 8.0 / 3.0]]) + np.testing.assert_allclose( + cell_mean.numpy(), + [ + [3.0 / 2.0, 3.0, 0.0, 2.0 / 2.0, 1.0, 0.0, 4.0 / 2.0, 4.0, 0.0], + [1.0, 2.0, 3.0, 2.0, 0.0, 1.0, 1.0, 3.0, 4.0], + ], + ) + + def test_reduce_max(self): + values = torch.as_tensor([2.0, 1.0, 0.0, 3.0]) + index = IndexMap(indices=torch.as_tensor([0, 1, 0, 1]), num_segments=2) + maximum, _ = reduce_max(values, index) + + # We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_array_equal(maximum.numpy(), [2, 3]) + + def test_reduce_sum_vectorized(self): + values = torch.as_tensor([[1.0, 2.0, 3.0], [2.0, 3.0, 4.0], [3.0, 4.0, 5.0]]) + index = IndexMap(indices=torch.as_tensor([0, 0, 1]), num_segments=2, batch_dims=0) + sums, new_index = reduce_sum(values, index) + + # We use np.testing.assert_allclose rather than Tensorflow's assertAllClose + np.testing.assert_allclose(sums.numpy(), [[3.0, 5.0, 7.0], [3.0, 4.0, 5.0]]) + # We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_array_equal(new_index.indices.numpy(), [0, 1]) + np.testing.assert_array_equal(new_index.num_segments.numpy(), 2) + np.testing.assert_array_equal(new_index.batch_dims, 0) + + def test_gather(self): + values, row_index, col_index = self._prepare_tables() + cell_index = ProductIndexMap(row_index, col_index) + + # Compute sums and then gather. The result should have the same shape as + # the original table and each element should contain the sum the values in + # its cell. + sums, _ = reduce_sum(values, cell_index) + cell_sum = gather(sums, cell_index) + assert cell_sum.size() == values.size() + + # We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_allclose( + cell_sum.numpy(), + [[[3.0, 3.0, 3.0], [2.0, 2.0, 1.0], [4.0, 4.0, 4.0]], [[1.0, 2.0, 3.0], [2.0, 0.0, 1.0], [1.0, 3.0, 4.0]]], + ) + + def test_gather_vectorized(self): + values = torch.as_tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + index = IndexMap(indices=torch.as_tensor([[0, 1], [1, 0]]), num_segments=2, batch_dims=1) + result = gather(values, index) + + # We use np.testing.assert_array_equal rather than Tensorflow's assertAllEqual + np.testing.assert_array_equal(result.numpy(), [[[1, 2], [3, 4]], [[7, 8], [5, 6]]]) diff --git a/tests/test_modeling_tf_albert.py b/tests/test_modeling_tf_albert.py index 3dec2837bdec..354e116671a8 100644 --- a/tests/test_modeling_tf_albert.py +++ b/tests/test_modeling_tf_albert.py @@ -272,6 +272,17 @@ def test_for_question_answering(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_albert_for_question_answering(*config_and_inputs) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + @slow def test_model_from_pretrained(self): for model_name in TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_bart.py b/tests/test_modeling_tf_bart.py index c31523612f1c..99c3d03eca9b 100644 --- a/tests/test_modeling_tf_bart.py +++ b/tests/test_modeling_tf_bart.py @@ -126,6 +126,17 @@ def test_saved_model_with_attentions_output(self): # Should be uncommented during patrick TF refactor pass + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + @require_tf class TFBartHeadTests(unittest.TestCase): diff --git a/tests/test_modeling_tf_bert.py b/tests/test_modeling_tf_bert.py index a8ca5e30220a..a1d2bb747acf 100644 --- a/tests/test_modeling_tf_bert.py +++ b/tests/test_modeling_tf_bert.py @@ -331,6 +331,25 @@ def test_model_from_pretrained(self): model = TFBertModel.from_pretrained("jplu/tiny-tf-bert-random") self.assertIsNotNone(model) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + list_lm_models = [TFBertForMaskedLM, TFBertForPreTraining, TFBertLMHeadModel] + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + + if model_class in list_lm_models: + x = model.get_output_layer_with_bias() + assert isinstance(x, tf.keras.layers.Layer) + name = model.get_prefix_bias_name() + assert isinstance(name, str) + else: + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert x is None + def test_custom_load_tf_weights(self): model, output_loading_info = TFBertForTokenClassification.from_pretrained( "jplu/tiny-tf-bert-random", output_loading_info=True diff --git a/tests/test_modeling_tf_blenderbot.py b/tests/test_modeling_tf_blenderbot.py index 2dbd14ef9e37..7b2f4196c89f 100644 --- a/tests/test_modeling_tf_blenderbot.py +++ b/tests/test_modeling_tf_blenderbot.py @@ -18,17 +18,17 @@ from tests.test_configuration_common import ConfigTester from tests.test_modeling_tf_bart import TFBartModelTester from tests.test_modeling_tf_common import TFModelTesterMixin -from transformers import ( - BlenderbotConfig, - BlenderbotSmallTokenizer, - TFAutoModelForSeq2SeqLM, - TFBlenderbotForConditionalGeneration, - is_tf_available, -) +from transformers import BlenderbotConfig, BlenderbotSmallTokenizer, is_tf_available from transformers.file_utils import cached_property from transformers.testing_utils import is_pt_tf_cross_test, require_tf, require_tokenizers, slow +if is_tf_available(): + import tensorflow as tf + + from transformers import TFAutoModelForSeq2SeqLM, TFBlenderbotForConditionalGeneration + + class TFBlenderbotModelTester(TFBartModelTester): config_updates = dict( normalize_before=True, @@ -65,6 +65,17 @@ def test_saved_model_with_attentions_output(self): # Should be uncommented during patrick TF refactor pass + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + @is_pt_tf_cross_test @require_tokenizers diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 0e06ee757c9a..5aa1e78e1734 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -592,12 +592,26 @@ def check_hidden_states_output(config, inputs_dict, model_class): def test_model_common_attributes(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + list_lm_models = ( + list(TF_MODEL_FOR_CAUSAL_LM_MAPPING.values()) + + list(TF_MODEL_FOR_MASKED_LM_MAPPING.values()) + + list(TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.values()) + ) for model_class in self.all_model_classes: model = model_class(config) assert isinstance(model.get_input_embeddings(), (tf.keras.layers.Layer, TFAdaptiveEmbedding)) - x = model.get_output_embeddings() - assert x is None or isinstance(x, tf.keras.layers.Layer) + + if model_class in list_lm_models: + x = model.get_output_layer_with_bias() + assert isinstance(x, tf.keras.layers.Layer) + name = model.get_prefix_bias_name() + assert isinstance(name, str) + else: + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert x is None def test_determinism(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/test_modeling_tf_gpt2.py b/tests/test_modeling_tf_gpt2.py index fe8eb7e4c822..07d2f8ae6587 100644 --- a/tests/test_modeling_tf_gpt2.py +++ b/tests/test_modeling_tf_gpt2.py @@ -353,6 +353,17 @@ def test_gpt2_double_head(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_gpt2_double_head(*config_and_inputs) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + def test_gpt2_sequence_classification_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_gpt2_for_sequence_classification(*config_and_inputs) diff --git a/tests/test_modeling_tf_lxmert.py b/tests/test_modeling_tf_lxmert.py index 047b71c32436..1c90ec5e18dd 100644 --- a/tests/test_modeling_tf_lxmert.py +++ b/tests/test_modeling_tf_lxmert.py @@ -678,6 +678,25 @@ def test_compile_tf_model(self): extended_model = tf.keras.Model(inputs=[input_ids, visual_feats, visual_pos], outputs=[outputs]) extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + list_lm_models = [TFLxmertForPreTraining] + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + + if model_class in list_lm_models: + x = model.get_output_layer_with_bias() + assert isinstance(x, tf.keras.layers.Layer) + name = model.get_prefix_bias_name() + assert isinstance(name, str) + else: + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert x is None + @slow def test_saved_model_with_hidden_states_output(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/tests/test_modeling_tf_marian.py b/tests/test_modeling_tf_marian.py index a713023d4f1f..b4ce498706b8 100644 --- a/tests/test_modeling_tf_marian.py +++ b/tests/test_modeling_tf_marian.py @@ -94,6 +94,17 @@ def test_compile_tf_model(self): extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs]) extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + class AbstractMarianIntegrationTest(unittest.TestCase): maxDiff = 1000 # show more chars for failing integration tests diff --git a/tests/test_modeling_tf_mbart.py b/tests/test_modeling_tf_mbart.py index d631971c43b6..80a7e9115459 100644 --- a/tests/test_modeling_tf_mbart.py +++ b/tests/test_modeling_tf_mbart.py @@ -93,6 +93,17 @@ def test_compile_tf_model(self): extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs]) extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + @is_pt_tf_cross_test @require_sentencepiece diff --git a/tests/test_modeling_tf_mobilebert.py b/tests/test_modeling_tf_mobilebert.py index e170798b374a..939d2a42356b 100644 --- a/tests/test_modeling_tf_mobilebert.py +++ b/tests/test_modeling_tf_mobilebert.py @@ -283,6 +283,25 @@ def test_for_token_classification(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_mobilebert_for_token_classification(*config_and_inputs) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + list_lm_models = [TFMobileBertForMaskedLM, TFMobileBertForPreTraining] + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + + if model_class in list_lm_models: + x = model.get_output_layer_with_bias() + assert isinstance(x, tf.keras.layers.Layer) + name = model.get_prefix_bias_name() + assert isinstance(name, str) + else: + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert x is None + @slow def test_model_from_pretrained(self): # for model_name in TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_openai.py b/tests/test_modeling_tf_openai.py index 4cb71670e034..7eb9e316c34d 100644 --- a/tests/test_modeling_tf_openai.py +++ b/tests/test_modeling_tf_openai.py @@ -202,6 +202,17 @@ def test_openai_gpt_double_head(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_openai_gpt_double_head(*config_and_inputs) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + @slow def test_model_from_pretrained(self): for model_name in TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_modeling_tf_pegasus.py b/tests/test_modeling_tf_pegasus.py index fab6c343735d..b6e16f75d57d 100644 --- a/tests/test_modeling_tf_pegasus.py +++ b/tests/test_modeling_tf_pegasus.py @@ -99,6 +99,17 @@ def test_compile_tf_model(self): extended_model = tf.keras.Model(inputs=[input_ids], outputs=[outputs]) extended_model.compile(optimizer=optimizer, loss=loss, metrics=[metric]) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + @is_pt_tf_cross_test @require_sentencepiece diff --git a/tests/test_modeling_tf_t5.py b/tests/test_modeling_tf_t5.py index 9854e28fd2b9..64bb41bef153 100644 --- a/tests/test_modeling_tf_t5.py +++ b/tests/test_modeling_tf_t5.py @@ -282,6 +282,17 @@ def test_t5_decoder_model_past_large_inputs(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_t5_decoder_model_past_large_inputs(*config_and_inputs) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + @slow def test_model_from_pretrained(self): model = TFT5Model.from_pretrained("t5-small") diff --git a/tests/test_modeling_tf_transfo_xl.py b/tests/test_modeling_tf_transfo_xl.py index f8a8cc4d24e3..94167bbac576 100644 --- a/tests/test_modeling_tf_transfo_xl.py +++ b/tests/test_modeling_tf_transfo_xl.py @@ -163,6 +163,17 @@ def test_transfo_xl_lm_head(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_transfo_xl_lm_head(*config_and_inputs) + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + x = model.get_output_layer_with_bias() + assert x is None + name = model.get_prefix_bias_name() + assert name is None + @slow def test_model_from_pretrained(self): for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 0095b9e2436c..dd4ae1a72981 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -584,7 +584,7 @@ def test_token_type_ids(self): # We want to have sequence 0 and sequence 1 are tagged # respectively with 0 and 1 token_ids - # (regardeless of weither the model use token type ids) + # (regardless of whether the model use token type ids) # We use this assumption in the QA pipeline among other place output = tokenizer(seq_0, return_token_type_ids=True) self.assertIn(0, output["token_type_ids"]) @@ -600,7 +600,7 @@ def test_sequence_ids(self): # We want to have sequence 0 and sequence 1 are tagged # respectively with 0 and 1 token_ids - # (regardeless of weither the model use token type ids) + # (regardless of whether the model use token type ids) # We use this assumption in the QA pipeline among other place output = tokenizer(seq_0) self.assertIn(0, output.sequence_ids()) diff --git a/tests/test_tokenization_tapas.py b/tests/test_tokenization_tapas.py new file mode 100644 index 000000000000..11028b2d2e27 --- /dev/null +++ b/tests/test_tokenization_tapas.py @@ -0,0 +1,1242 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import os +import shutil +import tempfile +import unittest +from typing import List + +import numpy as np +import pandas as pd + +from transformers import AddedToken +from transformers.models.tapas.tokenization_tapas import ( + VOCAB_FILES_NAMES, + BasicTokenizer, + TapasTokenizer, + WordpieceTokenizer, + _is_control, + _is_punctuation, + _is_whitespace, +) +from transformers.testing_utils import is_pt_tf_cross_test, require_pandas, require_tokenizers, require_torch, slow + +from .test_tokenization_common import TokenizerTesterMixin, filter_non_english, merge_model_tokenizer_mappings + + +@require_tokenizers +@require_pandas +class TapasTokenizationTest(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = TapasTokenizer + test_rust_tokenizer = False + space_between_special_tokens = True + from_pretrained_filter = filter_non_english + + def get_table( + self, + tokenizer: TapasTokenizer, + length=5, + ): + toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))] + + if length == 0: + data = {} + else: + data = {toks[0]: [toks[tok] for tok in range(1, length)]} + + table = pd.DataFrame.from_dict(data) + + return table + + def get_table_and_query( + self, + tokenizer: TapasTokenizer, + length=5, + ): + toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))] + table = self.get_table(tokenizer, length=length - 3) + query = " ".join(toks[:3]) + + return table, query + + def get_clean_sequence( + self, + tokenizer: TapasTokenizer, + with_prefix_space=False, + max_length=20, + min_length=5, + empty_table: bool = False, + add_special_tokens: bool = True, + return_table_and_query: bool = False, + ): + + toks = [tokenizer.decode([i], clean_up_tokenization_spaces=False) for i in range(len(tokenizer))] + + if empty_table: + table = pd.DataFrame.from_dict({}) + query = " ".join(toks[:min_length]) + else: + data = {toks[0]: [toks[tok] for tok in range(1, min_length - 3)]} + table = pd.DataFrame.from_dict(data) + query = " ".join(toks[:3]) + + output_ids = tokenizer.encode(table, query, add_special_tokens=add_special_tokens) + output_txt = tokenizer.decode(output_ids) + + assert len(output_ids) >= min_length, "Update the code to generate the sequences so that they are larger" + assert len(output_ids) <= max_length, "Update the code to generate the sequences so that they are smaller" + + if return_table_and_query: + return output_txt, output_ids, table, query + + return output_txt, output_ids + + def setUp(self): + super().setUp() + + vocab_tokens = [ + "[UNK]", + "[CLS]", + "[SEP]", + "[PAD]", + "[MASK]", + "want", + "##want", + "##ed", + "wa", + "un", + "runn", + "##ing", + ",", + "low", + "lowest", + ] + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + + def get_input_output_texts(self, tokenizer): + input_text = "UNwant\u00E9d,running" + output_text = "unwanted, running" + return input_text, output_text + + def test_rust_and_python_full_tokenizers(self): + if not self.test_rust_tokenizer: + return + + tokenizer = self.get_tokenizer() + rust_tokenizer = self.get_rust_tokenizer() + + sequence = "UNwant\u00E9d,running" + + tokens = tokenizer.tokenize(sequence) + rust_tokens = rust_tokenizer.tokenize(sequence) + self.assertListEqual(tokens, rust_tokens) + + ids = tokenizer.encode(sequence, add_special_tokens=False) + rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False) + self.assertListEqual(ids, rust_ids) + + rust_tokenizer = self.get_rust_tokenizer() + ids = tokenizer.encode(sequence) + rust_ids = rust_tokenizer.encode(sequence) + self.assertListEqual(ids, rust_ids) + + # With lower casing + tokenizer = self.get_tokenizer(do_lower_case=True) + rust_tokenizer = self.get_rust_tokenizer(do_lower_case=True) + + sequence = "UNwant\u00E9d,running" + + tokens = tokenizer.tokenize(sequence) + rust_tokens = rust_tokenizer.tokenize(sequence) + self.assertListEqual(tokens, rust_tokens) + + ids = tokenizer.encode(sequence, add_special_tokens=False) + rust_ids = rust_tokenizer.encode(sequence, add_special_tokens=False) + self.assertListEqual(ids, rust_ids) + + rust_tokenizer = self.get_rust_tokenizer() + ids = tokenizer.encode(sequence) + rust_ids = rust_tokenizer.encode(sequence) + self.assertListEqual(ids, rust_ids) + + def test_chinese(self): + tokenizer = BasicTokenizer() + + self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"), ["ah", "\u535A", "\u63A8", "zz"]) + + def test_basic_tokenizer_lower(self): + tokenizer = BasicTokenizer(do_lower_case=True) + + self.assertListEqual( + tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "), ["hello", "!", "how", "are", "you", "?"] + ) + self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"]) + + def test_basic_tokenizer_lower_strip_accents_false(self): + tokenizer = BasicTokenizer(do_lower_case=True, strip_accents=False) + + self.assertListEqual( + tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["hällo", "!", "how", "are", "you", "?"] + ) + self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["h\u00E9llo"]) + + def test_basic_tokenizer_lower_strip_accents_true(self): + tokenizer = BasicTokenizer(do_lower_case=True, strip_accents=True) + + self.assertListEqual( + tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["hallo", "!", "how", "are", "you", "?"] + ) + self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"]) + + def test_basic_tokenizer_lower_strip_accents_default(self): + tokenizer = BasicTokenizer(do_lower_case=True) + + self.assertListEqual( + tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["hallo", "!", "how", "are", "you", "?"] + ) + self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"]) + + def test_basic_tokenizer_no_lower(self): + tokenizer = BasicTokenizer(do_lower_case=False) + + self.assertListEqual( + tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "), ["HeLLo", "!", "how", "Are", "yoU", "?"] + ) + + def test_basic_tokenizer_no_lower_strip_accents_false(self): + tokenizer = BasicTokenizer(do_lower_case=False, strip_accents=False) + + self.assertListEqual( + tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["HäLLo", "!", "how", "Are", "yoU", "?"] + ) + + def test_basic_tokenizer_no_lower_strip_accents_true(self): + tokenizer = BasicTokenizer(do_lower_case=False, strip_accents=True) + + self.assertListEqual( + tokenizer.tokenize(" \tHäLLo!how \n Are yoU? "), ["HaLLo", "!", "how", "Are", "yoU", "?"] + ) + + def test_basic_tokenizer_respects_never_split_tokens(self): + tokenizer = BasicTokenizer(do_lower_case=False, never_split=["[UNK]"]) + + self.assertListEqual( + tokenizer.tokenize(" \tHeLLo!how \n Are yoU? [UNK]"), ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"] + ) + + def test_wordpiece_tokenizer(self): + vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = WordpieceTokenizer(vocab=vocab, unk_token="[UNK]") + + self.assertListEqual(tokenizer.tokenize(""), []) + + self.assertListEqual(tokenizer.tokenize("unwanted running"), ["un", "##want", "##ed", "runn", "##ing"]) + + self.assertListEqual(tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) + + def test_is_whitespace(self): + self.assertTrue(_is_whitespace(" ")) + self.assertTrue(_is_whitespace("\t")) + self.assertTrue(_is_whitespace("\r")) + self.assertTrue(_is_whitespace("\n")) + self.assertTrue(_is_whitespace("\u00A0")) + + self.assertFalse(_is_whitespace("A")) + self.assertFalse(_is_whitespace("-")) + + def test_is_control(self): + self.assertTrue(_is_control("\u0005")) + + self.assertFalse(_is_control("A")) + self.assertFalse(_is_control(" ")) + self.assertFalse(_is_control("\t")) + self.assertFalse(_is_control("\r")) + + def test_is_punctuation(self): + self.assertTrue(_is_punctuation("-")) + self.assertTrue(_is_punctuation("$")) + self.assertTrue(_is_punctuation("`")) + self.assertTrue(_is_punctuation(".")) + + self.assertFalse(_is_punctuation("A")) + self.assertFalse(_is_punctuation(" ")) + + def test_clean_text(self): + tokenizer = self.get_tokenizer() + + # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340 + self.assertListEqual( + [tokenizer.tokenize(t) for t in ["Test", "\xad", "test"]], [["[UNK]"], ["[EMPTY]"], ["[UNK]"]] + ) + + @slow + def test_sequence_builders(self): + tokenizer = self.tokenizer_class.from_pretrained("nielsr/tapas-base-finetuned-wtq") + + empty_table = self.get_table(tokenizer, length=0) + table = self.get_table(tokenizer, length=10) + + text = tokenizer.encode(table, add_special_tokens=False) + text_2 = tokenizer.encode(empty_table, "multi-sequence build", add_special_tokens=False) + + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + assert encoded_pair == [101] + text + [102] + text_2 + + def test_offsets_with_special_characters(self): + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: + with self.subTest("{} ({})".format(tokenizer.__class__.__name__, pretrained_name)): + tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) + + sentence = f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence." + tokens = tokenizer_r.encode_plus( + sentence, + return_attention_mask=False, + return_token_type_ids=False, + return_offsets_mapping=True, + add_special_tokens=True, + ) + + do_lower_case = tokenizer_r.do_lower_case if hasattr(tokenizer_r, "do_lower_case") else False + expected_results = ( + [ + ((0, 0), tokenizer_r.cls_token), + ((0, 1), "A"), + ((1, 2), ","), + ((3, 5), "na"), + ((5, 6), "##ï"), + ((6, 8), "##ve"), + ((9, 15), tokenizer_r.mask_token), + ((16, 21), "Allen"), + ((21, 23), "##NL"), + ((23, 24), "##P"), + ((25, 33), "sentence"), + ((33, 34), "."), + ((0, 0), tokenizer_r.sep_token), + ] + if not do_lower_case + else [ + ((0, 0), tokenizer_r.cls_token), + ((0, 1), "a"), + ((1, 2), ","), + ((3, 8), "naive"), + ((9, 15), tokenizer_r.mask_token), + ((16, 21), "allen"), + ((21, 23), "##nl"), + ((23, 24), "##p"), + ((25, 33), "sentence"), + ((33, 34), "."), + ((0, 0), tokenizer_r.sep_token), + ] + ) + + self.assertEqual( + [e[1] for e in expected_results], tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]) + ) + self.assertEqual([e[0] for e in expected_results], tokens["offset_mapping"]) + + def test_add_special_tokens(self): + tokenizers: List[TapasTokenizer] = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + input_table = self.get_table(tokenizer, length=0) + + special_token = "[SPECIAL_TOKEN]" + + tokenizer.add_special_tokens({"cls_token": special_token}) + encoded_special_token = tokenizer.encode(input_table, special_token, add_special_tokens=False) + self.assertEqual(len(encoded_special_token), 1) + + decoded = tokenizer.decode(encoded_special_token, skip_special_tokens=True) + self.assertTrue(special_token not in decoded) + + def test_add_tokens_tokenizer(self): + tokenizers: List[TapasTokenizer] = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + vocab_size = tokenizer.vocab_size + all_size = len(tokenizer) + + self.assertNotEqual(vocab_size, 0) + + # We usually have added tokens from the start in tests because our vocab fixtures are + # smaller than the original vocabs - let's not assert this + # self.assertEqual(vocab_size, all_size) + + new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd"] + added_toks = tokenizer.add_tokens(new_toks) + vocab_size_2 = tokenizer.vocab_size + all_size_2 = len(tokenizer) + + self.assertNotEqual(vocab_size_2, 0) + self.assertEqual(vocab_size, vocab_size_2) + self.assertEqual(added_toks, len(new_toks)) + self.assertEqual(all_size_2, all_size + len(new_toks)) + + tokens = tokenizer.encode(table, "aaaaa bbbbbb low cccccccccdddddddd l", add_special_tokens=False) + + self.assertGreaterEqual(len(tokens), 4) + self.assertGreater(tokens[0], tokenizer.vocab_size - 1) + self.assertGreater(tokens[-2], tokenizer.vocab_size - 1) + + new_toks_2 = {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"} + added_toks_2 = tokenizer.add_special_tokens(new_toks_2) + vocab_size_3 = tokenizer.vocab_size + all_size_3 = len(tokenizer) + + self.assertNotEqual(vocab_size_3, 0) + self.assertEqual(vocab_size, vocab_size_3) + self.assertEqual(added_toks_2, len(new_toks_2)) + self.assertEqual(all_size_3, all_size_2 + len(new_toks_2)) + + tokens = tokenizer.encode( + table, + ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l", + add_special_tokens=False, + ) + + self.assertGreaterEqual(len(tokens), 6) + self.assertGreater(tokens[0], tokenizer.vocab_size - 1) + self.assertGreater(tokens[0], tokens[1]) + self.assertGreater(tokens[-2], tokenizer.vocab_size - 1) + self.assertGreater(tokens[-2], tokens[-3]) + self.assertEqual(tokens[0], tokenizer.eos_token_id) + self.assertEqual(tokens[-2], tokenizer.pad_token_id) + + @require_tokenizers + def test_encode_decode_with_spaces(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + + # new_toks = ["[ABC]", "[DEF]"] # TODO(thom) add this one back when Rust toks are ready: , "GHI IHG"] + new_toks = [AddedToken("[ABC]", normalized=False), AddedToken("[DEF]", normalized=False)] + tokenizer.add_tokens(new_toks) + input = "[ABC][DEF][ABC][DEF]" # TODO(thom) add back cf above: "[ABC] [DEF] [ABC] GHI IHG [DEF]" + if self.space_between_special_tokens: + output = "[ABC] [DEF] [ABC] [DEF]" + else: + output = input + encoded = tokenizer.encode(table, input, add_special_tokens=False) + decoded = tokenizer.decode(encoded, spaces_between_special_tokens=self.space_between_special_tokens) + self.assertIn(decoded, [output, output.lower()]) + + def test_encode_plus_with_padding(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + sequence = "Sequence" + + # check correct behaviour if no pad_token_id exists and add it eventually + self._check_no_pad_token_padding(tokenizer, sequence) + + padding_size = 10 + padding_idx = tokenizer.pad_token_id + token_type_padding_idx = tokenizer.pad_token_type_id + + encoded_sequence = tokenizer.encode_plus(table, sequence, return_special_tokens_mask=True) + input_ids = encoded_sequence["input_ids"] + special_tokens_mask = encoded_sequence["special_tokens_mask"] + sequence_length = len(input_ids) + + # Test 'longest' and 'no_padding' don't do anything + tokenizer.padding_side = "right" + + not_padded_sequence = tokenizer.encode_plus( + table, + sequence, + padding=False, + return_special_tokens_mask=True, + ) + not_padded_input_ids = not_padded_sequence["input_ids"] + + not_padded_special_tokens_mask = not_padded_sequence["special_tokens_mask"] + not_padded_sequence_length = len(not_padded_input_ids) + + assert sequence_length == not_padded_sequence_length + assert input_ids == not_padded_input_ids + assert special_tokens_mask == not_padded_special_tokens_mask + + not_padded_sequence = tokenizer.encode_plus( + table, + sequence, + padding=False, + return_special_tokens_mask=True, + ) + not_padded_input_ids = not_padded_sequence["input_ids"] + + not_padded_special_tokens_mask = not_padded_sequence["special_tokens_mask"] + not_padded_sequence_length = len(not_padded_input_ids) + + assert sequence_length == not_padded_sequence_length + assert input_ids == not_padded_input_ids + assert special_tokens_mask == not_padded_special_tokens_mask + + # Test right padding + tokenizer.padding_side = "right" + + right_padded_sequence = tokenizer.encode_plus( + table, + sequence, + max_length=sequence_length + padding_size, + padding="max_length", + return_special_tokens_mask=True, + ) + right_padded_input_ids = right_padded_sequence["input_ids"] + + right_padded_special_tokens_mask = right_padded_sequence["special_tokens_mask"] + right_padded_sequence_length = len(right_padded_input_ids) + + assert sequence_length + padding_size == right_padded_sequence_length + assert input_ids + [padding_idx] * padding_size == right_padded_input_ids + assert special_tokens_mask + [1] * padding_size == right_padded_special_tokens_mask + + # Test left padding + tokenizer.padding_side = "left" + left_padded_sequence = tokenizer.encode_plus( + table, + sequence, + max_length=sequence_length + padding_size, + padding="max_length", + return_special_tokens_mask=True, + ) + left_padded_input_ids = left_padded_sequence["input_ids"] + left_padded_special_tokens_mask = left_padded_sequence["special_tokens_mask"] + left_padded_sequence_length = len(left_padded_input_ids) + + assert sequence_length + padding_size == left_padded_sequence_length + assert [padding_idx] * padding_size + input_ids == left_padded_input_ids + assert [1] * padding_size + special_tokens_mask == left_padded_special_tokens_mask + + if "token_type_ids" in tokenizer.model_input_names: + token_type_ids = encoded_sequence["token_type_ids"] + left_padded_token_type_ids = left_padded_sequence["token_type_ids"] + right_padded_token_type_ids = right_padded_sequence["token_type_ids"] + + assert ( + token_type_ids + [[token_type_padding_idx] * 7] * padding_size == right_padded_token_type_ids + ) + assert [[token_type_padding_idx] * 7] * padding_size + token_type_ids == left_padded_token_type_ids + + if "attention_mask" in tokenizer.model_input_names: + attention_mask = encoded_sequence["attention_mask"] + right_padded_attention_mask = right_padded_sequence["attention_mask"] + left_padded_attention_mask = left_padded_sequence["attention_mask"] + + assert attention_mask + [0] * padding_size == right_padded_attention_mask + assert [0] * padding_size + attention_mask == left_padded_attention_mask + + def test_internal_consistency(self): + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + input_text, output_text = self.get_input_output_texts(tokenizer) + + tokens = tokenizer.tokenize(input_text) + ids = tokenizer.convert_tokens_to_ids(tokens) + ids_2 = tokenizer.encode(table, input_text, add_special_tokens=False) + self.assertListEqual(ids, ids_2) + + tokens_2 = tokenizer.convert_ids_to_tokens(ids) + self.assertNotEqual(len(tokens_2), 0) + text_2 = tokenizer.decode(ids) + self.assertIsInstance(text_2, str) + + self.assertEqual(text_2, output_text) + + def test_mask_output(self): + tokenizers = self.get_tokenizers(fast=False, do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table, query = self.get_table_and_query(tokenizer) + + if ( + tokenizer.build_inputs_with_special_tokens.__qualname__.split(".")[0] != "PreTrainedTokenizer" + and "token_type_ids" in tokenizer.model_input_names + ): + information = tokenizer.encode_plus(table, query, add_special_tokens=True) + sequences, mask = information["input_ids"], information["token_type_ids"] + self.assertEqual(len(sequences), len(mask)) + + @unittest.skip("TAPAS tokenizer only handles two sequences.") + def test_maximum_encoding_length_pair_input(self): + pass + + @unittest.skip("TAPAS tokenizer only handles two sequences.") + def test_maximum_encoding_length_single_input(self): + pass + + def test_number_of_added_tokens(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + + table, query = self.get_table_and_query(tokenizer) + + sequences = tokenizer.encode(table, query, add_special_tokens=False) + attached_sequences = tokenizer.encode(table, query, add_special_tokens=True) + + # Method is implemented (e.g. not GPT-2) + if len(attached_sequences) != 2: + self.assertEqual( + tokenizer.num_special_tokens_to_add(pair=True), len(attached_sequences) - len(sequences) + ) + + def test_padding_to_max_length(self): + """We keep this test for backward compatibility but it should be removed when `pad_to_max_length` will be deprecated""" + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer) + sequence = "Sequence" + padding_size = 10 + + # check correct behaviour if no pad_token_id exists and add it eventually + self._check_no_pad_token_padding(tokenizer, sequence) + + padding_idx = tokenizer.pad_token_id + + # Check that it correctly pads when a maximum length is specified along with the padding flag set to True + tokenizer.padding_side = "right" + encoded_sequence = tokenizer.encode(table, sequence) + sequence_length = len(encoded_sequence) + # FIXME: the next line should be padding(max_length) to avoid warning + padded_sequence = tokenizer.encode( + table, sequence, max_length=sequence_length + padding_size, padding=True + ) + padded_sequence_length = len(padded_sequence) + assert sequence_length + padding_size == padded_sequence_length + assert encoded_sequence + [padding_idx] * padding_size == padded_sequence + + # Check that nothing is done when a maximum length is not specified + encoded_sequence = tokenizer.encode(table, sequence) + sequence_length = len(encoded_sequence) + + tokenizer.padding_side = "right" + padded_sequence_right = tokenizer.encode(table, sequence, pad_to_max_length=True) + padded_sequence_right_length = len(padded_sequence_right) + assert sequence_length == padded_sequence_right_length + assert encoded_sequence == padded_sequence_right + + def test_call(self): + # Tests that all call wrap to encode_plus and batch_encode_plus + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + sequences = [ + "Testing batch encode plus", + "Testing batch encode plus with different sequence lengths", + "Testing batch encode plus with different sequence lengths correctly pads", + ] + + # Test not batched + table = self.get_table(tokenizer, length=0) + encoded_sequences_1 = tokenizer.encode_plus(table, sequences[0]) + encoded_sequences_2 = tokenizer(table, sequences[0]) + self.assertEqual(encoded_sequences_1, encoded_sequences_2) + + # Test not batched pairs + table = self.get_table(tokenizer, length=10) + encoded_sequences_1 = tokenizer.encode_plus(table, sequences[1]) + encoded_sequences_2 = tokenizer(table, sequences[1]) + self.assertEqual(encoded_sequences_1, encoded_sequences_2) + + # Test batched + table = self.get_table(tokenizer, length=0) + encoded_sequences_1 = tokenizer.batch_encode_plus(table, sequences) + encoded_sequences_2 = tokenizer(table, sequences) + self.assertEqual(encoded_sequences_1, encoded_sequences_2) + + def test_batch_encode_plus_batch_sequence_length(self): + # Tests that all encoded values have the correct size + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + sequences = [ + "Testing batch encode plus", + "Testing batch encode plus with different sequence lengths", + "Testing batch encode plus with different sequence lengths correctly pads", + ] + + encoded_sequences = [tokenizer.encode_plus(table, sequence) for sequence in sequences] + encoded_sequences_batch = tokenizer.batch_encode_plus(table, sequences, padding=False) + self.assertListEqual( + encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch) + ) + + maximum_length = len( + max([encoded_sequence["input_ids"] for encoded_sequence in encoded_sequences], key=len) + ) + + # check correct behaviour if no pad_token_id exists and add it eventually + self._check_no_pad_token_padding(tokenizer, sequences) + + encoded_sequences_padded = [ + tokenizer.encode_plus(table, sequence, max_length=maximum_length, padding="max_length") + for sequence in sequences + ] + + encoded_sequences_batch_padded = tokenizer.batch_encode_plus(table, sequences, padding=True) + self.assertListEqual( + encoded_sequences_padded, + self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch_padded), + ) + + # check 'longest' is unsensitive to a max length + encoded_sequences_batch_padded_1 = tokenizer.batch_encode_plus(table, sequences, padding=True) + encoded_sequences_batch_padded_2 = tokenizer.batch_encode_plus( + table, sequences, max_length=maximum_length + 10, padding="longest" + ) + for key in encoded_sequences_batch_padded_1.keys(): + self.assertListEqual( + encoded_sequences_batch_padded_1[key], + encoded_sequences_batch_padded_2[key], + ) + + # check 'no_padding' is unsensitive to a max length + encoded_sequences_batch_padded_1 = tokenizer.batch_encode_plus(table, sequences, padding=False) + encoded_sequences_batch_padded_2 = tokenizer.batch_encode_plus( + table, sequences, max_length=maximum_length + 10, padding=False + ) + for key in encoded_sequences_batch_padded_1.keys(): + self.assertListEqual( + encoded_sequences_batch_padded_1[key], + encoded_sequences_batch_padded_2[key], + ) + + @unittest.skip("batch_encode_plus does not handle overflowing tokens.") + def test_batch_encode_plus_overflowing_tokens(self): + pass + + def test_batch_encode_plus_padding(self): + # Test that padded sequences are equivalent between batch_encode_plus and encode_plus + + # Right padding tests + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + sequences = [ + "Testing batch encode plus", + "Testing batch encode plus with different sequence lengths", + "Testing batch encode plus with different sequence lengths correctly pads", + ] + + max_length = 100 + + # check correct behaviour if no pad_token_id exists and add it eventually + self._check_no_pad_token_padding(tokenizer, sequences) + + encoded_sequences = [ + tokenizer.encode_plus(table, sequence, max_length=max_length, padding="max_length") + for sequence in sequences + ] + encoded_sequences_batch = tokenizer.batch_encode_plus( + table, sequences, max_length=max_length, padding="max_length" + ) + self.assertListEqual( + encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch) + ) + + # Left padding tests + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + tokenizer.padding_side = "left" + sequences = [ + "Testing batch encode plus", + "Testing batch encode plus with different sequence lengths", + "Testing batch encode plus with different sequence lengths correctly pads", + ] + + max_length = 100 + + # check correct behaviour if no pad_token_id exists and add it eventually + self._check_no_pad_token_padding(tokenizer, sequences) + + encoded_sequences = [ + tokenizer.encode_plus(table, sequence, max_length=max_length, padding="max_length") + for sequence in sequences + ] + encoded_sequences_batch = tokenizer.batch_encode_plus( + table, sequences, max_length=max_length, padding="max_length" + ) + self.assertListEqual( + encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch) + ) + + def test_padding_to_multiple_of(self): + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + if tokenizer.pad_token is None: + self.skipTest("No padding token.") + else: + empty_tokens = tokenizer(table, padding=True, pad_to_multiple_of=8) + normal_tokens = tokenizer(table, "This is a sample input", padding=True, pad_to_multiple_of=8) + for key, value in empty_tokens.items(): + self.assertEqual(len(value) % 8, 0, "BatchEncoding.{} is not multiple of 8".format(key)) + for key, value in normal_tokens.items(): + self.assertEqual(len(value) % 8, 0, "BatchEncoding.{} is not multiple of 8".format(key)) + + normal_tokens = tokenizer(table, "This", pad_to_multiple_of=8) + for key, value in normal_tokens.items(): + self.assertNotEqual(len(value) % 8, 0, "BatchEncoding.{} is not multiple of 8".format(key)) + + # Should also work with truncation + normal_tokens = tokenizer(table, "This", padding=True, truncation=True, pad_to_multiple_of=8) + for key, value in normal_tokens.items(): + self.assertEqual(len(value) % 8, 0, "BatchEncoding.{} is not multiple of 8".format(key)) + + @unittest.skip("TAPAS cannot handle `prepare_for_model` without passing by `encode_plus` or `batch_encode_plus`") + def test_prepare_for_model(self): + pass + + def test_tokenizer_slow_store_full_signature(self): + signature = inspect.signature(self.tokenizer_class.__init__) + tokenizer = self.get_tokenizer() + + for parameter_name, parameter in signature.parameters.items(): + if parameter.default != inspect.Parameter.empty: + self.assertIn(parameter_name, tokenizer.init_kwargs) + + def test_special_tokens_mask_input_pairs(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + sequence_0 = "Encode this." + empty_table = self.get_table(tokenizer, length=0) + table = self.get_table(tokenizer, length=10) + encoded_sequence = tokenizer.encode(empty_table, sequence_0, add_special_tokens=False) + encoded_sequence += tokenizer.encode(table, "", add_special_tokens=False) + encoded_sequence_dict = tokenizer.encode_plus( + table, + sequence_0, + add_special_tokens=True, + return_special_tokens_mask=True, + # add_prefix_space=False, + ) + encoded_sequence_w_special = encoded_sequence_dict["input_ids"] + special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] + self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) + + filtered_sequence = [ + (x if not special_tokens_mask[i] else None) for i, x in enumerate(encoded_sequence_w_special) + ] + filtered_sequence = [x for x in filtered_sequence if x is not None] + self.assertEqual(encoded_sequence, filtered_sequence) + + def test_special_tokens_mask(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + sequence_0 = "Encode this." + # Testing single inputs + encoded_sequence = tokenizer.encode(table, sequence_0, add_special_tokens=False) + encoded_sequence_dict = tokenizer.encode_plus( + table, sequence_0, add_special_tokens=True, return_special_tokens_mask=True + ) + encoded_sequence_w_special = encoded_sequence_dict["input_ids"] + special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] + self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) + + filtered_sequence = [x for i, x in enumerate(encoded_sequence_w_special) if not special_tokens_mask[i]] + self.assertEqual(encoded_sequence, filtered_sequence) + + def test_save_and_load_tokenizer(self): + # safety check on max_len default value so we are sure the test works + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + self.assertNotEqual(tokenizer.model_max_length, 42) + + # Now let's start the test + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + # Isolate this from the other tests because we save additional tokens/etc + table = self.get_table(tokenizer, length=0) + tmpdirname = tempfile.mkdtemp() + + sample_text = " He is very happy, UNwant\u00E9d,running" + before_tokens = tokenizer.encode(table, sample_text, add_special_tokens=False) + before_vocab = tokenizer.get_vocab() + tokenizer.save_pretrained(tmpdirname) + + after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname) + after_tokens = after_tokenizer.encode(table, sample_text, add_special_tokens=False) + after_vocab = after_tokenizer.get_vocab() + self.assertListEqual(before_tokens, after_tokens) + self.assertDictEqual(before_vocab, after_vocab) + + shutil.rmtree(tmpdirname) + + def test_right_and_left_padding(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + table = self.get_table(tokenizer, length=0) + sequence = "Sequence" + padding_size = 10 + + # check correct behaviour if no pad_token_id exists and add it eventually + self._check_no_pad_token_padding(tokenizer, sequence) + + padding_idx = tokenizer.pad_token_id + + # RIGHT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True + tokenizer.padding_side = "right" + encoded_sequence = tokenizer.encode(table, sequence) + sequence_length = len(encoded_sequence) + padded_sequence = tokenizer.encode( + table, sequence, max_length=sequence_length + padding_size, padding="max_length" + ) + padded_sequence_length = len(padded_sequence) + assert sequence_length + padding_size == padded_sequence_length + assert encoded_sequence + [padding_idx] * padding_size == padded_sequence + + # LEFT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True + tokenizer.padding_side = "left" + encoded_sequence = tokenizer.encode(table, sequence) + sequence_length = len(encoded_sequence) + padded_sequence = tokenizer.encode( + table, sequence, max_length=sequence_length + padding_size, padding="max_length" + ) + padded_sequence_length = len(padded_sequence) + assert sequence_length + padding_size == padded_sequence_length + assert [padding_idx] * padding_size + encoded_sequence == padded_sequence + + # RIGHT & LEFT PADDING - Check that nothing is done for 'longest' and 'no_padding' + encoded_sequence = tokenizer.encode(table, sequence) + sequence_length = len(encoded_sequence) + + tokenizer.padding_side = "right" + padded_sequence_right = tokenizer.encode(table, sequence, padding=True) + padded_sequence_right_length = len(padded_sequence_right) + assert sequence_length == padded_sequence_right_length + assert encoded_sequence == padded_sequence_right + + tokenizer.padding_side = "left" + padded_sequence_left = tokenizer.encode(table, sequence, padding="longest") + padded_sequence_left_length = len(padded_sequence_left) + assert sequence_length == padded_sequence_left_length + assert encoded_sequence == padded_sequence_left + + tokenizer.padding_side = "right" + padded_sequence_right = tokenizer.encode(table, sequence) + padded_sequence_right_length = len(padded_sequence_right) + assert sequence_length == padded_sequence_right_length + assert encoded_sequence == padded_sequence_right + + tokenizer.padding_side = "left" + padded_sequence_left = tokenizer.encode(table, sequence, padding=False) + padded_sequence_left_length = len(padded_sequence_left) + assert sequence_length == padded_sequence_left_length + assert encoded_sequence == padded_sequence_left + + def test_token_type_ids(self): + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + empty_table = self.get_table(tokenizer, length=0) + seq_0 = "Test this method." + + # We want to have sequence 0 and sequence 1 are tagged + # respectively with 0 and 1 token_ids + # (regardless of whether the model use token type ids) + # We use this assumption in the QA pipeline among other place + output = tokenizer(empty_table, seq_0, return_token_type_ids=True) + + # Assert that the token type IDs have the same length as the input IDs + self.assertEqual(len(output["token_type_ids"]), len(output["input_ids"])) + + # Assert that each token type ID has 7 values + self.assertTrue(all(len(token_type_ids) == 7 for token_type_ids in output["token_type_ids"])) + + # Do the same test as modeling common. + self.assertIn(0, output["token_type_ids"][0]) + + # TODO: Check if require_torch is the best to test for numpy here ... Maybe move to require_flax when available + @require_torch + @slow + def test_np_encode_plus_sent_to_model(self): + from transformers import MODEL_MAPPING, TOKENIZER_MAPPING + + MODEL_TOKENIZER_MAPPING = merge_model_tokenizer_mappings(MODEL_MAPPING, TOKENIZER_MAPPING) + + tokenizer = self.get_tokenizer() + if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING: + return + + config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__] + config = config_class() + + if config.is_encoder_decoder or config.pad_token_id is None: + return + + # Build sequence + first_ten_tokens = list(tokenizer.get_vocab().keys())[:10] + table = self.get_table(tokenizer, length=0) + sequence = " ".join(first_ten_tokens) + encoded_sequence = tokenizer.encode_plus(table, sequence, return_tensors="np") + batch_encoded_sequence = tokenizer.batch_encode_plus(table, [sequence, sequence], return_tensors="np") + + # TODO: add forward through JAX/Flax when PR is merged + # This is currently here to make flake8 happy ! + if encoded_sequence is None: + raise ValueError("Cannot convert list to numpy tensor on encode_plus()") + + if batch_encoded_sequence is None: + raise ValueError("Cannot convert list to numpy tensor on batch_encode_plus()") + + if self.test_rust_tokenizer: + fast_tokenizer = self.get_rust_tokenizer() + encoded_sequence_fast = fast_tokenizer.encode_plus(table, sequence, return_tensors="np") + batch_encoded_sequence_fast = fast_tokenizer.batch_encode_plus( + table, [sequence, sequence], return_tensors="np" + ) + + # TODO: add forward through JAX/Flax when PR is merged + # This is currently here to make flake8 happy ! + if encoded_sequence_fast is None: + raise ValueError("Cannot convert list to numpy tensor on encode_plus() (fast)") + + if batch_encoded_sequence_fast is None: + raise ValueError("Cannot convert list to numpy tensor on batch_encode_plus() (fast)") + + @require_torch + @slow + def test_torch_encode_plus_sent_to_model(self): + import torch + + from transformers import MODEL_MAPPING, TOKENIZER_MAPPING + + MODEL_TOKENIZER_MAPPING = merge_model_tokenizer_mappings(MODEL_MAPPING, TOKENIZER_MAPPING) + + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + + if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING: + return + + config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__] + config = config_class() + + if config.is_encoder_decoder or config.pad_token_id is None: + return + + model = model_class(config) + + # Make sure the model contains at least the full vocabulary size in its embedding matrix + is_using_common_embeddings = hasattr(model.get_input_embeddings(), "weight") + assert ( + (model.get_input_embeddings().weight.shape[0] >= len(tokenizer)) + if is_using_common_embeddings + else True + ) + + # Build sequence + first_ten_tokens = list(tokenizer.get_vocab().keys())[:10] + sequence = " ".join(first_ten_tokens) + table = self.get_table(tokenizer, length=0) + encoded_sequence = tokenizer.encode_plus(table, sequence, return_tensors="pt") + batch_encoded_sequence = tokenizer.batch_encode_plus(table, [sequence, sequence], return_tensors="pt") + # This should not fail + + with torch.no_grad(): # saves some time + model(**encoded_sequence) + model(**batch_encoded_sequence) + + @unittest.skip("TAPAS doesn't handle pre-tokenized inputs.") + def test_pretokenized_inputs(self): + pass + + @slow + def test_tapas_truncation_integration_test(self): + data = { + "Actors": ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], + "Age": ["56", "45", "59"], + "Number of movies": ["87", "53", "69"], + "Date of birth": ["18 december 1963", "11 november 1974", "6 may 1961"], + } + queries = [ + "When was Brad Pitt born?", + "Which actor appeared in the least number of movies?", + "What is the average number of movies?", + ] + table = pd.DataFrame.from_dict(data) + + # TODO: Should update this in the future + tokenizer = TapasTokenizer.from_pretrained("lysandre/tapas-temporary-repo", model_max_length=512) + + for i in range(12): + # The table cannot even encode the headers, so raise an error + with self.assertRaises(ValueError): + tokenizer.encode(table=table, query=queries[0], max_length=i, truncation="drop_rows_to_fit") + + for i in range(12, 512): + new_encoded_inputs = tokenizer.encode( + table=table, query=queries[0], max_length=i, truncation="drop_rows_to_fit" + ) + + # Ensure that the input IDs are less than the max length defined. + self.assertLessEqual(len(new_encoded_inputs), i) + + tokenizer.model_max_length = 20 + new_encoded_inputs = tokenizer.encode(table=table, query=queries[0], truncation=True) + dropped_encoded_inputs = tokenizer.encode(table=table, query=queries[0], truncation="drop_rows_to_fit") + + # Ensure that the input IDs are still truncated when no max_length is specified + self.assertListEqual(new_encoded_inputs, dropped_encoded_inputs) + self.assertLessEqual(len(new_encoded_inputs), 20) + + @is_pt_tf_cross_test + def test_batch_encode_plus_tensors(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + sequences = [ + "Testing batch encode plus", + "Testing batch encode plus with different sequence lengths", + "Testing batch encode plus with different sequence lengths correctly pads", + ] + + table = self.get_table(tokenizer, length=0) + + # A Tensor cannot be build by sequences which are not the same size + self.assertRaises(ValueError, tokenizer.batch_encode_plus, table, sequences, return_tensors="pt") + self.assertRaises(ValueError, tokenizer.batch_encode_plus, table, sequences, return_tensors="tf") + + if tokenizer.pad_token_id is None: + self.assertRaises( + ValueError, + tokenizer.batch_encode_plus, + table, + sequences, + padding=True, + return_tensors="pt", + ) + self.assertRaises( + ValueError, + tokenizer.batch_encode_plus, + table, + sequences, + padding="longest", + return_tensors="tf", + ) + else: + pytorch_tensor = tokenizer.batch_encode_plus(table, sequences, padding=True, return_tensors="pt") + tensorflow_tensor = tokenizer.batch_encode_plus( + table, sequences, padding="longest", return_tensors="tf" + ) + encoded_sequences = tokenizer.batch_encode_plus(table, sequences, padding=True) + + for key in encoded_sequences.keys(): + pytorch_value = pytorch_tensor[key].tolist() + tensorflow_value = tensorflow_tensor[key].numpy().tolist() + encoded_value = encoded_sequences[key] + + self.assertEqual(pytorch_value, tensorflow_value, encoded_value) + + @slow + def test_tapas_integration_test(self): + data = { + "Actors": ["Brad Pitt", "Leonardo Di Caprio", "George Clooney"], + "Age": ["56", "45", "59"], + "Number of movies": ["87", "53", "69"], + "Date of birth": ["18 december 1963", "11 november 1974", "6 may 1961"], + } + queries = [ + "When was Brad Pitt born?", + "Which actor appeared in the least number of movies?", + "What is the average number of movies?", + ] + table = pd.DataFrame.from_dict(data) + + # TODO: Should update this in the future + tokenizer = TapasTokenizer.from_pretrained("lysandre/tapas-temporary-repo", model_max_length=512) + + # fmt: off + expected_results = {'input_ids':[101,2043,2001,8226,15091,2141,1029,102,5889,2287,2193,1997,5691,3058,1997,4182,8226,15091,5179,6584,2324,2285,3699,14720,4487,6178,9488,3429,5187,2340,2281,3326,2577,18856,7828,3240,5354,6353,1020,2089,3777],'attention_mask':[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1],'token_type_ids':[[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[0,0,0,0,0,0,0],[1,1,0,0,0,0,0],[1,2,0,0,0,0,0],[1,3,0,0,0,0,0],[1,3,0,0,0,0,0],[1,3,0,0,0,0,0],[1,4,0,0,0,0,0],[1,4,0,0,0,0,0],[1,4,0,0,0,0,0],[1,1,1,0,0,0,0],[1,1,1,0,0,0,0],[1,2,1,0,2,2,0],[1,3,1,0,3,1,0],[1,4,1,0,2,2,0],[1,4,1,0,2,2,0],[1,4,1,0,2,2,0],[1,1,2,0,0,0,0],[1,1,2,0,0,0,0],[1,1,2,0,0,0,0],[1,1,2,0,0,0,0],[1,2,2,0,1,3,0],[1,3,2,0,1,3,0],[1,4,2,0,3,1,0],[1,4,2,0,3,1,0],[1,4,2,0,3,1,0],[1,1,3,0,0,0,0],[1,1,3,0,0,0,0],[1,1,3,0,0,0,0],[1,1,3,0,0,0,0],[1,2,3,0,3,1,0],[1,3,3,0,2,2,0],[1,4,3,0,1,3,0],[1,4,3,0,1,3,0],[1,4,3,0,1,3,0]]} # noqa: E231 + # fmt: on + + new_encoded_inputs = tokenizer.encode_plus(table=table, query=queries[0]) + + self.assertDictEqual(dict(new_encoded_inputs), expected_results) + + @slow + def test_full_tokenizer(self): + data = [ + ["Pos", "No", "Driver", "Team", "Laps", "Time/Retired", "Grid", "Points"], + ["1", "32", "Patrick Carpentier", "Team Player's", "87", "1:48:11.023", "1", "22"], + ["2", "1", "Bruno Junqueira", "Newman/Haas Racing", "87", "+0.8 secs", "2", "17"], + ["3", "3", "Paul Tracy", "Team Player's", "87", "+28.6 secs", "3", "14"], + ["4", "9", "Michel Jourdain, Jr.", "Team Rahal", "87", "+40.8 secs", "13", "12"], + ["5", "34", "Mario Haberfeld", "Mi-Jack Conquest Racing", "87", "+42.1 secs", "6", "10"], + ["6", "20", "Oriol Servia", "Patrick Racing", "87", "+1:00.2", "10", "8"], + ["7", "51", "Adrian Fernandez", "Fernandez Racing", "87", "+1:01.4", "5", "6"], + ["8", "12", "Jimmy Vasser", "American Spirit Team Johansson", "87", "+1:01.8", "8", "5"], + ["9", "7", "Tiago Monteiro", "Fittipaldi-Dingman Racing", "86", "+ 1 Lap", "15", "4"], + ["10", "55", "Mario Dominguez", "Herdez Competition", "86", "+ 1 Lap", "11", "3"], + ["11", "27", "Bryan Herta", "PK Racing", "86", "+ 1 Lap", "12", "2"], + ["12", "31", "Ryan Hunter-Reay", "American Spirit Team Johansson", "86", "+ 1 Lap", "17", "1"], + ["13", "19", "Joel Camathias", "Dale Coyne Racing", "85", "+ 2 Laps", "18", "0"], + ["14", "33", "Alex Tagliani", "Rocketsports Racing", "85", "+ 2 Laps", "14", "0"], + ["15", "4", "Roberto Moreno", "Herdez Competition", "85", "+ 2 Laps", "9", "0"], + ["16", "11", "Geoff Boss", "Dale Coyne Racing", "83", "Mechanical", "19", "0"], + ["17", "2", "Sebastien Bourdais", "Newman/Haas Racing", "77", "Mechanical", "4", "0"], + ["18", "15", "Darren Manning", "Walker Racing", "12", "Mechanical", "7", "0"], + ["19", "5", "Rodolfo Lavin", "Walker Racing", "10", "Mechanical", "16", "0"], + ] + query = "what were the drivers names?" + table = pd.DataFrame.from_records(data[1:], columns=data[0]) + + # TODO: Will update when moving to Google namespace + tokenizer = TapasTokenizer.from_pretrained("lysandre/tapas-temporary-repo", model_max_length=512) + model_inputs = tokenizer(table, query, padding="max_length") + + input_ids = model_inputs["input_ids"] + token_type_ids = np.array(model_inputs["token_type_ids"]) + segment_ids = token_type_ids[:, 0] + column_ids = token_type_ids[:, 1] + row_ids = token_type_ids[:, 2] + + # fmt: off + expected_results = {'input_ids':[101,2054,2020,1996,6853,3415,1029,102,13433,2015,2053,4062,2136,10876,2051,1013,3394,8370,2685,1015,3590,4754,29267,4765,3771,2136,2447,1005,1055,6584,1015,1024,4466,1024,2340,1012,6185,2509,1015,2570,1016,1015,10391,12022,4226,7895,10625,1013,22996,3868,6584,1009,1014,1012,1022,10819,2015,1016,2459,1017,1017,2703,10555,2136,2447,1005,1055,6584,1009,2654,1012,1020,10819,2015,1017,2403,1018,1023,8709,8183,3126,21351,2078,1010,3781,1012,2136,10958,8865,6584,1009,2871,1012,1022,10819,2015,2410,2260,1019,4090,7986,5292,5677,8151,2771,1011,2990,9187,3868,6584,1009,4413,1012,1015,10819,2015,1020,2184,1020,2322,2030,20282,14262,9035,4754,3868,6584,1009,1015,1024,4002,1012,1016,2184,1022,1021,4868,7918,12023,12023,3868,6584,1009,1015,1024,5890,1012,1018,1019,1020,1022,2260,5261,12436,18116,2137,4382,2136,26447,6584,1009,1015,1024,5890,1012,1022,1022,1019,1023,1021,27339,3995,10125,9711,4906,25101,24657,1011,22033,2386,3868,6564,1009,1015,5001,2321,1018,2184,4583,7986,14383,2075,29488,14906,9351,2971,6564,1009,1015,5001,2340,1017,2340,2676,8527,2014,2696,1052,2243,3868,6564,1009,1015,5001,2260,1016,2260,2861,4575,4477,1011,2128,4710,2137,4382,2136,26447,6564,1009,1015,5001,2459,1015,2410,2539,8963,11503,25457,3022,8512,2522,9654,3868,5594,1009,1016,10876,2324,1014,2403,3943,4074,6415,15204,2072,12496,25378,3868,5594,1009,1016,10876,2403,1014,2321,1018,10704,17921,14906,9351,2971,5594,1009,1016,10876,1023,1014,2385,2340,14915,5795,8512,2522,9654,3868,6640,6228,2539,1014,2459,1016,28328,8945,3126,21351,2015,10625,1013,22996,3868,6255,6228,1018,1014,2324,2321,12270,11956,5232,3868,2260,6228,1021,1014,2539,1019,8473,28027,2080,2474,6371,5232,3868,2184,6228,2385,1014,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],'column_ids':[0,0,0,0,0,0,0,0,1,1,2,3,4,5,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,5,6,6,6,6,6,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,4,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,3,3,3,3,3,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,3,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,4,4,4,4,5,6,6,6,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,4,4,4,5,6,6,6,7,8,1,2,3,3,3,3,4,4,4,5,6,6,6,7,8,1,2,3,3,3,4,4,4,5,6,6,6,7,8,1,2,3,3,3,3,3,4,4,4,4,5,6,6,6,7,8,1,2,3,3,3,3,4,4,4,4,5,6,6,6,7,8,1,2,3,3,3,3,4,4,4,5,6,6,6,7,8,1,2,3,3,4,4,4,5,6,6,6,7,8,1,2,3,3,4,4,4,4,5,6,7,8,1,2,3,3,3,3,3,4,4,4,4,5,6,7,8,1,2,3,3,4,4,5,6,7,8,1,2,3,3,3,3,3,4,4,5,6,7,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],'row_ids':[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,19,19,19,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],'segment_ids':[0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]} # noqa: E231 + # fmt: on + + self.assertListEqual(input_ids, expected_results["input_ids"]) + self.assertListEqual(segment_ids.tolist(), expected_results["segment_ids"]) + self.assertListEqual(column_ids.tolist(), expected_results["column_ids"]) + self.assertListEqual(row_ids.tolist(), expected_results["row_ids"]) + + @unittest.skip("Skip this test while all models are still to be uploaded.") + def test_pretrained_model_lists(self): + pass