merge latest #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

iedmrc merged 32 commits into iedmrc:master from huggingface:master

Nov 16, 2019

.circleci/deploy.sh

-Original file line number
+Diff line change
@@ Expand Up / @@ -5,8 +5,12 @@ function deploy_doc(){ @@
     	git checkout $1
     	if [ ! -z "$2" ]
     	then
-    		echo "Pushing version" $2
-    		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2
+    		if [ -d "$dir/$2" ]; then
+    			echo "Directory" $2 "already exists"
+    		else
+    			echo "Pushing version" $2
+    			make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2
+    		fi
     	else
     		echo "Pushing master"
     		make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir
@@ Expand Down @@

README.md

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -520,12 +520,12 @@ Here is a conversion examples from `BertAdam` with a linear warmup and decay sch
  
    # Parameters:

    lr = 1e-3

    max_grad_norm = 1.0

    num_total_steps = 1000

    num_training_steps = 1000

    num_warmup_steps = 100

    warmup_proportion = float(num_warmup_steps) / float(num_total_steps)  # 0.1

    warmup_proportion = float(num_warmup_steps) / float(num_training_steps)  # 0.1

    ### Previously BertAdam optimizer was instantiated like this:

    optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_total_steps)

    optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_training_steps)

    ### and used like this:

    for batch in train_data:

        loss = model(batch)

    @@ -534,7 +534,7 @@ for batch in train_data:
  
    ### In Transformers, optimizer and schedules are splitted and instantiated like this:

    optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False

    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler

    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)  # PyTorch scheduler

    ### and used like this:

    for batch in train_data:

        model.train()

docs/source/main_classes/optimizer_schedules.rst

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -18,37 +18,33 @@ Schedules
  
    Learning Rate Schedules

    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

    .. autoclass:: transformers.ConstantLRSchedule

        :members:

    .. autofunction:: transformers.get_constant_schedule

    .. autoclass:: transformers.WarmupConstantSchedule

        :members:

    .. autofunction:: transformers.get_constant_schedule_with_warmup

    .. image:: /imgs/warmup_constant_schedule.png

        :target: /imgs/warmup_constant_schedule.png

        :alt:

    .. autoclass:: transformers.WarmupCosineSchedule

    .. autofunction:: transformers.get_cosine_schedule_with_warmup

        :members:

    .. image:: /imgs/warmup_cosine_schedule.png

        :target: /imgs/warmup_cosine_schedule.png

        :alt:

    .. autoclass:: transformers.WarmupCosineWithHardRestartsSchedule

        :members:

    .. autofunction:: transformers.get_cosine_with_hard_restarts_schedule_with_warmup

    .. image:: /imgs/warmup_cosine_hard_restarts_schedule.png

        :target: /imgs/warmup_cosine_hard_restarts_schedule.png

        :alt:

    .. autoclass:: transformers.WarmupLinearSchedule

        :members:

    .. autofunction:: transformers.get_linear_schedule_with_warmup

    .. image:: /imgs/warmup_linear_schedule.png

        :target: /imgs/warmup_linear_schedule.png

docs/source/migration.md

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -84,12 +84,12 @@ Here is a conversion examples from `BertAdam` with a linear warmup and decay sch
  
    # Parameters:

    lr = 1e-3

    max_grad_norm = 1.0

    num_total_steps = 1000

    num_training_steps = 1000

    num_warmup_steps = 100

    warmup_proportion = float(num_warmup_steps) / float(num_total_steps)  # 0.1

    warmup_proportion = float(num_warmup_steps) / float(num_training_steps)  # 0.1

    ### Previously BertAdam optimizer was instantiated like this:

    optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_total_steps)

    optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, num_training_steps=num_training_steps)

    ### and used like this:

    for batch in train_data:

        loss = model(batch)

    @@ -98,7 +98,7 @@ for batch in train_data:
  
    ### In Transformers, optimizer and schedules are splitted and instantiated like this:

    optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False

    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler

    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)  # PyTorch scheduler

    ### and used like this:

    for batch in train_data:

        loss = model(batch)

docs/source/pretrained_models.rst

-Original file line number
+Diff line change
@@ Expand Up @@
     | CTRL              | ``ctrl``                                                   | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters                                                                                    |
     |                   |                                                            | | Salesforce's Large-sized CTRL English model                                                                                         |
     +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
+    | CamemBERT         | ``camembert-base``                                         | | 12-layer, 768-hidden, 12-heads, 110M parameters                                                                                     |
+    |                   |                                                            | | CamemBERT using the BERT-base architecture                                                                                          |
+    |                   |                                                            | (see `details <https://github.com/pytorch/fairseq/tree/master/examples/camembert>`__)                                                 |
+    +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+
-    .. <https://huggingface.co/transformers/examples.html>`__
+    .. <https://huggingface.co/transformers/examples.html>`__

docs/source/quickstart.md

-Original file line number
+Diff line change
@@ Expand Up @@
     ```
     Examples for each model class of each model architecture (Bert, GPT, GPT-2, Transformer-XL, XLNet and XLM) can be found in the [documentation](#documentation).
+    #### Using the past
+    GPT-2 as well as some other models (GPT, XLNet, Transfo-XL, CTRL) make use of a `past` or `mems` attribute which can be used to prevent re-computing the key/value pairs when using sequential decoding. It is useful when generating sequences as a big part of the attention mechanism benefits from previous computations.
+    Here is a fully-working example using the `past` with `GPT2LMHeadModel` and argmax decoding (which should only be used as an example, as argmax decoding introduces a lot of repetition):
+    ```python
+    from transformers import GPT2LMHeadModel, GPT2Tokenizer
+    import torch
+    tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+    model = GPT2LMHeadModel.from_pretrained('gpt2')
+    generated = tokenizer.encode("The Manhattan bridge")
+    context = torch.tensor([generated])
+    past = None
+    for i in range(100):
+        print(i)
+        output, past = model(context, past=past)
+        token = torch.argmax(output[0, :])
+        generated += [token.tolist()]
+        context = token.unsqueeze(0)
+    sequence = tokenizer.decode(generated)
+    print(sequence)
+    ```
+    The model only requires a single token as input as all the previous tokens' key/value pairs are contained in the `past`.

examples/README.md

-Original file line number
+Diff line change
@@ Expand Up @@
 /04/2019 00:42:42 - INFO - __main__ -     recall = 0.8624150210424085
     ```
+    ### Comparing BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased)
+    Here is a small comparison between BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) with the same hyperparameters as specified in the [example documentation](https://huggingface.co/transformers/examples.html#named-entity-recognition) (one run):
+    | Model | F-Score Dev | F-Score Test
+    | --------------------------------- | ------- | --------
+    | `bert-large-cased`            | 95.59 | 91.70
+    | `roberta-large`                  | 95.96 | 91.87
+    | `distilbert-base-uncased` | 94.34 | 90.32
     ## Abstractive summarization
     Based on the script
@@ Expand Down @@

examples/contrib/run_camembert.py

-Original file line number
+Diff line change
@@ -0,0 +1,48 @@
+    from pathlib import Path
+    import tarfile
+    import urllib.request
+    import torch
+    from transformers.tokenization_camembert import CamembertTokenizer
+    from transformers.modeling_camembert import CamembertForMaskedLM
+    def fill_mask(masked_input, model, tokenizer, topk=5):
+        # Adapted from https://github.com/pytorch/fairseq/blob/master/fairseq/models/roberta/hub_interface.py
+        assert masked_input.count('<mask>') == 1
+        input_ids = torch.tensor(tokenizer.encode(masked_input, add_special_tokens=True)).unsqueeze(0)  # Batch size 1
+        logits = model(input_ids)[0]  # The last hidden-state is the first element of the output tuple
+        masked_index = (input_ids.squeeze() == tokenizer.mask_token_id).nonzero().item()
+        logits = logits[0, masked_index, :]
+        prob = logits.softmax(dim=0)
+        values, indices = prob.topk(k=topk, dim=0)
+        topk_predicted_token_bpe = ' '.join([tokenizer.convert_ids_to_tokens(indices[i].item())
+                                             for i in range(len(indices))])
+        masked_token = tokenizer.mask_token
+        topk_filled_outputs = []
+        for index, predicted_token_bpe in enumerate(topk_predicted_token_bpe.split(' ')):
+            predicted_token = predicted_token_bpe.replace('\u2581', ' ')
+            if " {0}".format(masked_token) in masked_input:
+                topk_filled_outputs.append((
+                    masked_input.replace(
+                        ' {0}'.format(masked_token), predicted_token
+                    ),
+                    values[index].item(),
+                    predicted_token,
+                ))
+            else:
+                topk_filled_outputs.append((
+                    masked_input.replace(masked_token, predicted_token),
+                    values[index].item(),
+                    predicted_token,
+                ))
+        return topk_filled_outputs
+    tokenizer = CamembertTokenizer.from_pretrained('camembert-base')
+    model = CamembertForMaskedLM.from_pretrained('camembert-base')
+    model.eval()
+    masked_input = "Le camembert est <mask> :)"
+    print(fill_mask(masked_input, model, tokenizer, topk=3))

examples/contrib/run_openai_gpt.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -41,7 +41,7 @@
  
    from transformers import (OpenAIGPTDoubleHeadsModel, OpenAIGPTTokenizer,

                                         AdamW, cached_path, WEIGHTS_NAME, CONFIG_NAME,

                                         WarmupLinearSchedule)

                                         get_linear_schedule_with_warmup)

    ROCSTORIES_URL = "https://s3.amazonaws.com/datasets.huggingface.co/ROCStories.tar.gz"

    @@ -211,7 +211,7 @@ def tokenize_and_encode(obj):
  
                {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}

                ]

            optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)

            scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)

            scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)

        if args.do_train:

            nb_tr_steps, tr_loss, exp_average_loss = 0, 0, None

examples/contrib/run_swag.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -42,7 +42,7 @@
  
    from transformers import (WEIGHTS_NAME, BertConfig,

                                      BertForMultipleChoice, BertTokenizer)

    from transformers import AdamW, WarmupLinearSchedule

    from transformers import AdamW, get_linear_schedule_with_warmup

    logger = logging.getLogger(__name__)

    @@ -322,7 +322,7 @@ def train(args, train_dataset, model, tokenizer):
  
            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}

            ]

        optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)

        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)

        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)

        if args.fp16:

            try:

                from apex import amp

examples/distillation/distiller.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -35,7 +35,7 @@
  
    except:

        from tensorboardX import SummaryWriter

    from transformers import WarmupLinearSchedule

    from transformers import get_linear_schedule_with_warmup

    from utils import logger

    from lm_seqs_dataset import LmSeqsDataset

    @@ -137,9 +137,9 @@ def __init__(self,
  
                                   betas=(0.9, 0.98))

            warmup_steps = math.ceil(num_train_optimization_steps * params.warmup_prop)

            self.scheduler = WarmupLinearSchedule(self.optimizer,

                                                    warmup_steps=warmup_steps,

                                                    t_total=num_train_optimization_steps)

            self.scheduler = get_linear_schedule_with_warmup(self.optimizer,

                                                    num_warmup_steps=warmup_steps,

                                                    num_training_steps=num_train_optimization_steps)

            if self.fp16:

                try:

examples/distillation/run_squad_w_distillation.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -46,7 +46,7 @@
  
                                      XLNetTokenizer,

                                      DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer)

    from transformers import AdamW, WarmupLinearSchedule

    from transformers import AdamW, get_linear_schedule_with_warmup

    from ..utils_squad import (read_squad_examples, convert_examples_to_features,

                             RawResult, write_predictions,

    @@ -101,7 +101,7 @@ def train(args, train_dataset, model, tokenizer, teacher=None):
  
            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}

            ]

        optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)

        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)

        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)

        if args.fp16:

            try:

                from apex import amp

examples/run_glue.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -49,7 +49,7 @@
  
                                      DistilBertForSequenceClassification,

                                      DistilBertTokenizer)

    from transformers import AdamW, WarmupLinearSchedule

    from transformers import AdamW, get_linear_schedule_with_warmup

    from transformers import glue_compute_metrics as compute_metrics

    from transformers import glue_output_modes as output_modes

    @@ -100,7 +100,7 @@ def train(args, train_dataset, model, tokenizer):
  
            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}

            ]

        optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)

        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)

        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)

        if args.fp16:

            try:

                from apex import amp

    @@ -224,6 +224,10 @@ def evaluate(args, model, tokenizer, prefix=""):
  
            eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)

            eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)

            # multi-gpu eval

            if args.n_gpu > 1:

                model = torch.nn.DataParallel(model)

            # Eval!

            logger.info("***** Running evaluation {} *****".format(prefix))

            logger.info("  Num examples = %d", len(eval_dataset))

examples/run_lm_finetuning.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -42,7 +42,7 @@
  
    from tqdm import tqdm, trange

    from transformers import (WEIGHTS_NAME, AdamW, WarmupLinearSchedule,

    from transformers import (WEIGHTS_NAME, AdamW, get_linear_schedule_with_warmup,

                                      BertConfig, BertForMaskedLM, BertTokenizer,

                                      GPT2Config, GPT2LMHeadModel, GPT2Tokenizer,

                                      OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer,

    @@ -63,10 +63,10 @@
  
    class TextDataset(Dataset):

        def __init__(self, tokenizer, file_path='train', block_size=512):

        def __init__(self, tokenizer, args, file_path='train', block_size=512):

            assert os.path.isfile(file_path)

            directory, filename = os.path.split(file_path)

            cached_features_file = os.path.join(directory, 'cached_lm_' + str(block_size) + '_' + filename)

            cached_features_file = os.path.join(directory, args.model_name_or_path + '_cached_lm_' + str(block_size) + '_' + filename)

            if os.path.exists(cached_features_file):

                logger.info("Loading features from cached file %s", cached_features_file)

    @@ -99,7 +99,7 @@ def __getitem__(self, item):
  
    def load_and_cache_examples(args, tokenizer, evaluate=False):

        dataset = TextDataset(tokenizer, file_path=args.eval_data_file if evaluate else args.train_data_file, block_size=args.block_size)

        dataset = TextDataset(tokenizer, args, file_path=args.eval_data_file if evaluate else args.train_data_file, block_size=args.block_size)

        return dataset

    @@ -185,7 +185,7 @@ def train(args, train_dataset, model, tokenizer):
  
            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}

            ]

        optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)

        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)

        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)

        if args.fp16:

            try:

                from apex import amp

    @@ -300,6 +300,10 @@ def evaluate(args, model, tokenizer, prefix=""):
  
        eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)

        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)

        # multi-gpu evaluate

        if args.n_gpu > 1:

            model = torch.nn.DataParallel(model)

        # Eval!

        logger.info("***** Running evaluation {} *****".format(prefix))

        logger.info("  Num examples = %d", len(eval_dataset))

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

merge latest #1

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!