2.0.12

Author: yangheng95

examples-v2/train_rna_tokenizers.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# file: train_tokenizers.py
+# time: 2022/11/19 15:30
+# author: yangheng <[email protected]>
+# github: https://github.com/yangheng95
+# GScholar: https://scholar.google.com/citations?user=NPq5a_0AAAAJ&hl=en
+# ResearchGate: https://www.researchgate.net/profile/Heng-Yang-17/research
+# Copyright (C) 2022. All Rights Reserved.
+import findfile
+from transformers import AutoTokenizer
+
+from pyabsa.utils import train_word2vec, train_bpe_tokenizer
+
+if __name__ == '__main__':
+    """
+    This script is used to train word2vec and bpe tokenizer for rna/protein classification/regression tasks.
+    For example:
+    MQFKVYTYKRESRYRLFCDVQSDIIDTPGRRMVIPLASARLLSDKVSRELYPVVHIGDESWRMMTTDMASVPVSVIGEEVADLSHRENDIKNAINLMFWGI
+    -> Tokenize
+    MQFK VYTYKR ESRY RLFCDV QSDIIDT PGRRM VIP LASARLLSD KVSRELYPV VHIGDESW RMMTTDM ASVPV SVIGEE VADLSH RENDI KNAIN LMFWGI
+    -> Word2Vec Embedding
+    [1*768] or [1*300]
+    This is a not a real protein sequence, just for example.
+    """
+    paths = findfile.find_cwd_files('.txt')
+
+    # train bpe tokenizer for protein or rna sequence
+    train_bpe_tokenizer(paths, save_path='bpe_tokenizer', base_tokenizer='roberta-base')
+
+    # then you can use the bpe_tokenizer to train a protein or rna sequence word2vec embedding
+    pre_tokenizer = AutoTokenizer.from_pretrained('bpe_tokenizer')
+    train_word2vec(paths, save_path='word2vec', pre_tokenizer=pre_tokenizer)

examples-v2/train_tokenizers.py

@@ -7,7 +7,7 @@
 # Copyright (C) 2021. All Rights Reserved.
 
 __name__ = 'pyabsa'
-__version__ = '2.0.11'
+__version__ = '2.0.12'
 
 from pyabsa.framework.flag_class import *
 

pyabsa/__init__.py

@@ -23,6 +23,7 @@ class TaskNameOption(dict):
         'TAD': 'Text Adversarial Defense',
         'RNAC': 'RNA Sequence Classification',
         'RNAR': 'RNA Sequence Regression',
+        'PR': 'Protein Sequence Regression',
     }
 
     def __init__(self):
@@ -40,6 +41,7 @@ class TaskCodeOption:
     Text_Adversarial_Defense = 'TAD'
     RNASequenceClassification = 'RNAC'
     RNASequenceRegression = 'RNAR'
+    ProteinSequenceRegression = 'PR'
 
 
 class LabelPaddingOption:

pyabsa/framework/flag_class/flag_template.py

@@ -164,6 +164,9 @@ def _prepare_dataloader(self):
                     DataLoader(dataset=val_set, batch_size=self.config.batch_size, sampler=val_sampler))
 
     def _prepare_env(self):
+        self.config.tokenizer = self.tokenizer
+        self.config.embedding_matrix = self.embedding_matrix
+
         if os.path.exists('init_state_dict.bin'):
             os.remove('init_state_dict.bin')
         if self.config.cross_validate_fold > 0:
@@ -180,8 +183,6 @@ def _prepare_env(self):
         if self.config.device.type == 'cuda':
             self.logger.info("cuda memory allocated:{}".format(torch.cuda.memory_allocated(device=self.config.device)))
 
-        self.config.tokenizer = self.tokenizer
-        self.config.embedding_matrix = self.embedding_matrix
 
         print_args(self.config, self.logger)
 

pyabsa/framework/instructor_class/instructor_template.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# file: bert_mean_pooler.py
+# time: 2022/11/24 17:46
+# author: yangheng <[email protected]>
+# github: https://github.com/yangheng95
+# GScholar: https://scholar.google.com/citations?user=NPq5a_0AAAAJ&hl=en
+# ResearchGate: https://www.researchgate.net/profile/Heng-Yang-17/research
+# Copyright (C) 2022. All Rights Reserved.
+import torch
+from torch import nn
+
+
+class BERTMeanPooler(nn.Module):
+    def __init__(self, ):
+        super(BERTMeanPooler, self).__init__()
+
+    def forward(self, model_output, attention_mask):
+
+        token_embeddings = model_output
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+        sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
+        sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+        return sum_embeddings / sum_mask

pyabsa/networks/bert_mean_pooler.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# file: MAELoss.py
+# time: 2022/11/24 20:11
+# author: yangheng <[email protected]>
+# github: https://github.com/yangheng95
+# GScholar: https://scholar.google.com/citations?user=NPq5a_0AAAAJ&hl=en
+# ResearchGate: https://www.researchgate.net/profile/Heng-Yang-17/research
+# Copyright (C) 2022. All Rights Reserved.
+import torch
+from torch import nn
+
+
+class MAELoss(nn.Module):
+
+    def __init__(self):
+        super(MAELoss, self).__init__()
+
+    def forward(self, y_pred, y_true):
+        return torch.mean(torch.abs(y_pred - y_true))

pyabsa/networks/losses/MAELoss.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# file: R2Loss.py
+# time: 2022/11/24 20:06
+# author: yangheng <[email protected]>
+# github: https://github.com/yangheng95
+# GScholar: https://scholar.google.com/citations?user=NPq5a_0AAAAJ&hl=en
+# ResearchGate: https://www.researchgate.net/profile/Heng-Yang-17/research
+# Copyright (C) 2022. All Rights Reserved.
+import torch
+from torch import nn
+
+
+class R2Loss(nn.Module):
+    def __init__(self):
+        super(R2Loss, self).__init__()
+        self.mse = nn.MSELoss()
+
+    def forward(self, y_pred, y_true):
+        y_true_mean = torch.mean(y_true, dim=0)
+        ss_tot = torch.sum((y_true - y_true_mean) ** 2, dim=0)
+        ss_res = torch.sum((y_true - y_pred) ** 2, dim=0)
+        r2 = 1 - ss_res / ss_tot
+        return 1 - torch.mean(r2)

pyabsa/networks/losses/R2Loss.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# file: RMSELoss.py
+# time: 2022/11/24 20:10
+# author: yangheng <[email protected]>
+# github: https://github.com/yangheng95
+# GScholar: https://scholar.google.com/citations?user=NPq5a_0AAAAJ&hl=en
+# ResearchGate: https://www.researchgate.net/profile/Heng-Yang-17/research
+# Copyright (C) 2022. All Rights Reserved.
+import torch
+from torch import nn
+
+
+class RMSELoss(nn.Module):
+    def __init__(self):
+        super(RMSELoss, self).__init__()
+
+    def forward(self, y_pred, y_true):
+        return torch.sqrt(nn.MSELoss()(y_pred, y_true))

pyabsa/networks/losses/RMSELoss.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+# file: __init__.py.py
+# time: 2022/11/24 20:05
+# author: yangheng <[email protected]>
+# github: https://github.com/yangheng95
+# GScholar: https://scholar.google.com/citations?user=NPq5a_0AAAAJ&hl=en
+# ResearchGate: https://www.researchgate.net/profile/Heng-Yang-17/research
+# Copyright (C) 2022. All Rights Reserved.

pyabsa/networks/losses/__init__.py

@@ -11,8 +11,8 @@
 from termcolor import colored
 
 from pyabsa.framework.dataset_class.dataset_template import PyABSADataset
-from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__plm__.classic_bert_apc_utils import prepare_input_for_apc, build_sentiment_window
-from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__plm__.dependency_graph import configure_spacy_model, prepare_dependency_graph
+from ...dataset_utils.__plm__.classic_bert_apc_utils import prepare_input_for_apc, build_sentiment_window
+from ...dataset_utils.__plm__.dependency_graph import configure_spacy_model, prepare_dependency_graph
 from pyabsa.utils.file_utils.file_utils import load_dataset_from_file
 from pyabsa.utils.pyabsa_utils import check_and_fix_labels, validate_example
 

pyabsa/tasks/AspectPolarityClassification/dataset_utils/__plm__/data_utils_for_training.py

@@ -19,7 +19,7 @@
 
 from pyabsa import DeviceTypeOption
 from pyabsa.framework.instructor_class.instructor_template import BaseTrainingInstructor
-from pyabsa.tasks.AspectPolarityClassification.instructor.ensembler import APCEnsembler
+from ..instructor.ensembler import APCEnsembler
 from pyabsa.utils.file_utils.file_utils import save_model
 from pyabsa.utils.pyabsa_utils import print_args, init_optimizer
 
@@ -480,15 +480,18 @@ def _init_misc(self):
                     {'params': self.model.models[0].eta2, 'lr': self.config.eta_lr, 'weight_decay': self.config.l2reg}
                 ],
                 lr=self.config.learning_rate,
-                weight_decay=self.config.l2reg
+                weight_decay=self.config.l2reg,
+                maximize=self.config.maximize_loss if self.config.get('maximize_loss') else False
             )
         else:
             self.optimizer = init_optimizer(self.config.optimizer)(
                 self.model.parameters(),
                 lr=self.config.learning_rate,
-                weight_decay=self.config.l2reg
+                weight_decay=self.config.l2reg,
+                maximize=self.config.maximize_loss if self.config.get('maximize_loss') else False
             )
 
+
     def _cache_or_load_dataset(self):
         pass
 

pyabsa/tasks/AspectPolarityClassification/instructor/apc_instructor.py

@@ -19,12 +19,12 @@
 from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from transformers import AutoTokenizer, AutoModel
 
-from pyabsa.tasks.AspectPolarityClassification.models.__classic__ import GloVeAPCModelList
-from pyabsa.tasks.AspectPolarityClassification.models.__lcf__ import APCModelList
-from pyabsa.tasks.AspectPolarityClassification.models.__plm__ import BERTBaselineAPCModelList
-from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__classic__.data_utils_for_training import GloVeABSADataset
-from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__lcf__.data_utils_for_training import ABSADataset
-from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__plm__.data_utils_for_training import BERTBaselineABSADataset
+from ..models.__classic__ import GloVeAPCModelList
+from ..models.__lcf__ import APCModelList
+from ..models.__plm__ import BERTBaselineAPCModelList
+from ..dataset_utils.__classic__.data_utils_for_training import GloVeABSADataset
+from ..dataset_utils.__lcf__.data_utils_for_training import ABSADataset
+from ..dataset_utils.__plm__.data_utils_for_training import BERTBaselineABSADataset
 from pyabsa.framework.tokenizer_class.tokenizer_class import PretrainedTokenizer, Tokenizer, build_embedding_matrix
 
 
@@ -119,7 +119,6 @@ def __init__(self, config, load_dataset=True, **kwargs):
                     self.valid_set = GloVeABSADataset(self.config, self.tokenizer, dataset_type='valid') if not self.valid_set else self.valid_set
 
                 self.models.append(models[i](copy.deepcopy(self.embedding_matrix) if self.config.deep_ensemble else self.embedding_matrix, self.config))
-                self.config.tokenizer = self.tokenizer
                 self.config.embedding_matrix = self.embedding_matrix
 
             if self.config.cache_dataset and not os.path.exists(cache_path) and not self.config.overwrite_cache:
@@ -137,6 +136,8 @@ def __init__(self, config, load_dataset=True, **kwargs):
                     valid_sampler = SequentialSampler(self.valid_set if not self.valid_set else self.valid_set)
                     self.valid_dataloader = DataLoader(self.valid_set, batch_size=self.config.batch_size, pin_memory=True, sampler=valid_sampler)
 
+            self.config.tokenizer = self.tokenizer
+
         self.dense = nn.Linear(config.output_dim * len(models), config.output_dim)
 
     def forward(self, inputs):

pyabsa/tasks/AspectPolarityClassification/instructor/ensembler.py

@@ -19,17 +19,16 @@
 
 from pyabsa import LabelPaddingOption, TaskCodeOption
 from pyabsa.framework.prediction_class.predictor_template import InferenceModel
-from pyabsa.tasks.AspectPolarityClassification.models.__plm__ import BERTBaselineAPCModelList
-from pyabsa.tasks.AspectPolarityClassification.models.__classic__ import GloVeAPCModelList
-from pyabsa.tasks.AspectPolarityClassification.models.__lcf__ import APCModelList
-from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__classic__.data_utils_for_inference import GloVeABSAInferenceDataset
-from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__lcf__.data_utils_for_inference import ABSAInferenceDataset
-from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__plm__.data_utils_for_inference import BERTABSAInferenceDataset
-from pyabsa.tasks.AspectPolarityClassification.instructor.ensembler import APCEnsembler
+from ..models.__plm__ import BERTBaselineAPCModelList
+from ..models.__classic__ import GloVeAPCModelList
+from ..models.__lcf__ import APCModelList
+from ..dataset_utils.__classic__.data_utils_for_inference import GloVeABSAInferenceDataset
+from ..dataset_utils.__lcf__.data_utils_for_inference import ABSAInferenceDataset
+from ..dataset_utils.__plm__.data_utils_for_inference import BERTABSAInferenceDataset
+from ..instructor.ensembler import APCEnsembler
 from pyabsa.utils.data_utils.dataset_manager import detect_infer_dataset
 from pyabsa.utils.pyabsa_utils import get_device, print_args
 from pyabsa.utils.text_utils.mlm import get_mlm_and_tokenizer
-from pyabsa.framework.tokenizer_class.tokenizer_class import Tokenizer
 
 
 class SentimentClassifier(InferenceModel):
@@ -41,7 +40,7 @@ def __init__(self, checkpoint=None, cal_perplexity=False, **kwargs):
         super().__init__(checkpoint, cal_perplexity, task_code=self.task_code, **kwargs)
 
         # load from a trainer
-        if not isinstance(self.checkpoint, str):
+        if self.checkpoint and not isinstance(self.checkpoint, str):
             print('Load sentiment classifier from trainer')
             self.model = self.checkpoint[0]
             self.config = self.checkpoint[1]

pyabsa/tasks/AspectPolarityClassification/prediction/sentiment_classifier.py

@@ -11,9 +11,9 @@
 
 from pyabsa import DeviceTypeOption, ModelSaveOption, TaskCodeOption, TaskNameOption
 from pyabsa.framework.trainer_class.trainer_template import Trainer
-from pyabsa.tasks.AspectPolarityClassification.configuration.apc_configuration import APCConfigManager
-from pyabsa.tasks.AspectPolarityClassification.prediction.sentiment_classifier import SentimentClassifier
-from pyabsa.tasks.AspectPolarityClassification.instructor.apc_instructor import APCTrainingInstructor
+from ..configuration.apc_configuration import APCConfigManager
+from ..prediction.sentiment_classifier import SentimentClassifier
+from ..instructor.apc_instructor import APCTrainingInstructor
 
 
 class APCTrainer(Trainer):

pyabsa/tasks/AspectPolarityClassification/trainer/apc_trainer.py

@@ -10,7 +10,7 @@
 import copy
 
 from pyabsa.framework.configuration_class.configuration_template import ConfigManager
-from pyabsa.tasks.AspectTermExtraction.models.__lcf__.lcf_atepc import LCF_ATEPC
+from ..models.__lcf__.lcf_atepc import LCF_ATEPC
 
 # if you find the optimal param set of some situation, e.g., some model on some datasets
 # please share the main use template main

pyabsa/tasks/AspectTermExtraction/configuration/atepc_configuration.py

@@ -9,7 +9,7 @@
 
 from pyabsa import LabelPaddingOption
 from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__lcf__.apc_utils import configure_spacy_model
-from pyabsa.tasks.AspectTermExtraction.dataset_utils.__lcf__.atepc_utils import simple_split_text, prepare_input_for_atepc
+from ...dataset_utils.__lcf__.atepc_utils import simple_split_text, prepare_input_for_atepc
 
 
 class InputExample(object):

pyabsa/tasks/AspectTermExtraction/dataset_utils/__lcf__/data_utils_for_inference.py

@@ -9,7 +9,7 @@
 
 from pyabsa import LabelPaddingOption
 from pyabsa.tasks.AspectPolarityClassification.dataset_utils.__lcf__.apc_utils import configure_spacy_model
-from pyabsa.tasks.AspectTermExtraction.dataset_utils.__lcf__.atepc_utils import prepare_input_for_atepc
+from ...dataset_utils.__lcf__.atepc_utils import prepare_input_for_atepc
 from pyabsa.utils.pyabsa_utils import validate_example, check_and_fix_labels, check_and_fix_IOB_labels
 
 Labels = set()

pyabsa/tasks/AspectTermExtraction/dataset_utils/__lcf__/data_utils_for_training.py

@@ -19,14 +19,13 @@
 import tqdm
 from seqeval.metrics import classification_report
 from sklearn.metrics import f1_score
-from termcolor import colored
 from torch import cuda
 from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset)
 from transformers import AutoTokenizer, AutoModel
 
 from pyabsa import DeviceTypeOption
 from pyabsa.framework.instructor_class.instructor_template import BaseTrainingInstructor
-from pyabsa.tasks.AspectTermExtraction.dataset_utils.__lcf__.data_utils_for_training import ATEPCProcessor, convert_examples_to_features
+from ..dataset_utils.__lcf__.data_utils_for_training import ATEPCProcessor, convert_examples_to_features
 from pyabsa.utils.file_utils.file_utils import save_model
 from pyabsa.utils.pyabsa_utils import print_args, init_optimizer
 
@@ -468,9 +467,12 @@ def _init_misc(self):
             self.model.to(self.config.device)
 
         if isinstance(self.config.optimizer, str):
-            self.optimizer = init_optimizer(self.config.optimizer)(self.optimizer_grouped_parameters,
-                                                                   lr=self.config.learning_rate,
-                                                                   weight_decay=self.config.l2reg)
+            self.optimizer = init_optimizer(self.config.optimizer)(
+                self.optimizer_grouped_parameters,
+                lr=self.config.learning_rate,
+                weight_decay=self.config.l2reg,
+                maximize=self.config.maximize_loss if self.config.get('maximize_loss') else False
+            )
         self.config.device = torch.device(self.config.device)
         if self.config.device.type == 'cuda':
             self.logger.info(