Add files via upload

Author: cjymz886

lora/ds_config.json

@@ -0,0 +1,22 @@
+{
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu" :"auto",
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": 1.0,
+    "bf16": {
+        "enabled": "auto"
+    },
+    "zero_optimization": {
+    "stage": 3,
+    "overlap_comm": true,
+    "stage3_gather_16bit_weights_on_model_save": true
+    },
+    "flops_profiler": {
+        "enabled": false,
+        "profile_step": 1,
+        "module_depth": -1,
+        "top_modules": 1,
+        "detailed": true,
+        "output_file": null
+    }
+}

lora/evaluate.sh

@@ -0,0 +1,4 @@
+
+python infer-tune.py  \
+    --data_path ./data/pkumod-ccks_query_list_test4.txt\
+    --output_dir output

lora/fine-tune.py

@@ -0,0 +1,252 @@
+import os
+import math
+import pathlib
+from typing import Optional, Dict
+from dataclasses import dataclass, field
+import json
+import time
+
+import torch
+from torch.utils.data import Dataset
+import transformers
+from transformers.training_args import TrainingArguments
+
+os.environ["PL_TORCH_DISTRIBUTED_BACKEND"] = "gloo"
+os.environ["WANDB_DISABLED"] = "true"
+
+
+@dataclass
+class ModelArguments:
+    model_name_or_path: Optional[str] = field(default=r"E:\pretraing_models\torch\baichuan2-7B-Chat")
+
+
+@dataclass
+class DataArguments:
+    data_path: str = field(
+        default=None, metadata={"help": "Path to the training data."}
+    )
+    max_source_length: int = field(
+        default=1000,
+        metadata={
+            "help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
+        },
+    )
+    max_target_length: int = field(
+        default=200,
+        metadata={
+            "help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
+        },
+    )
+
+
+
+@dataclass
+class TrainingArguments(transformers.TrainingArguments):
+    cache_dir: Optional[str] = field(default=None)
+    optim: str = field(default="adamw_torch")
+    use_lora: bool = field(default=True)
+    model_max_length: int = field(
+        default=1201,
+        metadata={
+            "help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
+        },
+    )
+    
+
+
+class SupervisedDataset(Dataset):
+    """Dataset for supervised fine-tuning."""
+
+    def __init__(
+        self,
+        data_path,
+        tokenizer,
+        model_source_length,
+        user_tokens=[195],
+        assistant_tokens=[196],
+    ):
+        super(SupervisedDataset, self).__init__()
+        self.data = json.load(open(data_path))
+        self.tokenizer = tokenizer
+        self.model_max_length = model_max_length
+        self.user_tokens = user_tokens
+        self.assistant_tokens = assistant_tokens
+        self.ignore_index = -100
+        item = self.preprocessing(self.data[120])
+        # print("input:", self.tokenizer.decode(item["input_ids"]))
+        labels = []
+        for id_ in item["labels"]:
+            if id_ == -100:
+                continue
+            labels.append(id_)
+        print("label:", self.tokenizer.decode(labels))
+
+    def __len__(self):
+        return len(self.data)
+
+    def preprocessing(self, example):
+        input_ids = []
+        labels = []
+
+        for message in example["conversations"]:
+            from_ = message["from"]
+            value = message["value"]
+            value_ids = self.tokenizer.encode(value)
+
+            if from_ == "human":
+                input_ids += self.user_tokens + value_ids
+                labels += [self.tokenizer.eos_token_id] + [self.ignore_index] * len(
+                    value_ids
+                )
+            else:
+                input_ids += self.assistant_tokens + value_ids
+                labels += [self.ignore_index] + value_ids
+        input_ids.append(self.tokenizer.eos_token_id)
+        labels.append(self.tokenizer.eos_token_id)
+        input_ids = input_ids[: self.model_max_length]
+        labels = labels[: self.model_max_length]
+        input_ids += [self.tokenizer.pad_token_id] * (
+            self.model_max_length - len(input_ids)
+        )
+        labels += [self.ignore_index] * (self.model_max_length - len(labels))
+        input_ids = torch.LongTensor(input_ids)
+        labels = torch.LongTensor(labels)
+        attention_mask = input_ids.ne(self.tokenizer.pad_token_id)
+        return {
+            "input_ids": input_ids,
+            "labels": labels,
+            "attention_mask": attention_mask,
+        }
+
+    def __getitem__(self, idx) -> Dict[str, torch.Tensor]:
+        return self.preprocessing(self.data[idx])
+
+
+
+class MySupervisedDataset(Dataset):
+    """Dataset for supervised fine-tuning."""
+
+    def __init__(
+        self,
+        data_path,
+        tokenizer,
+        max_source_length,
+        max_target_length,
+        max_seq_length
+    ):
+        super(MySupervisedDataset, self).__init__()
+        self.data = self.load_data(data_path)
+        self.tokenizer = tokenizer
+        self.max_source_length = max_source_length
+        self.max_target_length = max_target_length
+        self.max_seq_length = max_seq_length
+        self.ignore_index = -100
+        item = self.preprocessing(self.data[1])
+        print("input:", self.tokenizer.decode(item["input_ids"]))
+        labels = []
+        for id_ in item["labels"]:
+            if id_ == -100:
+                continue
+            labels.append(id_)
+        print("label:", self.tokenizer.decode(labels))
+
+    def load_data(self,data_path):
+        D = []
+        with open(data_path,'r',encoding='utf-8') as f:
+            for line in f :
+                line = json.loads(line)
+                D.append(line)
+        return D
+
+    def __len__(self):
+        return len(self.data)
+
+    def preprocessing(self, example):
+        input_ids = []
+        labels = []
+
+        prompt, answer = example['instruction'], example['output']
+        
+        a_ids = self.tokenizer.encode(text=prompt, add_special_tokens=True, truncation=True,
+                                 max_length=self.max_source_length)
+        b_ids = self.tokenizer.encode(text=answer, add_special_tokens=False, truncation=True,
+                                 max_length=self.max_target_length)
+
+        context_length = len(a_ids)
+        input_ids = a_ids + b_ids + [self.tokenizer.eos_token_id]
+        labels = [self.tokenizer.pad_token_id] * context_length + b_ids + [self.tokenizer.eos_token_id]
+
+        pad_len = self.max_seq_length - len(input_ids)
+        input_ids = input_ids + [self.tokenizer.pad_token_id] * pad_len
+        labels = labels + [self.tokenizer.pad_token_id] * pad_len
+        labels = [(l if l != self.tokenizer.pad_token_id else -100) for l in labels]
+
+        input_ids = torch.LongTensor(input_ids)
+        labels = torch.LongTensor(labels)
+        attention_mask = input_ids.ne(self.tokenizer.pad_token_id)
+        return {
+            "input_ids": input_ids,
+            "labels": labels,
+            "attention_mask": attention_mask,
+        }
+
+
+    def __getitem__(self, idx) -> Dict[str, torch.Tensor]:
+        return self.preprocessing(self.data[idx])
+
+
+
+
+
+
+
+def train():
+    parser = transformers.HfArgumentParser(
+        (ModelArguments, DataArguments, TrainingArguments)
+    )
+    model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    model = transformers.AutoModelForCausalLM.from_pretrained(
+        model_args.model_name_or_path,
+        trust_remote_code=True,
+        cache_dir=training_args.cache_dir,
+    ).half()
+
+    tokenizer = transformers.AutoTokenizer.from_pretrained(
+        model_args.model_name_or_path,
+        use_fast=False,
+        trust_remote_code=True,
+        model_max_length=training_args.model_max_length,
+        cache_dir=training_args.cache_dir,
+    )
+    if training_args.use_lora:
+        from peft import LoraConfig, TaskType, get_peft_model
+
+        peft_config = LoraConfig(
+            task_type=TaskType.CAUSAL_LM,
+            target_modules=["W_pack"],
+            inference_mode=False,
+            r=8,
+            lora_alpha=32,
+            lora_dropout=0.1,
+        )
+        model.enable_input_require_grads()
+        model = get_peft_model(model, peft_config)
+        model.print_trainable_parameters()
+
+    dataset = MySupervisedDataset(
+        data_args.data_path, tokenizer, data_args.max_source_length,data_args.max_target_length,training_args.model_max_length
+    )
+
+    trainer = transformers.Trainer(
+        model=model, args=training_args, train_dataset=dataset, tokenizer=tokenizer
+    )
+
+    trainer.train()
+    trainer.save_state()
+    trainer.save_model(output_dir=training_args.output_dir)
+
+
+if __name__ == "__main__":
+    train()
+