Skaulintel/kvcache mistral

Makefile

@@ -22,10 +22,12 @@ REAL_CLONE_URL = $(if $(CLONE_URL),$(CLONE_URL),$(DEFAULT_CLONE_URL))
 
 # Run code quality checks
 style_check: clean
+	pip install -U pip ruff
 	ruff check . setup.py
 	ruff format --check . setup.py
 
 style: clean
+	pip install -U pip ruff
 	ruff check . setup.py --fix
 	ruff format . setup.py
 
@@ -53,8 +55,11 @@ slow_tests_deepspeed: test_installs
 	python -m pytest tests/test_examples.py -v -s -k "deepspeed"
 
 slow_tests_diffusers: test_installs
+	python -m pip install git+https://github.com/huggingface/diffusers.git
 	python -m pytest tests/test_diffusers.py -v -s -k "test_no_"
 	python -m pytest tests/test_diffusers.py -v -s -k "test_textual_inversion"
+	python -m pip install peft==0.7.0
+	python -m pytest tests/test_diffusers.py -v -s -k "test_train_text_to_image_"
 
 # Run text-generation non-regression tests
 slow_tests_text_generation_example: test_installs
@@ -109,4 +114,3 @@ clean:
 
 test_installs:
 	python -m pip install .[tests]
-	python -m pip install git+https://github.com/huggingface/accelerate.git

README.md

@@ -31,32 +31,45 @@ Check out [this blog post about BERT pre-training](https://huggingface.co/blog/p
 If you are not familiar with HPUs and would like to know more about them, we recommend you take a look at [our conceptual guide](https://huggingface.co/docs/optimum/habana/concept_guides/hpu).
 
 
-## Install
-To install the latest stable release of this package:
+## Install the library and get example scripts
 
-```bash
-pip install --upgrade-strategy eager optimum[habana]
-```
+### Option 1: Use the latest stable release
 
-The `--upgrade-strategy eager` option is needed to ensure `optimum-habana` is upgraded to the latest stable release.
-
-> To use DeepSpeed on HPUs, you also need to run the following command:
+To install the latest stable release of this package
 >```bash
->pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0
+>pip install --upgrade-strategy eager optimum[habana]
 >```
 
-Optimum Habana is a fast-moving project, and you may want to install it from source:
+The `--upgrade-strategy eager` option is needed to ensure `optimum-habana` is upgraded to the latest stable release.
+
+To use the example associated with the latest stable release, run:
+> ```
+> git clone https://github.com/huggingface/optimum-habana
+> cd optimum-habana && git checkout v1.10.2
+> ```
+> with `v1.10.2` the version number of this release.
+
+### Option 2: Use the latest main branch under development
+
+Optimum Habana is a fast-moving project, and you may want to install it from source and get the latest scripts :
 
 ```bash
 pip install git+https://github.com/huggingface/optimum-habana.git
+git clone https://github.com/huggingface/optimum-habana
 ```
 
-Last but not least, don't forget to install the requirements for every example:
+## Install dependencies
 
-```bash
-cd <example-folder>
-pip install -r requirements.txt
-```
+To use DeepSpeed on HPUs, you also need to run the following command:
+>```bash
+>pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0
+>```
+
+To install the requirements for every example:
+>```bash
+>cd <example-folder>
+>pip install -r requirements.txt
+>```
 
 
 ## How to use it?
@@ -164,11 +177,12 @@ The following model architectures, tasks and device distributions have been vali
 | OPT |   | <div style="text-align:left"><li>DeepSpeed</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
 | Llama 2 / CodeLlama | <div style="text-align:left"><li>DeepSpeed</li><li>LoRA</li></div> | :heavy_check_mark: | <li>[language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)</li><li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
 | StableLM |   | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
-| Falcon | <div style="text-align:left"><li>LoRA</li></div> | :heavy_check_mark: | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
+| Falcon | <div style="text-align:left"><li>LoRA</li></div> | :heavy_check_mark: | <li>[language modeling](https://github.com/huggingface/optimum-habana/tree/main/examples/language-modeling)</li><li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
 | CodeGen |   | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
 | MPT |   | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
 | Mistral |   | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
-| T5 | :heavy_check_mark: | :heavy_check_mark: | <li>[summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)</li><li>[translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)</li><li>[question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)</li> |
+| Mixtral |   | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
+| T5 / Flan T5 | :heavy_check_mark: | :heavy_check_mark: | <li>[summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)</li><li>[translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)</li><li>[question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)</li> |
 | BART |   | <div style="text-align:left"><li>Single card</li></div> | <li>[summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)</li><li>[translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)</li><li>[question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)</li> |
 | ViT | :heavy_check_mark: | :heavy_check_mark: | <li>[image classification](https://github.com/huggingface/optimum-habana/tree/main/examples/image-classification)</li> |
 | Swin | :heavy_check_mark: | :heavy_check_mark: | <li>[image classification](https://github.com/huggingface/optimum-habana/tree/main/examples/image-classification)</li> |

docs/source/index.mdx

@@ -53,7 +53,8 @@ In the tables below, ✅ means single-card, multi-card and DeepSpeed have all be
 | CodeGen      |          | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
 | MPT          |          | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
 | Mistral      |          | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
-| T5           | ✅       | ✅        | <li>[summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)</li><li>[translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)</li><li>[question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)</li> |
+| Mixtral      |          | <div style="text-align:left"><li>Single card</li></div> | <li>[text generation](https://github.com/huggingface/optimum-habana/tree/main/examples/text-generation)</li> |
+| T5 / Flan T5 | ✅       | ✅        | <li>[summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)</li><li>[translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)</li><li>[question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)</li> |
 | BART         |          | <div style="text-align:left"><li>Single card</li></div> | <li>[summarization](https://github.com/huggingface/optimum-habana/tree/main/examples/summarization)</li><li>[translation](https://github.com/huggingface/optimum-habana/tree/main/examples/translation)</li><li>[question answering](https://github.com/huggingface/optimum-habana/tree/main/examples/question-answering#fine-tuning-t5-on-squad20)</li> |
 | ViT          | ✅       | ✅        | <li>[image classification](https://github.com/huggingface/optimum-habana/tree/main/examples/image-classification)</li> |
 | Swin         | ✅       | ✅        | <li>[image classification](https://github.com/huggingface/optimum-habana/tree/main/examples/image-classification)</li> |

examples/audio-classification/README.md

@@ -20,6 +20,7 @@ The following examples showcase how to fine-tune `Wav2Vec2` for audio classifica
 
 Speech recognition models that have been pretrained in an unsupervised fashion on audio data alone, *e.g.* [Wav2Vec2](https://huggingface.co/transformers/main/model_doc/wav2vec2.html), have shown to require only very little annotated data to yield good performance on speech classification datasets.
 
+
 ## Single-HPU
 
 The following command shows how to fine-tune [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base) on the 🗣️ [Keyword Spotting subset](https://huggingface.co/datasets/superb#ks) of the SUPERB dataset on a single HPU.

examples/audio-classification/run_audio_classification.py

@@ -47,8 +47,8 @@ def check_optimum_habana_min_version(*a, **b):
 logger = logging.getLogger(__name__)
 
 # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks.
-check_min_version("4.34.0")
-check_optimum_habana_min_version("1.8.1")
+check_min_version("4.37.0")
+check_optimum_habana_min_version("1.10.0")
 
 require_version("datasets>=1.14.0", "To fix: pip install -r examples/pytorch/audio-classification/requirements.txt")
 

examples/contrastive-image-text/README.md

@@ -250,5 +250,8 @@ python run_clip.py \
     --use_lazy_mode \
     --use_hpu_graphs_for_inference \
     --gaudi_config_name Habana/clip \
-    --bf16
+    --bf16 \
+    --mediapipe_dataloader
 ```
+
+> `--mediapipe_dataloader` only works on Gaudi2.

examples/contrastive-image-text/clip_media_pipe.py

@@ -24,29 +24,37 @@
 
 try:
     from habana_frameworks.mediapipe import fn
-    from habana_frameworks.mediapipe.backend.nodes import opnode_tensor_info
-    from habana_frameworks.mediapipe.backend.operator_specs import schema
     from habana_frameworks.mediapipe.media_types import dtype, ftype, imgtype, randomCropType, readerOutType
     from habana_frameworks.mediapipe.mediapipe import MediaPipe
-    from habana_frameworks.mediapipe.operators.media_nodes import MediaReaderNode
     from habana_frameworks.mediapipe.operators.reader_nodes.read_image_from_dir import get_max_file
+    from habana_frameworks.mediapipe.operators.reader_nodes.reader_nodes import (
+        media_ext_reader_op_impl,
+        media_ext_reader_op_tensor_info,
+    )
     from habana_frameworks.torch.hpu import get_device_name
 except ImportError:
     pass
 
+read_image_text_from_dataset_params = {
+    "label_dtype": dtype.UINT32,
+    "dataset": None,
+}
+
 
-class read_image_text_from_dataset(MediaReaderNode):
+class read_image_text_from_dataset(media_ext_reader_op_impl):
     """
-    Class defining read image/text from directory node.
+    Class defining read image/text from clip dataset.
 
     """
 
-    def __init__(self, name, guid, device, inputs, params, cparams, node_attr):
-        super().__init__(name, guid, device, inputs, params, cparams, node_attr)
+    def __init__(self, params):
+        self.batch_size = 1
+        params = params["priv_params"]
         self.meta_dtype = params["label_dtype"]
         self.dataset = params["dataset"]
         self.epoch = 0
-
+        self.batch_sampler_iter = None
+        self.iter_loc = 0
         self.num_imgs_slice = len(ClipMediaPipe.batch_sampler.sampler)
         self.num_batches_slice = len(ClipMediaPipe.batch_sampler)
 
@@ -62,13 +70,13 @@ def set_params(self, params):
 
     def gen_output_info(self):
         out_info = []
-        o = opnode_tensor_info(dtype.NDT, np.array([self.batch_size], dtype=np.uint32), "")
+        o = media_ext_reader_op_tensor_info(dtype.NDT, np.array([self.batch_size], dtype=np.uint32), "")
         out_info.append(o)
-        o = opnode_tensor_info(
+        o = media_ext_reader_op_tensor_info(
             self.meta_dtype, np.array([self.dataset.text_max_length, self.batch_size], dtype=np.uint32), ""
         )
         out_info.append(o)
-        o = opnode_tensor_info(
+        o = media_ext_reader_op_tensor_info(
             self.meta_dtype, np.array([self.dataset.text_max_length, self.batch_size], dtype=np.uint32), ""
         )
         out_info.append(o)
@@ -112,27 +120,6 @@ def __next__(self):
         return img_list, input_id_list, attention_mask_list
 
 
-read_image_text_from_dataset_params = {
-    "label_dtype": dtype.UINT64,
-    "dataset": None,
-}
-schema.add_operator(
-    "ClipDataReader",
-    None,
-    0,
-    0,
-    [],
-    3,
-    read_image_text_from_dataset_params,
-    None,
-    read_image_text_from_dataset,
-    dtype.NDT,
-)
-op_class = fn.operator_add("ClipDataReader")
-op_class.__module__ = fn.__name__
-setattr(fn, "ClipDataReader", op_class)
-
-
 class ClipMediaPipe(MediaPipe):
     """
     Class defining clip media pipe:
@@ -160,8 +147,13 @@ def __init__(self, dataset=None, sampler=None, batch_size=512, drop_last=False,
         super(ClipMediaPipe, self).__init__(
             device=self.device, batch_size=batch_size, prefetch_depth=queue_depth, pipe_name=pipe_name
         )
-
-        self.input = fn.ClipDataReader(label_dtype=dtype.UINT32, dataset=self.dataset)
+        params = read_image_text_from_dataset_params.copy()
+        params["dataset"] = self.dataset
+        self.input = fn.MediaExtReaderOp(
+            impl=read_image_text_from_dataset,
+            num_outputs=3,
+            priv_params=params,
+        )
         def_output_image_size = [self.image_size, self.image_size]
         res_pp_filter = ftype.BICUBIC
         self.decode = fn.ImageDecoder(

examples/contrastive-image-text/run_bridgetower.py

@@ -57,8 +57,8 @@ def check_optimum_habana_min_version(*a, **b):
 logger = logging.getLogger(__name__)
 
 # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks.
-check_min_version("4.34.0")
-check_optimum_habana_min_version("1.8.1")
+check_min_version("4.37.0")
+check_optimum_habana_min_version("1.10.0")
 
 require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/contrastive-image-text/requirements.txt")
 

examples/contrastive-image-text/run_clip.py

@@ -62,8 +62,8 @@ def check_optimum_habana_min_version(*a, **b):
 logger = logging.getLogger(__name__)
 
 # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks.
-check_min_version("4.34.0")
-check_optimum_habana_min_version("1.8.1")
+check_min_version("4.37.0")
+check_optimum_habana_min_version("1.10.0")
 
 require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/contrastive-image-text/requirements.txt")
 

examples/image-classification/requirements.txt

@@ -1,5 +1,5 @@
 torch>=1.5.0
 torchvision>=0.6.0
-datasets>=2.4.0
+datasets>=2.14.0
 evaluate
 scikit-learn

examples/image-classification/run_image_classification.py

@@ -64,10 +64,10 @@ def check_optimum_habana_min_version(*a, **b):
 logger = logging.getLogger(__name__)
 
 # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks.
-check_min_version("4.34.0")
-check_optimum_habana_min_version("1.8.1")
+check_min_version("4.37.0")
+check_optimum_habana_min_version("1.10.0")
 
-require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/image-classification/requirements.txt")
+require_version("datasets>=2.14.0", "To fix: pip install -r examples/pytorch/image-classification/requirements.txt")
 
 MODEL_CONFIG_CLASSES = list(MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING.keys())
 MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)

examples/language-modeling/README.md

@@ -370,6 +370,7 @@ python3 run_lora_clm.py \
     --max_grad_norm  0.3 \
     --logging_steps 1 \
     --do_train \
+    --do_eval \
     --use_habana \
     --use_lazy_mode \
     --throughput_warmup_steps 3 \
@@ -380,6 +381,7 @@ python3 run_lora_clm.py \
     --dataset_concatenation \
     --max_seq_length 512 \
     --low_cpu_mem_usage True \
+    --validation_split_percentage 4 \
     --adam_epsilon 1e-08
 ```
 
@@ -414,7 +416,8 @@ LOWER_LIST=ops_bf16.txt python3 run_lora_clm.py \
     --max_seq_length 256 \
     --low_cpu_mem_usage True \
     --adam_epsilon 1e-08 \
-    --do_eval
+    --do_eval \
+    --validation_split_percentage 5
 ```
 
 - Multi-card finetuning of Llama1-7B:
@@ -436,6 +439,7 @@ python ../gaudi_spawn.py \
     --max_grad_norm  0.3 \
     --logging_steps 1 \
     --do_train \
+    --do_eval \
     --use_habana \
     --use_lazy_mode \
     --throughput_warmup_steps 3 \
@@ -447,6 +451,7 @@ python ../gaudi_spawn.py \
     --max_seq_length 512 \
     --ddp_bucket_cap_mb 50 \
     --adam_epsilon 1e-08 \
+    --validation_split_percentage 4 \
     --low_cpu_mem_usage True
 ```
 
@@ -512,7 +517,8 @@ LOWER_LIST=ops_bf16.txt python3 ../gaudi_spawn.py \
     --ddp_bucket_cap_mb 50 \
     --adam_epsilon 1e-08 \
     --do_eval \
-    --low_cpu_mem_usage True
+    --low_cpu_mem_usage True \
+    --validation_split_percentage 6
 ```
 
 - Multi-card finetuning of Llama2-70B with DeepSpeed ZeRO-3 optimization and LoRA:
@@ -550,7 +556,8 @@ python3 ../gaudi_spawn.py --use_deepspeed  --world_size 8  run_lora_clm.py \
   --lora_rank 4 \
   --lora_target_modules "q_proj" "v_proj" "k_proj" "o_proj" \
   --validation_split_percentage 4 \
-  --use_flash_attention True
+  --use_flash_attention True \
+  --flash_attention_causal_mask True
 ```
 
 - Multi-card finetuning of Falcon-180B:
@@ -587,6 +594,7 @@ DEEPSPEED_HPU_ZERO3_SYNC_MARK_STEP_REQUIRED=1 LOWER_LIST=ops_bf16.txt python3 ..
     --max_seq_length 256 \
     --adam_epsilon 1e-08 \
     --do_eval \
+    --validation_split_percentage 5 \
     --deepspeed ds_falcon_180b_z3.json
 ```
 ## Streaming

examples/language-modeling/run_clm.py

@@ -63,8 +63,8 @@ def check_optimum_habana_min_version(*a, **b):
 logger = logging.getLogger(__name__)
 
 # Will error if the minimal version of Transformers and Optimum Habana are not installed. Remove at your own risks.
-check_min_version("4.34.0")
-check_optimum_habana_min_version("1.8.1")
+check_min_version("4.37.0")
+check_optimum_habana_min_version("1.10.0")
 
 require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
 
@@ -243,6 +243,9 @@ class DataTrainingArguments:
     keep_linebreaks: bool = field(
         default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
     )
+    save_last_ckpt: bool = field(
+        default=True, metadata={"help": "Whether to save checkpoint at the end of the training."}
+    )
 
     def __post_init__(self):
         if self.streaming:
@@ -643,7 +646,8 @@ def compute_metrics(eval_preds):
         elif last_checkpoint is not None:
             checkpoint = last_checkpoint
         train_result = trainer.train(resume_from_checkpoint=checkpoint)
-        trainer.save_model()  # Saves the tokenizer too for easy upload
+        if data_args.save_last_ckpt:
+            trainer.save_model()  # Saves the tokenizer too for easy upload
 
         metrics = train_result.metrics