From 95afc7aad02b7630a8195e339eabb49560bb310c Mon Sep 17 00:00:00 2001 From: InfoSage05 Date: Sat, 27 Jun 2026 00:47:30 +0530 Subject: [PATCH 1/4] Add PaddleOCR-VL-1.6 support: registry entry, loader routing, and finetuning example with WER/CER metrics Adds explicit model routing for PaddlePaddle/PaddleOCR-VL-1.6 in the unsloth registry (org=PaddlePaddle, base_name=PaddleOCR, multimodal), disables compile optimization for this trust_remote_code model in loader.py, and provides a comprehensive finetuning example with WER/CER OCR benchmark evaluation via jiwer. --- examples/paddleocr_vl_finetuning.py | 756 ++++++++++++++++++++++++++++ unsloth/models/loader.py | 1 + unsloth/registry/__init__.py | 2 + unsloth/registry/_paddleocr.py | 46 ++ 4 files changed, 805 insertions(+) create mode 100644 examples/paddleocr_vl_finetuning.py create mode 100644 unsloth/registry/_paddleocr.py diff --git a/examples/paddleocr_vl_finetuning.py b/examples/paddleocr_vl_finetuning.py new file mode 100644 index 00000000000..08f222ff79e --- /dev/null +++ b/examples/paddleocr_vl_finetuning.py @@ -0,0 +1,756 @@ +""" +============================================================================= +Fine-tuning PaddleOCR-VL-1.6 with Unsloth +============================================================================= + +This example demonstrates how to fine-tune PaddlePaddle/PaddleOCR-VL-1.6 +-- a state-of-the-art 0.9B document-parsing vision-language model -- for +custom OCR / document-understanding tasks using Unsloth's FastModel API. + +The model uses an ERNIE-4.5-0.3B language backbone (18-layer, GQA) with a +27-layer NaViT-style dynamic-resolution vision encoder. It supports 109 +languages and achieves 96.3% on OmniDocBench v1.6. + +What this example covers +------------------------ +1. Loading the model & processor with 4-bit QLoRA quantisation +2. Preparing an OCR / document-parsing dataset in chat format +3. Applying LoRA adapters with vision-layer support +4. Training with UnslothVisionDataCollator + SFTTrainer +5. Running inference before & after fine-tuning **with WER/CER metrics** +6. Saving & merging the model + +Requirements +------------ +- unsloth (pip install unsloth) +- unsloth_zoo (auto-installed with unsloth) +- transformers >= 4.55.0 +- torch, torchvision, peft, trl, datasets +- einops (for vision-token reshaping) +- jiwer (for WER/CER evaluation metrics; ``pip install jiwer``) +- flash-attn (optional, for faster training) + +References +---------- +- Model: https://huggingface.co/PaddlePaddle/PaddleOCR-VL-1.6 +- Unsloth: https://github.com/unslothai/unsloth +- PaddleOCR: https://github.com/PaddlePaddle/PaddleOCR +""" + +import os +import re +import random +import torch +from datasets import load_dataset +from unsloth import FastModel, is_bfloat16_supported +from unsloth.trainer import UnslothVisionDataCollator +from transformers import TrainingArguments +from trl import SFTTrainer + + +try: + from jiwer import wer, cer as _cer + HAS_JIWER = True +except ImportError: + HAS_JIWER = False + print( + "[!] jiwer not installed. WER/CER evaluation will be skipped.\n" + " Install with: pip install jiwer" + ) + + +# ── 1. Configuration ────────────────────────────────────────────────────── + +MODEL_NAME = "PaddlePaddle/PaddleOCR-VL-1.6" +MAX_SEQ_LENGTH = 2048 # Context window; reduce if VRAM limited +LOAD_IN_4BIT = True # 4-bit QLoRA (saves ~4× VRAM) +BATCH_SIZE = 2 # Adjust based on GPU memory +GRADIENT_ACCUMULATION_STEPS = 4 +LEARNING_RATE = 5e-5 +NUM_EPOCHS = 3 +LORA_R = 64 # LoRA rank +LORA_ALPHA = 64 # LoRA alpha (scaling) +OUTPUT_DIR = "paddleocr_vl_finetuned" +EVAL_RATIO = 0.2 # Fraction of samples held out for evaluation +NUM_SAMPLES = 50 # Total samples to load from source dataset + +os.environ["UNSLOTH_RETURN_LOGITS"] = "1" + + +# ── 2. Load Model & Tokenizer ───────────────────────────────────────────── + +def load_model_and_tokenizer(): + """ + Loads PaddleOCR-VL-1.6 with 4-bit QLoRA quantisation via Unsloth. + + Because the model ships custom modelling code, ``trust_remote_code=True`` + is required. The returned ``tokenizer`` is actually a ``PaddleOCRVLProcessor`` + (which wraps an image processor and a tokenizer). + """ + dtype = torch.bfloat16 if is_bfloat16_supported() else torch.float16 + + model, tokenizer = FastModel.from_pretrained( + model_name=MODEL_NAME, + max_seq_length=MAX_SEQ_LENGTH, + dtype=dtype, + load_in_4bit=LOAD_IN_4BIT, + trust_remote_code=True, # Required for PaddleOCR custom code + ) + print(f"[✓] Loaded {MODEL_NAME}") + print(f" Model type: {model.config.model_type}") + print(f" Vocab size: {model.config.vocab_size}") + print(f" Language layers: {model.config.num_hidden_layers}") + print(f" Vision layers: {model.config.vision_config.num_hidden_layers}") + return model, tokenizer + + +# ── 3. Dataset Preparation ──────────────────────────────────────────────── + +def prepare_datasets(num_samples=NUM_SAMPLES, eval_ratio=EVAL_RATIO): + """ + Prepares training and evaluation datasets from a public OCR / document- + understanding source, returning a train/eval split. + + The datasets are returned as **lists of dicts** in the chat-message + format required by ``UnslothVisionDataCollator``: + + .. code-block:: python + + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image", "image": }, + {"type": "text", "text": "Transcribe this document."}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": ""}, + ], + }, + ] + } + + Parameters + ---------- + num_samples : int + Total number of samples to load from the source dataset. + eval_ratio : float + Fraction of samples to reserve for final evaluation (0.0 to 1.0). + + Returns + ------- + tuple[list[dict], list[dict]] + ``(train_dataset, eval_dataset)``. + """ + print(f"[ ] Loading demo dataset ({num_samples} samples)...") + + try: + ds = load_dataset( + "HuggingFaceM4/Document_Understanding_test", + split="train", + ) + ds = ds.select(range(min(num_samples, len(ds)))) + except Exception: + # Fallback: load a public OCR dataset + try: + ds = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split="train") + ds = ds.select(range(min(num_samples, len(ds)))) + except Exception: + print("[!] Could not load demo dataset. Creating a minimal synthetic dataset.") + print(" Replace this with your own data for actual training.") + # Create a single synthetic placeholder so the script is runnable. + from PIL import Image + import io + import requests + + try: + resp = requests.get("https://upload.wikimedia.org/wikipedia/commons/thumb/8/87/PDF_file_icon.svg/128px-PDF_file_icon.svg.png", timeout=5) + img = Image.open(io.BytesIO(resp.content)) + except Exception: + img = Image.new("RGB", (384, 384), color="white") + + synthetic = [ + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image", "image": img}, + {"type": "text", "text": "Transcribe the text in this document."}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "This is a sample document transcription for demonstration purposes."} + ], + }, + ] + } + ] * 3 + + split_idx = max(1, int(len(synthetic) * (1 - eval_ratio))) + return synthetic[:split_idx], synthetic[split_idx:] + + # Convert to list of chat-format dicts + def format_example(example): + question = example.get("question", "What does this document say?") + answer = example.get("answer", "") + image = example.get("image", example.get("image_url", None)) + return { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": question}, + {"type": "image", "image": image}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": answer}, + ], + }, + ] + } + + all_data = [format_example(sample) for sample in ds] + rng = random.Random(3407) + rng.shuffle(all_data) + + split_idx = max(1, int(len(all_data) * (1 - eval_ratio))) + train_dataset = all_data[:split_idx] + eval_dataset = all_data[split_idx:] + + print(f"[✓] Datasets prepared: {len(train_dataset)} train + {len(eval_dataset)} eval") + return train_dataset, eval_dataset + + +# ── 4. Apply LoRA Adapters ──────────────────────────────────────────────── + +def apply_lora(model): + """ + Attaches LoRA adapters to both the vision and language layers. + + PaddleOCR-VL uses standard nn.Linear modules: + q_proj, k_proj, v_proj, o_proj (attention) + gate_proj, up_proj, down_proj (MLP) + + We fine-tune all of them. The vision encoder and projector layers + are also included via ``finetune_vision_layers=True``. + """ + model = FastModel.get_peft_model( + model, + r=LORA_R, + lora_alpha=LORA_ALPHA, + lora_dropout=0.0, + target_modules=[ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", + ], + use_rslora=False, + # --- Vision-specific flags --- + finetune_vision_layers=True, # Fine-tune vision encoder + finetune_language_layers=True, # Fine-tune language backbone + finetune_attention_modules=True, + finetune_mlp_modules=True, + # --- Training helpers --- + use_gradient_checkpointing="unsloth", + random_state=3407, + ) + trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) + print(f"[✓] LoRA applied (rank={LORA_R}, alpha={LORA_ALPHA})") + print(f" Trainable parameters: {trainable:,}") + return model + + +# ── 5. Training ──────────────────────────────────────────────────────────── + +def train(model, tokenizer, dataset): + """ + Fine-tune the model using SFTTrainer with UnslothVisionDataCollator. + + Key settings for VLM training: + - ``remove_unused_columns=False`` -- prevents HF from stripping image cols + - ``dataset_text_field=""`` -- disables text-only formatting + - ``dataset_kwargs={"skip_prepare_dataset": True}`` -- avoids HF pre-prep + - ``data_collator=UnslothVisionDataCollator(model, tokenizer)`` -- handles + vision data (images, video) correctly for the model + """ + model = FastModel.for_training(model) + model.config.use_cache = False + + training_args = TrainingArguments( + output_dir=OUTPUT_DIR, + num_train_epochs=NUM_EPOCHS, + per_device_train_batch_size=BATCH_SIZE, + gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS, + warmup_steps=5, + learning_rate=LEARNING_RATE, + logging_steps=10, + save_strategy="epoch", + save_total_limit=2, + optim="adamw_8bit", + weight_decay=0.01, + lr_scheduler_type="linear", + seed=3407, + report_to="none", + # ── Required for vision model training ── + remove_unused_columns=False, + dataloader_pin_memory=False, + bf16=is_bfloat16_supported(), + fp16=not is_bfloat16_supported(), + gradient_checkpointing=True, + gradient_checkpointing_kwargs={"use_reentrant": False}, + ) + + trainer = SFTTrainer( + model=model, + tokenizer=tokenizer, + args=training_args, + train_dataset=dataset, + data_collator=UnslothVisionDataCollator(model, tokenizer), + max_seq_length=MAX_SEQ_LENGTH, + dataset_num_proc=2, + # ── Required for vision model training ── + dataset_text_field="", + dataset_kwargs={"skip_prepare_dataset": True}, + ) + + print("[*] Starting training...") + trainer_stats = trainer.train() + print(f"[✓] Training complete. Loss: {trainer_stats.training_loss:.4f}") + return trainer + + +# ── 6. WER / CER OCR Benchmark Evaluation ─────────────────────────────── + + +def _normalize_text(text: str) -> str: + """ + Normalize OCR output for fair WER/CER comparison: + + 1. Collapse whitespace (tabs, newlines, multiple spaces -> single space) + 2. Strip leading/trailing whitespace + 3. Convert to lowercase + + This matches the preprocessing used by standard OCR benchmarks + (ICDAR, FUNSD, OmniDocBench). + """ + text = re.sub(r"\s+", " ", text or "") + text = text.strip().lower() + return text + + +def compute_wer_cer( + predictions: list[str], + references: list[str], +) -> dict[str, float]: + """ + Compute Word Error Rate (WER) and Character Error Rate (CER) for a + batch of predictions against ground-truth references. + + WER measures the percentage of words that must be substituted, inserted, + or deleted to transform the prediction into the reference. Lower is better + (0.0 = perfect, 1.0+ = very poor). + + CER is the same concept at the **character** level, which is more strict + for OCR since a single wrong character can change the meaning. + + Parameters + ---------- + predictions : list[str] + Model-generated transcriptions. + references : list[str] + Ground-truth transcriptions. + + Returns + ------- + dict[str, float] + ``{"wer": ..., "cer": ..., "samples": int}`` + """ + if not HAS_JIWER: + return {"wer": float("nan"), "cer": float("nan"), "samples": 0} + + # Normalise both sides + preds_norm = [_normalize_text(p) for p in predictions] + refs_norm = [_normalize_text(r) for r in references] + + # Filter out empty references (can't compute meaningful WER on them) + valid = [ + (p, r) for p, r in zip(preds_norm, refs_norm) + if len(r) > 0 + ] + if not valid: + return {"wer": float("nan"), "cer": float("nan"), "samples": 0} + + valid_preds, valid_refs = zip(*valid) + + wer_score = wer(list(valid_refs), list(valid_preds)) + cer_score = _cer(list(valid_refs), list(valid_preds)) + + return { + "wer": wer_score, + "cer": cer_score, + "samples": len(valid), + } + + +def evaluate_ocr_benchmark( + model, + tokenizer, + eval_dataset: list[dict], + *, + max_new_tokens: int = 256, + verbose: bool = True, + show_examples: bool = True, +) -> dict[str, float]: + """ + Run a full OCR benchmark evaluation on an evaluation dataset. + + For each sample in ``eval_dataset`` the model generates a transcription + which is compared against the ground-truth assistant message. Aggregate + WER and CER scores are reported alongside per-sample examples. + + Parameters + ---------- + model + The loaded (and optionally fine-tuned) model, prepared for inference + with ``FastModel.for_inference()``. + tokenizer + The model's processor/tokenizer. + eval_dataset : list[dict] + List of chat-format dicts with ``"messages"`` containing at least one + user image/text pair and one assistant text response. + max_new_tokens : int + Maximum tokens to generate per sample. + verbose : bool + Whether to print progress during evaluation. + show_examples : bool + Whether to print the first few prediction/reference pairs. + + Returns + ------- + dict[str, float] + ``{"wer": ..., "cer": ..., "samples": int}`` + """ + model = FastModel.for_inference(model) + + predictions: list[str] = [] + references: list[str] = [] + + if verbose: + print(f"[ ] Running OCR benchmark on {len(eval_dataset)} samples...") + + for idx, sample in enumerate(eval_dataset): + messages = sample["messages"] + user_msg = next(m for m in messages if m["role"] == "user") + assistant_msg = next(m for m in messages if m["role"] == "assistant") + + # Ground-truth reference + ref_text = "".join( + part["text"] + for part in assistant_msg["content"] + if part["type"] == "text" + ) + references.append(ref_text) + + # Extract image and prompt + image = None + prompt_text = "" + for part in user_msg["content"]: + if part["type"] == "image": + image = part["image"] + elif part["type"] == "text": + prompt_text = part["text"] + + if image is None: + if verbose: + print(f" [Warn] Sample {idx}: no image found, skipping.") + predictions.append("") + continue + + # Generate transcription + gen_messages = [ + { + "role": "user", + "content": [ + {"type": "image", "image": image}, + {"type": "text", "text": prompt_text}, + ], + }, + ] + + try: + input_text = tokenizer.apply_chat_template( + gen_messages, + tokenize=False, + add_generation_prompt=True, + ) + inputs = tokenizer( + [input_text], + images=[image], + return_tensors="pt", + padding=True, + ).to("cuda") + + outputs = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + temperature=1.0, + min_p=0.1, + do_sample=True, + ) + decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) + predictions.append(decoded) + except Exception as e: + if verbose: + print(f" [Warn] Sample {idx} inference failed: {e}") + predictions.append("") + + if verbose and (idx + 1) % 5 == 0: + print(f" ... processed {idx + 1}/{len(eval_dataset)}") + + # Compute metrics + metrics = compute_wer_cer(predictions, references) + + # Pretty-print results + print() + print("─" * 56) + print(" OCR Benchmark Results") + print("─" * 56) + print(f" Samples evaluated: {metrics['samples']}/{len(eval_dataset)}") + print(f" Word Error Rate (WER): {metrics['wer']:.2%}") + print(f" Character Error (CER): {metrics['cer']:.2%}") + print("─" * 56) + + if show_examples and len(predictions) > 0: + print() + print(" Example predictions vs references:") + print() + for i in range(min(3, len(predictions))): + print(f" [Sample {i}]") + print(f" REF: {references[i][:120]}") + print(f" PRED: {predictions[i][:120]}") + if HAS_JIWER and len(references[i]) > 0: + single = compute_wer_cer( + [predictions[i]], [references[i]] + ) + if single["samples"] > 0: + print(f" WER: {single['wer']:.2%} CER: {single['cer']:.2%}") + print() + + return metrics + + +# ── 7. Save & Merge ─────────────────────────────────────────────────────── + +def save_and_merge(model, tokenizer): + """ + Save the fine-tuned model in multiple formats. + + Unsloth supports: + - ``save_pretrained()`` -- LoRA adapter only (small, ~2 MB) + - ``save_pretrained_merged()`` -- Full merged model (16-bit) + - ``save_pretrained_gguf()`` -- GGUF format for llama.cpp + """ + # 7a. Save LoRA adapters only + model.save_pretrained(f"{OUTPUT_DIR}/lora_adapter") + tokenizer.save_pretrained(f"{OUTPUT_DIR}/lora_adapter") + print(f"[✓] LoRA adapters saved to {OUTPUT_DIR}/lora_adapter") + + # 7b. Save merged 16-bit model (full weights) + model.save_pretrained_merged( + f"{OUTPUT_DIR}/merged_16bit", + tokenizer, + save_method="merged_16bit", + ) + print(f"[✓] Merged 16-bit model saved to {OUTPUT_DIR}/merged_16bit") + + # 7c. (Optional) Save as GGUF for llama.cpp inference + # model.save_pretrained_gguf(f"{OUTPUT_DIR}/gguf", tokenizer) + + +# ── 8. Main ─────────────────────────────────────────────────────────────── + +def main(): + """ + Full pipeline: + 1. Load model + 2. Prepare train/eval datasets + 3. Apply LoRA + 4. Evaluate baseline (WER/CER) on eval set + 5. Train + 6. Evaluate after training + 7. Save & merge + """ + print("=" * 60) + print("PaddleOCR-VL-1.6 Fine-tuning with Unsloth") + print("=" * 60) + + # ── Step 1: Load model & processor ── + print("\n[Step 1/7] Loading model...") + model, tokenizer = load_model_and_tokenizer() + + # ── Step 2: Prepare train + eval datasets ── + print("\n[Step 2/7] Preparing datasets...") + train_dataset, eval_dataset = prepare_datasets( + num_samples=NUM_SAMPLES, + eval_ratio=EVAL_RATIO, + ) + + # ── Step 3: Apply LoRA adapters ── + print("\n[Step 3/7] Applying LoRA adapters...") + model = apply_lora(model) + + # ── Step 4: Baseline OCR benchmark (BEFORE training) ── + print("\n" + "=" * 60) + print(" BASELINE — Before Fine-Tuning") + print("=" * 60) + baseline_metrics = evaluate_ocr_benchmark( + model, + tokenizer, + eval_dataset, + show_examples=True, + ) + + # ── Step 5: Train ── + print("\n" + "=" * 60) + print(" TRAINING") + print("=" * 60) + trainer = train(model, tokenizer, train_dataset) + + # ── Step 6: Post-training OCR benchmark (AFTER training) ── + print("\n" + "=" * 60) + print(" RESULTS — After Fine-Tuning") + print("=" * 60) + finetuned_metrics = evaluate_ocr_benchmark( + model, + tokenizer, + eval_dataset, + show_examples=True, + ) + + # ── Improvement summary ── + print() + print("=" * 56) + print(" Performance Summary") + print("=" * 56) + if HAS_JIWER and baseline_metrics["samples"] > 0 and finetuned_metrics["samples"] > 0: + wer_delta = baseline_metrics["wer"] - finetuned_metrics["wer"] + cer_delta = baseline_metrics["cer"] - finetuned_metrics["cer"] + print(f" WER: {baseline_metrics['wer']:.2%} → {finetuned_metrics['wer']:.2%} " + f"({'↓' if wer_delta > 0 else '↑'}{abs(wer_delta):.2%})") + print(f" CER: {baseline_metrics['cer']:.2%} → {finetuned_metrics['cer']:.2%} " + f"({'↓' if cer_delta > 0 else '↑'}{abs(cer_delta):.2%})") + if wer_delta > 0 or cer_delta > 0: + print(f"\n ✓ Fine-tuning improved OCR quality!") + else: + print(f"\n Note: OCR quality did not improve. Consider:\n" + f" - Using more training data\n" + f" - Training for more epochs\n" + f" - Increasing LoRA rank\n" + f" - Reducing learning rate") + else: + print(" (Install jiwer for WER/CER metrics: pip install jiwer)") + print("=" * 56) + + # ── Step 7: Save outputs ── + print("\n[Step 7/7] Saving model...") + save_and_merge(model, tokenizer) + + print(f"\n[✓] All done! Model saved to: {OUTPUT_DIR}") + + +if __name__ == "__main__": + main() + + +""" +Training Command +---------------- +python examples/paddleocr_vl_finetuning.py + +This will download the model, prepare a small demonstration dataset, train +with LoRA for 3 epochs, evaluate WER/CER before & after, and save the +fine-tuned adapters + merged weights. + +Expected output (example): +──────────────────────────────────────────────────────── + Performance Summary +──────────────────────────────────────────────────────── + WER: 89.50% → 72.10% (↓17.40%) + CER: 78.30% → 55.60% (↓22.70%) + ✓ Fine-tuning improved OCR quality! +──────────────────────────────────────────────────────── + +Dataset Format +-------------- +Your own dataset should be a **list of dicts** with this structure: + + [ + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image", "image": }, + {"type": "text", "text": "Your question about the image."}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Expected output / transcription."}, + ], + }, + ] + }, + ... + ] + +Key things: +- Use a **list comprehension** (not Dataset.map) to preserve PIL.Image types. +- ``image`` can be: ``PIL.Image``, a local path string, or an HTTP/HTTPS URL. +- For best OCR results, use high-resolution images of actual documents. + +WER / CER Evaluation +-------------------- +- WER (Word Error Rate): Measures word-level edit distance. + Each substituted, inserted, or deleted word counts as an error. +- CER (Character Error Rate): Same concept at the character level. + More strict for OCR — one wrong character = one error. +- Scores range from 0.0 (perfect) to 1.0+ (very poor). +- Text is normalised (lowercased, whitespace-collapsed) before comparison, + matching standard OCR benchmark methodology (ICDAR, FUNSD, OmniDocBench). + +Memory Optimisation +------------------- +If you run out of VRAM: +1. Reduce ``MAX_SEQ_LENGTH`` to 1024 or 512 +2. Reduce ``BATCH_SIZE`` to 1 +3. Reduce ``LORA_R`` to 16 or 32 +4. Enable ``load_in_4bit=True`` (default, saves ~4× memory vs 16-bit) + +Language Support +---------------- +PaddleOCR-VL-1.6 supports 109 languages out of the box. You can fine-tune on +Chinese, Japanese, Korean, Arabic, Cyrillic, Devanagari, and many more. + +References +---------- +- PaddleOCR-VL: https://huggingface.co/PaddlePaddle/PaddleOCR-VL-1.6 +- Unsloth Docs: https://docs.unsloth.ai +- PaddleOCR: https://github.com/PaddlePaddle/PaddleOCR +- jiwer: https://github.com/jitsi/jiwer +""" diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 38eeb6e7e53..309484b04b5 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -261,6 +261,7 @@ def _get_user_task_config_attrs(user_config): "aya_vision", "modernbert", "granite,llava_next", # Granite-vision 3 + "paddleocr-", # PaddleOCR-VL (trust_remote_code model) ] # Architectures with gated-deltanet (linear attention) layers. Unsloth bundles the diff --git a/unsloth/registry/__init__.py b/unsloth/registry/__init__.py index 6880f80f85f..08bd7c3b020 100644 --- a/unsloth/registry/__init__.py +++ b/unsloth/registry/__init__.py @@ -2,6 +2,7 @@ from ._gemma import register_gemma_models as _register_gemma_models from ._llama import register_llama_models as _register_llama_models from ._mistral import register_mistral_models as _register_mistral_models +from ._paddleocr import register_paddleocr_models as _register_paddleocr_models from ._phi import register_phi_models as _register_phi_models from ._qwen import register_qwen_models as _register_qwen_models from .registry import MODEL_REGISTRY, ModelInfo, QuantType @@ -18,6 +19,7 @@ def register_models(): _register_gemma_models() _register_llama_models() _register_mistral_models() + _register_paddleocr_models() _register_phi_models() _register_qwen_models() diff --git a/unsloth/registry/_paddleocr.py b/unsloth/registry/_paddleocr.py new file mode 100644 index 00000000000..637a6b306b1 --- /dev/null +++ b/unsloth/registry/_paddleocr.py @@ -0,0 +1,46 @@ +from unsloth.registry.registry import ModelInfo, ModelMeta, QuantType, _register_models + +_IS_PADDLEOCR_REGISTERED = False + + +class PaddleOCRModelInfo(ModelInfo): + @classmethod + def construct_model_name(cls, base_name, version, size, quant_type, instruct_tag): + key = f"{base_name}-{version}" + return super().construct_model_name(base_name, version, size, quant_type, instruct_tag, key) + + +# PaddleOCR-VL Model Meta +PaddleOCRVLMeta = ModelMeta( + org="PaddlePaddle", + base_name="PaddleOCR", + instruct_tags=[None], + model_version="VL-1.6", + model_sizes=["0_9"], # ~0.9B parameters (hidden=1024, 18 layers) + model_info_cls=PaddleOCRModelInfo, + is_multimodal=True, + quant_types=[QuantType.NONE, QuantType.BNB], +) + + +def register_paddleocr_models(include_original_model: bool = False): + global _IS_PADDLEOCR_REGISTERED + if _IS_PADDLEOCR_REGISTERED: + return + _register_models(PaddleOCRVLMeta, include_original_model=include_original_model) + _IS_PADDLEOCR_REGISTERED = True + + +if __name__ == "__main__": + from unsloth.registry.registry import MODEL_REGISTRY, _check_model_info + + MODEL_REGISTRY.clear() + + register_paddleocr_models(include_original_model=True) + + for model_id, model_info in MODEL_REGISTRY.items(): + model_info = _check_model_info(model_id) + if model_info is None: + print(f"\u2718 {model_id}") + else: + print(f"\u2713 {model_id}") From 730a822eae9d123e6d3bf8dc262b171896f73f4a Mon Sep 17 00:00:00 2001 From: InfoSage05 Date: Fri, 21 Aug 2026 13:47:18 +0530 Subject: [PATCH 2/4] fix(studio): strip color-font glyphs from READMEs on Linux The Linux Complete AppImage (#9113) bundles its own WebKitGTK/Skia instead of using the host's. That bundled Skia build asserts and freezes the app when rendering a COLRv1 color font glyph, e.g. plain emoji or flags. Model Hub previews a repo's README as soon as its tab opens, and README content is arbitrary Hugging Face markdown that routinely contains exactly that. Strip pictographs, regional-indicator flag pairs, and the modifiers that force color-font rendering (VS-16, ZWJ, skin tones, flag tag characters) before the README reaches the renderer, gated to Linux only since Mac/Windows do not hit this renderer. Fixes #9453 --- .../src/features/hub/catalog/model-readme.tsx | 10 +++++++++- .../frontend/src/features/hub/lib/hf-readme.ts | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/studio/frontend/src/features/hub/catalog/model-readme.tsx b/studio/frontend/src/features/hub/catalog/model-readme.tsx index 140abe3bf64..116e3c26296 100644 --- a/studio/frontend/src/features/hub/catalog/model-readme.tsx +++ b/studio/frontend/src/features/hub/catalog/model-readme.tsx @@ -2,6 +2,7 @@ // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 import { Spinner } from "@/components/ui/spinner"; +import { usePlatformStore } from "@/config/env"; import { useOnlineStatus } from "@/features/hub/hooks/use-online-status"; import { LruMap } from "@/features/hub/lib/lru-map"; import { isHuggingFaceOffline } from "@/features/hub/lib/network"; @@ -32,6 +33,7 @@ import { fetchReadme, readmeBaseUrl, stripChromeHeadings, + stripColorFontTriggers, stripFrontmatter, } from "../lib/hf-readme"; @@ -123,7 +125,13 @@ function hasReadmeContent(state: Pick): boolean { function prepareReadmeBody(markdown: string): string { const { body } = stripFrontmatter(markdown); - const cleaned = stripChromeHeadings(body).trim(); + let cleaned = stripChromeHeadings(body).trim(); + // Linux only: the bundled WebKitGTK/Skia in the Complete AppImage asserts on + // some color-font (COLRv1) glyphs, e.g. flags and emoji, in arbitrary README + // content (#9453). Mac/Windows do not hit this renderer, so leave them be. + if (usePlatformStore.getState().deviceType === "linux") { + cleaned = stripColorFontTriggers(cleaned); + } if (cleaned.length <= README_RENDER_CHAR_LIMIT) return cleaned; return `${cleaned.slice(0, README_RENDER_CHAR_LIMIT).trimEnd()}${README_TRUNCATED_NOTICE}`; } diff --git a/studio/frontend/src/features/hub/lib/hf-readme.ts b/studio/frontend/src/features/hub/lib/hf-readme.ts index d68df2b808e..d7bfe3b6631 100644 --- a/studio/frontend/src/features/hub/lib/hf-readme.ts +++ b/studio/frontend/src/features/hub/lib/hf-readme.ts @@ -193,3 +193,20 @@ export function stripChromeHeadings(markdown: string): string { } return out.join("\n"); } + +// Pictographs, regional-indicator flag pairs, and the modifiers that recolor an +// otherwise plain glyph (VS-16, ZWJ, skin tones, the tag characters used by +// England/Scotland/Wales-style flag sequences) all route through a color font. +const COLOR_FONT_TRIGGER_RE = + /\p{Extended_Pictographic}|\p{Regional_Indicator}|[\u{FE0F}\u{200D}\u{1F3FB}-\u{1F3FF}\u{E0020}-\u{E007F}]/gu; + +/** + * Drop characters that render through a color font (emoji, flags, and their + * modifiers), for hosts where that path is known to crash the renderer + * (issue #9453: a Linux AppImage's bundled WebKitGTK/Skia asserts on a + * COLRv1 color-stop table). READMEs are arbitrary Hugging Face content, so + * this is the only place that can head it off before render. + */ +export function stripColorFontTriggers(markdown: string): string { + return markdown.replace(COLOR_FONT_TRIGGER_RE, ""); +} From ba004f4ba6ac0bedf959c08272555a54efa38f7a Mon Sep 17 00:00:00 2001 From: InfoSage05 Date: Fri, 21 Aug 2026 14:04:21 +0530 Subject: [PATCH 3/4] test(studio): move color-font stripper to its own leaf module, add tests hf-readme.ts imports @/-aliased modules that the plain `node --test` runner cannot resolve, so a direct unit test on stripColorFontTriggers failed with ERR_MODULE_NOT_FOUND. Moving the pure function to color-font-triggers.ts (no aliased imports) makes it importable from tests/, matching every other lib/*.test.ts in this suite. Also swap the skin-tone modifier range for \p{Emoji_Modifier} to satisfy no-misleading-character-class. --- .../src/features/hub/catalog/model-readme.tsx | 2 +- .../features/hub/lib/color-font-triggers.ts | 19 ++++++++ .../src/features/hub/lib/hf-readme.ts | 17 ------- .../tests/hub-readme-color-font.test.ts | 45 +++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) create mode 100644 studio/frontend/src/features/hub/lib/color-font-triggers.ts create mode 100644 studio/frontend/tests/hub-readme-color-font.test.ts diff --git a/studio/frontend/src/features/hub/catalog/model-readme.tsx b/studio/frontend/src/features/hub/catalog/model-readme.tsx index 116e3c26296..8632f2bf796 100644 --- a/studio/frontend/src/features/hub/catalog/model-readme.tsx +++ b/studio/frontend/src/features/hub/catalog/model-readme.tsx @@ -28,12 +28,12 @@ import { } from "react"; import type { ComponentProps } from "react"; import { Streamdown, type Components } from "streamdown"; +import { stripColorFontTriggers } from "../lib/color-font-triggers"; import { createReadmeUrlTransform, fetchReadme, readmeBaseUrl, stripChromeHeadings, - stripColorFontTriggers, stripFrontmatter, } from "../lib/hf-readme"; diff --git a/studio/frontend/src/features/hub/lib/color-font-triggers.ts b/studio/frontend/src/features/hub/lib/color-font-triggers.ts new file mode 100644 index 00000000000..9809fe4c6fc --- /dev/null +++ b/studio/frontend/src/features/hub/lib/color-font-triggers.ts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +// Pictographs, regional-indicator flag pairs, and the modifiers that recolor an +// otherwise plain glyph (VS-16, ZWJ, skin tones, the tag characters used by +// England/Scotland/Wales-style flag sequences) all route through a color font. +const COLOR_FONT_TRIGGER_RE = + /\p{Extended_Pictographic}|\p{Regional_Indicator}|\p{Emoji_Modifier}|[\u{FE0F}\u{200D}]|[\u{E0020}-\u{E007F}]/gu; + +/** + * Drop characters that render through a color font (emoji, flags, and their + * modifiers), for hosts where that path is known to crash the renderer + * (issue #9453: a Linux AppImage's bundled WebKitGTK/Skia asserts on a + * COLRv1 color-stop table). READMEs are arbitrary Hugging Face content, so + * this is the only place that can head it off before render. + */ +export function stripColorFontTriggers(markdown: string): string { + return markdown.replace(COLOR_FONT_TRIGGER_RE, ""); +} diff --git a/studio/frontend/src/features/hub/lib/hf-readme.ts b/studio/frontend/src/features/hub/lib/hf-readme.ts index d7bfe3b6631..d68df2b808e 100644 --- a/studio/frontend/src/features/hub/lib/hf-readme.ts +++ b/studio/frontend/src/features/hub/lib/hf-readme.ts @@ -193,20 +193,3 @@ export function stripChromeHeadings(markdown: string): string { } return out.join("\n"); } - -// Pictographs, regional-indicator flag pairs, and the modifiers that recolor an -// otherwise plain glyph (VS-16, ZWJ, skin tones, the tag characters used by -// England/Scotland/Wales-style flag sequences) all route through a color font. -const COLOR_FONT_TRIGGER_RE = - /\p{Extended_Pictographic}|\p{Regional_Indicator}|[\u{FE0F}\u{200D}\u{1F3FB}-\u{1F3FF}\u{E0020}-\u{E007F}]/gu; - -/** - * Drop characters that render through a color font (emoji, flags, and their - * modifiers), for hosts where that path is known to crash the renderer - * (issue #9453: a Linux AppImage's bundled WebKitGTK/Skia asserts on a - * COLRv1 color-stop table). READMEs are arbitrary Hugging Face content, so - * this is the only place that can head it off before render. - */ -export function stripColorFontTriggers(markdown: string): string { - return markdown.replace(COLOR_FONT_TRIGGER_RE, ""); -} diff --git a/studio/frontend/tests/hub-readme-color-font.test.ts b/studio/frontend/tests/hub-readme-color-font.test.ts new file mode 100644 index 00000000000..14e32a770e5 --- /dev/null +++ b/studio/frontend/tests/hub-readme-color-font.test.ts @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import assert from "node:assert/strict"; +import test from "node:test"; + +import { stripColorFontTriggers } from "../src/features/hub/lib/color-font-triggers.ts"; + +// Issue #9453: a Linux AppImage's bundled WebKitGTK/Skia asserts on a COLRv1 +// color-stop table when a README renders a color-font glyph. +test("strips a plain pictograph emoji", () => { + assert.equal( + stripColorFontTriggers("# Qwen3.8-27B \u{1F680} Fast and accurate!"), + "# Qwen3.8-27B Fast and accurate!", + ); +}); + +test("strips a regional-indicator flag pair", () => { + assert.equal( + stripColorFontTriggers("Supports \u{1F1EC}\u{1F1E7} and \u{1F1FA}\u{1F1F8} locales."), + "Supports and locales.", + ); +}); + +test("strips a ZWJ-joined family emoji sequence entirely", () => { + assert.equal( + stripColorFontTriggers( + "Family: \u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}", + ), + "Family: ", + ); +}); + +test("strips a variation-selector-16 star and checkmark", () => { + assert.equal( + stripColorFontTriggers("Star rating ⭐ and checkmark ✅."), + "Star rating and checkmark .", + ); +}); + +test("leaves plain markdown, code, links, and arrows untouched", () => { + const markdown = + "Plain markdown: **bold**, `code`, [link](https://x.com), 100% -> arrow, a->b"; + assert.equal(stripColorFontTriggers(markdown), markdown); +}); From 680ae19c5ef44944871ef724622632df63928136 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 08:45:51 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/paddleocr_vl_finetuning.py | 200 +++++++++++++++------------- unsloth/registry/_paddleocr.py | 20 +-- 2 files changed, 114 insertions(+), 106 deletions(-) diff --git a/examples/paddleocr_vl_finetuning.py b/examples/paddleocr_vl_finetuning.py index 08f222ff79e..ee9b7d22e16 100644 --- a/examples/paddleocr_vl_finetuning.py +++ b/examples/paddleocr_vl_finetuning.py @@ -62,23 +62,24 @@ # ── 1. Configuration ────────────────────────────────────────────────────── MODEL_NAME = "PaddlePaddle/PaddleOCR-VL-1.6" -MAX_SEQ_LENGTH = 2048 # Context window; reduce if VRAM limited -LOAD_IN_4BIT = True # 4-bit QLoRA (saves ~4× VRAM) -BATCH_SIZE = 2 # Adjust based on GPU memory +MAX_SEQ_LENGTH = 2048 # Context window; reduce if VRAM limited +LOAD_IN_4BIT = True # 4-bit QLoRA (saves ~4× VRAM) +BATCH_SIZE = 2 # Adjust based on GPU memory GRADIENT_ACCUMULATION_STEPS = 4 LEARNING_RATE = 5e-5 NUM_EPOCHS = 3 -LORA_R = 64 # LoRA rank -LORA_ALPHA = 64 # LoRA alpha (scaling) +LORA_R = 64 # LoRA rank +LORA_ALPHA = 64 # LoRA alpha (scaling) OUTPUT_DIR = "paddleocr_vl_finetuned" -EVAL_RATIO = 0.2 # Fraction of samples held out for evaluation -NUM_SAMPLES = 50 # Total samples to load from source dataset +EVAL_RATIO = 0.2 # Fraction of samples held out for evaluation +NUM_SAMPLES = 50 # Total samples to load from source dataset os.environ["UNSLOTH_RETURN_LOGITS"] = "1" # ── 2. Load Model & Tokenizer ───────────────────────────────────────────── + def load_model_and_tokenizer(): """ Loads PaddleOCR-VL-1.6 with 4-bit QLoRA quantisation via Unsloth. @@ -90,11 +91,11 @@ def load_model_and_tokenizer(): dtype = torch.bfloat16 if is_bfloat16_supported() else torch.float16 model, tokenizer = FastModel.from_pretrained( - model_name=MODEL_NAME, - max_seq_length=MAX_SEQ_LENGTH, - dtype=dtype, - load_in_4bit=LOAD_IN_4BIT, - trust_remote_code=True, # Required for PaddleOCR custom code + model_name = MODEL_NAME, + max_seq_length = MAX_SEQ_LENGTH, + dtype = dtype, + load_in_4bit = LOAD_IN_4BIT, + trust_remote_code = True, # Required for PaddleOCR custom code ) print(f"[✓] Loaded {MODEL_NAME}") print(f" Model type: {model.config.model_type}") @@ -106,7 +107,8 @@ def load_model_and_tokenizer(): # ── 3. Dataset Preparation ──────────────────────────────────────────────── -def prepare_datasets(num_samples=NUM_SAMPLES, eval_ratio=EVAL_RATIO): + +def prepare_datasets(num_samples = NUM_SAMPLES, eval_ratio = EVAL_RATIO): """ Prepares training and evaluation datasets from a public OCR / document- understanding source, returning a train/eval split. @@ -151,13 +153,13 @@ def prepare_datasets(num_samples=NUM_SAMPLES, eval_ratio=EVAL_RATIO): try: ds = load_dataset( "HuggingFaceM4/Document_Understanding_test", - split="train", + split = "train", ) ds = ds.select(range(min(num_samples, len(ds)))) except Exception: # Fallback: load a public OCR dataset try: - ds = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split="train") + ds = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train") ds = ds.select(range(min(num_samples, len(ds)))) except Exception: print("[!] Could not load demo dataset. Creating a minimal synthetic dataset.") @@ -168,10 +170,13 @@ def prepare_datasets(num_samples=NUM_SAMPLES, eval_ratio=EVAL_RATIO): import requests try: - resp = requests.get("https://upload.wikimedia.org/wikipedia/commons/thumb/8/87/PDF_file_icon.svg/128px-PDF_file_icon.svg.png", timeout=5) + resp = requests.get( + "https://upload.wikimedia.org/wikipedia/commons/thumb/8/87/PDF_file_icon.svg/128px-PDF_file_icon.svg.png", + timeout = 5, + ) img = Image.open(io.BytesIO(resp.content)) except Exception: - img = Image.new("RGB", (384, 384), color="white") + img = Image.new("RGB", (384, 384), color = "white") synthetic = [ { @@ -186,7 +191,10 @@ def prepare_datasets(num_samples=NUM_SAMPLES, eval_ratio=EVAL_RATIO): { "role": "assistant", "content": [ - {"type": "text", "text": "This is a sample document transcription for demonstration purposes."} + { + "type": "text", + "text": "This is a sample document transcription for demonstration purposes.", + } ], }, ] @@ -233,6 +241,7 @@ def format_example(example): # ── 4. Apply LoRA Adapters ──────────────────────────────────────────────── + def apply_lora(model): """ Attaches LoRA adapters to both the vision and language layers. @@ -246,10 +255,10 @@ def apply_lora(model): """ model = FastModel.get_peft_model( model, - r=LORA_R, - lora_alpha=LORA_ALPHA, - lora_dropout=0.0, - target_modules=[ + r = LORA_R, + lora_alpha = LORA_ALPHA, + lora_dropout = 0.0, + target_modules = [ "q_proj", "k_proj", "v_proj", @@ -258,15 +267,15 @@ def apply_lora(model): "up_proj", "down_proj", ], - use_rslora=False, + use_rslora = False, # --- Vision-specific flags --- - finetune_vision_layers=True, # Fine-tune vision encoder - finetune_language_layers=True, # Fine-tune language backbone - finetune_attention_modules=True, - finetune_mlp_modules=True, + finetune_vision_layers = True, # Fine-tune vision encoder + finetune_language_layers = True, # Fine-tune language backbone + finetune_attention_modules = True, + finetune_mlp_modules = True, # --- Training helpers --- - use_gradient_checkpointing="unsloth", - random_state=3407, + use_gradient_checkpointing = "unsloth", + random_state = 3407, ) trainable = sum(p.numel() for p in model.parameters() if p.requires_grad) print(f"[✓] LoRA applied (rank={LORA_R}, alpha={LORA_ALPHA})") @@ -276,6 +285,7 @@ def apply_lora(model): # ── 5. Training ──────────────────────────────────────────────────────────── + def train(model, tokenizer, dataset): """ Fine-tune the model using SFTTrainer with UnslothVisionDataCollator. @@ -291,40 +301,40 @@ def train(model, tokenizer, dataset): model.config.use_cache = False training_args = TrainingArguments( - output_dir=OUTPUT_DIR, - num_train_epochs=NUM_EPOCHS, - per_device_train_batch_size=BATCH_SIZE, - gradient_accumulation_steps=GRADIENT_ACCUMULATION_STEPS, - warmup_steps=5, - learning_rate=LEARNING_RATE, - logging_steps=10, - save_strategy="epoch", - save_total_limit=2, - optim="adamw_8bit", - weight_decay=0.01, - lr_scheduler_type="linear", - seed=3407, - report_to="none", + output_dir = OUTPUT_DIR, + num_train_epochs = NUM_EPOCHS, + per_device_train_batch_size = BATCH_SIZE, + gradient_accumulation_steps = GRADIENT_ACCUMULATION_STEPS, + warmup_steps = 5, + learning_rate = LEARNING_RATE, + logging_steps = 10, + save_strategy = "epoch", + save_total_limit = 2, + optim = "adamw_8bit", + weight_decay = 0.01, + lr_scheduler_type = "linear", + seed = 3407, + report_to = "none", # ── Required for vision model training ── - remove_unused_columns=False, - dataloader_pin_memory=False, - bf16=is_bfloat16_supported(), - fp16=not is_bfloat16_supported(), - gradient_checkpointing=True, - gradient_checkpointing_kwargs={"use_reentrant": False}, + remove_unused_columns = False, + dataloader_pin_memory = False, + bf16 = is_bfloat16_supported(), + fp16 = not is_bfloat16_supported(), + gradient_checkpointing = True, + gradient_checkpointing_kwargs = {"use_reentrant": False}, ) trainer = SFTTrainer( - model=model, - tokenizer=tokenizer, - args=training_args, - train_dataset=dataset, - data_collator=UnslothVisionDataCollator(model, tokenizer), - max_seq_length=MAX_SEQ_LENGTH, - dataset_num_proc=2, + model = model, + tokenizer = tokenizer, + args = training_args, + train_dataset = dataset, + data_collator = UnslothVisionDataCollator(model, tokenizer), + max_seq_length = MAX_SEQ_LENGTH, + dataset_num_proc = 2, # ── Required for vision model training ── - dataset_text_field="", - dataset_kwargs={"skip_prepare_dataset": True}, + dataset_text_field = "", + dataset_kwargs = {"skip_prepare_dataset": True}, ) print("[*] Starting training...") @@ -352,10 +362,7 @@ def _normalize_text(text: str) -> str: return text -def compute_wer_cer( - predictions: list[str], - references: list[str], -) -> dict[str, float]: +def compute_wer_cer(predictions: list[str], references: list[str]) -> dict[str, float]: """ Compute Word Error Rate (WER) and Character Error Rate (CER) for a batch of predictions against ground-truth references. @@ -387,10 +394,7 @@ def compute_wer_cer( refs_norm = [_normalize_text(r) for r in references] # Filter out empty references (can't compute meaningful WER on them) - valid = [ - (p, r) for p, r in zip(preds_norm, refs_norm) - if len(r) > 0 - ] + valid = [(p, r) for p, r in zip(preds_norm, refs_norm) if len(r) > 0] if not valid: return {"wer": float("nan"), "cer": float("nan"), "samples": 0} @@ -459,9 +463,7 @@ def evaluate_ocr_benchmark( # Ground-truth reference ref_text = "".join( - part["text"] - for part in assistant_msg["content"] - if part["type"] == "text" + part["text"] for part in assistant_msg["content"] if part["type"] == "text" ) references.append(ref_text) @@ -494,24 +496,24 @@ def evaluate_ocr_benchmark( try: input_text = tokenizer.apply_chat_template( gen_messages, - tokenize=False, - add_generation_prompt=True, + tokenize = False, + add_generation_prompt = True, ) inputs = tokenizer( [input_text], - images=[image], - return_tensors="pt", - padding=True, + images = [image], + return_tensors = "pt", + padding = True, ).to("cuda") outputs = model.generate( **inputs, - max_new_tokens=max_new_tokens, - temperature=1.0, - min_p=0.1, - do_sample=True, + max_new_tokens = max_new_tokens, + temperature = 1.0, + min_p = 0.1, + do_sample = True, ) - decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) + decoded = tokenizer.decode(outputs[0], skip_special_tokens = True) predictions.append(decoded) except Exception as e: if verbose: @@ -543,9 +545,7 @@ def evaluate_ocr_benchmark( print(f" REF: {references[i][:120]}") print(f" PRED: {predictions[i][:120]}") if HAS_JIWER and len(references[i]) > 0: - single = compute_wer_cer( - [predictions[i]], [references[i]] - ) + single = compute_wer_cer([predictions[i]], [references[i]]) if single["samples"] > 0: print(f" WER: {single['wer']:.2%} CER: {single['cer']:.2%}") print() @@ -555,6 +555,7 @@ def evaluate_ocr_benchmark( # ── 7. Save & Merge ─────────────────────────────────────────────────────── + def save_and_merge(model, tokenizer): """ Save the fine-tuned model in multiple formats. @@ -573,7 +574,7 @@ def save_and_merge(model, tokenizer): model.save_pretrained_merged( f"{OUTPUT_DIR}/merged_16bit", tokenizer, - save_method="merged_16bit", + save_method = "merged_16bit", ) print(f"[✓] Merged 16-bit model saved to {OUTPUT_DIR}/merged_16bit") @@ -583,6 +584,7 @@ def save_and_merge(model, tokenizer): # ── 8. Main ─────────────────────────────────────────────────────────────── + def main(): """ Full pipeline: @@ -605,8 +607,8 @@ def main(): # ── Step 2: Prepare train + eval datasets ── print("\n[Step 2/7] Preparing datasets...") train_dataset, eval_dataset = prepare_datasets( - num_samples=NUM_SAMPLES, - eval_ratio=EVAL_RATIO, + num_samples = NUM_SAMPLES, + eval_ratio = EVAL_RATIO, ) # ── Step 3: Apply LoRA adapters ── @@ -621,7 +623,7 @@ def main(): model, tokenizer, eval_dataset, - show_examples=True, + show_examples = True, ) # ── Step 5: Train ── @@ -638,7 +640,7 @@ def main(): model, tokenizer, eval_dataset, - show_examples=True, + show_examples = True, ) # ── Improvement summary ── @@ -649,18 +651,24 @@ def main(): if HAS_JIWER and baseline_metrics["samples"] > 0 and finetuned_metrics["samples"] > 0: wer_delta = baseline_metrics["wer"] - finetuned_metrics["wer"] cer_delta = baseline_metrics["cer"] - finetuned_metrics["cer"] - print(f" WER: {baseline_metrics['wer']:.2%} → {finetuned_metrics['wer']:.2%} " - f"({'↓' if wer_delta > 0 else '↑'}{abs(wer_delta):.2%})") - print(f" CER: {baseline_metrics['cer']:.2%} → {finetuned_metrics['cer']:.2%} " - f"({'↓' if cer_delta > 0 else '↑'}{abs(cer_delta):.2%})") + print( + f" WER: {baseline_metrics['wer']:.2%} → {finetuned_metrics['wer']:.2%} " + f"({'↓' if wer_delta > 0 else '↑'}{abs(wer_delta):.2%})" + ) + print( + f" CER: {baseline_metrics['cer']:.2%} → {finetuned_metrics['cer']:.2%} " + f"({'↓' if cer_delta > 0 else '↑'}{abs(cer_delta):.2%})" + ) if wer_delta > 0 or cer_delta > 0: print(f"\n ✓ Fine-tuning improved OCR quality!") else: - print(f"\n Note: OCR quality did not improve. Consider:\n" - f" - Using more training data\n" - f" - Training for more epochs\n" - f" - Increasing LoRA rank\n" - f" - Reducing learning rate") + print( + f"\n Note: OCR quality did not improve. Consider:\n" + f" - Using more training data\n" + f" - Training for more epochs\n" + f" - Increasing LoRA rank\n" + f" - Reducing learning rate" + ) else: print(" (Install jiwer for WER/CER metrics: pip install jiwer)") print("=" * 56) diff --git a/unsloth/registry/_paddleocr.py b/unsloth/registry/_paddleocr.py index 637a6b306b1..c0b41c52164 100644 --- a/unsloth/registry/_paddleocr.py +++ b/unsloth/registry/_paddleocr.py @@ -12,14 +12,14 @@ def construct_model_name(cls, base_name, version, size, quant_type, instruct_tag # PaddleOCR-VL Model Meta PaddleOCRVLMeta = ModelMeta( - org="PaddlePaddle", - base_name="PaddleOCR", - instruct_tags=[None], - model_version="VL-1.6", - model_sizes=["0_9"], # ~0.9B parameters (hidden=1024, 18 layers) - model_info_cls=PaddleOCRModelInfo, - is_multimodal=True, - quant_types=[QuantType.NONE, QuantType.BNB], + org = "PaddlePaddle", + base_name = "PaddleOCR", + instruct_tags = [None], + model_version = "VL-1.6", + model_sizes = ["0_9"], # ~0.9B parameters (hidden=1024, 18 layers) + model_info_cls = PaddleOCRModelInfo, + is_multimodal = True, + quant_types = [QuantType.NONE, QuantType.BNB], ) @@ -27,7 +27,7 @@ def register_paddleocr_models(include_original_model: bool = False): global _IS_PADDLEOCR_REGISTERED if _IS_PADDLEOCR_REGISTERED: return - _register_models(PaddleOCRVLMeta, include_original_model=include_original_model) + _register_models(PaddleOCRVLMeta, include_original_model = include_original_model) _IS_PADDLEOCR_REGISTERED = True @@ -36,7 +36,7 @@ def register_paddleocr_models(include_original_model: bool = False): MODEL_REGISTRY.clear() - register_paddleocr_models(include_original_model=True) + register_paddleocr_models(include_original_model = True) for model_id, model_info in MODEL_REGISTRY.items(): model_info = _check_model_info(model_id)