From 51d2e43ee9cc0eb68722dc158b680c8ef3f4e49e Mon Sep 17 00:00:00 2001 From: JartX Date: Wed, 28 Jan 2026 11:59:14 +0100 Subject: [PATCH 1/5] added qwen3-omni-30b-a3b-example Signed-off-by: JartX --- .../qwen3-omni-30b-a3b-instruct-example.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 examples/awq/qwen3-omni-30b-a3b-instruct-example.py diff --git a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py new file mode 100644 index 0000000000..54e0da4ce0 --- /dev/null +++ b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py @@ -0,0 +1,113 @@ +import base64 +from io import BytesIO +import torch +from datasets import load_dataset +from transformers import AutoProcessor, Qwen3OmniMoeForConditionalGeneration +from llmcompressor import oneshot +from llmcompressor.modifiers.awq import AWQModifier, AWQMapping +from llmcompressor.modeling.patch.qwen3_omni_patch import fast_pos_embed_interpolate +from llmcompressor.transformers.compression.compressed_tensors_utils import modify_save_pretrained + +MODEL_ID = "Qwen/Qwen3-Omni-30B-A3B-Instruct" +OUTPUT_DIR = MODEL_ID.split("/")[-1] + "-AWQ-W4A16" +NUM_CALIBRATION_SAMPLES = 512 +MAX_SEQUENCE_LENGTH = 2048 + +model = Qwen3OmniMoeForConditionalGeneration.from_pretrained( + MODEL_ID, + torch_dtype="auto", + device_map=None, + trust_remote_code=True, +) + +model.thinker.visual.fast_pos_embed_interpolate = fast_pos_embed_interpolate.__get__( + model.thinker.visual +) + +processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True) + +DATASET_ID = "lmms-lab/flickr30k" +DATASET_SPLIT = f"test[:{NUM_CALIBRATION_SAMPLES}]" + +ds = load_dataset(DATASET_ID, split=DATASET_SPLIT) +ds = ds.shuffle(seed=42) + +def preprocess_and_tokenize(example): + buffered = BytesIO() + example["image"].save(buffered, format="PNG") + encoded_image = base64.b64encode(buffered.getvalue()) + base64_image = f"data:image;base64,{encoded_image.decode('utf-8')}" + messages = [{ + "role": "user", + "content": [ + {"type": "image", "image": base64_image}, + {"type": "text", "text": "What does the image show?"} + ] + }] + text = processor.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True + ) + inputs = processor( + text=[text], + images=[example["image"]], + padding=False, + max_length=MAX_SEQUENCE_LENGTH, + truncation=True, + ) + return inputs + +ds = ds.map(preprocess_and_tokenize, remove_columns=ds.column_names) + +def data_collator(batch): + assert len(batch) == 1 + return {key: torch.tensor(value) for key, value in batch[0].items()} + +recipe = AWQModifier( + ignore=[ + "lm_head", + "re:.*visual.*", + "re:.*code2wav.*", + "re:.*audio_tower.*", + "re:^talker\..*", + "re:.*embed_tokens", + "re:.*mlp\.gate$", + "re:.*shared_expert_gate$", + "re:.*input_layernorm$", + "re:.*post_attention_layernorm$", + "re:.*norm$", + "re:.*lm_head$" + ], + duo_scaling=False, + config_groups={ + "group_0": { + "targets": ["Linear"], + "input_activations": None, + "output_activations": None, + "weights": { + "num_bits": 4, + "type": "int", + "symmetric": True, + "strategy": "group", + "group_size": 32, + "observer": "mse", + } + } + } +) + +oneshot( + model=model.thinker.model, + processor=processor, + recipe=recipe, + dataset=ds, + max_seq_length=MAX_SEQUENCE_LENGTH, + num_calibration_samples=NUM_CALIBRATION_SAMPLES, + data_collator=data_collator, + pipeline="sequential", +) + +modify_save_pretrained(model) +model.save_pretrained(OUTPUT_DIR, save_compressed=True) +processor.save_pretrained(OUTPUT_DIR) \ No newline at end of file From e9c40325c3664ae951db59044e0d45cd456eb2a1 Mon Sep 17 00:00:00 2001 From: JartX Date: Wed, 28 Jan 2026 12:18:43 +0100 Subject: [PATCH 2/5] remove unused code Signed-off-by: JartX --- examples/awq/qwen3-omni-30b-a3b-instruct-example.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py index 54e0da4ce0..bf50035317 100644 --- a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py +++ b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py @@ -4,18 +4,18 @@ from datasets import load_dataset from transformers import AutoProcessor, Qwen3OmniMoeForConditionalGeneration from llmcompressor import oneshot -from llmcompressor.modifiers.awq import AWQModifier, AWQMapping +from llmcompressor.modifiers.awq import AWQModifier from llmcompressor.modeling.patch.qwen3_omni_patch import fast_pos_embed_interpolate from llmcompressor.transformers.compression.compressed_tensors_utils import modify_save_pretrained MODEL_ID = "Qwen/Qwen3-Omni-30B-A3B-Instruct" OUTPUT_DIR = MODEL_ID.split("/")[-1] + "-AWQ-W4A16" NUM_CALIBRATION_SAMPLES = 512 -MAX_SEQUENCE_LENGTH = 2048 +MAX_SEQUENCE_LENGTH = 4096 model = Qwen3OmniMoeForConditionalGeneration.from_pretrained( MODEL_ID, - torch_dtype="auto", + torch_dtype=torch.bfloat16, device_map=None, trust_remote_code=True, ) @@ -110,4 +110,4 @@ def data_collator(batch): modify_save_pretrained(model) model.save_pretrained(OUTPUT_DIR, save_compressed=True) -processor.save_pretrained(OUTPUT_DIR) \ No newline at end of file +processor.save_pretrained(OUTPUT_DIR) From f9619e26ac5775f79df8382fb15b85571e781db1 Mon Sep 17 00:00:00 2001 From: JartX Date: Wed, 28 Jan 2026 12:20:00 +0100 Subject: [PATCH 3/5] remove redundant ignore Signed-off-by: JartX --- examples/awq/qwen3-omni-30b-a3b-instruct-example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py index bf50035317..ae4f51d3ef 100644 --- a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py +++ b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py @@ -66,7 +66,6 @@ def data_collator(batch): recipe = AWQModifier( ignore=[ - "lm_head", "re:.*visual.*", "re:.*code2wav.*", "re:.*audio_tower.*", From f0a2d7ce189e2099f6090c172af2fe7c512b773c Mon Sep 17 00:00:00 2001 From: JartX Date: Wed, 28 Jan 2026 17:20:01 +0100 Subject: [PATCH 4/5] remove mse Signed-off-by: JartX --- examples/awq/qwen3-omni-30b-a3b-instruct-example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py index ae4f51d3ef..1db0e5a78a 100644 --- a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py +++ b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py @@ -90,7 +90,6 @@ def data_collator(batch): "symmetric": True, "strategy": "group", "group_size": 32, - "observer": "mse", } } } From 5beb0ae8ac32da483ed62cf9a42860c3d1a7510e Mon Sep 17 00:00:00 2001 From: JartX Date: Wed, 28 Jan 2026 18:57:06 +0100 Subject: [PATCH 5/5] added comment to explain the group_size 32 Signed-off-by: JartX --- examples/awq/qwen3-omni-30b-a3b-instruct-example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py index 1db0e5a78a..602ce1a868 100644 --- a/examples/awq/qwen3-omni-30b-a3b-instruct-example.py +++ b/examples/awq/qwen3-omni-30b-a3b-instruct-example.py @@ -80,6 +80,7 @@ def data_collator(batch): ], duo_scaling=False, config_groups={ + # Using W4A16 group 32 scheme to work with Exllama kernel "group_0": { "targets": ["Linear"], "input_activations": None,