[Example] Add Mixed W2A16 and W4A16 MoE Example

examples/autoround/quantization_wNa16/qwen3_moe_example_mixed_w2a16_w4a16.py

@@ -0,0 +1,94 @@
+from auto_round.calib_dataset import get_dataset
+from compressed_tensors.offload import dispatch_model
+from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from llmcompressor import oneshot
+from llmcompressor.modifiers.autoround import AutoRoundModifier
+from llmcompressor.utils import load_context
+
+# Select model and load it.
+model_id = "Qwen/Qwen3-30B-A3B-Instruct-2507"
+with load_context():
+    model = AutoModelForCausalLM.from_pretrained(model_id)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+# Select calibration dataset.
+NUM_CALIBRATION_SAMPLES = 128
+MAX_SEQUENCE_LENGTH = 2048
+ITERS = 200
+# Get aligned calibration dataset.
+
+ds = get_dataset(
+    tokenizer=tokenizer,
+    seqlen=MAX_SEQUENCE_LENGTH,
+    nsamples=NUM_CALIBRATION_SAMPLES,
+)
+
+
+# Configure the quantization algorithm to run.
+#   * quantize the attention weights to 4 bit with a group size 64
+#   * mlp weights to 2 bit with a group size 64
+
+recipe = AutoRoundModifier(
+    config_groups={
+        "attention": QuantizationScheme(
+            targets=[
+                r"re:.*self_attn\.(q|k|v|o)_proj$",
+            ],
+            weights=QuantizationArgs(
+                num_bits=4,
+                type="int",
+                strategy="group",
+                symmetric=True,
+                dynamic=False,
+                group_size=64,
+            ),
+        ),
+        "mlp": QuantizationScheme(
+            targets=[
+                r"re:.*mlp\..*(gate|up|down)_proj$",
+            ],
+            weights=QuantizationArgs(
+                num_bits=2,
+                type="int",
+                strategy="group",
+                symmetric=True,
+                dynamic=False,
+                group_size=64,
+            ),
+        ),
+    },
+    ignore=["lm_head"],
+    iters=ITERS,
+    enable_torch_compile=False,
+)
+
+
+# Apply algorithms.
+oneshot(
+    model=model,
+    dataset=ds,
+    recipe=recipe,
+    max_seq_length=MAX_SEQUENCE_LENGTH,
+    num_calibration_samples=NUM_CALIBRATION_SAMPLES,
+    shuffle_calibration_samples=False,
+)
+
+# Confirm generations of the quantized model look sane.
+print("\n\n")
+print("========== SAMPLE GENERATION ==============")
+dispatch_model(model)
+sample = tokenizer("Hello my name is", return_tensors="pt")
+sample = {key: value.to(model.device) for key, value in sample.items()}
+output = model.generate(**sample, max_new_tokens=100)
+print(tokenizer.decode(output[0]))
+print("==========================================\n\n")
+
+# Save to disk compressed.
+SAVE_DIR = (
+    model_id.rstrip("/").split("/")[-1] + f"-Attn4bits-Mlp2bits-AutoRound-{ITERS}iters"
+)
+print(f"save to {SAVE_DIR}")
+model.save_pretrained(SAVE_DIR, save_compressed=True)
+tokenizer.save_pretrained(SAVE_DIR)