implement R3

Signed-off-by: Kyle Sayers <[email protected]>

Author: kylesayrs

src/llmcompressor/modifiers/transform/spinquant/base.py

@@ -236,8 +236,22 @@ def _create_r2_scheme(self, head_dim: int) -> TransformScheme:
         )
 
     def _create_r3_scheme(self, head_dim: int) -> TransformScheme:
-        raise NotImplementedError(
-            "SpinQuant R3 rotations will be added in a future release"
+        return TransformScheme(
+            type=self.transform_type,
+            randomize=self.randomize,
+            requires_grad=self.learnable,
+            precision=self.precision,
+            head_dim=head_dim,
+            apply=[
+                TransformArgs(
+                    targets=[self.mappings.attn],
+                    location="q_attn",
+                ),
+                TransformArgs(
+                    targets=[self.mappings.attn],
+                    location="k_cache",
+                ),
+            ],
         )
 
     def _create_r4_scheme(self) -> TransformScheme:

src/llmcompressor/modifiers/transform/spinquant/mappings.py

@@ -14,6 +14,7 @@ class SpinQuantMapping(BaseModel):
     layers (https://arxiv.org/pdf/2405.16406 Fig. 1).
 
     :param embedding: name or regex of embedding layer
+    :param attn: name or regex of attention block in decoder layer
     :param attn_q: name or regex of q_proj layer in attention block
     :param attn_k: name or regex of k_proj layer in attention block
     :param attn_v: name or regex of v_proj layer in attention block
@@ -29,6 +30,7 @@ class SpinQuantMapping(BaseModel):
 
     embedding: str
 
+    attn: str
     attn_q: str
     attn_k: str
     attn_v: str
@@ -50,6 +52,7 @@ def cast_to_list(cls, value):
 
 _default_mappings = SpinQuantMapping(
     embedding="re:.*embed_tokens$",
+    attn="re:.*self_attn$",
     attn_q="re:.*q_proj$",
     attn_k="re:.*k_proj$",
     attn_v="re:.*v_proj$",