Legalize load.i128 and store.i128 with arbitrary offsets

Author: bjorn3

cranelift-codegen/meta/src/shared/legalize.rs

@@ -148,7 +148,6 @@ pub fn define(insts: &InstructionGroup, immediates: &OperandKinds) -> TransformG
     let ieee32 = immediates.by_name("ieee32");
     let ieee64 = immediates.by_name("ieee64");
     let intcc = immediates.by_name("intcc");
-    let offset32 = immediates.by_name("offset32");
 
     // List of variables to reuse in patterns.
     let x = var("x");
@@ -287,29 +286,8 @@ pub fn define(insts: &InstructionGroup, immediates: &OperandKinds) -> TransformG
         ],
     );
 
-    // FIXME generalize to any offset once offset+8 can be represented
-    narrow.legalize(
-        def!(a = load.I128(flags, ptr, Literal::constant(offset32, 0))),
-        vec![
-            def!(al = load.I64(flags, ptr, Literal::constant(offset32, 0))),
-            def!(ah = load.I64(flags, ptr, Literal::constant(offset32, 8))),
-            // `iconcat` expects the same byte order as stored in memory,
-            // so no need to swap depending on endianness.
-            def!(a = iconcat(al, ah)),
-        ],
-    );
-
-    // FIXME generalize to any offset once offset+8 can be represented
-    narrow.legalize(
-        def!(store.I128(flags, a, ptr, Literal::constant(offset32, 0))),
-        vec![
-            // `isplit` gives the same byte order as stored in memory,
-            // so no need to swap depending on endianness.
-            def!((al, ah) = isplit(a)),
-            def!(store.I64(flags, al, ptr, Literal::constant(offset32, 0))),
-            def!(store.I64(flags, ah, ptr, Literal::constant(offset32, 8))),
-        ],
-    );
+    narrow.custom_legalize(load, "narrow_load");
+    narrow.custom_legalize(store, "narrow_store");
 
     // Widen instructions with one input operand.
     for &op in &[bnot, popcnt] {

cranelift-codegen/src/legalizer/mod.rs

@@ -559,3 +559,63 @@ fn expand_stack_store(
     mflags.set_aligned();
     pos.func.dfg.replace(inst).store(mflags, val, addr, 0);
 }
+
+/// Split a load into two parts before `iconcat`ing the result together.
+fn narrow_load(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let (ptr, offset, flags) = match pos.func.dfg[inst] {
+        ir::InstructionData::Load {
+            opcode: ir::Opcode::Load,
+            arg,
+            offset,
+            flags,
+        } => (arg, offset, flags),
+        _ => panic!("Expected load: {}", pos.func.dfg.display_inst(inst, None)),
+    };
+
+    let al = pos.ins().load(ir::types::I64, flags, ptr, offset);
+    let ah = pos.ins().load(
+        ir::types::I64,
+        flags,
+        ptr,
+        offset.try_add_i64(8).expect("load offset overflow"),
+    );
+    pos.func.dfg.replace(inst).iconcat(al, ah);
+}
+
+/// Split a store into two parts after `isplit`ing the value.
+fn narrow_store(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let (val, ptr, offset, flags) = match pos.func.dfg[inst] {
+        ir::InstructionData::Store {
+            opcode: ir::Opcode::Store,
+            args,
+            offset,
+            flags,
+        } => (args[0], args[1], offset, flags),
+        _ => panic!("Expected store: {}", pos.func.dfg.display_inst(inst, None)),
+    };
+
+    let (al, ah) = pos.func.dfg.replace(inst).isplit(val);
+    pos.ins().store(flags, al, ptr, offset);
+    pos.ins().store(
+        flags,
+        ah,
+        ptr,
+        offset.try_add_i64(8).expect("store offset overflow"),
+    );
+}

filetests/isa/x86/i128.clif

@@ -28,19 +28,19 @@ ebb0(v0: i128):
 }
 
 function u0:2(i64, i128) fast {
-; check: ebb0(v0: i64 [%rdi], v2: i64 [%rsi], v3: i64 [%rdx], v4: i64 [%rbp]):
+; check: ebb0(v0: i64 [%rdi], v2: i64 [%rsi], v3: i64 [%rdx], v6: i64 [%rbp]):
 ebb0(v0: i64, v1: i128):
-    ; check: store v2, v0
-    ; check: store v3, v0+8
-    store v1, v0
+    ; check: store v2, v0+8
+    ; check: store v3, v0+16
+    store v1, v0+8
     return
 }
 
 function u0:3(i64) -> i128 fast {
 ebb0(v0: i64):
-    ; check: v2 = load.i64 v0
-    ; check: v3 = load.i64 v0+8
-    v1 = load.i128 v0
+    ; check: v2 = load.i64 v0+8
+    ; check: v3 = load.i64 v0+16
+    v1 = load.i128 v0+8
     ; check: return v2, v3, v5
     return v1
 }