fix(rocke): f64 follow-ups: fabs for abs, 8-byte scalar alignment, mean as a division

dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/ir/ir_mem.cpp

@@ -189,6 +189,14 @@ rocke_value_t* rocke_b_global_load_bf16(rocke_ir_builder_t* b,
     return rocke_b_global_load(b, ptr, idx, rocke_bf16(), align <= 0 ? 2 : align);
 }
 
+rocke_value_t* rocke_b_global_load_f64(rocke_ir_builder_t* b,
+                                       rocke_value_t* ptr,
+                                       rocke_value_t* idx,
+                                       int align)
+{
+    return rocke_b_global_load(b, ptr, idx, rocke_f64(), align <= 0 ? 8 : align);
+}
+
 rocke_value_t*
     rocke_b_global_load_i8(rocke_ir_builder_t* b, rocke_value_t* ptr, rocke_value_t* idx, int align)
 {

dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_hip/lower_hip_arith.cpp

@@ -520,7 +520,9 @@ static rocke_status_t rocke_h_op_arith_fneg(rocke_h_lowerer_t* lw, const rocke_o
 /* def _op_arith_fabs(self, op):
  *     (v,) = op.operands
  *     ty = _type_to_hip(op.result.type)
- *     _reject_f64(op, op.result.type.name)
+ *     if op.result.type.name == "f64":
+ *         self._emit(f"{ty} {_name(op.result)} = fabs({_name(v)});")
+ *         return
  *     helper = {"f32":"fabsf","f16":"__builtin_fabsf","bf16":"__builtin_fabsf"}
  *              .get(op.result.type.name, "fabsf")
  *     self._emit(f"{ty} {_name(op.result)} = ({ty}){helper}((float){_name(v)});") */
@@ -530,7 +532,13 @@ static rocke_status_t rocke_h_op_arith_fabs(rocke_h_lowerer_t* lw, const rocke_o
     const rocke_value_t* r = h_res(op);
     const char* ty = rocke_h_type_to_hip(lw, r->type);
     const char* tname = r->type->name;
-    rocke_h_reject_f64(lw, op, tname);
+    if(tname && strcmp(tname, "f64") == 0)
+    {
+        /* The double overload stays in double; the float helper below would
+         * round the value to f32. */
+        rocke_h_emitf(lw, "%s %s = fabs(%s);", ty, rocke_h_name(lw, r), rocke_h_name(lw, v));
+        return lw->status;
+    }
     const char* helper = "fabsf";
     if(tname && (strcmp(tname, "f16") == 0 || strcmp(tname, "bf16") == 0))
     {

dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/arith.cpp

@@ -231,7 +231,9 @@ static void _op_arith_fneg(rocke_lower_t* L, const rocke_op_t* op)
         L, "  %s = fneg %s %s", res->name, rocke_ll_llvm_type(L, v->type), rocke_ll_operand(L, v));
 }
 
-/* Python _op_arith_fabs -> llvm.fabs.<ty>. */
+/* Python _op_arith_fabs -> llvm.fabs.<ty>. Unlike fma / fmax3, fabs also
+ * accepts f64 (a sign-bit clear), so f64 is admitted here rather than in
+ * the shared ll_fp_llvm_ty table. */
 static void _op_arith_fabs(rocke_lower_t* L, const rocke_op_t* op)
 {
     const rocke_value_t* res = ll_result(op);
@@ -241,7 +243,8 @@ static void _op_arith_fabs(rocke_lower_t* L, const rocke_op_t* op)
     }
     const rocke_value_t* v = op->operands[0];
     const char* ty_name = v->type ? v->type->name : NULL;
-    const char* llvm_ty = ll_fp_llvm_ty(ty_name);
+    const char* llvm_ty
+        = (ty_name && strcmp(ty_name, "f64") == 0) ? "double" : ll_fp_llvm_ty(ty_name);
     if(llvm_ty == NULL)
     {
         rocke_ll_fail(

dnn-providers/hip-kernel-provider/rocke/platform/cpp/core/lower_llvm/data.cpp

@@ -158,6 +158,7 @@ const rocke_ll_decl_t ROCKE_LL_INTRINSIC_DECLS[] = {
     {"minnum.f32", "declare float @llvm.minnum.f32(float, float)"},
     {"minnum.f16", "declare half @llvm.minnum.f16(half, half)"},
     {"minnum.bf16", "declare bfloat @llvm.minnum.bf16(bfloat, bfloat)"},
+    {"fabs.f64", "declare double @llvm.fabs.f64(double)"},
     {"fabs.f32", "declare float @llvm.fabs.f32(float)"},
     {"fabs.f16", "declare half @llvm.fabs.f16(half)"},
     {"fabs.bf16", "declare bfloat @llvm.fabs.bf16(bfloat)"},

dnn-providers/hip-kernel-provider/rocke/platform/cpp/helpers/tensor_view.cpp

@@ -232,6 +232,8 @@ rocke_value_t* rocke_tensor_view_load_scalar(rocke_ir_builder_t* b,
         return rocke_b_global_load_i32(b, v->base, off, 0);
     if(rocke_tv_name_is(dt, "i64"))
         return rocke_b_global_load_i64(b, v->base, off, 0);
+    if(rocke_tv_name_is(dt, "f64"))
+        return rocke_b_global_load_f64(b, v->base, off, 0);
     return rocke_b_global_load(b, v->base, off, dt, 0);
 }
 
@@ -273,6 +275,8 @@ void rocke_tensor_view_store_scalar(rocke_ir_builder_t* b,
         return;
     }
     off = rocke_tensor_descriptor_offset(b, &v->desc, indices, num_indices);
+    if(align <= 0 && rocke_tv_name_is(dt, "f64"))
+        align = 8; /* Same 8-byte element alignment as the f64 load_scalar. */
     if(align <= 0)
         rocke_b_global_store(b, v->base, off, value, 0); /* Python: align default 1 */
     else

dnn-providers/hip-kernel-provider/rocke/platform/cpp/include/rocke/ir.h

@@ -999,6 +999,10 @@ rocke_value_t* rocke_b_global_load_bf16(rocke_ir_builder_t* b,
                                         rocke_value_t* ptr,
                                         rocke_value_t* idx,
                                         int align);
+rocke_value_t* rocke_b_global_load_f64(rocke_ir_builder_t* b,
+                                       rocke_value_t* ptr,
+                                       rocke_value_t* idx,
+                                       int align);
 rocke_value_t* rocke_b_global_load_i8(rocke_ir_builder_t* b,
                                       rocke_value_t* ptr,
                                       rocke_value_t* idx,

dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/elementwise.cpp

@@ -240,6 +240,11 @@ static rocke_value_t* rocke_ew_apply_unary(rocke_ir_builder_t* b, rocke_value_t*
     }
     if(rocke_ew_streq(op, "abs"))
     {
+        /* f64: one sign-bit clear (llvm.fabs.f64) instead of max(x, -x). */
+        if(strcmp(x->type->name, "f64") == 0)
+        {
+            return rocke_b_fabs(b, x);
+        }
         return rocke_b_fmax(b, x, rocke_b_fneg(b, x));
     }
     if(rocke_ew_streq(op, "relu"))

dnn-providers/hip-kernel-provider/rocke/platform/cpp/instances/common/reduce.cpp

@@ -550,21 +550,26 @@ rocke_kernel_def_t*
             total = rocke_block_lds_reduce(b, acc, lds, tid, BS, combine);
         }
 
-        /* if spec.op == "mean": total = total * rcp(const_c(float(N))) */
+        /* if spec.op == "mean":
+         *     f64: total = fdiv(total, const_c(float(N)))  (one rounding)
+         *     else: total = total * rcp(const_c(float(N))) */
         if(strcmp(spec->op, "mean") == 0)
         {
-            total = rocke_b_fmul(
-                b, total, rocke_b_rcp(b, rocke_reduce2d_const_c(b, is_f64, (double)N)));
+            rocke_value_t* n_value = rocke_reduce2d_const_c(b, is_f64, (double)N);
+            if(is_f64)
+                total = rocke_b_fdiv(b, total, n_value);
+            else
+                total = rocke_b_fmul(b, total, rocke_b_rcp(b, n_value));
         }
 
         /* with b.scf_if(b.cmp_eq(tid, const_i32(0))):
-         *     if is_f64: b.global_store(Y, row, total)
+         *     if is_f64: b.global_store(Y, row, total, align=8)
          *     else: store_scalar_from_f32(b, Y, row, total, dtype=spec.dtype) */
         {
             rocke_if_t iff = rocke_b_scf_if(b, rocke_b_cmp_eq(b, tid, rocke_b_const_i32(b, 0)));
             rocke_b_region_enter(b, iff.then_region);
             if(is_f64)
-                rocke_b_global_store(b, Y, row, total, 0);
+                rocke_b_global_store(b, Y, row, total, 8);
             else
                 rocke_b_store_scalar_from_f32(b, Y, row, total, spec->dtype);
             rocke_b_region_leave(b);

dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/ir_lowering/ir_model.md

@@ -128,7 +128,7 @@ cvt_f32_to_i8_sat(v)     # round + saturate to i8
 
 ### Math intrinsics
 
-LLVM target intrinsics; lowering is f32-centric. `exp2`, `exp2_fast`, `log2`, `sqrt`, `rsqrt`, `rcp_fast`, `tanh`, `fabs`, `fma`, `fmax3`, `fmin3` and `vector_fma` have no f64 lowering and reject f64 inputs in both the LLVM and HIP lowerings; `rcp` is a plain `fdiv` and accepts f64.
+LLVM target intrinsics; lowering is f32-centric. `exp2`, `log2`, `sqrt`, `rsqrt`, `rcp_fast`, `fma`, `fmax3`, `fmin3` and `vector_fma` have no f64 lowering and reject f64 inputs in both the LLVM and HIP lowerings. `exp2_fast` rejects f64 in the LLVM lowering and has no HIP lowering for any dtype. `tanh` accepts only f32: the builder rejects any other operand type. `rcp` is a plain `fdiv` and accepts f64, and `fabs` lowers f64 to `llvm.fabs.f64`.
 
 ```text
 exp2(v)        # llvm.exp2.f32
@@ -176,6 +176,7 @@ global_load_bf16(ptr, idx, align=2)
 global_load_f32(ptr, idx, align=4)
 global_load_i32(ptr, idx, align=4)
 global_load_i64(ptr, idx, align=8)
+global_load_f64(ptr, idx, align=8)
 global_load_fp8e4m3(ptr, idx, align=1)
 masked_global_load(ptr, idx, mask, other, dtype, align=1)   # clamps false-lane idx to 0
 global_store(ptr, idx, value, align=1)

dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/dtype_support.md

@@ -386,16 +386,18 @@ mode. The missing selector and precision contract are described in §2.5.
 - **`u16-64` — 0 at every layer.** Absent from the hipDNN enum, rejected by the
   MIOpen provider, and not requested by any family.
 - **`f64` — IR, memory, and elementwise/reduce2d only.** The IR type,
-  `const_f64`, f64 `fadd`/`fsub`/`fmul`/`fdiv`/`fneg`/`fmax`/`fmin`/`fcmp`/
-  `select` and `vector_add`/`vector_sub`/`vector_mul`, global/LDS memory up to
-  2-element vectors, and HIP lowering are present. Elementwise
+  `const_f64`, f64 `fadd`/`fsub`/`fmul`/`fdiv`/`fneg`/`fabs`/`fmax`/`fmin`/
+  `fcmp`/`select` and `vector_add`/`vector_sub`/`vector_mul`, global/LDS
+  memory up to 2-element vectors, and HIP lowering are present. Elementwise
   (non-transcendental ops) and reduce2d accept f64 with `vec=2`
   ([SUPPORT_MATRIX](../../python/rocke/instances/SUPPORT_MATRIX.md)).
-  `exp2`, `exp2_fast`, `log2`, `sqrt`, `rsqrt`, `rcp_fast`, `tanh`, `fabs`,
-  `fma`, `fmax3`, `fmin3`, and `vector_fma` reject f64 in both lowerings.
-  `cvt_f32_to_f64` (exact) and `cvt_f64_to_f32` (round to nearest even) are the
-  only conversions; `cast_to_f32` / `cast_f32_to` refuse f64, and there are no
-  direct f16/bf16 conversions, MMA atoms, or atomics. f64 is in the
+  `exp2`, `log2`, `sqrt`, `rsqrt`, `rcp_fast`, `fma`, `fmax3`, `fmin3`, and
+  `vector_fma` reject f64 in both lowerings. `exp2_fast` rejects f64 in the
+  LLVM lowering and has no HIP lowering for any dtype. `tanh` accepts only
+  f32: the builder rejects any other operand type. `cvt_f32_to_f64` (exact)
+  and `cvt_f64_to_f32` (round to nearest even) are the only conversions;
+  `cast_to_f32` / `cast_f32_to` refuse f64, and there are no direct f16/bf16
+  conversions, MMA atoms, or atomics. f64 is in the
   hipDNN enum (`DOUBLE`) but rejected by the MIOpen provider.
   `V_MFMA_F64_*` is present on gfx90a/gfx942/gfx950 but undeclared (§4C.6).
   gfx1250 has no F64 matrix instruction: LLVM gates `V_WMMA_F64_16X16X4_F64` on
@@ -405,7 +407,7 @@ mode. The missing selector and precision contract are described in §2.5.
 
 Every score above is traceable to: compiler `ir.py:43-52`,
 `lower_llvm.py:1093-1129`,
-FP-op guards `lower_llvm.py:2068-2150` (FMA/FABS/FMAX3 accept only f32/f16/bf16;
+FP-op guards `lower_llvm.py:2068-2150` (FMA/FMAX3 accept only f32/f16/bf16, FABS also f64;
 the dtype rejection is at `:2122`);
 helpers `ir.py:838-1170` (casts), `helpers/io.py:48-211,260-315`,
 `helpers/quant.py:58-357`, `helpers/i4_dequant.py`, `helpers/mx_scale.py`;

dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/reference/op_vocabulary.md

@@ -53,7 +53,7 @@ Complete reference of operations recognized by `core/ir.py` and lowered to AMDGP
 | `math.tanh`    | `tanh`       | piecewise f32 expansion: OCML polynomial for `abs(x) < 0.625`, otherwise `exp2` + reciprocal; bitwise sign restoration |
 | `math.rcp`     | `rcp`        | `1.0 / v` (hardware reciprocal) |
 
-`exp2`, `exp2_fast`, `log2`, `sqrt`, `rsqrt`, `rcp_fast`, `tanh`, `fabs`, `fma`, `fmax3`, `fmin3` and `vector_fma` have no f64 lowering and reject f64 inputs in both the LLVM and HIP lowerings; `rcp` accepts f64.
+`exp2`, `log2`, `sqrt`, `rsqrt`, `rcp_fast`, `fma`, `fmax3`, `fmin3` and `vector_fma` have no f64 lowering and reject f64 inputs in both the LLVM and HIP lowerings. `exp2_fast` rejects f64 in the LLVM lowering and has no HIP lowering for any dtype. `tanh` accepts only f32: the builder rejects any other operand type. `rcp` accepts f64, and `fabs` accepts f64 (`llvm.fabs.f64`, HIP `fabs`).
 
 `clamp_f32(v, lo, hi)` is `fmin(hi, fmax(lo, v))` — folds to `v_med3_f32`.
 
@@ -93,7 +93,7 @@ Complete reference of operations recognized by `core/ir.py` and lowered to AMDGP
 | Op name                       | Builder                       | LLVM emission |
 |-------------------------------|-------------------------------|---------------|
 | `memref.global_load`          | `global_load_f16`             | typed `load addrspace(1)` |
-| `memref.global_load_typed`    | `global_load`, `global_load_i8`, `global_load_i16`, `global_load_i32`, `global_load_i64`, `global_load_fp8e4m3`, `global_load_bf8e5m2`, `global_load_bf16`, `global_load_f32` | typed `load addrspace(1)` |
+| `memref.global_load_typed`    | `global_load`, `global_load_i8`, `global_load_i16`, `global_load_i32`, `global_load_i64`, `global_load_fp8e4m3`, `global_load_bf8e5m2`, `global_load_bf16`, `global_load_f32`, `global_load_f64` | typed `load addrspace(1)` |
 | `memref.global_load_vN`       | `global_load_vN`, `global_load_vN_f16` | vector `load addrspace(1)` (n in {2,4,8}) |
 | `memref.global_store_typed`   | `global_store`                | typed `store addrspace(1)` |
 | `memref.global_store_vN`      | `global_store_vN`             | vector `store addrspace(1)` |

dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/intrinsic_arch_domain.llvm20.json

@@ -26,6 +26,7 @@
     "fabs.bf16": "llvm.fabs.bf16",
     "fabs.f16": "llvm.fabs.f16",
     "fabs.f32": "llvm.fabs.f32",
+    "fabs.f64": "llvm.fabs.f64",
     "fmuladd.bf16": "llvm.fmuladd.bf16",
     "fmuladd.f16": "llvm.fmuladd.f16",
     "fmuladd.f32": "llvm.fmuladd.f32",
@@ -2154,6 +2155,41 @@
         "verified_on": "llvm20"
       }
     },
+    "fabs.f64": {
+      "gfx11-generic": {
+        "status": "ok",
+        "verified_on": "llvm20"
+      },
+      "gfx1151": {
+        "status": "ok",
+        "verified_on": "llvm20"
+      },
+      "gfx12-generic": {
+        "status": "ok",
+        "verified_on": "llvm20"
+      },
+      "gfx1201": {
+        "status": "ok",
+        "verified_on": "llvm20"
+      },
+      "gfx1250": {
+        "evidence": "invalid target ID",
+        "status": "target_unsupported",
+        "verified_on": "llvm20"
+      },
+      "gfx90a": {
+        "status": "ok",
+        "verified_on": "llvm20"
+      },
+      "gfx942": {
+        "status": "ok",
+        "verified_on": "llvm20"
+      },
+      "gfx950": {
+        "status": "ok",
+        "verified_on": "llvm20"
+      }
+    },
     "flat.prefetch": {
       "gfx11-generic": {
         "evidence": "not an intrinsic in llvm20",

dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/intrinsic_arch_domain.llvm22.json

@@ -46,6 +46,7 @@
     "fabs.bf16": "llvm.fabs.bf16",
     "fabs.f16": "llvm.fabs.f16",
     "fabs.f32": "llvm.fabs.f32",
+    "fabs.f64": "llvm.fabs.f64",
     "flat.prefetch": "llvm.amdgcn.flat.prefetch",
     "fmuladd.bf16": "llvm.fmuladd.bf16",
     "fmuladd.f16": "llvm.fmuladd.f16",
@@ -2085,6 +2086,40 @@
         "verified_on": "llvm22"
       }
     },
+    "fabs.f64": {
+      "gfx11-generic": {
+        "status": "ok",
+        "verified_on": "llvm22"
+      },
+      "gfx1151": {
+        "status": "ok",
+        "verified_on": "llvm22"
+      },
+      "gfx12-generic": {
+        "status": "ok",
+        "verified_on": "llvm22"
+      },
+      "gfx1201": {
+        "status": "ok",
+        "verified_on": "llvm22"
+      },
+      "gfx1250": {
+        "status": "ok",
+        "verified_on": "llvm22"
+      },
+      "gfx90a": {
+        "status": "ok",
+        "verified_on": "llvm22"
+      },
+      "gfx942": {
+        "status": "ok",
+        "verified_on": "llvm22"
+      },
+      "gfx950": {
+        "status": "ok",
+        "verified_on": "llvm22"
+      }
+    },
     "flat.prefetch": {
       "gfx11-generic": {
         "evidence": "fatal error: error in backend: Cannot select: intrinsic %llvm.amdgcn.flat.prefetch",

dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/arch/data/intrinsic_arch_domain.llvm23.json

@@ -51,6 +51,7 @@
     "fabs.bf16": "llvm.fabs.bf16",
     "fabs.f16": "llvm.fabs.f16",
     "fabs.f32": "llvm.fabs.f32",
+    "fabs.f64": "llvm.fabs.f64",
     "flat.prefetch": "llvm.amdgcn.flat.prefetch",
     "fmuladd.bf16": "llvm.fmuladd.bf16",
     "fmuladd.f16": "llvm.fmuladd.f16",
@@ -2064,6 +2065,40 @@
         "verified_on": "llvm23"
       }
     },
+    "fabs.f64": {
+      "gfx11-generic": {
+        "status": "ok",
+        "verified_on": "llvm23"
+      },
+      "gfx1151": {
+        "status": "ok",
+        "verified_on": "llvm23"
+      },
+      "gfx12-generic": {
+        "status": "ok",
+        "verified_on": "llvm23"
+      },
+      "gfx1201": {
+        "status": "ok",
+        "verified_on": "llvm23"
+      },
+      "gfx1250": {
+        "status": "ok",
+        "verified_on": "llvm23"
+      },
+      "gfx90a": {
+        "status": "ok",
+        "verified_on": "llvm23"
+      },
+      "gfx942": {
+        "status": "ok",
+        "verified_on": "llvm23"
+      },
+      "gfx950": {
+        "status": "ok",
+        "verified_on": "llvm23"
+      }
+    },
     "flat.prefetch": {
       "gfx11-generic": {
         "evidence": "fatal error: error in backend: Cannot select: intrinsic %llvm.amdgcn.flat.prefetch",

dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/ir.py

@@ -1103,8 +1103,8 @@ def fneg(self, a: Value) -> Value:
     def fabs(self, a: Value) -> Value:
         """Floating-point absolute value (single-instruction on AMDGPU).
 
-        Lowers to ``llvm.fabs.f32`` (or the matching f16 / bf16
-        variant). On the AMDGPU backend this becomes a free input
+        Lowers to ``llvm.fabs.f32`` (or the matching f64 / f16 / bf16
+        variant; HIP uses ``fabs`` for f64). On the AMDGPU backend this becomes a free input
         modifier on the consumer or a single-cycle ``v_max_f32 a, -a``
         — strictly cheaper than the historical ``fmax(a, fneg(a))``
         idiom which materialises ``-a`` as an SSA value and prevents
@@ -2169,6 +2169,9 @@ def global_load_i64(self, ptr: Value, idx: Value, *, align: int = 8) -> Value:
     def global_load_bf16(self, ptr: Value, idx: Value, *, align: int = 2) -> Value:
         return self.global_load(ptr, idx, BF16, align=align)
 
+    def global_load_f64(self, ptr: Value, idx: Value, *, align: int = 8) -> Value:
+        return self.global_load(ptr, idx, F64, align=align)
+
     def global_load_i8(self, ptr: Value, idx: Value, *, align: int = 1) -> Value:
         return self.global_load(ptr, idx, I8, align=align)
 

dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/core/lower_hip.py

@@ -1436,7 +1436,11 @@ def _op_arith_fneg(self, op: Op) -> None:
     def _op_arith_fabs(self, op: Op) -> None:
         (v,) = op.operands
         ty = _type_to_hip(op.result.type)
-        _reject_f64(op, op.result.type.name)
+        if op.result.type.name == "f64":
+            # The double overload stays in double; routing it through the
+            # float helper below would round the value to f32.
+            self._emit(f"{ty} {_name(op.result)} = fabs({_name(v)});")
+            return
         # `__builtin_fabsf` lowers to a free `abs` modifier on AMDGPU
         # (exactly what the LLVM `llvm.fabs.f32` intrinsic produces);
         # for f16/bf16 the matching builtin yields the same modifier.