From 2c96a5cd8be58dc7a59995b1dd7230342e2e321a Mon Sep 17 00:00:00 2001 From: "zhou.weiguo" Date: Mon, 10 Jun 2024 11:43:03 +0800 Subject: [PATCH 1/6] examples: refine tensor dump --- examples/benchmark/benchmark-matmult.cpp | 53 +++++++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp index 47cb16c69d53..7446568eefb2 100644 --- a/examples/benchmark/benchmark-matmult.cpp +++ b/examples/benchmark/benchmark-matmult.cpp @@ -31,15 +31,64 @@ static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * ggml_graph_compute(graph, &plan); } +#define QK8_0 32 + +typedef struct { + uint16_t d; // delta + int8_t qs[QK8_0]; // quants +} block_q8_0; + +static inline float ggml_compute_fp16_to_fp32(uint16_t h) { + uint16_t tmp; + memcpy(&tmp, &h, sizeof(uint16_t)); + return (float) tmp; +} + static float tensor_sum_elements(const ggml_tensor * tensor) { - double sum = 0; + double sum = 0; + float floatvalue = 0; + unsigned short shortvalue = 0; + if (tensor->type == GGML_TYPE_F32) { for (int j = 0; j < tensor->ne[1]; j++) { for (int k = 0; k < tensor->ne[0]; k++) { - sum += ((float *) tensor->data)[j*tensor->ne[0] + k]; + sum += ((float *) tensor->data)[j * tensor->ne[0] + k]; + } + } + } + + if (tensor->type == GGML_TYPE_I8) { + for (int j = 0; j < tensor->ne[1]; j++) { + for (int k = 0; k < tensor->ne[0]; k++) { + sum += ((int8_t *) tensor->data)[j * tensor->ne[0] + k]; } } } + + if (tensor->type == GGML_TYPE_F16) { + for (int j = 0; j < tensor->ne[1]; j++) { + for (int k = 0; k < tensor->ne[0]; k++) { + shortvalue = ((unsigned short *) tensor->data)[j * tensor->ne[0] + k]; + floatvalue = ggml_compute_fp16_to_fp32(shortvalue); + sum += floatvalue; + } + } + } + + if (tensor->type == GGML_TYPE_Q8_0) { + int blocks = 0; + block_q8_0 * quant_datas = (block_q8_0 *)tensor->data; + for (int j = 0; j < tensor->ne[1]; j++) { + blocks = tensor->ne[0] / QK8_0; + for (int i = 0; i < blocks; i++) { + floatvalue = ggml_compute_fp16_to_fp32(quant_datas[j * blocks + i].d); + for (int k = 0; k < QK8_0; k++) { + sum += (quant_datas[j * blocks + i].qs[k] * floatvalue); + } + } + } + } + return sum; } From 3695a2b00fb20ab04092a2c4cac9d904ffc58382 Mon Sep 17 00:00:00 2001 From: "zhou.weiguo" Date: Mon, 10 Jun 2024 12:37:27 +0800 Subject: [PATCH 2/6] examples: refine tensor dump --- examples/benchmark/benchmark-matmult.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp index 7446568eefb2..11bd370359d8 100644 --- a/examples/benchmark/benchmark-matmult.cpp +++ b/examples/benchmark/benchmark-matmult.cpp @@ -39,9 +39,15 @@ typedef struct { } block_q8_0; static inline float ggml_compute_fp16_to_fp32(uint16_t h) { +#if defined(__ARM_NEON) + __fp16 tmp; + memcpy(&tmp, &h, sizeof(uint16_t)); + return (float) tmp; +#else uint16_t tmp; memcpy(&tmp, &h, sizeof(uint16_t)); return (float) tmp; +#endif } static float tensor_sum_elements(const ggml_tensor * tensor) { From b7a9d40e51eab9a177d11a7dc3f97b9476e7aa3d Mon Sep 17 00:00:00 2001 From: "zhou.weiguo" Date: Thu, 13 Jun 2024 20:54:06 +0800 Subject: [PATCH 3/6] examples: refine tensor dump in examples/benchmark/benchmark-matmult.cpp --- examples/benchmark/benchmark-matmult.cpp | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp index 11bd370359d8..8dc8bd4da4e8 100644 --- a/examples/benchmark/benchmark-matmult.cpp +++ b/examples/benchmark/benchmark-matmult.cpp @@ -1,5 +1,6 @@ #include "common.h" #include "ggml.h" +#include "ggml-impl.h" #include #include @@ -38,18 +39,6 @@ typedef struct { int8_t qs[QK8_0]; // quants } block_q8_0; -static inline float ggml_compute_fp16_to_fp32(uint16_t h) { -#if defined(__ARM_NEON) - __fp16 tmp; - memcpy(&tmp, &h, sizeof(uint16_t)); - return (float) tmp; -#else - uint16_t tmp; - memcpy(&tmp, &h, sizeof(uint16_t)); - return (float) tmp; -#endif -} - static float tensor_sum_elements(const ggml_tensor * tensor) { double sum = 0; float floatvalue = 0; @@ -75,7 +64,7 @@ static float tensor_sum_elements(const ggml_tensor * tensor) { for (int j = 0; j < tensor->ne[1]; j++) { for (int k = 0; k < tensor->ne[0]; k++) { shortvalue = ((unsigned short *) tensor->data)[j * tensor->ne[0] + k]; - floatvalue = ggml_compute_fp16_to_fp32(shortvalue); + floatvalue = GGML_FP16_TO_FP32(shortvalue); sum += floatvalue; } } @@ -87,7 +76,7 @@ static float tensor_sum_elements(const ggml_tensor * tensor) { for (int j = 0; j < tensor->ne[1]; j++) { blocks = tensor->ne[0] / QK8_0; for (int i = 0; i < blocks; i++) { - floatvalue = ggml_compute_fp16_to_fp32(quant_datas[j * blocks + i].d); + floatvalue = GGML_FP16_TO_FP32(quant_datas[j * blocks + i].d); for (int k = 0; k < QK8_0; k++) { sum += (quant_datas[j * blocks + i].qs[k] * floatvalue); } From fe59684e322a8f1e743ab88fe4f999020907b726 Mon Sep 17 00:00:00 2001 From: "zhou.weiguo" Date: Fri, 14 Jun 2024 10:54:15 +0800 Subject: [PATCH 4/6] review: modify codes as review comments --- examples/benchmark/benchmark-matmult.cpp | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp index 8dc8bd4da4e8..66c0b54430b0 100644 --- a/examples/benchmark/benchmark-matmult.cpp +++ b/examples/benchmark/benchmark-matmult.cpp @@ -71,8 +71,23 @@ static float tensor_sum_elements(const ggml_tensor * tensor) { } if (tensor->type == GGML_TYPE_Q8_0) { - int blocks = 0; block_q8_0 * quant_datas = (block_q8_0 *)tensor->data; +#if 1 + ggml_type_traits_t qtype = ggml_internal_get_type_traits(tensor->type); + float * float32 = (float*)malloc((tensor->ne[0] * tensor->ne[1]) * sizeof(float)); + if (NULL == float32) { + printf("malloc failed\n"); + return 0.0; + } + qtype.to_float(quant_datas, float32, tensor->ne[0] * tensor->ne[1]); + for (int j = 0; j < tensor->ne[1]; j++) { + for (int k = 0; k < tensor->ne[0]; k++) { + sum += float32[j * tensor->ne[0] + k]; + } + } + free(float32); +#else + int blocks = 0; for (int j = 0; j < tensor->ne[1]; j++) { blocks = tensor->ne[0] / QK8_0; for (int i = 0; i < blocks; i++) { @@ -82,6 +97,7 @@ static float tensor_sum_elements(const ggml_tensor * tensor) { } } } +#endif } return sum; From 11c7b1e25a506676768f38b3672e7e2191cec9bc Mon Sep 17 00:00:00 2001 From: "zhou.weiguo" Date: Fri, 14 Jun 2024 23:04:13 +0800 Subject: [PATCH 5/6] review: modify codes as review suggestion --- examples/benchmark/benchmark-matmult.cpp | 48 +++++------------------- 1 file changed, 10 insertions(+), 38 deletions(-) diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp index 66c0b54430b0..7c8b6a639c39 100644 --- a/examples/benchmark/benchmark-matmult.cpp +++ b/examples/benchmark/benchmark-matmult.cpp @@ -32,26 +32,11 @@ static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * ggml_graph_compute(graph, &plan); } -#define QK8_0 32 - -typedef struct { - uint16_t d; // delta - int8_t qs[QK8_0]; // quants -} block_q8_0; - static float tensor_sum_elements(const ggml_tensor * tensor) { double sum = 0; float floatvalue = 0; unsigned short shortvalue = 0; - if (tensor->type == GGML_TYPE_F32) { - for (int j = 0; j < tensor->ne[1]; j++) { - for (int k = 0; k < tensor->ne[0]; k++) { - sum += ((float *) tensor->data)[j * tensor->ne[0] + k]; - } - } - } - if (tensor->type == GGML_TYPE_I8) { for (int j = 0; j < tensor->ne[1]; j++) { for (int k = 0; k < tensor->ne[0]; k++) { @@ -70,34 +55,21 @@ static float tensor_sum_elements(const ggml_tensor * tensor) { } } - if (tensor->type == GGML_TYPE_Q8_0) { - block_q8_0 * quant_datas = (block_q8_0 *)tensor->data; -#if 1 - ggml_type_traits_t qtype = ggml_internal_get_type_traits(tensor->type); - float * float32 = (float*)malloc((tensor->ne[0] * tensor->ne[1]) * sizeof(float)); - if (NULL == float32) { - printf("malloc failed\n"); - return 0.0; - } - qtype.to_float(quant_datas, float32, tensor->ne[0] * tensor->ne[1]); + if (tensor->type == GGML_TYPE_F32) { for (int j = 0; j < tensor->ne[1]; j++) { for (int k = 0; k < tensor->ne[0]; k++) { - sum += float32[j * tensor->ne[0] + k]; + sum += ((float *) tensor->data)[j * tensor->ne[0] + k]; } } - free(float32); -#else - int blocks = 0; - for (int j = 0; j < tensor->ne[1]; j++) { - blocks = tensor->ne[0] / QK8_0; - for (int i = 0; i < blocks; i++) { - floatvalue = GGML_FP16_TO_FP32(quant_datas[j * blocks + i].d); - for (int k = 0; k < QK8_0; k++) { - sum += (quant_datas[j * blocks + i].qs[k] * floatvalue); - } - } + } + + if (ggml_is_quantized(tensor->type)) { + std::vector f32out(ggml_nelements(tensor)); + ggml_type_traits_t qtype = ggml_internal_get_type_traits(tensor->type); + qtype.to_float((void *)tensor->data, f32out.data(), f32out.size()); + for (const float & value : f32out) { + sum += value; } -#endif } return sum; From 181c0e3b0fec8d1f24ea56291b684f6efff6382a Mon Sep 17 00:00:00 2001 From: "zhou.weiguo" Date: Fri, 14 Jun 2024 23:18:43 +0800 Subject: [PATCH 6/6] review: modify codes as review suggestion --- examples/benchmark/benchmark-matmult.cpp | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp index 7c8b6a639c39..424504375f17 100644 --- a/examples/benchmark/benchmark-matmult.cpp +++ b/examples/benchmark/benchmark-matmult.cpp @@ -33,18 +33,10 @@ static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * } static float tensor_sum_elements(const ggml_tensor * tensor) { - double sum = 0; - float floatvalue = 0; + double sum = 0.0; + float floatvalue = 0.0f; unsigned short shortvalue = 0; - if (tensor->type == GGML_TYPE_I8) { - for (int j = 0; j < tensor->ne[1]; j++) { - for (int k = 0; k < tensor->ne[0]; k++) { - sum += ((int8_t *) tensor->data)[j * tensor->ne[0] + k]; - } - } - } - if (tensor->type == GGML_TYPE_F16) { for (int j = 0; j < tensor->ne[1]; j++) { for (int k = 0; k < tensor->ne[0]; k++) { @@ -53,17 +45,13 @@ static float tensor_sum_elements(const ggml_tensor * tensor) { sum += floatvalue; } } - } - - if (tensor->type == GGML_TYPE_F32) { + } else if (tensor->type == GGML_TYPE_F32) { for (int j = 0; j < tensor->ne[1]; j++) { for (int k = 0; k < tensor->ne[0]; k++) { sum += ((float *) tensor->data)[j * tensor->ne[0] + k]; } } - } - - if (ggml_is_quantized(tensor->type)) { + } else if (ggml_is_quantized(tensor->type)) { std::vector f32out(ggml_nelements(tensor)); ggml_type_traits_t qtype = ggml_internal_get_type_traits(tensor->type); qtype.to_float((void *)tensor->data, f32out.data(), f32out.size());