Skip to content

add hardware fp16_to_fp32 #27452

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lexasub wants to merge 2 commits into
base: master
Choose a base branch
from
+20 −2
Open

add hardware fp16_to_fp32 #27452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e5fcb51
ggml : add hardware fp16_to_fp32
Aug 13, 2026
0b05747
batch : avoid zero-filling embd buffer in ubatch_add
Aug 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions ggml/src/ggml-impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
#include <immintrin.h>
#endif

#ifdef __ARM_FEATURE_SVE
#include <arm_sve.h>
Expand Down Expand Up @@ -382,6 +385,12 @@ static inline uint32_t fp32_to_bits(float f) {
}

static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
#ifdef __F16C__
return _cvtsh_ss(h);
#elif defined(__aarch64__) && defined(__ARM_FP) && (__ARM_FP & 2)
union { uint16_t u; __fp16 f; } u = { .u = h };
return (float)u.f;
#else
const uint32_t w = (uint32_t) h << 16;
const uint32_t sign = w & UINT32_C(0x80000000);
const uint32_t two_w = w + w;
Expand All @@ -402,6 +411,7 @@ static inline float ggml_compute_fp16_to_fp32(ggml_fp16_t h) {
const uint32_t result = sign |
(two_w < denormalized_cutoff ? fp32_to_bits(denormalized_value) : fp32_to_bits(normalized_value));
return fp32_from_bits(result);
#endif
}

static inline ggml_fp16_t ggml_compute_fp32_to_fp16(float f) {
Expand Down
12 changes: 10 additions & 2 deletions src/llama-batch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -757,7 +757,6 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
const int64_t n_pos_all = (int64_t) n_tokens*n_pos_per_embd;

udata->token .resize(n_tokens);
udata->embd .resize(n_embd_all);
udata->pos .resize(n_pos_all);
udata->n_seq_id .resize(n_tokens);
udata->seq_id .resize(n_tokens);
Expand All @@ -766,6 +765,12 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->output .resize(n_tokens);

udata->seq_id_data.reserve(n_tokens);
if (batch.embd) {
udata->embd.clear();
udata->embd.reserve(n_embd_all);
} else {
udata->embd.resize(n_embd_all); // fill all size..new_size elems by 0.0f

@lexasub lexasub Aug 20, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

guaranted save prev code logic - always alloc n_embd_all and fill 0.0f when batch.embd doesn't present

}

seq_set_t seq_set_unq;

Expand All @@ -775,7 +780,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
}

if (batch.embd) {
memcpy(udata->embd.data() + i*n_embd, batch.embd + (int64_t) idxs[i]*n_embd, n_embd*sizeof(float));
auto src = batch.embd + (int64_t) idxs[1] * n_embd;
// use safe method for auto increase size
// next improvements - write own vector without automatic filling float)
udata->embd.insert(udata->embd.end(), src, src + n_embd);
}

for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) {
Expand Down