Skip to content

Commit

Permalink
Merge branch 'master' into small-fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
anon998 authored Jun 2, 2023
2 parents 47efbb5 + 88cc7bb commit abb7782
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions examples/server/server.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -533,6 +533,7 @@ json format_generation_settings(llama_server_context &llama) {
const auto eos_bias = llama.params.logit_bias.find(llama_token_eos());
const bool ignore_eos = eos_bias != llama.params.logit_bias.end() &&
eos_bias->second < 0.0f && std::isinf(eos_bias->second);

return json {
{ "seed", llama.params.seed },
{ "temp", llama.params.temp },
Expand Down Expand Up @@ -653,10 +654,15 @@ bool parse_options_completion(json body, llama_server_context& llama, Response &
if (body["logit_bias"].is_array()) {
int n_vocab = llama_n_vocab(llama.ctx);
for (const auto &el : body["logit_bias"]) {
if (el.is_array() && el.size() == 2 && el[0].is_number_integer() && el[1].is_number_float()) {
if (el.is_array() && el.size() == 2 && el[0].is_number_integer()) {
llama_token tok = el[0].get<llama_token>();
if (tok < 0 || tok >= n_vocab) continue;
llama.params.logit_bias[tok] = el[1].get<float>();
if (tok >= 0 && tok < n_vocab) {
if (el[1].is_number_float()) {
llama.params.logit_bias[tok] = el[1].get<float>();
} else if (el[1].is_boolean() && !el[1].get<bool>()) {
llama.params.logit_bias[tok] = -INFINITY;
}
}
}
}
}
Expand Down

0 comments on commit abb7782

Please sign in to comment.