Skip to content

llama.vim: filter server response fields #24

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jan 15, 2025
Merged

llama.vim: filter server response fields #24

Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
aa91f6a
filter response fields
VJHack Jan 14, 2025
c57ac19
clean up
VJHack Jan 14, 2025
5bce411
response fields for ring update
VJHack Jan 14, 2025
32169e8
formatting change
VJHack Jan 14, 2025
fc6f90d
Update autoload/llama.vim
VJHack Jan 14, 2025
79ee0d1
removed n_ctx
VJHack Jan 14, 2025
4185ae4
merge
VJHack Jan 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 28 additions & 16 deletions autoload/llama.vim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,9 +307,9 @@ function! s:ring_update()
\ 'samplers': ["temperature"],
\ 'cache_prompt': v:true,
\ 't_max_prompt_ms': 1,
\ 't_max_predict_ms': 1
\ 't_max_predict_ms': 1,
\ 'response_fields': [""]
\ })

let l:curl_command = [
\ "curl",
\ "--silent",
Expand Down Expand Up @@ -420,7 +420,21 @@ function! llama#fim(is_auto, cache) abort
\ 'samplers': ["top_k", "top_p", "infill"],
\ 'cache_prompt': v:true,
\ 't_max_prompt_ms': g:llama_config.t_max_prompt_ms,
\ 't_max_predict_ms': g:llama_config.t_max_predict_ms
\ 't_max_predict_ms': g:llama_config.t_max_predict_ms,
\ 'response_fields': [
\ "content",
\ "timings/prompt_n",
\ "timings/prompt_ms",
\ "timings/prompt_per_token_ms",
\ "timings/prompt_per_second",
\ "timings/predicted_n",
\ "timings/predicted_ms",
\ "timings/predicted_per_token_ms",
\ "timings/predicted_per_second",
\ "truncated",
\ "tokens_cached",
\ "generation_settings/n_ctx",
\ ],
\ })

let l:curl_command = [
Expand Down Expand Up @@ -662,24 +676,22 @@ function! s:fim_on_stdout(hash, cache, pos_x, pos_y, is_auto, job_id, data, even
call remove(s:content, -1)
endwhile

let l:generation_settings = get(l:response, 'generation_settings', {})
let l:n_ctx = get(l:generation_settings, 'n_ctx', 0)

let l:n_cached = get(l:response, 'tokens_cached', 0)
let l:truncated = get(l:response, 'truncated', v:false)
let l:n_ctx = get(l:response, 'generation_settings/n_ctx', 0)
Copy link
Member

@ggerganov ggerganov Jan 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After recent refactoring, the server no longer provides the n_ctx field in the responses. We should remove it from the llama.vim client and instead of displaying:

c: 1234 / 0 | ...

We should simply display:

c: 1234 | ...

Copy link
Collaborator Author

@VJHack VJHack Jan 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did notice that. I removed all instances of n_ctx field and adjusted the info message accordingly as shown below.
Screen Shot 2025-01-14 at 3 22 19 PM

Thank you!

let l:n_cached = get(l:response, 'timings/tokens_cached', 0)
let l:truncated = get(l:response, 'timings/truncated', v:false)

" if response.timings is available
if len(get(l:response, 'timings', {})) > 0
if has_key(l:response, 'timings/prompt_n') && has_key(l:response, 'timings/prompt_ms') && has_key(l:response, 'timings/prompt_per_second')
\ && has_key(l:response, 'timings/predicted_n') && has_key(l:response, 'timings/predicted_ms') && has_key(l:response, 'timings/predicted_per_second')
let l:has_info = v:true
let l:timings = get(l:response, 'timings', {})

let l:n_prompt = get(l:timings, 'prompt_n', 0)
let l:t_prompt_ms = get(l:timings, 'prompt_ms', 1)
let l:s_prompt = get(l:timings, 'prompt_per_second', 0)
let l:n_prompt = get(l:response, 'timings/prompt_n', 0)
let l:t_prompt_ms = get(l:response, 'timings/prompt_ms', 1)
let l:s_prompt = get(l:response, 'timings/prompt_per_second', 0)

let l:n_predict = get(l:timings, 'predicted_n', 0)
let l:t_predict_ms = get(l:timings, 'predicted_ms', 1)
let l:s_predict = get(l:timings, 'predicted_per_second', 0)
let l:n_predict = get(l:response, 'timings/predicted_n', 0)
let l:t_predict_ms = get(l:response, 'timings/predicted_ms', 1)
let l:s_predict = get(l:response, 'timings/predicted_per_second', 0)
endif

" if response was pulled from cache
Expand Down