Skip to content

info: cached info message #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jan 6, 2025
Merged

info: cached info message #21

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1cfa96a
added cached info message
VJHack Jan 5, 2025
11cf5e9
optimize space is cache
VJHack Jan 5, 2025
c1543c5
comment clarification
VJHack Jan 5, 2025
322b9e6
handle empty caches
ggerganov Jan 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 31 additions & 11 deletions autoload/llama.vim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,30 +586,35 @@ function! s:insert_cache(key, value)
let l:hash = l:keys[rand() % len(l:keys)]
call remove(g:result_cache, l:hash)
endif
let g:result_cache[a:key] = a:value
" put just the raw content in the cache without metrics
let l:parsed_value = json_decode(a:value)
let l:stripped_content = get(l:parsed_value, 'content', '')
let g:result_cache[a:key] = json_encode({'content': l:stripped_content})
endfunction

" callback that processes the FIM result from the server and displays the suggestion
function! s:fim_on_stdout(hash, cache, pos_x, pos_y, is_auto, job_id, data, event = v:null)
" make sure cursor position hasn't changed since fim_on_stdout was triggered
if a:pos_x != col('.') - 1 || a:pos_y != line('.')
return
endif

" show the suggestion only in insert mode
if mode() !=# 'i'
return
endif

" Retrieve the FIM result from cache
if a:cache && has_key(g:result_cache, a:hash)
let l:raw = get(g:result_cache, a:hash)
let l:is_cached = v:true
else
if s:ghost_text_nvim
let l:raw = join(a:data, "\n")
elseif s:ghost_text_vim
let l:raw = a:data
endif
call s:insert_cache(a:hash, l:raw)
endif

if a:pos_x != col('.') - 1 || a:pos_y != line('.')
return
endif

" show the suggestion only in insert mode
if mode() !=# 'i'
return
let l:is_cached = v:false
endif

" TODO: this does not seem to work as expected, so disabling for now
Expand All @@ -623,6 +628,10 @@ function! s:fim_on_stdout(hash, cache, pos_x, pos_y, is_auto, job_id, data, even
return
endif

if !l:is_cached
call s:insert_cache(a:hash, l:raw)
endif

let s:pos_x = a:pos_x
let s:pos_y = a:pos_y

Expand Down Expand Up @@ -669,6 +678,11 @@ function! s:fim_on_stdout(hash, cache, pos_x, pos_y, is_auto, job_id, data, even
let l:t_predict_ms = get(l:timings, 'predicted_ms', 1)
let l:s_predict = get(l:timings, 'predicted_per_second', 0)
endif

" if response was pulled from cache
if l:is_cached
let l:has_info = v:true
endif
endif

if len(s:content) == 0
Expand Down Expand Up @@ -759,6 +773,12 @@ function! s:fim_on_stdout(hash, cache, pos_x, pos_y, is_auto, job_id, data, even
\ g:llama_config.show_info == 2 ? l:prefix : 'llama.vim',
\ l:n_cached, l:n_ctx
\ )
elseif l:is_cached
let l:info = printf("%s | C: %d / %d, | t: %.2f ms",
\ g:llama_config.show_info == 2 ? l:prefix : 'llama.vim',
\ len(keys(g:result_cache)), g:llama_config.max_cache_keys,
\ 1000.0 * reltimefloat(reltime(s:t_fim_start))
\ )
else
let l:info = printf("%s | c: %d / %d, r: %d / %d, e: %d, q: %d / 16 | p: %d (%.2f ms, %.2f t/s) | g: %d (%.2f ms, %.2f t/s) | t: %.2f ms",
\ g:llama_config.show_info == 2 ? l:prefix : 'llama.vim',
Expand Down