Skip to content

Commit

Permalink
sheet: Only notify the UI of progress after 32MB
Browse files Browse the repository at this point in the history
This reduces the indexing time by 3-4x on a 13GB file with a Ryzen 7 6800U.
  • Loading branch information
richiejp committed Dec 11, 2024
1 parent 813f29b commit d8cc28b
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 0 deletions.
5 changes: 5 additions & 0 deletions app/sheet/index.c
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,12 @@ enum zsv_index_status build_memory_index(struct zsvsheet_index_opts *optsp) {
optsp->uib->index = ixr.ix;

char cancelled = 0;
size_t committed_bytes = 0;
while (!cancelled && (zst = zsv_parse_more(ixr.parser)) == zsv_status_ok) {
if (zsv_cum_scanned_length(ixr.parser) - committed_bytes < 32 * 1024 * 1024)
continue;
committed_bytes = zsv_cum_scanned_length(ixr.parser);

pthread_mutex_lock(&optsp->uib->mutex);
if (optsp->uib->worker_cancelled) {
cancelled = 1;
Expand Down
1 change: 1 addition & 0 deletions app/sheet/transformation.c
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ enum zsvsheet_status zsvsheet_push_transformation(zsvsheet_proc_context_t ctx,
zopts.ctx = opts.user_context;
zopts.row_handler = (void (*)(void *))opts.row_handler;
zopts.stream = fopen(filename, "rb");
zopts.buffsize = 2 * 1024 * 1024;

if (!zopts.stream)
goto error;
Expand Down

0 comments on commit d8cc28b

Please sign in to comment.