From e5d0d237a8790ca1181c9d84499687b74381f990 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Wed, 27 Nov 2024 10:40:23 +0000 Subject: [PATCH] sheet: Use the transformation API to do filtering --- app/ext_example/Makefile | 26 +++- app/ext_example/my_extension.c | 19 ++- app/ext_example/test/expected/.gitignore | 1 + ...tension.out => test-sheet-extension-1.out} | 0 .../test/expected/test-sheet-extension-2.out | 5 + app/sheet.c | 39 ++--- app/sheet/file.c | 6 +- app/sheet/file.h | 2 +- app/sheet/filter.c | 89 +++++++++++ app/sheet/handlers.c | 130 ---------------- app/sheet/handlers_internal.h | 4 +- app/sheet/index.c | 113 +------------- app/sheet/index.h | 1 - app/sheet/read-data.c | 84 +++++------ app/sheet/transformation.c | 139 +++++++++++++++++- app/sheet/transformation.h | 2 + app/sheet/ui_buffer.c | 11 +- app/test/Makefile | 10 +- include/zsv/ext.h | 6 +- include/zsv/ext/sheet.h | 40 +++++ 20 files changed, 375 insertions(+), 352 deletions(-) create mode 100644 app/ext_example/test/expected/.gitignore rename app/ext_example/test/expected/{test-sheet-extension.out => test-sheet-extension-1.out} (100%) create mode 100644 app/ext_example/test/expected/test-sheet-extension-2.out create mode 100644 app/sheet/filter.c diff --git a/app/ext_example/Makefile b/app/ext_example/Makefile index c73821e9..019f69c2 100644 --- a/app/ext_example/Makefile +++ b/app/ext_example/Makefile @@ -115,7 +115,7 @@ ${BUILD_DIR}/objs/utils/%.o: TESTS=test-1 test-2 test-3 test-4 test-5 test-thirdparty ifeq ($(ZSVSHEET_BUILD),1) - TESTS+=test-sheet-extension + TESTS+=test-sheet-extension-1 test-sheet-extension-2 endif test: ${TESTS} @@ -147,7 +147,7 @@ test-3: test-%: ${CLI} ${TARGET} TMP_DIR=/tmp TMUX_TERM=xterm-256color -test-sheet-extension: ../test/worldcitiespop_mil.csv +test-sheet-extension-1: ${CLI} ${TARGET} ../test/worldcitiespop_mil.csv @${TEST_INIT} @rm -f ${TMP_DIR}/zsvext-$@.out tmux-*.log @tmux kill-session -t $@ || echo 'No tmux session to kill' @@ -156,7 +156,7 @@ test-sheet-extension: ../test/worldcitiespop_mil.csv @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf @(ZSV_CONFIG_DIR=/tmp tmux -v new-session -x 80 -y 5 -d -s $@ && \ sleep 0.5 && \ - tmux send-keys -t $@ "${CLI} sheet ../test/worldcitiespop_mil.csv" ENTER && \ + tmux send-keys -t $@ "${RUN_CLI} sheet ../test/worldcitiespop_mil.csv" ENTER && \ sleep 0.5 && \ tmux send-keys -t $@ "t" "hello" Enter && \ tmux capture-pane -t $@ -p > ${TMP_DIR}/$@.out && \ @@ -164,6 +164,26 @@ test-sheet-extension: ../test/worldcitiespop_mil.csv cmp ${TMP_DIR}/$@.out test/expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && \ if [ -f ${TMP_DIR}/$@.out ]; then cat ${TMP_DIR}/$@.out; fi && ${TEST_FAIL})) +test-sheet-extension-2: ${CLI} ${TARGET} + @${TEST_INIT} + @rm -f ${TMP_DIR}/zsvext-$@.out tmux-*.log + @tmux kill-session -t $@ || echo 'No tmux session to kill' + @${RUN_CLI} unregister my 2>/dev/null 1>/dev/null || [ 1==1 ] + @${RUN_CLI} register my 2>&1 | grep -v [.]so || [ 1==1 ] + @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf + @(ZSV_CONFIG_DIR=/tmp tmux -v new-session -x 120 -y 5 -d -s $@ && \ + sleep 0.5 && \ + tmux send-keys -t $@ "${RUN_CLI} sheet -d 3 ../../data/test/mixed-line-endings.csv" ENTER && \ + sleep 0.5 && \ + tmux send-keys -t $@ T Enter && \ + sleep 0.5 && \ + tmux send-keys -t $@ G && \ + sleep 0.5 && \ + tmux capture-pane -t $@ -p > ${TMP_DIR}/$@.out && \ + tmux kill-session -t $@ && \ + cmp ${TMP_DIR}/$@.out test/expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && \ + if [ -f ${TMP_DIR}/$@.out ]; then cat ${TMP_DIR}/$@.out; fi && ${TEST_FAIL})) + test-4: test-%: ${CLI} ${TARGET} @${TEST_INIT} @${RUN_CLI} unregister my 2>/dev/null 1>/dev/null || [ 1==1 ] diff --git a/app/ext_example/my_extension.c b/app/ext_example/my_extension.c index 64a92eca..e9b94d07 100644 --- a/app/ext_example/my_extension.c +++ b/app/ext_example/my_extension.c @@ -98,7 +98,7 @@ zsvsheet_status my_test_command_handler(zsvsheet_proc_context_t ctx) { // print a list of open buffers and filenames for (zsvsheet_buffer_t buff = zsv_cb.ext_sheet_buffer_current(ctx); buff; buff = zsv_cb.ext_sheet_buffer_prior(buff), i--) { - const char *buff_filename = zsv_cb.ext_sheet_buffer_filename(buff); + const char *buff_filename = zsv_cb.ext_sheet_buffer_data_filename(buff); if (buff_filename) fprintf(f, "%i,%s\n", i, buff_filename); // assumes no need for quoting or escaping buff_filename... } @@ -109,13 +109,13 @@ zsvsheet_status my_test_command_handler(zsvsheet_proc_context_t ctx) { } struct transformation_context { + zsvsheet_proc_context_t proc_ctx; size_t col_count; size_t row_count; }; // Similar to a regular ZSV row handler used in ext_parse_all -void my_transformation_row_handler(void *ctx) { - zsvsheet_transformation trn = ctx; +void my_transformation_row_handler(zsvsheet_transformation trn) { struct transformation_context *priv = zsv_cb.ext_sheet_transformation_user_context(trn); zsv_parser parser = zsv_cb.ext_sheet_transformation_parser(trn); zsv_csv_writer writer = zsv_cb.ext_sheet_transformation_writer(trn); @@ -137,11 +137,14 @@ void my_transformation_row_handler(void *ctx) { } zsvsheet_status my_transformation_command_handler(zsvsheet_proc_context_t ctx) { - struct transformation_context *my_ctx = calloc(1, sizeof(*my_ctx)); - - // TODO: This probably should happen in another worker thread and while that is happening the status should display - // that some work is in progress. The extension author will maybe want to have control over the status message. - return zsv_cb.ext_sheet_push_transformation(ctx, my_ctx, my_transformation_row_handler); + struct zsvsheet_buffer_transformation_opts opts = { + // Gets freed automatically + .user_context = calloc(1, sizeof(struct transformation_context)), + .row_handler = my_transformation_row_handler, + .on_done = NULL, + }; + + return zsv_cb.ext_sheet_push_transformation(ctx, opts); } #endif diff --git a/app/ext_example/test/expected/.gitignore b/app/ext_example/test/expected/.gitignore new file mode 100644 index 00000000..6aa6b80a --- /dev/null +++ b/app/ext_example/test/expected/.gitignore @@ -0,0 +1 @@ +!*.out diff --git a/app/ext_example/test/expected/test-sheet-extension.out b/app/ext_example/test/expected/test-sheet-extension-1.out similarity index 100% rename from app/ext_example/test/expected/test-sheet-extension.out rename to app/ext_example/test/expected/test-sheet-extension-1.out diff --git a/app/ext_example/test/expected/test-sheet-extension-2.out b/app/ext_example/test/expected/test-sheet-extension-2.out new file mode 100644 index 00000000..79ca0beb --- /dev/null +++ b/app/ext_example/test/expected/test-sheet-extension-2.out @@ -0,0 +1,5 @@ +Row # HA1 HA2 HA3 HB1 HB2 HB3 HC1 HC2 HC3 Column count +4094 A4094 B4094 C4094 12285 +4095 A4095 B4095 C4095 12288 +4096 A4096 B4096 C4096 12291 +? for help 4096 diff --git a/app/sheet.c b/app/sheet.c index 68d88945..b2c4503a 100644 --- a/app/sheet.c +++ b/app/sheet.c @@ -41,7 +41,6 @@ struct zsvsheet_opts { #include "sheet/utf8-width.c" #include "sheet/ui_buffer.c" -#include "sheet/transformation.c" #include "sheet/index.c" #include "sheet/read-data.c" #include "sheet/key-bindings.c" @@ -194,6 +193,7 @@ static void zsvsheet_priv_set_status(const struct zsvsheet_display_dimensions *d #include "sheet/handlers.c" #include "sheet/file.c" #include "sheet/usage.c" +#include "sheet/transformation.c" struct zsvsheet_key_data *zsvsheet_key_handlers = NULL; struct zsvsheet_key_data **zsvsheet_next_key_handler = &zsvsheet_key_handlers; @@ -382,7 +382,7 @@ static zsvsheet_status zsvsheet_open_file_handler(struct zsvsheet_proc_context * goto no_input; const char *opts_used = NULL; - if ((err = zsvsheet_ui_buffer_open_file(prompt_buffer, NULL, NULL, state->custom_prop_handler, opts_used, + if ((err = zsvsheet_ui_buffer_open_file(prompt_buffer, NULL, state->custom_prop_handler, opts_used, di->ui_buffers.base, di->ui_buffers.current))) { if (err > 0) zsvsheet_priv_set_status(di->dimensions, 1, "%s: %s", prompt_buffer, strerror(err)); @@ -396,14 +396,14 @@ static zsvsheet_status zsvsheet_open_file_handler(struct zsvsheet_proc_context * return zsvsheet_status_ok; } -// TODO: Use same API as transformation extensions +#include "sheet/filter.c" + static zsvsheet_status zsvsheet_filter_handler(struct zsvsheet_proc_context *ctx) { char prompt_buffer[256] = {0}; struct zsvsheet_builtin_proc_state *state = (struct zsvsheet_builtin_proc_state *)ctx->subcommand_context; struct zsvsheet_display_info *di = &state->display_info; struct zsvsheet_ui_buffer *current_ui_buffer = *state->display_info.ui_buffers.current; int prompt_footer_row = (int)(di->dimensions->rows - di->dimensions->footer_span); - int err; struct zsvsheet_buffer_info binfo = zsvsheet_buffer_get_info(current_ui_buffer); if (binfo.transform_started && !binfo.transform_done) @@ -416,27 +416,7 @@ static zsvsheet_status zsvsheet_filter_handler(struct zsvsheet_proc_context *ctx if (*prompt_buffer == '\0') goto out; - const char *data_filename = zsvsheet_buffer_data_filename(current_ui_buffer); - char is_filtered_file = !(data_filename == current_ui_buffer->filename); - struct zsv_opts *zsv_opts = is_filtered_file ? NULL : ¤t_ui_buffer->zsv_opts; - - if ((err = zsvsheet_ui_buffer_open_file(data_filename, zsv_opts, prompt_buffer, state->custom_prop_handler, NULL, - di->ui_buffers.base, di->ui_buffers.current))) { - if (err > 0) - zsvsheet_priv_set_status(di->dimensions, 1, "%s: %s", current_ui_buffer->filename, strerror(err)); - else if (err < 0) - zsvsheet_priv_set_status(di->dimensions, 1, "Unexpected error"); - else - zsvsheet_priv_set_status(di->dimensions, 1, "Not found: %s", prompt_buffer); - return zsvsheet_status_ignore; - } - - struct zsvsheet_ui_buffer *new_ui_buffer = *state->display_info.ui_buffers.current; - if (is_filtered_file) { - // TO DO: move this into zsvsheet_ui_buffer_open_file() - free(new_ui_buffer->filename); - new_ui_buffer->filename = strdup(current_ui_buffer->filename); - } + return zsvsheet_filter_file(ctx, prompt_buffer); out: return zsvsheet_status_ok; } @@ -453,6 +433,7 @@ static zsvsheet_status zsvsheet_help_handler(struct zsvsheet_proc_context *ctx) struct zsvsheet_ui_buffer_opts uibopts = { .buff_opts = &bopts, .filename = NULL, + .data_filename = NULL, .no_rownum_col_offset = 1, .transform = 0, }; @@ -651,7 +632,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op if (argc > 1) { const char *filename = argv[1]; - if ((err = zsvsheet_ui_buffer_open_file(filename, optsp, NULL, custom_prop_handler, opts_used, &ui_buffers, + if ((err = zsvsheet_ui_buffer_open_file(filename, optsp, custom_prop_handler, opts_used, &ui_buffers, ¤t_ui_buffer))) { if (err > 0) perror(filename); @@ -716,13 +697,13 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op if (ub->transform_progressed) { handler_state.display_info.update_buffer = true; ub->transform_progressed = 0; - } else if (ub->index_ready && ub->dimensions.row_count != current_ui_buffer->index->row_count + 1) { - ub->dimensions.row_count = current_ui_buffer->index->row_count + 1; + } else if (ub->index_ready && ub->dimensions.row_count != ub->index->row_count + 1) { + ub->dimensions.row_count = ub->index->row_count + 1; handler_state.display_info.update_buffer = true; } pthread_mutex_unlock(&ub->mutex); - if (handler_state.display_info.update_buffer && ub->filename) { + if (handler_state.display_info.update_buffer && zsvsheet_buffer_data_filename(ub)) { struct zsvsheet_opts zsvsheet_opts = {0}; if (read_data(&ub, NULL, current_ui_buffer->input_offset.row, current_ui_buffer->input_offset.col, header_span, &zsvsheet_opts, custom_prop_handler)) { diff --git a/app/sheet/file.c b/app/sheet/file.c index 71f800f8..32aaba75 100644 --- a/app/sheet/file.c +++ b/app/sheet/file.c @@ -23,7 +23,7 @@ int zsvsheet_ui_buffer_open_file_opts(struct zsvsheet_ui_buffer_opts *uibopts, return 0; } -int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zsv_optsp, const char *row_filter, +int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zsv_optsp, struct zsv_prop_handler *custom_prop_handler, const char *opts_used, struct zsvsheet_ui_buffer **ui_buffer_stack_bottom, struct zsvsheet_ui_buffer **ui_buffer_stack_top) { @@ -33,7 +33,6 @@ int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zs if (zsv_optsp) uibopts.zsv_opts = *zsv_optsp; uibopts.opts_used = opts_used; - uibopts.row_filter = row_filter; return zsvsheet_ui_buffer_open_file_opts(&uibopts, custom_prop_handler, ui_buffer_stack_bottom, ui_buffer_stack_top); } @@ -48,8 +47,7 @@ zsvsheet_status zsvsheet_open_file(struct zsvsheet_proc_context *ctx, const char struct zsvsheet_display_info *di = &state->display_info; if (!di || !di->ui_buffers.base || !di->ui_buffers.current) return zsvsheet_status_error; - int err = - zsvsheet_ui_buffer_open_file(filepath, zopts, NULL, NULL, NULL, di->ui_buffers.base, di->ui_buffers.current); + int err = zsvsheet_ui_buffer_open_file(filepath, zopts, NULL, NULL, di->ui_buffers.base, di->ui_buffers.current); if (err) return zsvsheet_status_error; return zsvsheet_status_ok; diff --git a/app/sheet/file.h b/app/sheet/file.h index ee43d678..d5478df3 100644 --- a/app/sheet/file.h +++ b/app/sheet/file.h @@ -1,7 +1,7 @@ #ifndef ZSVSHEET_FILE_H #define ZSVSHEET_FILE_H -int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zsv_optsp, const char *row_filter, +int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zsv_optsp, struct zsv_prop_handler *custom_prop_handler, const char *opts_used, struct zsvsheet_ui_buffer **ui_buffer_stack_bottom, struct zsvsheet_ui_buffer **ui_buffer_stack_top); diff --git a/app/sheet/filter.c b/app/sheet/filter.c new file mode 100644 index 00000000..f67883bb --- /dev/null +++ b/app/sheet/filter.c @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include "transformation.h" +#include "handlers_internal.h" + +struct filtered_file_ctx { + char *filter; + size_t filter_len; + size_t row_num; // 1-based row number (1 = header row, 2 = first data row) + size_t passed; + unsigned char seen_header : 1; + unsigned char has_row_num : 1; + unsigned char _ : 7; +}; + +static void zsvsheet_save_filtered_file_row_handler(zsvsheet_transformation trn) { + struct filtered_file_ctx *ctx = zsvsheet_transformation_user_context(trn); + zsv_parser parser = zsvsheet_transformation_parser(trn); + zsv_csv_writer writer = zsvsheet_transformation_writer(trn); + size_t col_count = zsv_cell_count(parser); + ctx->row_num++; + if (col_count == 0) + return; + + struct zsv_cell first_cell = zsv_get_cell(parser, 0); + if (ctx->seen_header) { + struct zsv_cell last_cell = zsv_get_cell(parser, col_count - 1); + if (!memmem(first_cell.str, last_cell.str - first_cell.str + last_cell.len, ctx->filter, ctx->filter_len)) + return; + // future enhancement: optionally, handle if row may have unusual quotes e.g. cell1,"ce"ll2,cell3 + } else { + ctx->seen_header = 1; + if (first_cell.len == ZSVSHEET_ROWNUM_HEADER_LEN && !memcmp(first_cell.str, ZSVSHEET_ROWNUM_HEADER, first_cell.len)) + ctx->has_row_num = 1; + } + + char row_started = 0; + if (!ctx->has_row_num) { + // create our own rownum column + row_started = 1; + if (ctx->row_num == 1) + zsv_writer_cell_s(writer, 1, (const unsigned char *)"Row #", 0); // to do: consolidate "Row #" + else + zsv_writer_cell_zu(writer, 1, ctx->row_num - 1); + } + for (size_t i = 0; i < col_count; i++) { + struct zsv_cell cell = zsv_get_cell(parser, i); + zsv_writer_cell(writer, i == 0 && row_started == 0, cell.str, cell.len, cell.quoted); + } + + ctx->passed++; +} + +static void zsvsheet_filter_file_on_done(zsvsheet_transformation trn) { + struct filtered_file_ctx *ctx = zsvsheet_transformation_user_context(trn); + struct zsvsheet_ui_buffer *uib = trn->ui_buffer; + + char *status = NULL; + asprintf(&status, "(%zu filtered rows) ", ctx->passed - 1); + + pthread_mutex_lock(&uib->mutex); + char *old_status = uib->status; + uib->status = status; + pthread_mutex_unlock(&uib->mutex); + + free(old_status); + free(ctx->filter); +} + +static enum zsvsheet_status zsvsheet_filter_file(zsvsheet_proc_context_t proc_ctx, const char *row_filter) { + struct filtered_file_ctx ctx = { + .seen_header = 0, + .row_num = 0, + .passed = 0, + .has_row_num = 0, + .filter = strdup(row_filter), + .filter_len = strlen(row_filter), + }; + struct zsvsheet_buffer_transformation_opts opts = { + .user_context = zsv_memdup(&ctx, sizeof(ctx)), + .row_handler = zsvsheet_save_filtered_file_row_handler, + .on_done = zsvsheet_filter_file_on_done, + }; + + return zsvsheet_push_transformation(proc_ctx, opts); +} diff --git a/app/sheet/handlers.c b/app/sheet/handlers.c index 6ea989cf..b64d50c1 100644 --- a/app/sheet/handlers.c +++ b/app/sheet/handlers.c @@ -161,133 +161,3 @@ struct zsvsheet_buffer_info zsvsheet_buffer_get_info(zsvsheet_buffer_t h) { return info; } - -struct buffer_transform_ctx { - zsvsheet_transformation trn; - struct zsvsheet_ui_buffer *buff; -}; - -static void *run_buffer_transformation(void *arg) { - struct buffer_transform_ctx *ctx = arg; - struct zsvsheet_ui_buffer *buff = ctx->buff; - struct zsvsheet_transformation *trn = ctx->trn; - zsv_parser parser = trn->parser; - pthread_mutex_t *mutex = &buff->mutex; - enum zsv_status zst; - - size_t c = trn->output_count; - char cancelled = 0; - while (!cancelled && (zst = zsv_parse_more(parser)) == zsv_status_ok) { - pthread_mutex_lock(mutex); - cancelled = buff->worker_cancelled; - if (trn->output_count != c) - buff->transform_progressed = 1; - pthread_mutex_unlock(mutex); - } - - if (zst == zsv_status_no_more_input || zst == zsv_status_cancelled) - zst = zsv_finish(parser); - - pthread_mutex_lock(mutex); - char *buff_status_old = buff->status; - buff->transform_done = 1; - buff->status = NULL; - pthread_mutex_unlock(mutex); - free(buff_status_old); - - free(trn->user_context); - zsvsheet_transformation_delete(trn); - free(ctx); - - return NULL; -} - -enum zsvsheet_status zsvsheet_push_transformation(zsvsheet_proc_context_t ctx, void *user_context, - void (*row_handler)(void *exec_ctx)) { - zsvsheet_buffer_t buff = zsvsheet_buffer_current(ctx); - const char *filename = zsvsheet_buffer_data_filename(buff); - enum zsvsheet_status stat = zsvsheet_status_error; - struct zsvsheet_buffer_info info = zsvsheet_buffer_get_info(buff); - - // TODO: Starting a second transformation before the first ends works, but if the second is faster - // than the first then it can end prematurely and read a partially written row. - // We could override the input stream reader to wait for more data when it sees EOF - if (info.transform_started && !info.transform_done) - return zsvsheet_status_busy; - - if (!filename) - filename = zsvsheet_buffer_filename(buff); - - // TODO: custom_prop_handler is not passed to extensions? - struct zsvsheet_transformation_opts opts = { - .custom_prop_handler = NULL, - .input_filename = filename, - }; - zsvsheet_transformation trn; - struct zsv_opts zopts = zsvsheet_buffer_get_zsv_opts(buff); - - zopts.ctx = user_context; - zopts.row_handler = row_handler; - zopts.stream = fopen(filename, "rb"); - - if (!zopts.stream) - goto out; - - opts.zsv_opts = zopts; - - enum zsv_status zst = zsvsheet_transformation_new(opts, &trn); - if (zst != zsv_status_ok) - return stat; - - // Transform part of the file to initially populate the UI buffer - // TODO: If the transformation is a reduction that doesn't output for some time this will caus a pause - zsv_parser parser = zsvsheet_transformation_parser(trn); - while ((zst = zsv_parse_more(parser)) == zsv_status_ok) { - if (trn->output_count > 0) - break; - } - - switch (zst) { - case zsv_status_no_more_input: - case zsv_status_cancelled: - if (zsv_finish(parser) != zsv_status_ok) - goto out; - zsv_writer_flush(trn->writer); - break; - case zsv_status_ok: - break; - default: - goto out; - } - - struct zsvsheet_ui_buffer_opts uibopts = {0}; - - uibopts.filename = zsvsheet_transformation_filename(trn); - uibopts.transform = 1; - - stat = zsvsheet_open_file_opts(ctx, &uibopts); - if (stat != zsvsheet_status_ok) - goto out; - - struct zsvsheet_ui_buffer *nbuff = zsvsheet_buffer_current(ctx); - - if (zst != zsv_status_ok) { - nbuff->transform_done = 1; - goto out; - } - - asprintf(&nbuff->status, "(working) Press ESC to cancel"); - - struct buffer_transform_ctx *bctx = malloc(sizeof(*bctx)); - bctx->trn = trn; - bctx->buff = nbuff; - - zsvsheet_ui_buffer_create_worker(nbuff, run_buffer_transformation, bctx); - return stat; - -out: - if (trn) - zsvsheet_transformation_delete(trn); - - return stat; -} diff --git a/app/sheet/handlers_internal.h b/app/sheet/handlers_internal.h index 40f31e68..deea9e17 100644 --- a/app/sheet/handlers_internal.h +++ b/app/sheet/handlers_internal.h @@ -108,6 +108,6 @@ zsvsheet_status zsvsheet_register_command(int ch, const char *long_name, /** * Transform the current buffer's underlying file into a new one and open the new file in a buffer */ -enum zsvsheet_status zsvsheet_push_transformation(zsvsheet_proc_context_t ctx, void *user_context, - void (*row_handler)(void *exec_ctx)); +enum zsvsheet_status zsvsheet_push_transformation(zsvsheet_proc_context_t ctx, + struct zsvsheet_buffer_transformation_opts opts); #endif diff --git a/app/sheet/index.c b/app/sheet/index.c index 2ab3ecbd..95ffab00 100644 --- a/app/sheet/index.c +++ b/app/sheet/index.c @@ -8,113 +8,6 @@ #include #include "index.h" -#include "transformation.h" - -struct filtered_file_ctx { - const char *filter; - size_t filter_len; - size_t row_num; // 1-based row number (1 = header row, 2 = first data row) - unsigned char seen_header : 1; - unsigned char has_row_num : 1; - unsigned char _ : 7; -}; - -static void save_filtered_file_row_handler(void *trn) { - struct filtered_file_ctx *ctx = zsvsheet_transformation_user_context(trn); - zsv_parser parser = zsvsheet_transformation_parser(trn); - zsv_csv_writer writer = zsvsheet_transformation_writer(trn); - size_t col_count = zsv_cell_count(parser); - ctx->row_num++; - if (col_count == 0) - return; - - struct zsv_cell first_cell = zsv_get_cell(parser, 0); - if (ctx->seen_header) { - struct zsv_cell last_cell = zsv_get_cell(parser, col_count - 1); - if (!memmem(first_cell.str, last_cell.str - first_cell.str + last_cell.len, ctx->filter, ctx->filter_len)) - return; - // future enhancement: optionally, handle if row may have unusual quotes e.g. cell1,"ce"ll2,cell3 - } else { - ctx->seen_header = 1; - if (first_cell.len == ZSVSHEET_ROWNUM_HEADER_LEN && !memcmp(first_cell.str, ZSVSHEET_ROWNUM_HEADER, first_cell.len)) - ctx->has_row_num = 1; - } - - char row_started = 0; - if (!ctx->has_row_num) { - // create our own rownum column - row_started = 1; - if (ctx->row_num == 1) - zsv_writer_cell_s(writer, 1, (const unsigned char *)"Row #", 0); // to do: consolidate "Row #" - else - zsv_writer_cell_zu(writer, 1, ctx->row_num - 1); - } - for (size_t i = 0; i < col_count; i++) { - struct zsv_cell cell = zsv_get_cell(parser, i); - zsv_writer_cell(writer, i == 0 && row_started == 0, cell.str, cell.len, cell.quoted); - } -} - -static enum zsv_status filter_file(struct zsvsheet_index_opts *optsp) { - struct filtered_file_ctx ctx = { - .seen_header = 0, - .row_num = 0, - .has_row_num = 0, - .filter = optsp->row_filter, - .filter_len = strlen(optsp->row_filter), - }; - struct zsvsheet_transformation_opts opts = { - .custom_prop_handler = optsp->custom_prop_handler, - .input_filename = optsp->filename, - }; - zsvsheet_transformation trn; - struct zsv_opts zopts = optsp->zsv_opts; - - zopts.ctx = &ctx; - zopts.row_handler = save_filtered_file_row_handler; - zopts.stream = fopen(optsp->filename, "rb"); - - if (!zopts.stream) - goto out; - - opts.zsv_opts = zopts; - - enum zsv_status zst = zsvsheet_transformation_new(opts, &trn); - if (zst != zsv_status_ok) - return zst; - - zsv_parser parser = zsvsheet_transformation_parser(trn); - - char cancelled = 0; - while (!cancelled && (zst = zsv_parse_more(parser)) == zsv_status_ok) { - pthread_mutex_lock(&optsp->uib->mutex); - if (optsp->uib->worker_cancelled) { - cancelled = 1; - zst = zsv_status_cancelled; - } - pthread_mutex_unlock(&optsp->uib->mutex); - } - - switch (zst) { - case zsv_status_no_more_input: - case zsv_status_cancelled: - break; - default: - goto out; - } - - zst = zsv_finish(parser); - if (zst != zsv_status_ok) - goto out; - - if (!(optsp->uib->data_filename = strdup(zsvsheet_transformation_filename(trn)))) - zst = zsv_status_memory; - -out: - zsvsheet_transformation_delete(trn); - return zst; -} - static void build_memory_index_row_handler(void *ctx) { struct zsvsheet_indexer *ixr = ctx; struct zsv_index *ix = ixr->ix; @@ -129,11 +22,7 @@ enum zsv_index_status build_memory_index(struct zsvsheet_index_opts *optsp) { enum zsv_index_status ret = zsv_index_status_error; struct zsv_opts ix_zopts = optsp->zsv_opts; - if (optsp->row_filter) { - enum zsv_status zst = filter_file(optsp); - if (zst != zsv_status_ok) - goto out; - + if (optsp->uib->data_filename) { ix_zopts.stream = fopen(optsp->uib->data_filename, "rb"); } else { ix_zopts.stream = fopen(optsp->filename, "rb"); diff --git a/app/sheet/index.h b/app/sheet/index.h index e4db74c0..647a1f77 100644 --- a/app/sheet/index.h +++ b/app/sheet/index.h @@ -13,7 +13,6 @@ struct zsvsheet_indexer { struct zsvsheet_index_opts { pthread_mutex_t *mutexp; const char *filename; - const char *row_filter; struct zsv_opts zsv_opts; struct zsvsheet_ui_buffer *uib; int *errp; diff --git a/app/sheet/read-data.c b/app/sheet/read-data.c index a4bbc70a..caa339df 100644 --- a/app/sheet/read-data.c +++ b/app/sheet/read-data.c @@ -29,14 +29,13 @@ static char *zsvsheet_found_in_row(zsv_parser parser, size_t col_count, const ch static void *get_data_index(void *d); static void get_data_index_async(struct zsvsheet_ui_buffer *uibuffp, const char *filename, struct zsv_opts *optsp, - const char *row_filter, struct zsv_prop_handler *custom_prop_handler, + struct zsv_prop_handler *custom_prop_handler, // const char *opts_used, pthread_mutex_t *mutexp) { struct zsvsheet_index_opts *ixopts = calloc(1, sizeof(*ixopts)); ixopts->mutexp = mutexp; ixopts->filename = filename; ixopts->zsv_opts = *optsp; - ixopts->row_filter = row_filter; ixopts->custom_prop_handler = custom_prop_handler; // ixopts->opts_used = opts_used; ixopts->uib = uibuffp; @@ -64,10 +63,22 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ size_t remaining_rows_to_skip = start_row; size_t remaining_header_to_skip = header_span; size_t original_row_num = 0; - const char *row_filter = uibuff ? uibuff->row_filter : NULL; - size_t row_filter_len = row_filter ? strlen(row_filter) : 0; FILE *fp; + if (uibuff) { + if (uibuff->data_filename) + filename = uibuff->data_filename; + else if (uibuff->filename) + filename = uibuff->filename; + } + + if (!filename && uibopts) { + if (uibopts->data_filename) + filename = uibopts->data_filename; + else if (uibopts->filename) + filename = uibopts->filename; + } + assert(filename != NULL); fp = fopen(filename, "rb"); if (!fp) @@ -85,30 +96,12 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ if (uibuff) { pthread_mutex_lock(&uibuff->mutex); - if (uibuff->index_ready && row_filter) { - fclose(fp); - fp = fopen(uibuff->data_filename, "rb"); - if (!fp) { - pthread_mutex_unlock(&uibuff->mutex); - return errno; - } - opts.stream = fp; - } enum zsv_index_status zst = zsv_index_status_ok; if (uibuff->index_ready) { opts.header_span = 0; opts.rows_to_ignore = 0; - if (uibuff->data_filename) { - struct zsv_opts filter_opts = {0}; - filter_opts.stream = opts.stream; - filter_opts.max_columns = opts.max_columns; - filter_opts.max_row_size = opts.max_row_size; - filter_opts.max_rows = opts.max_rows; - opts = filter_opts; - uibuff->has_row_num = 1; // move this to coincide with when data_filename is assigned - } zst = zsv_index_seek_row(uibuff->index, &opts, start_row); zsv_delete(parser); @@ -147,8 +140,6 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ return -1; } *uibufferp = uibuff = tmp_uibuff; - row_filter = uibuff ? uibuff->row_filter : NULL; - row_filter_len = row_filter ? strlen(row_filter) : 0; } // row number @@ -176,10 +167,6 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ if (col_count > uibuff->dimensions.col_count) uibuff->dimensions.col_count = col_count; } - if (rows_read > 0 && row_filter) { - if (!zsvsheet_found_in_row(parser, col_count, row_filter, row_filter_len)) - continue; - } if (remaining_rows_to_skip > 0) { remaining_rows_to_skip--; @@ -222,29 +209,21 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ if (!uibuff) return 0; - char *ui_status = NULL; pthread_mutex_lock(&uibuff->mutex); char need_index = !uibuff->index_started && (!uibuff->transform_started || uibuff->transform_done); pthread_mutex_unlock(&uibuff->mutex); if (need_index) { uibuff->buff_used_rows = rows_read; + uibuff->dimensions.row_count = rows_read; uibuff->index_started = 1; - if (original_row_num > 1 && (row_filter == NULL || rows_read > 0)) { + if (original_row_num > 1 && rows_read > 0) { opts.stream = NULL; - get_data_index_async(uibuff, filename, &opts, row_filter, custom_prop_handler, /* opts_used, */ &uibuff->mutex); - asprintf(&ui_status, "(building index) "); + get_data_index_async(uibuff, filename, &opts, custom_prop_handler, /* opts_used, */ &uibuff->mutex); } } else if (rows_read > uibuff->buff_used_rows) { uibuff->buff_used_rows = rows_read; - } - - if (ui_status) { - pthread_mutex_lock(&uibuff->mutex); - if (uibuff->status) - free(uibuff->status); - uibuff->status = ui_status; - pthread_mutex_unlock(&uibuff->mutex); + uibuff->dimensions.row_count = rows_read; } return 0; @@ -255,6 +234,18 @@ static void *get_data_index(void *gdi) { struct zsvsheet_index_opts *d = gdi; pthread_mutex_t *mutexp = d->mutexp; int *errp = d->errp; + struct zsvsheet_ui_buffer *uib = d->uib; + + pthread_mutex_lock(&uib->mutex); + char *old_ui_status = uib->status; + pthread_mutex_unlock(&uib->mutex); + + char *ui_status; + asprintf(&ui_status, "%s(building index) ", old_ui_status ? old_ui_status : ""); + + pthread_mutex_lock(&uib->mutex); + uib->status = ui_status; + pthread_mutex_unlock(&uib->mutex); enum zsv_index_status ix_status = build_memory_index(d); @@ -267,18 +258,13 @@ static void *get_data_index(void *gdi) { return NULL; } - char *buff_status = NULL; - if (d->uib->row_filter && d->uib->index->row_count > 0) - asprintf(&buff_status, "(%" PRIu64 " filtered rows) ", d->uib->index->row_count); - pthread_mutex_lock(mutexp); - d->uib->index_ready = 1; - char *old_buff_status = d->uib->status; - d->uib->status = buff_status; - d->uib->ixopts = NULL; + uib->index_ready = 1; + uib->status = old_ui_status; + uib->ixopts = NULL; pthread_mutex_unlock(mutexp); - free(old_buff_status); + free(ui_status); free(d); return NULL; diff --git a/app/sheet/transformation.c b/app/sheet/transformation.c index 4e130ac6..41645f09 100644 --- a/app/sheet/transformation.c +++ b/app/sheet/transformation.c @@ -1,7 +1,9 @@ #include #include +#include "handlers_internal.h" #include "transformation.h" +#include "pthread.h" #include "zsv/utils/file.h" #include "zsv/utils/prop.h" @@ -15,6 +17,10 @@ struct zsvsheet_transformation { size_t output_count; struct zsvsheet_transformation_opts opts; void *user_context; + + struct zsvsheet_ui_buffer *ui_buffer; + char *default_status; + void (*on_done)(zsvsheet_transformation trn); }; static size_t transformation_write(const void *restrict ptr, size_t size, size_t nitems, void *restrict stream) { @@ -37,6 +43,7 @@ enum zsv_status zsvsheet_transformation_new(struct zsvsheet_transformation_opts if (trn == NULL) return zst; + trn->ui_buffer = opts.ui_buffer; zst = zsv_status_error; temp_filename = zsv_get_temp_filename("zsvsheet_filter_XXXXXXXX"); @@ -60,7 +67,7 @@ enum zsv_status zsvsheet_transformation_new(struct zsvsheet_transformation_opts if (!(temp_file_writer = zsv_writer_new(&writer_opts))) goto free; - const size_t temp_buff_size = 8192; + const size_t temp_buff_size = 2 * 1024 * 1024; temp_buff = malloc(temp_buff_size); if (!temp_buff) goto free; @@ -117,3 +124,133 @@ const char *zsvsheet_transformation_filename(zsvsheet_transformation trn) { void *zsvsheet_transformation_user_context(zsvsheet_transformation trn) { return trn->user_context; } + +static void *zsvsheet_run_buffer_transformation(void *arg) { + struct zsvsheet_transformation *trn = arg; + struct zsvsheet_ui_buffer *uib = trn->ui_buffer; + zsv_parser parser = trn->parser; + pthread_mutex_t *mutex = &uib->mutex; + enum zsv_status zst; + + size_t c = trn->output_count; + char cancelled = 0; + while (!cancelled && (zst = zsv_parse_more(parser)) == zsv_status_ok) { + pthread_mutex_lock(mutex); + cancelled = uib->worker_cancelled; + if (trn->output_count != c) + uib->transform_progressed = 1; + pthread_mutex_unlock(mutex); + } + + if (zst == zsv_status_no_more_input || zst == zsv_status_cancelled) + zst = zsv_finish(parser); + + if (trn->on_done) + trn->on_done(trn); + + pthread_mutex_lock(mutex); + char *buff_status_old = uib->status; + uib->transform_progressed = 1; + uib->transform_done = 1; + if (buff_status_old == trn->default_status) + uib->status = NULL; + pthread_mutex_unlock(mutex); + + if (buff_status_old == trn->default_status) + free(buff_status_old); + if (trn->user_context) + free(trn->user_context); + zsvsheet_transformation_delete(trn); + + return NULL; +} + +enum zsvsheet_status zsvsheet_push_transformation(zsvsheet_proc_context_t ctx, + struct zsvsheet_buffer_transformation_opts opts) { + zsvsheet_buffer_t buff = zsvsheet_buffer_current(ctx); + const char *filename = zsvsheet_buffer_data_filename(buff); + enum zsvsheet_status stat = zsvsheet_status_error; + struct zsvsheet_buffer_info info = zsvsheet_buffer_get_info(buff); + + // TODO: Starting a second transformation before the first ends works, but if the second is faster + // than the first then it can end prematurely and read a partially written row. + // We could override the input stream reader to wait for more data when it sees EOF + if (info.transform_started && !info.transform_done) + return zsvsheet_status_busy; + + // TODO: custom_prop_handler is not passed to extensions? + struct zsvsheet_transformation_opts trn_opts = { + .custom_prop_handler = NULL, + .input_filename = filename, + .on_done = opts.on_done, + .ui_buffer = NULL, + }; + zsvsheet_transformation trn; + struct zsv_opts zopts = zsvsheet_buffer_get_zsv_opts(buff); + + zopts.ctx = opts.user_context; + zopts.row_handler = (void (*)(void *))opts.row_handler; + zopts.stream = fopen(filename, "rb"); + + if (!zopts.stream) + goto out; + + trn_opts.zsv_opts = zopts; + + enum zsv_status zst = zsvsheet_transformation_new(trn_opts, &trn); + if (zst != zsv_status_ok) + return stat; + + // Transform part of the file to initially populate the UI buffer + // TODO: If the transformation is a reduction that doesn't output for some time this will caus a pause + zsv_parser parser = zsvsheet_transformation_parser(trn); + while ((zst = zsv_parse_more(parser)) == zsv_status_ok) { + if (trn->output_count > 0) + break; + } + + switch (zst) { + case zsv_status_no_more_input: + case zsv_status_cancelled: + if (zsv_finish(parser) != zsv_status_ok) + goto out; + zsv_writer_flush(trn->writer); + break; + case zsv_status_ok: + break; + default: + goto out; + } + + struct zsvsheet_ui_buffer_opts uibopts = {0}; + + uibopts.data_filename = zsvsheet_transformation_filename(trn); + uibopts.transform = 1; + + stat = zsvsheet_open_file_opts(ctx, &uibopts); + if (stat != zsvsheet_status_ok) + goto out; + + struct zsvsheet_ui_buffer *nbuff = zsvsheet_buffer_current(ctx); + trn->ui_buffer = nbuff; + nbuff->transform_progressed = 1; + + if (zst != zsv_status_ok) { + nbuff->transform_done = 1; + goto out; + } + + asprintf(&trn->default_status, "(working) Press ESC to cancel"); + nbuff->status = trn->default_status; + + zsvsheet_ui_buffer_create_worker(nbuff, zsvsheet_run_buffer_transformation, trn); + return stat; + +out: + if (trn && opts.on_done) + opts.on_done(trn); + if (trn) + zsvsheet_transformation_delete(trn); + + return stat; +} diff --git a/app/sheet/transformation.h b/app/sheet/transformation.h index 9d5807b4..87ea7226 100644 --- a/app/sheet/transformation.h +++ b/app/sheet/transformation.h @@ -13,6 +13,8 @@ struct zsvsheet_transformation_opts { struct zsv_opts zsv_opts; struct zsv_prop_handler *custom_prop_handler; const char *input_filename; + void (*on_done)(zsvsheet_transformation trn); + struct zsvsheet_ui_buffer *ui_buffer; }; enum zsv_status zsvsheet_transformation_new(struct zsvsheet_transformation_opts, zsvsheet_transformation *); diff --git a/app/sheet/ui_buffer.c b/app/sheet/ui_buffer.c index d9bd2cee..0f083f8b 100644 --- a/app/sheet/ui_buffer.c +++ b/app/sheet/ui_buffer.c @@ -27,7 +27,6 @@ struct zsvsheet_ui_buffer { struct zsvsheet_rowcol buff_offset; size_t buff_used_rows; char *status; - char *row_filter; void *ext_ctx; // extension context via zsvsheet_ext_set_ctx() zsvsheet_ext_get_ctx() @@ -81,7 +80,6 @@ void zsvsheet_ui_buffer_delete(struct zsvsheet_ui_buffer *ub) { pthread_mutex_destroy(&ub->mutex); if (ub->ixopts) ub->ixopts->uib = NULL; - free(ub->row_filter); zsv_index_delete(ub->index); free(ub->status); if (ub->data_filename) @@ -94,8 +92,8 @@ void zsvsheet_ui_buffer_delete(struct zsvsheet_ui_buffer *ub) { struct zsvsheet_ui_buffer_opts { struct zsvsheet_screen_buffer_opts *buff_opts; - const char *row_filter; const char *filename; + const char *data_filename; struct zsv_opts zsv_opts; // options to use when opening this file const char *opts_used; char no_rownum_col_offset; @@ -113,8 +111,11 @@ struct zsvsheet_ui_buffer *zsvsheet_ui_buffer_new(zsvsheet_screen_buffer_t buffe if (!(uibopts && uibopts->no_rownum_col_offset)) uib->rownum_col_offset = 1; if (uibopts) { - if ((uibopts->row_filter && !(uib->row_filter = strdup(uibopts->row_filter))) || - (uibopts->filename && !(uib->filename = strdup(uibopts->filename)))) { + if (uibopts->filename && !(uib->filename = strdup(uibopts->filename))) { + zsvsheet_ui_buffer_delete(uib); + return NULL; + } + if (uibopts->data_filename && !(uib->data_filename = strdup(uibopts->data_filename))) { zsvsheet_ui_buffer_delete(uib); return NULL; } diff --git a/app/test/Makefile b/app/test/Makefile index d086d29e..1520c63e 100644 --- a/app/test/Makefile +++ b/app/test/Makefile @@ -646,18 +646,18 @@ test-sheet-6: ${BUILD_DIR}/bin/zsv_sheet${EXE} @${TEST_INIT} @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf @(tmux new-session -x 80 -y 50 -d -s $@ "${PREFIX} $< -d 3 ${TEST_DATA_DIR}/test/mixed-line-endings.csv" && \ - sleep 0.5 && \ + sleep 1 && \ tmux send-keys -t $@ "G" "C-u" "C-u" "C-u" && \ sleep 0.5 && \ tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \ tmux send-keys -t $@ "q" && \ - ${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL}) + ${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL})) test-sheet-7: ${BUILD_DIR}/bin/zsv_sheet${EXE} @${TEST_INIT} @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf @(tmux new-session -x 80 -y 5 -d -s $@ "${PREFIX} $< -d 3 ${TEST_DATA_DIR}/test/mixed-line-endings.csv" && \ - sleep 0.5 && \ + sleep 1 && \ tmux send-keys -t $@ "G" "g" "g" "C-u" "/" "1234" "Enter" && \ sleep 0.5 && \ tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \ @@ -681,7 +681,7 @@ test-sheet-9: ${BUILD_DIR}/bin/zsv_sheet${EXE} @${TEST_INIT} @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf @(tmux new-session -x 80 -y 6 -d -s $@ "${PREFIX} $< -d 3 ${TEST_DATA_DIR}/test/mixed-line-endings.csv" && \ - sleep 0.5 && \ + sleep 1 && \ tmux send-keys -t $@ "G" && \ tmux send-keys -t $@ -N 4096 "k" && \ sleep 2 && \ @@ -705,7 +705,7 @@ test-sheet-11: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv @${TEST_INIT} @echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf @(tmux new-session -x 80 -y 50 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \ - sleep 0.5 && \ + sleep 1 && \ tmux send-keys -t $@ -N 21 "C-d" && \ sleep 1 && \ tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \ diff --git a/include/zsv/ext.h b/include/zsv/ext.h index cbd9dc93..ed70b603 100644 --- a/include/zsv/ext.h +++ b/include/zsv/ext.h @@ -263,9 +263,11 @@ struct zsv_ext_callbacks { * * Note that the transformation is performed in a seperate thread so the user_context * must not be a stack variable + * + * See struct zsvsheet_buffer_transformation_opts in zsv/ext/sheet.h */ - zsvsheet_status (*ext_sheet_push_transformation)(zsvsheet_proc_context_t ctx, void *user_context, - void (*row_handler)(void *ctx)); + zsvsheet_status (*ext_sheet_push_transformation)(zsvsheet_proc_context_t ctx, + struct zsvsheet_buffer_transformation_opts opts); /** * Get the writer associated with a transformation. diff --git a/include/zsv/ext/sheet.h b/include/zsv/ext/sheet.h index a9ae7154..df83530a 100644 --- a/include/zsv/ext/sheet.h +++ b/include/zsv/ext/sheet.h @@ -23,6 +23,46 @@ typedef void *zsvsheet_buffer_t; // int zsvsheet_ext_keypress(zsvsheet_proc_context_t); typedef struct zsvsheet_transformation *zsvsheet_transformation; +/** + * Transformation options passed to zsvsheet_push_transformation + */ +struct zsvsheet_buffer_transformation_opts { + /** + * Caller supplied context; can be retrieved with ext_sheet_transformation_user_context(trn). + * + * This is free'd by the library after on_done is called unless it is NULL. + */ + void *user_context; + /** + * This is called each time a row is parsed and available. + * + * It can be used to write out a new row immediately or contribute to a calculation + * where the partial results are stored in the context. + * + * Note that when doing a reduction which does not produce output until the end; + * it is best to write out a header row in the row_handler and flush the writer. Otherwise the + * main thread will be blocked waiting for something to display. + * + * Once some data has been written to the output file execution is moved to a background thread. + * + * The parser can be retrieved with ext_sheet_transformation_parser(trn), + * the output file writer can be retrieved with ext_sheet_transformation_writer(trn), + * the context can be retrieved with ext_sheet_transformation_user_context(trn), + * and so on + */ + void (*row_handler)(zsvsheet_transformation trn); + /** + * Called when the parser has no more data or was cancelled. + * + * This can be used to free resources, complete a calculation or to write out one + * or more final rows. + * + * For small input files this may be executed in the main thread, for larger ones + * it will be done in a background thread. So if an operation done inside this handler + * can take a long time even on small inputs then it could block the main thread. + */ + void (*on_done)(zsvsheet_transformation trn); +}; struct zsvsheet_buffer_info { unsigned char index_started : 1;