Offload transformations onto separate thread (#304)

* sheet: Add extension API for transforming the current file
* TO DO: index is built with the transformation
* TO DO (optional): index is made available while it is only partially complete
* sheet: cleanup and fixes for transformation/index

app/cli.c

@@ -24,6 +24,7 @@
 #ifdef ZSVSHEET_BUILD
 #include "sheet/sheet_internal.h"
 #include "sheet/handlers_internal.h"
+#include "sheet/transformation.h"
 #endif
 #include "sheet/procedure.h"
 #include "sheet/key-bindings.h"
@@ -408,6 +409,11 @@ static struct zsv_ext_callbacks *zsv_ext_callbacks_init(struct zsv_ext_callbacks
     e->ext_sheet_open_file = zsvsheet_open_file;
     e->ext_sheet_register_proc = zsvsheet_register_proc;
     e->ext_sheet_register_proc_key_binding = zsvsheet_register_proc_key_binding;
+    e->ext_sheet_push_transformation = zsvsheet_push_transformation;
+    e->ext_sheet_transformation_writer = zsvsheet_transformation_writer;
+    e->ext_sheet_transformation_parser = zsvsheet_transformation_parser;
+    e->ext_sheet_transformation_filename = zsvsheet_transformation_filename;
+    e->ext_sheet_transformation_user_context = zsvsheet_transformation_user_context;
 #endif
   }
   return e;

app/ext_example/Makefile

@@ -115,7 +115,7 @@ ${BUILD_DIR}/objs/utils/%.o:
 
 TESTS=test-1 test-2 test-3 test-4 test-5 test-thirdparty
 ifeq ($(ZSVSHEET_BUILD),1)
-  TESTS+=test-sheet-extension
+  TESTS+=test-sheet-extension-1 test-sheet-extension-2
 endif
 
 test: ${TESTS}
@@ -147,7 +147,7 @@ test-3: test-%: ${CLI} ${TARGET}
 
 TMP_DIR=/tmp
 TMUX_TERM=xterm-256color
-test-sheet-extension: ../test/worldcitiespop_mil.csv
+test-sheet-extension-1: ${CLI} ${TARGET} ../test/worldcitiespop_mil.csv
 	@${TEST_INIT}
 	@rm -f ${TMP_DIR}/zsvext-$@.out tmux-*.log
 	@tmux kill-session -t $@ || echo 'No tmux session to kill'
@@ -164,6 +164,26 @@ test-sheet-extension: ../test/worldcitiespop_mil.csv
 	cmp ${TMP_DIR}/$@.out test/expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && \
 	if [ -f ${TMP_DIR}/$@.out ]; then cat ${TMP_DIR}/$@.out; fi && ${TEST_FAIL}))
 
+test-sheet-extension-2: ${CLI} ${TARGET}
+	@${TEST_INIT}
+	@rm -f ${TMP_DIR}/zsvext-$@.out tmux-*.log
+	@tmux kill-session -t $@ || echo 'No tmux session to kill'
+	@${RUN_CLI} unregister my 2>/dev/null 1>/dev/null || [ 1==1 ]
+	@${RUN_CLI} register my 2>&1 | grep -v [.]so || [ 1==1 ]
+	@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
+	@(ZSV_CONFIG_DIR=/tmp tmux -v new-session -x 120 -y 5 -d -s $@ && \
+	sleep 0.5 && \
+	tmux send-keys -t $@ "${CLI} sheet -d 3 ../../data/test/mixed-line-endings.csv" ENTER && \
+	sleep 0.5 && \
+	tmux send-keys -t $@ T Enter && \
+	sleep 0.5 && \
+	tmux send-keys -t $@ G && \
+	sleep 0.5 && \
+	tmux capture-pane -t $@ -p > ${TMP_DIR}/$@.out && \
+	tmux kill-session -t $@ && \
+	cmp ${TMP_DIR}/$@.out test/expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && \
+	if [ -f ${TMP_DIR}/$@.out ]; then cat ${TMP_DIR}/$@.out; fi && ${TEST_FAIL}))
+
 test-4: test-%: ${CLI} ${TARGET}
 	@${TEST_INIT}
 	@${RUN_CLI} unregister my 2>/dev/null 1>/dev/null || [ 1==1 ]

app/ext_example/my_extension.c

@@ -98,7 +98,7 @@ zsvsheet_status my_test_command_handler(zsvsheet_proc_context_t ctx) {
     // print a list of open buffers and filenames
     for (zsvsheet_buffer_t buff = zsv_cb.ext_sheet_buffer_current(ctx); buff;
          buff = zsv_cb.ext_sheet_buffer_prior(buff), i--) {
-      const char *buff_filename = zsv_cb.ext_sheet_buffer_filename(buff);
+      const char *buff_filename = zsv_cb.ext_sheet_buffer_data_filename(buff);
       if (buff_filename)
         fprintf(f, "%i,%s\n", i, buff_filename); // assumes no need for quoting or escaping buff_filename...
     }
@@ -107,6 +107,45 @@ zsvsheet_status my_test_command_handler(zsvsheet_proc_context_t ctx) {
   }
   return zsvsheet_status_ok;
 }
+
+struct transformation_context {
+  zsvsheet_proc_context_t proc_ctx;
+  size_t col_count;
+  size_t row_count;
+};
+
+// Similar to a regular ZSV row handler used in ext_parse_all
+void my_transformation_row_handler(zsvsheet_transformation trn) {
+  struct transformation_context *priv = zsv_cb.ext_sheet_transformation_user_context(trn);
+  zsv_parser parser = zsv_cb.ext_sheet_transformation_parser(trn);
+  zsv_csv_writer writer = zsv_cb.ext_sheet_transformation_writer(trn);
+
+  size_t j = zsv_cb.cell_count(parser);
+  for (size_t i = 0; i < j; i++) {
+    struct zsv_cell c = zsv_cb.get_cell(parser, i);
+    zsv_writer_cell(writer, i == 0, c.str, c.len, c.quoted);
+  }
+
+  priv->col_count += j;
+
+  if (!priv->row_count)
+    zsv_writer_cell_s(writer, 0, (const unsigned char *)"Column count", 0);
+  else
+    zsv_writer_cell_zu(writer, 0, priv->col_count);
+
+  priv->row_count++;
+}
+
+zsvsheet_status my_transformation_command_handler(zsvsheet_proc_context_t ctx) {
+  struct zsvsheet_buffer_transformation_opts opts = {
+    // Gets freed automatically
+    .user_context = calloc(1, sizeof(struct transformation_context)),
+    .row_handler = my_transformation_row_handler,
+    .on_done = NULL,
+  };
+
+  return zsv_cb.ext_sheet_push_transformation(ctx, opts);
+}
 #endif
 
 /**
@@ -141,6 +180,11 @@ enum zsv_ext_status zsv_ext_init(struct zsv_ext_callbacks *cb, zsv_execution_con
   if (proc_id < 0)
     return zsv_ext_status_error;
   zsv_cb.ext_sheet_register_proc_key_binding('t', proc_id);
+
+  proc_id = zsv_cb.ext_sheet_register_proc("my-transformation", "my transformation", my_transformation_command_handler);
+  if (proc_id < 0)
+    return zsv_ext_status_error;
+  zsv_cb.ext_sheet_register_proc_key_binding('T', proc_id);
 #endif
   return zsv_ext_status_ok;
 }

app/ext_example/test/expected/.gitignore

@@ -0,0 +1 @@
+!*.out

app/ext_example/test/expected/test-sheet-extension-2.out

@@ -0,0 +1,5 @@
+Row #                   HA1 HA2 HA3              HB1  HB2  HB3           HC1  HC2  HC3          Column count
+4094                    A4094                    B4094                   C4094                  12285
+4095                    A4095                    B4095                   C4095                  12288
+4096                    A4096                    B4096                   C4096                  12291
+? for help 4096

app/sheet.c

@@ -193,6 +193,7 @@ static void zsvsheet_priv_set_status(const struct zsvsheet_display_dimensions *d
 #include "sheet/handlers.c"
 #include "sheet/file.c"
 #include "sheet/usage.c"
+#include "sheet/transformation.c"
 
 struct zsvsheet_key_data *zsvsheet_key_handlers = NULL;
 struct zsvsheet_key_data **zsvsheet_next_key_handler = &zsvsheet_key_handlers;
@@ -381,7 +382,7 @@ static zsvsheet_status zsvsheet_open_file_handler(struct zsvsheet_proc_context *
     goto no_input;
 
   const char *opts_used = NULL;
-  if ((err = zsvsheet_ui_buffer_open_file(prompt_buffer, NULL, NULL, state->custom_prop_handler, opts_used,
+  if ((err = zsvsheet_ui_buffer_open_file(prompt_buffer, NULL, state->custom_prop_handler, opts_used,
                                           di->ui_buffers.base, di->ui_buffers.current))) {
     if (err > 0)
       zsvsheet_priv_set_status(di->dimensions, 1, "%s: %s", prompt_buffer, strerror(err));
@@ -395,42 +396,27 @@ static zsvsheet_status zsvsheet_open_file_handler(struct zsvsheet_proc_context *
   return zsvsheet_status_ok;
 }
 
+#include "sheet/filter.c"
+
 static zsvsheet_status zsvsheet_filter_handler(struct zsvsheet_proc_context *ctx) {
   char prompt_buffer[256] = {0};
   struct zsvsheet_builtin_proc_state *state = (struct zsvsheet_builtin_proc_state *)ctx->subcommand_context;
   struct zsvsheet_display_info *di = &state->display_info;
   struct zsvsheet_ui_buffer *current_ui_buffer = *state->display_info.ui_buffers.current;
   int prompt_footer_row = (int)(di->dimensions->rows - di->dimensions->footer_span);
-  int err;
+  struct zsvsheet_buffer_info binfo = zsvsheet_buffer_get_info(current_ui_buffer);
 
-  if (!current_ui_buffer->filename)
+  if (binfo.transform_started && !binfo.transform_done)
+    return zsvsheet_status_busy;
+
+  if (!zsvsheet_buffer_data_filename(current_ui_buffer))
     goto out;
 
   get_subcommand("Filter", prompt_buffer, sizeof(prompt_buffer), prompt_footer_row);
   if (*prompt_buffer == '\0')
     goto out;
 
-  const char *data_filename = zsvsheet_buffer_data_filename(current_ui_buffer);
-  char is_filtered_file = !(data_filename == current_ui_buffer->filename);
-  struct zsv_opts *zsv_opts = is_filtered_file ? NULL : &current_ui_buffer->zsv_opts;
-
-  if ((err = zsvsheet_ui_buffer_open_file(data_filename, zsv_opts, prompt_buffer, state->custom_prop_handler, NULL,
-                                          di->ui_buffers.base, di->ui_buffers.current))) {
-    if (err > 0)
-      zsvsheet_priv_set_status(di->dimensions, 1, "%s: %s", current_ui_buffer->filename, strerror(err));
-    else if (err < 0)
-      zsvsheet_priv_set_status(di->dimensions, 1, "Unexpected error");
-    else
-      zsvsheet_priv_set_status(di->dimensions, 1, "Not found: %s", prompt_buffer);
-    return zsvsheet_status_ignore;
-  }
-
-  struct zsvsheet_ui_buffer *new_ui_buffer = *state->display_info.ui_buffers.current;
-  if (is_filtered_file) {
-    // TO DO: move this into zsvsheet_ui_buffer_open_file()
-    free(new_ui_buffer->filename);
-    new_ui_buffer->filename = strdup(current_ui_buffer->filename);
-  }
+  return zsvsheet_filter_file(ctx, prompt_buffer);
 out:
   return zsvsheet_status_ok;
 }
@@ -447,9 +433,11 @@ static zsvsheet_status zsvsheet_help_handler(struct zsvsheet_proc_context *ctx)
   struct zsvsheet_ui_buffer_opts uibopts = {
     .buff_opts = &bopts,
     .filename = NULL,
+    .data_filename = NULL,
     .no_rownum_col_offset = 1,
+    .transform = 0,
   };
-  struct zsvsheet_ui_buffer *uib;
+  struct zsvsheet_ui_buffer *uib = NULL;
   zsvsheet_screen_buffer_t buffer;
   enum zsvsheet_priv_status pstat;
   enum zsvsheet_status stat = zsvsheet_status_error;
@@ -505,7 +493,10 @@ static zsvsheet_status zsvsheet_help_handler(struct zsvsheet_proc_context *ctx)
   goto out;
 
 free_buffer:
-  zsvsheet_screen_buffer_delete(buffer);
+  if (uib)
+    zsvsheet_ui_buffer_delete(uib);
+  else
+    zsvsheet_screen_buffer_delete(buffer);
 out:
   return stat;
 }
@@ -644,7 +635,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
 
   if (argc > 1) {
     const char *filename = argv[1];
-    if ((err = zsvsheet_ui_buffer_open_file(filename, optsp, NULL, custom_prop_handler, opts_used, &ui_buffers,
+    if ((err = zsvsheet_ui_buffer_open_file(filename, optsp, custom_prop_handler, opts_used, &ui_buffers,
                                             &current_ui_buffer))) {
       if (err > 0)
         perror(filename);
@@ -689,20 +680,9 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
   halfdelay(2); // now ncurses getch() will fire every 2-tenths of a second so we can check for status update
 
   while (true) {
-    char *status_msg = NULL;
     ch = getch();
 
     handler_state.display_info.update_buffer = false;
-
-    pthread_mutex_lock(&current_ui_buffer->mutex);
-    status_msg = current_ui_buffer->status;
-    if (current_ui_buffer->index_ready &&
-        current_ui_buffer->dimensions.row_count != current_ui_buffer->index->row_count + 1) {
-      current_ui_buffer->dimensions.row_count = current_ui_buffer->index->row_count + 1;
-      handler_state.display_info.update_buffer = true;
-    }
-    pthread_mutex_unlock(&current_ui_buffer->mutex);
-
     zsvsheet_priv_set_status(&display_dims, 1, "");
 
     if (ch != ERR) {
@@ -713,19 +693,30 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
         continue;
     }
 
-    if (handler_state.display_info.update_buffer && current_ui_buffer->filename) {
+    struct zsvsheet_ui_buffer *ub = current_ui_buffer;
+    pthread_mutex_lock(&ub->mutex);
+    if (ub->status)
+      zsvsheet_priv_set_status(&display_dims, 1, ub->status);
+    if (ub->transform_progressed) {
+      handler_state.display_info.update_buffer = true;
+      ub->transform_progressed = 0;
+    }
+    if (ub->index_ready && ub->dimensions.row_count != ub->index->row_count + 1) {
+      ub->dimensions.row_count = ub->index->row_count + 1;
+      handler_state.display_info.update_buffer = true;
+    }
+    pthread_mutex_unlock(&ub->mutex);
+
+    if (handler_state.display_info.update_buffer && zsvsheet_buffer_data_filename(ub)) {
       struct zsvsheet_opts zsvsheet_opts = {0};
-      if (read_data(&current_ui_buffer, NULL, current_ui_buffer->input_offset.row, current_ui_buffer->input_offset.col,
-                    header_span, &zsvsheet_opts, custom_prop_handler)) {
+      if (read_data(&ub, NULL, current_ui_buffer->input_offset.row, current_ui_buffer->input_offset.col, header_span,
+                    &zsvsheet_opts, custom_prop_handler)) {
         zsvsheet_priv_set_status(&display_dims, 1, "Unexpected error!"); // to do: better error message
         continue;
       }
     }
 
-    if (status_msg)
-      zsvsheet_priv_set_status(&display_dims, 1, status_msg);
-
-    display_buffer_subtable(current_ui_buffer, header_span, &display_dims);
+    display_buffer_subtable(ub, header_span, &display_dims);
   }
 
   endwin();

app/sheet/file.c

@@ -1,30 +1,42 @@
-int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zsv_optsp, const char *row_filter,
-                                 struct zsv_prop_handler *custom_prop_handler, const char *opts_used,
-                                 struct zsvsheet_ui_buffer **ui_buffer_stack_bottom,
-                                 struct zsvsheet_ui_buffer **ui_buffer_stack_top) {
-  struct zsvsheet_screen_buffer_opts bopts = {0};
-  struct zsvsheet_ui_buffer_opts uibopts = {0};
-  uibopts.filename = filename;
-  if (zsv_optsp)
-    uibopts.zsv_opts = *zsv_optsp;
-  uibopts.opts_used = opts_used;
-  uibopts.buff_opts = &bopts;
+int zsvsheet_ui_buffer_open_file_opts(struct zsvsheet_ui_buffer_opts *uibopts,
+                                      struct zsv_prop_handler *custom_prop_handler,
+                                      struct zsvsheet_ui_buffer **ui_buffer_stack_bottom,
+                                      struct zsvsheet_ui_buffer **ui_buffer_stack_top) {
   struct zsvsheet_opts zsvsheet_opts = {0};
-  int err = 0;
+  struct zsvsheet_screen_buffer_opts bopts = {0};
   struct zsvsheet_ui_buffer *tmp_ui_buffer = NULL;
-  uibopts.row_filter = row_filter;
-  if ((err = read_data(&tmp_ui_buffer, &uibopts, 0, 0, 0, &zsvsheet_opts, custom_prop_handler)) != 0 ||
-      !tmp_ui_buffer || !tmp_ui_buffer->buff_used_rows) {
+
+  if (!uibopts->buff_opts)
+    uibopts->buff_opts = &bopts;
+
+  int err = 0;
+  if ((err = read_data(&tmp_ui_buffer, uibopts, 0, 0, 0, &zsvsheet_opts, custom_prop_handler)) != 0 || !tmp_ui_buffer ||
+      !tmp_ui_buffer->buff_used_rows) {
     zsvsheet_ui_buffer_delete(tmp_ui_buffer);
     if (err)
       return err;
     return -1;
   }
   tmp_ui_buffer->cursor_row = 1; // first row is header
   zsvsheet_ui_buffer_push(ui_buffer_stack_bottom, ui_buffer_stack_top, tmp_ui_buffer);
+
   return 0;
 }
 
+int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zsv_optsp,
+                                 struct zsv_prop_handler *custom_prop_handler, const char *opts_used,
+                                 struct zsvsheet_ui_buffer **ui_buffer_stack_bottom,
+                                 struct zsvsheet_ui_buffer **ui_buffer_stack_top) {
+  struct zsvsheet_ui_buffer_opts uibopts = {0};
+
+  uibopts.filename = filename;
+  if (zsv_optsp)
+    uibopts.zsv_opts = *zsv_optsp;
+  uibopts.opts_used = opts_used;
+
+  return zsvsheet_ui_buffer_open_file_opts(&uibopts, custom_prop_handler, ui_buffer_stack_bottom, ui_buffer_stack_top);
+}
+
 /****** API ******/
 
 /**
@@ -35,8 +47,18 @@ zsvsheet_status zsvsheet_open_file(struct zsvsheet_proc_context *ctx, const char
   struct zsvsheet_display_info *di = &state->display_info;
   if (!di || !di->ui_buffers.base || !di->ui_buffers.current)
     return zsvsheet_status_error;
-  int err =
-    zsvsheet_ui_buffer_open_file(filepath, zopts, NULL, NULL, NULL, di->ui_buffers.base, di->ui_buffers.current);
+  int err = zsvsheet_ui_buffer_open_file(filepath, zopts, NULL, NULL, di->ui_buffers.base, di->ui_buffers.current);
+  if (err)
+    return zsvsheet_status_error;
+  return zsvsheet_status_ok;
+}
+
+zsvsheet_status zsvsheet_open_file_opts(struct zsvsheet_proc_context *ctx, struct zsvsheet_ui_buffer_opts *opts) {
+  struct zsvsheet_builtin_proc_state *state = (struct zsvsheet_builtin_proc_state *)ctx->subcommand_context;
+  struct zsvsheet_display_info *di = &state->display_info;
+  if (!di || !di->ui_buffers.base || !di->ui_buffers.current)
+    return zsvsheet_status_error;
+  int err = zsvsheet_ui_buffer_open_file_opts(opts, NULL, di->ui_buffers.base, di->ui_buffers.current);
   if (err)
     return zsvsheet_status_error;
   return zsvsheet_status_ok;

app/sheet/file.h

@@ -1,9 +1,11 @@
 #ifndef ZSVSHEET_FILE_H
 #define ZSVSHEET_FILE_H
 
-int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zsv_optsp, const char *row_filter,
+int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zsv_optsp,
                                  struct zsv_prop_handler *custom_prop_handler, const char *opts_used,
                                  struct zsvsheet_ui_buffer **ui_buffer_stack_bottom,
                                  struct zsvsheet_ui_buffer **ui_buffer_stack_top);
 
+zsvsheet_status zsvsheet_open_file_opts(struct zsvsheet_proc_context *ctx, struct zsvsheet_ui_buffer_opts *opts);
+
 #endif