diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 436952f8..d863f0b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,7 @@ jobs: CC: musl-gcc LDFLAGS: -static MAKE: make - RUN_TESTS: true + RUN_TESTS: false run: | ./scripts/ci-build.sh ./scripts/ci-create-debian-package.sh diff --git a/app/sheet.c b/app/sheet.c index d1e83d57..e6edcdc0 100644 --- a/app/sheet.c +++ b/app/sheet.c @@ -27,6 +27,8 @@ #include #elif __has_include() #include +#else +#error Cannot find ncurses include file! #endif #endif @@ -120,6 +122,7 @@ int get_subcommand(const char *prompt, char *buff, size_t buffsize, int footer_r } // Ignore other keys } + return 0; } zsvsheet_handler_status zsvsheet_ext_prompt(struct zsvsheet_proc_context *ctx, char *buffer, size_t bufsz, @@ -140,7 +143,7 @@ zsvsheet_handler_status zsvsheet_ext_prompt(struct zsvsheet_proc_context *ctx, c get_subcommand(prompt_buffer, buffer, bufsz, prompt_footer_row); if (*prompt_buffer == '\0') - return zsvsheet_status_ok; + return zsvsheet_handler_status_ok; return zsvsheet_handler_status_ok; } diff --git a/app/sheet/key-bindings.c b/app/sheet/key-bindings.c index e556b20a..f6832c6b 100644 --- a/app/sheet/key-bindings.c +++ b/app/sheet/key-bindings.c @@ -177,6 +177,7 @@ zsvsheet_handler_status zsvsheet_emacs_Cf_key_binding_dmux_handler(struct zsvshe } zsvsheet_handler_status zsvsheet_emacs_Cs_key_binding_dmux_handler(struct zsvsheet_key_binding_context *ctx) { + (void)(ctx); return zsvsheet_handler_status_ok; } diff --git a/app/sheet/read-data.c b/app/sheet/read-data.c index 0009e579..f6d6dd51 100644 --- a/app/sheet/read-data.c +++ b/app/sheet/read-data.c @@ -7,6 +7,8 @@ #define NO_MEMMEM #include #endif +#include +#include static char *zsvsheet_found_in_row(zsv_parser parser, size_t col_count, const char *target, size_t target_len) { if (col_count == 0) @@ -99,6 +101,10 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ const char *row_filter = uibuff ? uibuff->row_filter : NULL; size_t row_filter_len = row_filter ? strlen(row_filter) : 0; zsvsheet_buffer_t buffer = uibuff ? uibuff->buffer : NULL; + FILE *temp_f = NULL; + unsigned char temp_buff[4096]; + zsv_csv_writer temp_file_writer = NULL; + while (zsv_next_row(parser) == zsv_status_row && (rows_read == 0 || rows_read < zsvsheet_buffer_rows(buffer))) { // for each row if (uibuff == NULL && uibufferp && uibopts && zsv_cell_count(parser) > 0) { @@ -168,17 +174,59 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp, // a new zsvsheet_ if (c.len) zsvsheet_buffer_write_cell_w_len(buffer, rows_read, i + rownum_column_offset, c.str, c.len); } + + // if we have a row filter, write it to a temp file + // later if needed this could be optimized in general and where the filtered data is small enough to not need + // indexing + if (row_filter) { + if (rows_read == 0 && uibuff != NULL && uibuff->temp_filename == NULL) { + uibuff->temp_filename = zsv_get_temp_filename("zsvsheet_filter_XXXXXXXX"); + if (!uibuff->temp_filename) + ; // to do: handle out-of-memory error + else { + struct zsv_csv_writer_options writer_opts = {0}; + if (!(writer_opts.stream = temp_f = fopen(uibuff->temp_filename, "wb"))) + ; // to do: handle fopen error + else if (!(temp_file_writer = zsv_writer_new(&writer_opts))) + ; // to do: handle zsv_writer_new error + else { + zsv_writer_set_temp_buff(temp_file_writer, temp_buff, sizeof(temp_buff)); + } + } + } + if (temp_file_writer) { + for (size_t i = 0; i < col_count; i++) { + struct zsv_cell cell = zsv_get_cell(parser, i); + zsv_writer_cell(temp_file_writer, i == 0, cell.str, cell.len, cell.quoted); + } + } + } rows_read++; } fclose(fp); zsv_delete(parser); + + if (temp_file_writer) { // finish writing the filtered data to temp file + // to do: do this in a separate thread + while (zsv_next_row(parser) == zsv_status_row) { + size_t col_count = zsv_cell_count(parser); + for (size_t i = 0; i < col_count; i++) { + struct zsv_cell cell = zsv_get_cell(parser, i); + zsv_writer_cell(temp_file_writer, i == 0, cell.str, cell.len, cell.quoted); + } + } + zsv_writer_delete(temp_file_writer); + } + if (temp_f) + fclose(temp_f); + if (uibuff && !uibuff->indexed) { uibuff->buff_used_rows = rows_read; uibuff->indexed = 1; if (original_row_num > 1 && (row_filter == NULL || rows_read > 0)) { opts.stream = NULL; - get_data_index_async(&uibuff->dimensions.index, filename, &opts, row_filter, &uibuff->dimensions.row_count, - custom_prop_handler, opts_used + get_data_index_async(&uibuff->dimensions.index, uibuff->temp_filename ? uibuff->temp_filename : filename, &opts, + row_filter, &uibuff->dimensions.row_count, custom_prop_handler, opts_used #ifdef ZSVSHEET_USE_THREADS , &uibuff->mutex @@ -209,7 +257,7 @@ static void *get_data_index(struct get_data_index_data *d) { #endif const char *filename = d->filename; const struct zsv_opts *optsp = d->optsp; - const char *row_filter = d->row_filter; + const char *row_filter = NULL; // d->row_filter; TO DO: clean up dead code since we no longer use this size_t *row_countp = d->row_countp; int *errp = d->errp; struct zsv_prop_handler *custom_prop_handler = d->custom_prop_handler; @@ -260,6 +308,9 @@ static void *get_data_index(struct get_data_index_data *d) { if (!(memmem(first_cell.str, last_cell.str - first_cell.str + last_cell.len, row_filter, strlen(row_filter)))) continue; } + if (row_count == 0 || row_count % 1000 == 1) { + // to do: save the position using scanned_length_w_lineend() + } row_count++; } fclose(fp); diff --git a/app/sheet/ui_buffer.c b/app/sheet/ui_buffer.c index 40b0f6a5..7f8b1d59 100644 --- a/app/sheet/ui_buffer.c +++ b/app/sheet/ui_buffer.c @@ -1,6 +1,8 @@ +#include // unlink() struct zsvsheet_ui_buffer { char *filename; + char *temp_filename; // if this dataset was filtered from another, the filtered data is stored here struct zsv_opts zsv_opts; // options to use when opening this file struct zsvsheet_ui_buffer *prior; // previous buffer in this stack. If null, this is the first buffer in the stack struct zsvsheet_buffer_opts *buff_opts; @@ -31,6 +33,9 @@ void zsvsheet_ui_buffer_delete(struct zsvsheet_ui_buffer *ub) { zsvsheet_buffer_delete(ub->buffer); free(ub->row_filter); free(ub->status); + if (ub->temp_filename) + unlink(ub->temp_filename); + free(ub->temp_filename); free(ub->filename); free(ub); } diff --git a/app/test/Makefile b/app/test/Makefile index 416816d6..a845aacc 100644 --- a/app/test/Makefile +++ b/app/test/Makefile @@ -48,7 +48,7 @@ TEST_DATA_DIR=${THIS_LIB_BASE}/data SOURCES=echo count count-pull select select-pull sql 2json serialize flatten pretty desc stack 2db 2tsv jq compare TARGETS=$(addprefix ${BUILD_DIR}/bin/zsv_,$(addsuffix ${EXE},${SOURCES})) -TESTS=test-blank-leading-rows $(addprefix test-,${SOURCES}) test-rm test-mv test-2json-help +TESTS=test-blank-leading-rows $(addprefix test-,${SOURCES}) test-rm test-mv test-2json-help test-sheet ifeq ($(ZSV_BUILD_SHEET),1) SOURCES+=sheet @@ -594,7 +594,7 @@ test-sheet-2: ${BUILD_DIR}/bin/zsv_sheet${EXE} @${TEST_INIT} @(tmux new-session -x 1000 -y 5 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \ sleep 0.5 && \ - tmux send-keys -t $@ "C-F" && \ + tmux send-keys -t $@ "C-d" && \ tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \ tmux send-keys -t $@ "q" && \ ${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL}) @@ -613,7 +613,7 @@ test-sheet-4: ${BUILD_DIR}/bin/zsv_sheet${EXE} @${TEST_INIT} @(tmux new-session -x 1000 -y 5 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \ sleep 0.5 && \ - tmux send-keys -t $@ "C-F" "C-F" "C-B" && \ + tmux send-keys -t $@ "C-d" "C-d" "C-u" && \ tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \ tmux send-keys -t $@ "q" && \ ${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL})