From 6f31d612e8062aa610c5d4d8c949c8ba3df604f8 Mon Sep 17 00:00:00 2001
From: Richard Palethorpe <io@richiejp.com>
Date: Wed, 11 Dec 2024 15:14:40 +0000
Subject: [PATCH 1/8] app: Move sql_internal.c into its own object

It needs to be included in the sheet command for the pivot builtin and
in the sql command.
---
 app/Makefile | 9 ++++++++-
 app/sql.c    | 4 ++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/app/Makefile b/app/Makefile
index a4c3b312..4bfd8282 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -285,6 +285,8 @@ SQLITE_SRC=${THIS_MAKEFILE_DIR}/external/sqlite3/sqlite3*.c
 SQLITE_EXT=${BUILD_DIR}-external/sqlite3/sqlite3_and_csv_vtab.o
 SQLITE_EXT_INCLUDE=-I${THIS_MAKEFILE_DIR}/external/sqlite3
 
+SQL_INTERNAL_OBJECT=${CLI_OBJ_PFX}sql_internal.o
+
 # everything uses prop, which in turn uses yajl and jq and json and sqlite3
 OBJECTS+= ${YAJL_OBJ} ${YAJL_HELPER_OBJ} ${BUILD_DIR}/objs/utils/json.o ${SQLITE_EXT}
 MORE_SOURCE+= ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -I${JQ_INCLUDE_DIR} ${SQLITE_EXT_INCLUDE}
@@ -398,12 +400,16 @@ ${INIH_OBJECT}: ${INIH_SRC}/ini.c
 	@mkdir -p `dirname "$@"`
 	${CC} ${CFLAGS} -I${INIH_INCLUDE} -DINI_HANDLER_LINENO=1 -DINI_CALL_HANDLER_ON_NEW_SECTION=1 -c $< -o $@
 
+${SQL_INTERNAL_OBJECT}: ${CLI_OBJ_PFX}%.o: %.c %.h
+	@mkdir -p `dirname "$@"`
+	${CC} ${CFLAGS} -I${INCLUDE_DIR} -c -o $@ $<
+
 ${CLI_APP_OBJECT} : cli_ini.c builtin/*.c ${JQ_LIB}
 ${CLI_APP_OBJECT} ${CLI_OBJECTS}: ${CLI_OBJ_PFX}%.o: %.c ${UTF8PROC_SRC}/utf8proc.c # ${MORE_OBJECTS}
 	@mkdir -p `dirname "$@"`
 	${CC} ${CFLAGS} -DVERSION=\"${VERSION}\" -DZSV_CLI ${CLI_INCLUDE} -I${THIS_MAKEFILE_DIR}/external/sglib -I${INCLUDE_DIR} -I${UTF8PROC_INCLUDE} -c $< -o $@ ${MORE_SOURCE}
 
-${CLI}: cli_internal.c.in cli_internal.h cli_internal.h.in ${CLI_APP_OBJECT} ${CLI_OBJECTS} ${OBJECTS} ${UTF8PROC_OBJECT} cli_ini.c ${INIH_OBJECT} ${LIBZSV_INSTALL} ${MORE_OBJECTS}
+${CLI}: cli_internal.c.in cli_internal.h cli_internal.h.in ${CLI_APP_OBJECT} ${CLI_OBJECTS} ${OBJECTS} ${UTF8PROC_OBJECT} cli_ini.c ${INIH_OBJECT} ${LIBZSV_INSTALL} ${MORE_OBJECTS} ${SQL_INTERNAL_OBJECT}
 	@mkdir -p `dirname "$@"`
 	${CC} ${CFLAGS} ${CFLAGS_EXE} -I${INCLUDE_DIR} -o $@ ${CLI_APP_OBJECT} ${CLI_OBJECTS} ${OBJECTS} ${UTF8PROC_OBJECT} ${INIH_OBJECT} -L${LIBDIR} ${LIBZSV_L} ${LDFLAGS} ${LDFLAGS_OPT} ${MORE_OBJECTS} ${MORE_SOURCE} ${MORE_LIBS} ${STATIC_LIB_FLAGS}
 	@echo Built $@
@@ -468,6 +474,7 @@ ${CLI} ${STANDALONE_PFX}2json${EXE}: MORE_OBJECTS+= ${BUILD_DIR}/objs/utils/db.o
 # pretty uses termcap
 ${CLI} ${STANDALONE_PFX}pretty${EXE}: MORE_LIBS+=${LDFLAGS_TERMCAP}
 
+${CLI} ${STANDALONE_PFX}sheet${EXE} ${STANDALONE_PFX}sql${EXE}: MORE_OBJECTS+=${SQL_INTERNAL_OBJECT}
 
 ${STANDALONE_PFX}%${EXE}: %.c ${OBJECTS} ${MORE_OBJECTS} ${LIBZSV_INSTALL} ${UTF8PROC_OBJECT}
 	@mkdir -p `dirname "$@"`
diff --git a/app/sql.c b/app/sql.c
index 816758bd..02ced918 100644
--- a/app/sql.c
+++ b/app/sql.c
@@ -19,6 +19,8 @@
 #include <zsv/utils/writer.h>
 #include <zsv/utils/file.h>
 #include <zsv/utils/string.h>
+#include <zsv/utils/sql.h>
+#include "sql_internal.h"
 
 #include <unistd.h> // unlink
 
@@ -109,8 +111,6 @@ static char is_select_sql(const char *s) {
                                                         (const unsigned char *)s, strlen("select "));
 }
 
-#include "sql_internal.c"
-
 int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *opts,
                                struct zsv_prop_handler *custom_prop_handler) {
   /**

From b52bdfbfe5f832a172e7f74faffda04b04d1438c Mon Sep 17 00:00:00 2001
From: Richard Palethorpe <io@richiejp.com>
Date: Mon, 16 Dec 2024 12:49:35 +0000
Subject: [PATCH 2/8] sheet: Mark sources included in app/sheet.c as
 dependencies in Make

---
 app/Makefile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/app/Makefile b/app/Makefile
index 4bfd8282..f18ee9ff 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -198,12 +198,14 @@ ZSV=$(BINDIR)/zsv${EXE}
 SOURCES=echo paste count count-pull select select-pull 2tsv 2json serialize flatten pretty stack desc sql 2db compare prop rm mv jq overwrite
 CLI_SOURCES=echo select desc count paste 2tsv pretty sql flatten 2json serialize stack 2db compare prop rm mv jq overwrite
 
+SHEET_INCLUDES=
 ifeq ($(ZSVSHEET_BUILD),1)
   SOURCES+=sheet
   CLI_SOURCES+=sheet
   CFLAGS+=-DZSVSHEET_BUILD
   CFLAGS+=${CFLAGS_NCURSES}
   LDFLAGS+=${LDFLAGS_NCURSES}
+  SHEET_INCLUDES=$(wildcard sheet/*.c sheet/*.h)
 endif
 
 CFLAGS+= -DUSE_JQ
@@ -404,6 +406,8 @@ ${SQL_INTERNAL_OBJECT}: ${CLI_OBJ_PFX}%.o: %.c %.h
 	@mkdir -p `dirname "$@"`
 	${CC} ${CFLAGS} -I${INCLUDE_DIR} -c -o $@ $<
 
+${CLI_OBJ_PFX}sheet.o: ${SHEET_INCLUDES}
+
 ${CLI_APP_OBJECT} : cli_ini.c builtin/*.c ${JQ_LIB}
 ${CLI_APP_OBJECT} ${CLI_OBJECTS}: ${CLI_OBJ_PFX}%.o: %.c ${UTF8PROC_SRC}/utf8proc.c # ${MORE_OBJECTS}
 	@mkdir -p `dirname "$@"`
@@ -474,6 +478,9 @@ ${CLI} ${STANDALONE_PFX}2json${EXE}: MORE_OBJECTS+= ${BUILD_DIR}/objs/utils/db.o
 # pretty uses termcap
 ${CLI} ${STANDALONE_PFX}pretty${EXE}: MORE_LIBS+=${LDFLAGS_TERMCAP}
 
+${STANDALONE_PFX}sheet${EXE}: ${SHEET_INCLUDES}
+
+${CLI} ${STANDALONE_PFX}sheet${EXE} ${STANDALONE_PFX}sql${EXE}: ${SQL_INTERNAL_OBJECT}
 ${CLI} ${STANDALONE_PFX}sheet${EXE} ${STANDALONE_PFX}sql${EXE}: MORE_OBJECTS+=${SQL_INTERNAL_OBJECT}
 
 ${STANDALONE_PFX}%${EXE}: %.c ${OBJECTS} ${MORE_OBJECTS} ${LIBZSV_INSTALL} ${UTF8PROC_OBJECT}

From 120c69123025db1c482012eec4f481071bdf4e7d Mon Sep 17 00:00:00 2001
From: Richard Palethorpe <io@richiejp.com>
Date: Mon, 16 Dec 2024 12:46:52 +0000
Subject: [PATCH 3/8] sheet: Return the correct column number under the cursor

---
 app/sheet/handlers.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/sheet/handlers.c b/app/sheet/handlers.c
index 4b6306c8..60f34f67 100644
--- a/app/sheet/handlers.c
+++ b/app/sheet/handlers.c
@@ -112,7 +112,7 @@ zsvsheet_status zsvsheet_buffer_get_selected_cell(zsvsheet_buffer_t h, struct zs
   if (!uib)
     return zsvsheet_status_error;
   rc->row = uib->cursor_row + uib->input_offset.row + uib->buff_offset.row;
-  rc->col = uib->cursor_col + uib->input_offset.col + uib->buff_offset.row;
+  rc->col = uib->cursor_col + uib->input_offset.col + uib->buff_offset.col;
   return zsvsheet_status_ok;
 }
 

From 5c0bfd56209fe7c831c3d0b6b322d791720fdbae Mon Sep 17 00:00:00 2001
From: Richard Palethorpe <io@richiejp.com>
Date: Tue, 10 Dec 2024 14:13:02 +0000
Subject: [PATCH 4/8] sheet: Add pivot builtin command

The user can either pivot on the column under the cursor with 'v' or
enter an SQL group-by expression with V
---
 app/sheet.c                                   |   5 +-
 app/sheet/file.h                              |   1 +
 app/sheet/key-bindings.c                      |   2 +
 app/sheet/pivot.c                             | 317 ++++++++++++++++++
 app/sheet/procedure.h                         |   2 +
 app/sheet/ui_buffer.c                         |   6 +
 app/test/Makefile                             |  39 ++-
 .../expected/test-sheet-pivot-1-drilldown.out |  25 ++
 .../expected/test-sheet-pivot-1-groups.out    |  25 ++
 .../expected/test-sheet-pivot-1-indexed.out   |  25 ++
 app/test/expected/test-sheet-pivot-1.out      |  25 ++
 .../expected/test-sheet-pivot-2-groups.out    |  25 ++
 .../expected/test-sheet-pivot-2-indexed.out   |  25 ++
 app/test/expected/test-sheet-pivot-2.out      |  25 ++
 .../expected/test-sheet-pivot-3-groups.out    |  25 ++
 .../expected/test-sheet-pivot-3-indexed.out   |  25 ++
 .../expected/test-sheet-pivot-3-prompt.out    |  25 ++
 app/test/expected/test-sheet-pivot-3.out      |  25 ++
 18 files changed, 645 insertions(+), 2 deletions(-)
 create mode 100644 app/sheet/pivot.c
 create mode 100644 app/test/expected/test-sheet-pivot-1-drilldown.out
 create mode 100644 app/test/expected/test-sheet-pivot-1-groups.out
 create mode 100644 app/test/expected/test-sheet-pivot-1-indexed.out
 create mode 100644 app/test/expected/test-sheet-pivot-1.out
 create mode 100644 app/test/expected/test-sheet-pivot-2-groups.out
 create mode 100644 app/test/expected/test-sheet-pivot-2-indexed.out
 create mode 100644 app/test/expected/test-sheet-pivot-2.out
 create mode 100644 app/test/expected/test-sheet-pivot-3-groups.out
 create mode 100644 app/test/expected/test-sheet-pivot-3-indexed.out
 create mode 100644 app/test/expected/test-sheet-pivot-3-prompt.out
 create mode 100644 app/test/expected/test-sheet-pivot-3.out

diff --git a/app/sheet.c b/app/sheet.c
index a34ed39d..ec8a76de 100644
--- a/app/sheet.c
+++ b/app/sheet.c
@@ -538,6 +538,7 @@ static zsvsheet_status zsvsheet_help_handler(struct zsvsheet_proc_context *ctx)
   return stat;
 }
 
+#include "sheet/pivot.c"
 #include "sheet/newline_handler.c"
 
 /* We do most procedures in one handler. More complex procedures can be
@@ -624,7 +625,9 @@ struct builtin_proc_desc {
   { zsvsheet_builtin_proc_filter,         "filter", "Hide rows that do not contain the specified text",                zsvsheet_filter_handler       },
   { zsvsheet_builtin_proc_subcommand,     "subcommand",  "Editor subcommand",                                          zsvsheet_subcommand_handler },
   { zsvsheet_builtin_proc_help,           "help",   "Display a list of actions and key-bindings",                      zsvsheet_help_handler         },
-  { zsvsheet_builtin_proc_newline,        "<Enter>","Follow hyperlink (if any)",                                       zsvsheet_newline_handler      },
+  { zsvsheet_builtin_proc_newline,        "<Enter>","Follow hyperlink (if any) or drill down",                         zsvsheet_newline_handler      },
+  { zsvsheet_builtin_proc_pivot_cur_col,  "pivotcur","Group rows by the column under the cursor",                      zsvsheet_pivot_handler        },
+  { zsvsheet_builtin_proc_pivot_expr,     "pivotexpr","Group rows with group-by SQL expression",                       zsvsheet_pivot_handler        },
   { -1, NULL, NULL, NULL }
 };
 /* clang-format on */
diff --git a/app/sheet/file.h b/app/sheet/file.h
index 482db958..d1df2227 100644
--- a/app/sheet/file.h
+++ b/app/sheet/file.h
@@ -7,5 +7,6 @@ int zsvsheet_ui_buffer_open_file(const char *filename, const struct zsv_opts *zs
                                  struct zsvsheet_ui_buffer **ui_buffer_stack_top);
 
 zsvsheet_status zsvsheet_open_file_opts(struct zsvsheet_proc_context *ctx, struct zsvsheet_ui_buffer_opts *opts);
+zsvsheet_status zsvsheet_open_file(struct zsvsheet_proc_context *ctx, const char *filepath, struct zsv_opts *zopts);
 
 #endif
diff --git a/app/sheet/key-bindings.c b/app/sheet/key-bindings.c
index c1b5bdc7..6eff5049 100644
--- a/app/sheet/key-bindings.c
+++ b/app/sheet/key-bindings.c
@@ -174,6 +174,8 @@ struct zsvsheet_key_binding zsvsheet_vim_key_bindings[] = {
   { .ch = '?',                 .proc_id = zsvsheet_builtin_proc_help,          },
   { .ch = '\n',                .proc_id = zsvsheet_builtin_proc_newline,       },
   { .ch = '\r',                .proc_id = zsvsheet_builtin_proc_newline,       },
+  { .ch = 'v',                 .proc_id = zsvsheet_builtin_proc_pivot_cur_col, },
+  { .ch = 'V',                 .proc_id = zsvsheet_builtin_proc_pivot_expr,    },
 
   { .ch = -1                                                          }
 };
diff --git a/app/sheet/pivot.c b/app/sheet/pivot.c
new file mode 100644
index 00000000..82a1a6d8
--- /dev/null
+++ b/app/sheet/pivot.c
@@ -0,0 +1,317 @@
+/*
+ * Copyright (C) 2021 Liquidaty and zsv contributors. All rights reserved.
+ * This file is part of zsv/lib, distributed under the license defined at
+ * https://opensource.org/licenses/MIT
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include "../external/sqlite3/sqlite3.h"
+#include <zsv/ext/implementation.h>
+#include <zsv/ext/sheet.h>
+#include <zsv/utils/writer.h>
+#include <zsv/utils/file.h>
+#include <zsv/utils/prop.h>
+#include "file.h"
+#include "handlers_internal.h"
+#include "../sql_internal.h"
+
+struct pivot_row {
+  char *value; // to do: this will be the drill-down criteria
+};
+
+struct pivot_data {
+  char *value_sql; // the sql expression entered by the user e.g. City
+  char *data_filename;
+  struct zsv_opts zopts;
+  struct {
+    struct pivot_row *data; // for each row, the value of the sql expression e.g. New York
+    size_t capacity;
+    size_t used;
+  } rows;
+};
+
+static void pivot_data_delete(void *h) {
+  struct pivot_data *pd = h;
+  if (pd) {
+    for (size_t i = 0; i < pd->rows.used; i++)
+      free(pd->rows.data[i].value);
+    free(pd->rows.data);
+    free(pd->value_sql);
+    free(pd->data_filename);
+    free(pd);
+  }
+}
+
+static struct pivot_data *pivot_data_new(const char *data_filename, const char *value_sql) {
+  struct pivot_data *pd = calloc(1, sizeof(*pd));
+  if (pd && (pd->value_sql = strdup(value_sql)) && (pd->data_filename = strdup(data_filename)))
+    return pd;
+  pivot_data_delete(pd);
+  return NULL;
+}
+
+#define ZSV_MYSHEET_PIVOT_DATA_ROWS_INITIAL 32
+static int pivot_data_grow(struct pivot_data *pd) {
+  if (pd->rows.used == pd->rows.capacity) {
+    size_t new_capacity = pd->rows.capacity == 0 ? ZSV_MYSHEET_PIVOT_DATA_ROWS_INITIAL : pd->rows.capacity * 2;
+    struct pivot_row *new_data = realloc(pd->rows.data, new_capacity * sizeof(*pd->rows.data));
+    if (!new_data)
+      return ENOMEM;
+    pd->rows.data = new_data;
+    pd->rows.capacity = new_capacity;
+  }
+  return 0;
+}
+
+static int add_pivot_row(struct pivot_data *pd, const char *value, size_t len) {
+  int err = pivot_data_grow(pd);
+  char *value_dup = NULL;
+  if (!err && value && len) {
+    value_dup = malloc(len + 1);
+    if (value_dup) {
+      memcpy(value_dup, value, len);
+      value_dup[len] = '\0';
+    }
+  }
+  pd->rows.data[pd->rows.used++].value = value_dup;
+  return err;
+}
+
+static struct pivot_row *get_pivot_row_data(struct pivot_data *pd, size_t row_ix) {
+  if (pd && row_ix < pd->rows.used)
+    return &pd->rows.data[row_ix];
+  return NULL;
+}
+
+// TO DO: return zsvsheet_status
+static enum zsv_ext_status get_cell_attrs(void *pdh, int *attrs, size_t start_row, size_t row_count, size_t cols) {
+  struct pivot_data *pd = pdh;
+  size_t end_row = start_row + row_count;
+  int attr = 0;
+
+#ifdef A_BOLD
+  attr |= A_BOLD;
+#endif
+// Absent on Mac OSX 13
+#ifdef A_ITALIC
+  attr |= A_ITALIC;
+#endif
+
+  if (end_row > pd->rows.used)
+    end_row = pd->rows.used;
+  for (size_t i = start_row; i < end_row; i++)
+    attrs[i * cols] = attr;
+  return zsv_ext_status_ok;
+}
+
+static void pivot_on_header_cell(void *ctx, size_t col_ix, const char *colname) {
+  (void)colname;
+  if (col_ix == 0)
+    add_pivot_row(ctx, NULL, 0);
+}
+
+static void pivot_on_data_cell(void *ctx, size_t col_ix, const char *text, size_t len) {
+  if (col_ix == 0)
+    add_pivot_row(ctx, text, len);
+}
+
+static zsvsheet_status zsv_sqlite3_to_csv(zsvsheet_proc_context_t pctx, struct zsv_sqlite3_db *zdb, const char *sql,
+                                          void *ctx, void (*on_header_cell)(void *, size_t, const char *),
+                                          void (*on_data_cell)(void *, size_t, const char *, size_t len)) {
+  const char *err_msg = NULL;
+  zsvsheet_status zst = zsvsheet_status_error;
+  sqlite3_stmt *stmt = NULL;
+
+  if ((zdb->rc = sqlite3_prepare_v2(zdb->db, sql, -1, &stmt, NULL)) == SQLITE_OK) {
+    char *tmp_fn = zsv_get_temp_filename("zsv_mysheet_ext_XXXXXXXX");
+    struct zsv_csv_writer_options writer_opts = zsv_writer_get_default_opts();
+    zsv_csv_writer cw = NULL;
+    if (!tmp_fn)
+      zst = zsvsheet_status_memory;
+    else if (!(writer_opts.stream = fopen(tmp_fn, "wb"))) {
+      zst = zsvsheet_status_error;
+      err_msg = strerror(errno);
+    } else if (!(cw = zsv_writer_new(&writer_opts)))
+      zst = zsvsheet_status_memory;
+    else {
+      zst = zsvsheet_status_ok;
+      unsigned char cw_buff[1024];
+      zsv_writer_set_temp_buff(cw, cw_buff, sizeof(cw_buff));
+
+      int col_count = sqlite3_column_count(stmt);
+      // write header row
+      for (int i = 0; i < col_count; i++) {
+        const char *colname = sqlite3_column_name(stmt, i);
+        zsv_writer_cell(cw, !i, (const unsigned char *)colname, colname ? strlen(colname) : 0, 1);
+        if (on_header_cell)
+          on_header_cell(ctx, i, colname);
+      }
+
+      // write sql results
+      while (sqlite3_step(stmt) == SQLITE_ROW) {
+        for (int i = 0; i < col_count; i++) {
+          const unsigned char *text = sqlite3_column_text(stmt, i);
+          int len = text ? sqlite3_column_bytes(stmt, i) : 0;
+          zsv_writer_cell(cw, !i, text, len, 1);
+          if (on_data_cell)
+            on_data_cell(ctx, i, (const char *)text, len);
+        }
+      }
+    }
+    if (cw)
+      zsv_writer_delete(cw);
+    if (writer_opts.stream)
+      fclose(writer_opts.stream);
+
+    if (tmp_fn && zsv_file_exists(tmp_fn)) {
+      struct zsvsheet_ui_buffer_opts uibopts = {0};
+      uibopts.data_filename = tmp_fn;
+      zst = zsvsheet_open_file_opts(pctx, &uibopts);
+    } else {
+      if (zst == zsvsheet_status_ok) {
+        zst = zsvsheet_status_error; // to do: make this more specific
+        if (!err_msg && zdb && zdb->rc != SQLITE_OK)
+          err_msg = sqlite3_errmsg(zdb->db);
+      }
+    }
+    if (zst != zsvsheet_status_ok)
+      free(tmp_fn);
+  } else {
+    err_msg = sqlite3_errmsg(zdb->db);
+  }
+  if (stmt)
+    sqlite3_finalize(stmt);
+  if (err_msg)
+    zsvsheet_set_status(pctx, "Error: %s", err_msg);
+  return zst;
+}
+
+zsvsheet_status pivot_drill_down(zsvsheet_proc_context_t ctx) {
+  enum zsvsheet_status zst = zsvsheet_status_ok;
+  zsvsheet_buffer_t buff = zsvsheet_buffer_current(ctx);
+  struct pivot_data *pd;
+  struct zsvsheet_rowcol rc;
+  if (zsvsheet_buffer_get_ctx(buff, (void **)&pd) != zsv_ext_status_ok ||
+      zsvsheet_buffer_get_selected_cell(buff, &rc) != zsvsheet_status_ok) {
+    return zsvsheet_status_error;
+  }
+  struct pivot_row *pr = get_pivot_row_data(pd, rc.row);
+  if (pd && pd->data_filename && pd->value_sql && pr) {
+    struct zsv_sqlite3_dbopts dbopts = {0};
+    sqlite3_str *sql_str = NULL;
+    struct zsv_sqlite3_db *zdb = zsv_sqlite3_db_new(&dbopts);
+
+    if (!zdb || !(sql_str = sqlite3_str_new(zdb->db)))
+      zst = zsvsheet_status_memory;
+    else if (zdb->rc == SQLITE_OK && zsv_sqlite3_add_csv(zdb, pd->data_filename, &pd->zopts, NULL) == SQLITE_OK) {
+      if (zsvsheet_buffer_info(buff).has_row_num)
+        sqlite3_str_appendf(sql_str, "select *");
+      else
+        sqlite3_str_appendf(sql_str, "select rowid as [Row #], *");
+      sqlite3_str_appendf(sql_str, " from data where %s = %Q", pd->value_sql, pr->value);
+      zst = zsv_sqlite3_to_csv(ctx, zdb, sqlite3_str_value(sql_str), NULL, NULL, NULL);
+    }
+
+    if (sql_str)
+      sqlite3_free(sqlite3_str_finish(sql_str));
+    if (zdb) {
+      if (zst != zsvsheet_status_ok) {
+        // to do: consolidate this with same code in sql.c
+        if (zdb->err_msg)
+          fprintf(stderr, "Error: %s\n", zdb->err_msg);
+        else if (!zdb->db)
+          fprintf(stderr, "Error (unable to open db, code %i): %s\n", zdb->rc, sqlite3_errstr(zdb->rc));
+        else if (zdb->rc != SQLITE_OK)
+          fprintf(stderr, "Error (code %i): %s\n", zdb->rc, sqlite3_errstr(zdb->rc));
+      }
+      zsv_sqlite3_db_delete(zdb);
+    }
+  }
+  return zst;
+}
+
+/**
+ * Here we define a custom command for the zsv `sheet` feature
+ */
+static zsvsheet_status zsvsheet_pivot_handler(struct zsvsheet_proc_context *ctx) {
+  char result_buffer[256] = {0};
+  const char *expr;
+  size_t trim_len;
+  struct zsvsheet_rowcol rc;
+  int ch = zsvsheet_ext_keypress(ctx);
+  if (ch < 0)
+    return zsvsheet_status_error;
+
+  zsvsheet_buffer_t buff = zsvsheet_buffer_current(ctx);
+  const char *data_filename = NULL;
+  if (buff)
+    data_filename = zsvsheet_buffer_data_filename(buff);
+
+  if (!data_filename) { // TO DO: check that the underlying data is a tabular file and we know how to parse
+    zsvsheet_set_status(ctx, "Pivot table only available for tabular data buffers");
+    return zsvsheet_status_ok;
+  }
+
+  switch (ctx->proc_id) {
+  case zsvsheet_builtin_proc_pivot_expr:
+    zsvsheet_ext_prompt(ctx, result_buffer, sizeof(result_buffer), "Pivot table: Enter group-by SQL expr");
+    if (*result_buffer == '\0')
+      return zsvsheet_status_ok;
+    expr = result_buffer;
+    break;
+  case zsvsheet_builtin_proc_pivot_cur_col:
+    if (zsvsheet_buffer_get_selected_cell(buff, &rc) != zsvsheet_status_ok)
+      return zsvsheet_status_error;
+
+    expr = zsvsheet_ui_buffer_get_header(buff, rc.col);
+    assert(expr);
+    trim_len = strlen(expr);
+    expr = (char *)zsv_strtrim((unsigned char *)expr, &trim_len);
+    assert(expr);
+    assert(trim_len <= sizeof(result_buffer) - 1);
+    assert(trim_len > 0);
+    expr = sqlite3_snprintf(sizeof(result_buffer), result_buffer, "\"%.*w\"", trim_len, expr);
+    assert(strlen(expr) > trim_len);
+    break;
+  default:
+    assert(0);
+    return zsvsheet_status_error;
+  }
+
+  assert(strlen(expr) > 0);
+
+  enum zsvsheet_status zst = zsvsheet_status_ok;
+  struct zsv_sqlite3_dbopts dbopts = {0};
+  struct zsv_opts zopts = zsvsheet_buffer_get_zsv_opts(buff);
+  struct zsv_sqlite3_db *zdb = zsv_sqlite3_db_new(&dbopts);
+  sqlite3_str *sql_str = NULL;
+  struct pivot_data *pd = NULL;
+  if (!zdb || !(sql_str = sqlite3_str_new(zdb->db)))
+    zst = zsvsheet_status_memory;
+  else if (zdb->rc == SQLITE_OK && zsv_sqlite3_add_csv(zdb, data_filename, &zopts, NULL) == SQLITE_OK) {
+    sqlite3_str_appendf(sql_str, "select %s as value, count(1) as Count from data group by %s", expr, expr);
+    if (!(pd = pivot_data_new(data_filename, expr)))
+      zst = zsvsheet_status_memory;
+    else {
+      zst = zsv_sqlite3_to_csv(ctx, zdb, sqlite3_str_value(sql_str), pd, pivot_on_header_cell, pivot_on_data_cell);
+      if (zst == zsvsheet_status_ok) {
+        buff = zsvsheet_buffer_current(ctx);
+        zsvsheet_buffer_set_ctx(buff, pd, pivot_data_delete);
+        zsvsheet_buffer_set_cell_attrs(buff, get_cell_attrs);
+        zsvsheet_buffer_on_newline(buff, pivot_drill_down);
+        pd->zopts = zopts;
+        pd = NULL; // so that it isn't cleaned up below
+      }
+    }
+    // TO DO: add param to ext_sheet_open_file to set filename vs data_filename, and set buffer type or proc owner
+    // TO DO: add way to attach custom context, and custom context destructor, to the new buffer
+    // TO DO: add cell highlighting
+  }
+
+  zsv_sqlite3_db_delete(zdb);
+  if (sql_str)
+    sqlite3_free(sqlite3_str_finish(sql_str));
+  pivot_data_delete(pd);
+  return zst;
+}
diff --git a/app/sheet/procedure.h b/app/sheet/procedure.h
index 5c36a309..d81ac0c8 100644
--- a/app/sheet/procedure.h
+++ b/app/sheet/procedure.h
@@ -34,6 +34,8 @@ enum {
   zsvsheet_builtin_proc_help,
   zsvsheet_builtin_proc_vim_g_key_binding_dmux,
   zsvsheet_builtin_proc_newline,
+  zsvsheet_builtin_proc_pivot_expr,
+  zsvsheet_builtin_proc_pivot_cur_col,
 };
 
 #define ZSVSHEET_PROC_INVALID 0
diff --git a/app/sheet/ui_buffer.c b/app/sheet/ui_buffer.c
index 8a4f4c8c..80314d59 100644
--- a/app/sheet/ui_buffer.c
+++ b/app/sheet/ui_buffer.c
@@ -192,3 +192,9 @@ int zsvsheet_ui_buffer_pop(struct zsvsheet_ui_buffer **base, struct zsvsheet_ui_
   }
   return 0;
 }
+
+static const char *zsvsheet_ui_buffer_get_header(struct zsvsheet_ui_buffer *uib, size_t col) {
+  struct zsvsheet_screen_buffer *sb = uib->buffer;
+
+  return (char *)zsvsheet_screen_buffer_cell_display(sb, 0, col);
+}
diff --git a/app/test/Makefile b/app/test/Makefile
index cf56d255..99ea1d2d 100644
--- a/app/test/Makefile
+++ b/app/test/Makefile
@@ -618,7 +618,7 @@ test-sheet-cleanup:
 	@rm -f tmux-*.log
 	@tmux kill-server || printf ''
 
-test-sheet-all: test-sheet-1 test-sheet-2 test-sheet-3 test-sheet-4 test-sheet-5 test-sheet-6 test-sheet-7 test-sheet-8 test-sheet-9 test-sheet-10 test-sheet-11 test-sheet-12 test-sheet-13 test-sheet-14 test-sheet-subcommand test-sheet-prop-cmd-opt
+test-sheet-all: test-sheet-1 test-sheet-2 test-sheet-3 test-sheet-4 test-sheet-5 test-sheet-6 test-sheet-7 test-sheet-8 test-sheet-9 test-sheet-10 test-sheet-11 test-sheet-12 test-sheet-13 test-sheet-14 test-sheet-subcommand test-sheet-prop-cmd-opt test-sheet-pivot
 	@(for SESSION in $^; do ! tmux kill-session -t "$$SESSION" 2>/dev/null; done && ${TEST_PASS} || ${TEST_FAIL})
 
 TMUX_TERM=xterm-256color
@@ -807,6 +807,43 @@ test-sheet-prop-cmd-opt: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${BUILD_DIR}/bin/zsv_p
 #	@tmux send-keys -t $@ "q"
 #	@${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL})
 
+test-sheet-pivot: test-sheet-pivot-1 test-sheet-pivot-2 test-sheet-pivot-3
+
+test-sheet-pivot-1: ${BUILD_DIR}/bin/zsv_sheet${EXE}
+	@${TEST_INIT}
+	@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
+	@(tmux new-session -x 80 -y 25 -d -s "$@" "${PREFIX} $< worldcitiespop_mil.csv" && \
+	${EXPECT} $@ indexed && \
+	tmux send-keys -t $@ l v && \
+	${EXPECT} $@ groups && \
+	tmux send-keys -t $@ j j Enter && \
+	${EXPECT} $@ drilldown && \
+	tmux send-keys -t $@ G && \
+	${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})
+
+test-sheet-pivot-2: ${BUILD_DIR}/bin/zsv_sheet${EXE}
+	@${TEST_INIT}
+	@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
+	@(tmux new-session -x 80 -y 25 -d -s "$@" "${PREFIX} $< -d 3 ../../data/test/mixed-line-endings.csv" && \
+	${EXPECT} $@ indexed && \
+	tmux send-keys -t $@ l l l v && \
+	${EXPECT} $@ groups && \
+	tmux send-keys -t $@ j j Enter && \
+	${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})
+
+test-sheet-pivot-3: ${BUILD_DIR}/bin/zsv_sheet${EXE}
+	@${TEST_INIT}
+	@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
+	@(tmux new-session -x 80 -y 25 -d -s "$@" "${PREFIX} $< worldcitiespop_mil.csv" && \
+	${EXPECT} $@ indexed && \
+	tmux send-keys -t $@ V && \
+	${EXPECT} $@ prompt && \
+	tmux send-keys -l -t $@ "iif(coalesce(Population, '') = '', 'Unknown', 'Populated')" && \
+	tmux send-keys -t $@ Enter && \
+	${EXPECT} $@ groups && \
+	tmux send-keys -t $@ Enter && \
+	${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})
+
 benchmark-sheet-index: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
 	@${TEST_INIT}
 	@if [ "${BIG_FILE}" = "none" ]; then \
diff --git a/app/test/expected/test-sheet-pivot-1-drilldown.out b/app/test/expected/test-sheet-pivot-1-drilldown.out
new file mode 100644
index 00000000..bd14fadf
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-1-drilldown.out
@@ -0,0 +1,25 @@
+Row #     Country   City      AccentCit Region    Populatio Latitude  Longitude
+546       af        abakeh    Abakeh    24                  36.67363  69.078194
+549       af        abakheyl' Abakheyl' 35                  34.744167 70.287778
+551       af        aband     Aband     42                  35.474953 69.871144
+556       af        abasali   Abasali   39                  32.813402 65.968042
+558       af        abaskhank Abaskhank 29                  32.503976 69.209795
+560       af        abas khel Abas Khel 28                  31.975853 67.59182
+561       af        abaskheyl Abaskheyl 28                  31.975853 67.59182
+563       af        `abaskust `Abaskust 27                  34.331309 67.640744
+576       af        ab barik  Ab Barik  09                  34.651024 66.254311
+577       af        `abbas    `Abbas    08                  33.333    67.643278
+582       af        `abbaskha `Abbaskha 29                  32.469231 69.20712
+583       af        `abbas kh `Abbas Kh 29                  32.469231 69.20712
+588       af        `abbas kh `Abbas Kh 29                  32.671879 69.134674
+589       af        `abbaskhe `Abbaskhe 29                  32.671879 69.134674
+590       af        abbaskhel Abbaskhel 29                  32.831993 69.12336
+593       af        `abbaskhe `Abbaskhe 29                  32.831389 69.260556
+594       af        abbaskhey Abbaskhey 29                  32.671879 69.134674
+596       af        `abbaskhe `Abbaskhe 29                  32.831993 69.12336
+600       af        `abbas ku `Abbas Ku 27                  34.331309 67.640744
+604       af        abbazha-y Abbazha-y 10                  31.791857 64.566416
+607       af        ab bazha- Ab Bazha- 10                  31.719049 64.600709
+608       af        abbazha-y Abbazha-y 10                  31.719049 64.600709
+611       af        abbedak   Abbedak   09                  33.482286 64.310608
+? for help 546
diff --git a/app/test/expected/test-sheet-pivot-1-groups.out b/app/test/expected/test-sheet-pivot-1-groups.out
new file mode 100644
index 00000000..f173628f
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-1-groups.out
@@ -0,0 +1,25 @@
+Row #                     value                     Count
+1                         ad                        27
+2                         ae                        147
+3                         af                        27822
+4                         ag                        55
+5                         ai                        14
+6                         al                        4730
+7                         am                        896
+8                         an                        75
+9                         ao                        6170
+10                        ar                        2817
+11                        at                        4740
+12                        au                        3403
+13                        aw                        37
+14                        az                        3522
+15                        ba                        5076
+16                        bb                        165
+17                        bd                        8280
+18                        be                        5056
+19                        bf                        3248
+20                        bg                        6258
+21                        bh                        102
+22                        bi                        646
+23                        bj                        1331
+? for help 1
diff --git a/app/test/expected/test-sheet-pivot-1-indexed.out b/app/test/expected/test-sheet-pivot-1-indexed.out
new file mode 100644
index 00000000..610fb180
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-1-indexed.out
@@ -0,0 +1,25 @@
+Row #     Country   City      AccentCit Region    Populatio Latitude  Longitude
+1         ir        sarmaj-e  Sarmaj-e  13                  34.3578   47.5207
+2         ad        aixirival Aixirival 06                  42.466666 1.5
+3         mm        mokho-atw Mokho-atw 09                  18.033333 96.75
+4         id        selingon  Selingon  17                  -8.8374   116.4914
+5         ir        berimvand Berimvand 13                  34.2953   47.1096
+6         pl        chomiaza  Chomiaza  73                  52.7508   17.841793
+7         mg        itona     Itona     02                  -23.86666 47.216666
+8         ad        andorra-v Andorra-V 07                  42.5      1.5166667
+9         mx        la tortug La Tortug 25                  25.75     -108.3333
+10        us        asaph     Asaph     PA                  41.770833 -77.40527
+11        ad        andorre-v Andorre-V 07                  42.5      1.5166667
+12        ru        dolmatova Dolmatova 71                  57.436791 63.279522
+13        ro        escu      Escu      13                  47.133333 23.533333
+14        us        la presa  La Presa  CA        35119     32.708055 -116.9963
+15        pk        makam khu Makam Khu 04                  33.650863 72.551536
+16        ad        aubinya   Aubinyà   06                  42.45     1.5
+17        mm        kiosong   Kiosöng   11                  22.583333 97.05
+18        tr        donencay  Dönençay  28                  40.266667 38.583333
+19        it        roncaglia Roncaglia 05                  45.05     9.8
+20        ml        kourmouss Kourmouss 04                  14.75     -5.033333
+21        id        lamogo    Lamogo    38                  -4.3945   119.9028
+22        ad        casas vil Casas Vil 03                  42.533333 1.5666667
+23        ru        otdeleniy Otdeleniy 86                  51.726473 39.714345
+? for help 1
diff --git a/app/test/expected/test-sheet-pivot-1.out b/app/test/expected/test-sheet-pivot-1.out
new file mode 100644
index 00000000..d2f20474
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-1.out
@@ -0,0 +1,25 @@
+Row #     Country   City      AccentCit Region    Populatio Latitude  Longitude
+89222     af        zungur ka Zungur Ka 27                  33.993056 68.658056
+89223     af        zunun     Zunun     01                  38.438492 70.904387
+89224     af        zu ol faq Zu ol Faq 03                  35.616782 68.670307
+89225     af        zu ol faq Zu ol Faq 27                  34.1575   68.501389
+89232     af        zur baraw Zur Baraw 34                  35.028791 71.507898
+89235     af        zuri      Zuri      11                  33.94062  62.160912
+89238     af        zur kalay Zur Kalay 35                  34.670345 70.165192
+89244     af        zur kowt  Zur Kowt  37                  33.542222 69.734167
+89246     af        zurmatiya Zurmatiya 28                  32.138117 66.835273
+89248     af        zurmat    Zurmat    36                  33.437782 69.02774
+89249     af        zurni     Zurni     09                  33.568987 63.289317
+89252     af        zurwam    Zurwam    23                  31.439167 66.936667
+89256     af        zutni khe Zutni Khe 36                  33.360477 68.973862
+89258     af        zutni khe Zutni Khe 36                  33.360477 68.973862
+89261     af        zu        Zu        01                  36.84154  71.069323
+89262     af        zvaka     Zvaka     29                  32.968334 68.809853
+89264     af        zwaka     Zwaka     29                  32.968334 68.809853
+89267     af        zyan say  Zyan Say  07                  35.864952 64.278152
+89279     af        zyaratjay Zyaratjay 11                  34.197386 62.140143
+89281     af        zyarat ka Zyarat Ka 18                  34.288611 71.138889
+89282     af        zyarat ka Zyarat Ka 23                  31.355826 66.535299
+89283     af        zyarat ka Zyarat Ka 35                  34.738611 70.269444
+89286     af        zyarat    Zyarat    39                  32.523283 65.905489
+? for help 89286
diff --git a/app/test/expected/test-sheet-pivot-2-groups.out b/app/test/expected/test-sheet-pivot-2-groups.out
new file mode 100644
index 00000000..71316b11
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-2-groups.out
@@ -0,0 +1,25 @@
+Row #                     value                     Count
+1                          C1                       1
+2                          C10                      1
+3                          C100                     1
+4                          C1000                    1
+5                          C1001                    1
+6                          C1002                    1
+7                          C1003                    1
+8                          C1004                    1
+9                          C1005                    1
+10                         C1006                    1
+11                         C1007                    1
+12                         C1008                    1
+13                         C1009                    1
+14                         C101                     1
+15                         C1010                    1
+16                         C1011                    1
+17                         C1012                    1
+18                         C1013                    1
+19                         C1014                    1
+20                         C1015                    1
+21                         C1016                    1
+22                         C1017                    1
+23                         C1018                    1
+? for help 1
diff --git a/app/test/expected/test-sheet-pivot-2-indexed.out b/app/test/expected/test-sheet-pivot-2-indexed.out
new file mode 100644
index 00000000..b7f86738
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-2-indexed.out
@@ -0,0 +1,25 @@
+Row #               HA1 HA2 HA3          HB1  HB2  HB3       HC1  HC2  HC3
+1                   A1                   B1                  C1
+2                   A2                   B2                  C2
+3                   A3                   B3                  C3
+4                   A4                   B4                  C4
+5                   A5                   B5                  C5
+6                   A6                   B6                  C6
+7                   A7                   B7                  C7
+8                   A8                   B8                  C8
+9                   A9                   B9                  C9
+10                  A10                  B10                 C10
+11                  A11                  B11                 C11
+12                  A12                  B12                 C12
+13                  A13                  B13                 C13
+14                  A14                  B14                 C14
+15                  A15                  B15                 C15
+16                  A16                  B16                 C16
+17                  A17                  B17                 C17
+18                  A18                  B18                 C18
+19                  A19                  B19                 C19
+20                  A20                  B20                 C20
+21                  A21                  B21                 C21
+22                  A22                  B22                 C22
+23                  A23                  B23                 C23
+? for help 1
diff --git a/app/test/expected/test-sheet-pivot-2.out b/app/test/expected/test-sheet-pivot-2.out
new file mode 100644
index 00000000..17285ddd
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-2.out
@@ -0,0 +1,25 @@
+Row #               HA1 HA2 HA3         HB1  HB2  HB3       HC1  HC2  HC3
+100                 A100                 B100                C100
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+? for help 100
diff --git a/app/test/expected/test-sheet-pivot-3-groups.out b/app/test/expected/test-sheet-pivot-3-groups.out
new file mode 100644
index 00000000..98a9d692
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-3-groups.out
@@ -0,0 +1,25 @@
+value                                   Count
+Populated                               15166
+Unknown                                 984834
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+? for help Populated
diff --git a/app/test/expected/test-sheet-pivot-3-indexed.out b/app/test/expected/test-sheet-pivot-3-indexed.out
new file mode 100644
index 00000000..610fb180
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-3-indexed.out
@@ -0,0 +1,25 @@
+Row #     Country   City      AccentCit Region    Populatio Latitude  Longitude
+1         ir        sarmaj-e  Sarmaj-e  13                  34.3578   47.5207
+2         ad        aixirival Aixirival 06                  42.466666 1.5
+3         mm        mokho-atw Mokho-atw 09                  18.033333 96.75
+4         id        selingon  Selingon  17                  -8.8374   116.4914
+5         ir        berimvand Berimvand 13                  34.2953   47.1096
+6         pl        chomiaza  Chomiaza  73                  52.7508   17.841793
+7         mg        itona     Itona     02                  -23.86666 47.216666
+8         ad        andorra-v Andorra-V 07                  42.5      1.5166667
+9         mx        la tortug La Tortug 25                  25.75     -108.3333
+10        us        asaph     Asaph     PA                  41.770833 -77.40527
+11        ad        andorre-v Andorre-V 07                  42.5      1.5166667
+12        ru        dolmatova Dolmatova 71                  57.436791 63.279522
+13        ro        escu      Escu      13                  47.133333 23.533333
+14        us        la presa  La Presa  CA        35119     32.708055 -116.9963
+15        pk        makam khu Makam Khu 04                  33.650863 72.551536
+16        ad        aubinya   Aubinyà   06                  42.45     1.5
+17        mm        kiosong   Kiosöng   11                  22.583333 97.05
+18        tr        donencay  Dönençay  28                  40.266667 38.583333
+19        it        roncaglia Roncaglia 05                  45.05     9.8
+20        ml        kourmouss Kourmouss 04                  14.75     -5.033333
+21        id        lamogo    Lamogo    38                  -4.3945   119.9028
+22        ad        casas vil Casas Vil 03                  42.533333 1.5666667
+23        ru        otdeleniy Otdeleniy 86                  51.726473 39.714345
+? for help 1
diff --git a/app/test/expected/test-sheet-pivot-3-prompt.out b/app/test/expected/test-sheet-pivot-3-prompt.out
new file mode 100644
index 00000000..edaa6c71
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-3-prompt.out
@@ -0,0 +1,25 @@
+Row #     Country   City      AccentCit Region    Populatio Latitude  Longitude
+1         ir        sarmaj-e  Sarmaj-e  13                  34.3578   47.5207
+2         ad        aixirival Aixirival 06                  42.466666 1.5
+3         mm        mokho-atw Mokho-atw 09                  18.033333 96.75
+4         id        selingon  Selingon  17                  -8.8374   116.4914
+5         ir        berimvand Berimvand 13                  34.2953   47.1096
+6         pl        chomiaza  Chomiaza  73                  52.7508   17.841793
+7         mg        itona     Itona     02                  -23.86666 47.216666
+8         ad        andorra-v Andorra-V 07                  42.5      1.5166667
+9         mx        la tortug La Tortug 25                  25.75     -108.3333
+10        us        asaph     Asaph     PA                  41.770833 -77.40527
+11        ad        andorre-v Andorre-V 07                  42.5      1.5166667
+12        ru        dolmatova Dolmatova 71                  57.436791 63.279522
+13        ro        escu      Escu      13                  47.133333 23.533333
+14        us        la presa  La Presa  CA        35119     32.708055 -116.9963
+15        pk        makam khu Makam Khu 04                  33.650863 72.551536
+16        ad        aubinya   Aubinyà   06                  42.45     1.5
+17        mm        kiosong   Kiosöng   11                  22.583333 97.05
+18        tr        donencay  Dönençay  28                  40.266667 38.583333
+19        it        roncaglia Roncaglia 05                  45.05     9.8
+20        ml        kourmouss Kourmouss 04                  14.75     -5.033333
+21        id        lamogo    Lamogo    38                  -4.3945   119.9028
+22        ad        casas vil Casas Vil 03                  42.533333 1.5666667
+23        ru        otdeleniy Otdeleniy 86                  51.726473 39.714345
+Pivot table: Enter group-by SQL expr:
diff --git a/app/test/expected/test-sheet-pivot-3.out b/app/test/expected/test-sheet-pivot-3.out
new file mode 100644
index 00000000..d4f0d5f8
--- /dev/null
+++ b/app/test/expected/test-sheet-pivot-3.out
@@ -0,0 +1,25 @@
+Row #     Country   City      AccentCit Region    Populatio Latitude  Longitude
+14        us        la presa  La Presa  CA        35119     32.708055 -116.9963
+54        ad        les escal Les Escal 08        15854     42.5      1.5333333
+72        us        lawrence  Lawrence  MA        72609     42.706944 -71.16361
+94        ae        abu dhabi Abu Dhabi 01        603687    24.466667 54.366667
+208       ph        general l General L H3        3132      10.833333 123.56666
+225       gr        mitikas   Mítikas   31        934       38.666666 20.933333
+307       ru        shudayag  Shudayag  34        3446      63.526652 53.605822
+385       ru        shcheglov Shcheglov 42        3095      60.001399 30.783441
+573       sk        prievidza Prievidza 01        52987     48.766666 18.633333
+640       jp        matsubara Matsubara 32        130857    34.566667 135.55
+660       gr        velventos Velventó  11        3378      40.251666 22.063055
+679       pt        arazede   Arazede   07        5906      40.286268 -8.649989
+716       ru        porkhov   Porkhov   60        11886     57.765016 29.556121
+751       us        winter sp Winter Sp FL        32658     28.698611 -81.30833
+824       ru        kubinka   Kubinka   47        25942     55.579569 36.703919
+830       no        alsvag    Alsvåg    09        335       68.9      15.283333
+913       pl        jozefow   Jozefow   78        17910     51.806452 21.220953
+914       us        del rio   Del Rio   TX        36068     29.362500 -100.8963
+1190      ru        kamenka   Kamenka   06        3258      65.887591 44.112612
+1330      lt        venta     Venta     40        3270      56.2      22.7
+1432      ro        farcasu   Farcasu   26        3620      44.6      23.75
+1532      sl        largo     Largo     03        4484      8.2666667 -12.15
+1562      pk        eminabad  Eminabad  04        22677     32.042188 74.259435
+? for help 14

From 3cdae96a1ccbc2ba4649ccd70d7c83525b0d6ec0 Mon Sep 17 00:00:00 2001
From: Richard Palethorpe <io@richiejp.com>
Date: Tue, 17 Dec 2024 16:13:02 +0000
Subject: [PATCH 5/8] sheet: Give status msgs a priority which decays with time

Allows errors to be displayed for longer without them blocking other
status messages after the user has been given time to view them.

Possibly errors should be displayed in a separate buffer or location,
but while we are overloading the status this gives the user chance to
view them.
---
 app/sheet.c          | 31 +++++++++++++++++++++----------
 app/sheet/handlers.c |  1 +
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/app/sheet.c b/app/sheet.c
index ec8a76de..ee005310 100644
--- a/app/sheet.c
+++ b/app/sheet.c
@@ -166,7 +166,8 @@ size_t display_data_rowcount(struct zsvsheet_display_dimensions *dims) {
   return dims->rows - dims->footer_span - dims->header_span;
 }
 
-char zsvsheet_status_text[256] = {0};
+static int zsvsheet_status_priority;
+static char zsvsheet_status_text[256];
 static void zsvsheet_display_status_text(const struct zsvsheet_display_dimensions *ddims) {
   // clear the entire line
   mvprintw(ddims->rows - ddims->footer_span, 0, "%-*s", (int)sizeof(zsvsheet_status_text), "");
@@ -177,15 +178,24 @@ static void zsvsheet_display_status_text(const struct zsvsheet_display_dimension
   attroff(A_REVERSE);
 }
 
-static void zsvsheet_priv_set_status(const struct zsvsheet_display_dimensions *ddims, int overwrite, const char *fmt,
+#define ZSVSHEET_STATUS_LOW_PRIO 1
+#define ZSVSHEET_STATUS_HIGH_PRIO 10
+
+static void zsvsheet_priv_set_status(const struct zsvsheet_display_dimensions *ddims, int priority, const char *fmt,
                                      ...) {
-  if (overwrite || !*zsvsheet_status_text) {
+  if (priority > zsvsheet_status_priority || !*zsvsheet_status_text) {
     va_list argv;
     va_start(argv, fmt);
     vsnprintf(zsvsheet_status_text, sizeof(zsvsheet_status_text), fmt, argv);
     va_end(argv);
     // note: if (n < (int)sizeof(zsvsheet_status_text)), then we just ignore
+    zsvsheet_status_priority = priority;
   }
+
+  // The priority decays with each call which is at least n times per second as set by halfdelay
+  if (zsvsheet_status_priority > 0)
+    zsvsheet_status_priority--;
+
   zsvsheet_display_status_text(ddims);
 }
 
@@ -331,7 +341,7 @@ char zsvsheet_handle_find_next(struct zsvsheet_ui_buffer *uib, const char *needl
     *update_buffer = zsvsheet_goto_input_raw_row(uib, zsvsheet_opts->found_rownum, header_span, ddims, (size_t)-1);
     return 1;
   }
-  zsvsheet_priv_set_status(ddims, 1, "Not found");
+  zsvsheet_priv_set_status(ddims, ZSVSHEET_STATUS_HIGH_PRIO, "Not found");
   return 0;
 }
 
@@ -394,11 +404,11 @@ static zsvsheet_status zsvsheet_open_file_handler(struct zsvsheet_proc_context *
   if ((err = zsvsheet_ui_buffer_open_file(filename, NULL, state->custom_prop_handler, di->ui_buffers.base,
                                           di->ui_buffers.current))) {
     if (err > 0)
-      zsvsheet_priv_set_status(di->dimensions, 1, "%s: %s", filename, strerror(err));
+      zsvsheet_priv_set_status(di->dimensions, ZSVSHEET_STATUS_HIGH_PRIO, "%s: %s", filename, strerror(err));
     else if (err < 0)
-      zsvsheet_priv_set_status(di->dimensions, 1, "Unexpected error");
+      zsvsheet_priv_set_status(di->dimensions, ZSVSHEET_STATUS_HIGH_PRIO, "Unexpected error");
     else
-      zsvsheet_priv_set_status(di->dimensions, 1, "Not found: %s", filename);
+      zsvsheet_priv_set_status(di->dimensions, ZSVSHEET_STATUS_HIGH_PRIO, "Not found: %s", filename);
     return zsvsheet_status_ignore;
   }
 no_input:
@@ -645,7 +655,7 @@ static void zsvsheet_check_buffer_worker_updates(struct zsvsheet_ui_buffer *ub,
                                                  struct zsvsheet_sheet_context *handler_state) {
   pthread_mutex_lock(&ub->mutex);
   if (ub->status)
-    zsvsheet_priv_set_status(display_dims, 1, ub->status);
+    zsvsheet_priv_set_status(display_dims, ZSVSHEET_STATUS_LOW_PRIO, ub->status);
   if (ub->index_ready && ub->dimensions.row_count != ub->index->row_count + 1) {
     ub->dimensions.row_count = ub->index->row_count + 1;
     handler_state->display_info.update_buffer = true;
@@ -741,7 +751,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
     ch = getch();
 
     handler_state.display_info.update_buffer = false;
-    zsvsheet_priv_set_status(&display_dims, 1, "");
+    zsvsheet_priv_set_status(&display_dims, ZSVSHEET_STATUS_LOW_PRIO, "");
 
     if (ch != ERR) {
       status = zsvsheet_key_press(ch, &handler_state);
@@ -758,7 +768,8 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
       struct zsvsheet_opts zsvsheet_opts = {0};
       if (read_data(&ub, NULL, current_ui_buffer->input_offset.row, current_ui_buffer->input_offset.col, header_span,
                     &zsvsheet_opts, custom_prop_handler)) {
-        zsvsheet_priv_set_status(&display_dims, 1, "Unexpected error!"); // to do: better error message
+        zsvsheet_priv_set_status(&display_dims, ZSVSHEET_STATUS_HIGH_PRIO,
+                                 "Unexpected error!"); // to do: better error message
         continue;
       }
     }
diff --git a/app/sheet/handlers.c b/app/sheet/handlers.c
index 60f34f67..8ff6b56f 100644
--- a/app/sheet/handlers.c
+++ b/app/sheet/handlers.c
@@ -45,6 +45,7 @@ zsvsheet_status zsvsheet_set_status(struct zsvsheet_proc_context *ctx, const cha
   va_end(argv);
   // note: if (n < (int)sizeof(zsvsheet_status_text)), then we just ignore
   zsvsheet_display_status_text(state->display_info.dimensions);
+  zsvsheet_status_priority = ZSVSHEET_STATUS_HIGH_PRIO;
   return zsvsheet_status_ok;
 }
 

From 25866e46cf6a8fbb1867681585794ecfac419391 Mon Sep 17 00:00:00 2001
From: Richard Palethorpe <io@richiejp.com>
Date: Wed, 18 Dec 2024 15:44:30 +0000
Subject: [PATCH 6/8] sheet: Hide row numbers on pivot groups and refactor ui
 args

---
 app/sheet.c                                   | 15 +++---
 app/sheet/file.c                              |  5 +-
 app/sheet/handlers.c                          |  2 +-
 app/sheet/pivot.c                             |  9 ++--
 app/sheet/read-data.c                         | 28 ++++++-----
 app/sheet/screen_buffer.c                     |  5 --
 app/sheet/screen_buffer.h                     |  7 ++-
 app/sheet/transformation.c                    |  2 +
 app/sheet/ui_buffer.c                         | 13 +++--
 .../expected/test-sheet-pivot-1-groups.out    | 50 +++++++++----------
 .../expected/test-sheet-pivot-2-groups.out    | 50 +++++++++----------
 11 files changed, 97 insertions(+), 89 deletions(-)

diff --git a/app/sheet.c b/app/sheet.c
index ee005310..ab10f77c 100644
--- a/app/sheet.c
+++ b/app/sheet.c
@@ -49,8 +49,8 @@ struct zsvsheet_opts {
 #define ZSVSHEET_CELL_DISPLAY_MIN_WIDTH 10
 static size_t zsvsheet_cell_display_width(struct zsvsheet_ui_buffer *ui_buffer,
                                           struct zsvsheet_display_dimensions *ddims) {
-  size_t width = ddims->columns /
-                 (ui_buffer->dimensions.col_count + (ui_buffer->rownum_col_offset && !ui_buffer->has_row_num ? 1 : 0));
+  size_t width = ddims->columns / (ui_buffer->dimensions.col_count +
+                                   (ui_buffer->rownum_col_offset && !ui_buffer->no_add_row_num ? 1 : 0));
   return width < ZSVSHEET_CELL_DISPLAY_MIN_WIDTH ? ZSVSHEET_CELL_DISPLAY_MIN_WIDTH : width;
 }
 
@@ -472,16 +472,17 @@ static zsvsheet_status zsvsheet_help_handler(struct zsvsheet_proc_context *ctx)
   struct zsvsheet_sheet_context *state = (struct zsvsheet_sheet_context *)ctx->subcommand_context;
   struct zsvsheet_display_info *di = &state->display_info;
   struct zsvsheet_screen_buffer_opts bopts = {
-    .no_rownum_column = 1,
     .cell_buff_len = 64,
     .max_cell_len = 0,
     .rows = 256,
   };
+  struct zsvsheet_opts zsvsheet_opts = {0};
+  zsvsheet_opts.hide_row_nums = 1;
   struct zsvsheet_ui_buffer_opts uibopts = {
+    .zsvsheet_opts = &zsvsheet_opts,
     .buff_opts = &bopts,
     .filename = NULL,
     .data_filename = NULL,
-    .no_rownum_col_offset = 1,
     .write_after_open = 0,
   };
   struct zsvsheet_ui_buffer *uib = NULL;
@@ -766,8 +767,10 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
 
     if (handler_state.display_info.update_buffer && zsvsheet_buffer_data_filename(ub)) {
       struct zsvsheet_opts zsvsheet_opts = {0};
-      if (read_data(&ub, NULL, current_ui_buffer->input_offset.row, current_ui_buffer->input_offset.col, header_span,
-                    &zsvsheet_opts, custom_prop_handler)) {
+      struct zsvsheet_ui_buffer_opts uibopts = {0};
+      uibopts.zsvsheet_opts = &zsvsheet_opts;
+      if (read_data(&ub, &uibopts, current_ui_buffer->input_offset.row, current_ui_buffer->input_offset.col,
+                    header_span, custom_prop_handler)) {
         zsvsheet_priv_set_status(&display_dims, ZSVSHEET_STATUS_HIGH_PRIO,
                                  "Unexpected error!"); // to do: better error message
         continue;
diff --git a/app/sheet/file.c b/app/sheet/file.c
index 5e4122e9..2ad41318 100644
--- a/app/sheet/file.c
+++ b/app/sheet/file.c
@@ -6,11 +6,14 @@ int zsvsheet_ui_buffer_open_file_opts(struct zsvsheet_ui_buffer_opts *uibopts,
   struct zsvsheet_screen_buffer_opts bopts = {0};
   struct zsvsheet_ui_buffer *tmp_ui_buffer = NULL;
 
+  if (!uibopts->zsvsheet_opts)
+    uibopts->zsvsheet_opts = &zsvsheet_opts;
+
   if (!uibopts->buff_opts)
     uibopts->buff_opts = &bopts;
 
   int err = 0;
-  if ((err = read_data(&tmp_ui_buffer, uibopts, 0, 0, 0, &zsvsheet_opts, custom_prop_handler)) != 0 || !tmp_ui_buffer ||
+  if ((err = read_data(&tmp_ui_buffer, uibopts, 0, 0, 0, custom_prop_handler)) != 0 || !tmp_ui_buffer ||
       !tmp_ui_buffer->buff_used_rows) {
     zsvsheet_ui_buffer_delete(tmp_ui_buffer);
     if (err)
diff --git a/app/sheet/handlers.c b/app/sheet/handlers.c
index 8ff6b56f..c3a3dd97 100644
--- a/app/sheet/handlers.c
+++ b/app/sheet/handlers.c
@@ -192,7 +192,7 @@ struct zsvsheet_buffer_data zsvsheet_buffer_info(zsvsheet_buffer_t h) {
   struct zsvsheet_buffer_data d = {0};
   struct zsvsheet_ui_buffer *b = h;
   if (b) {
-    d.has_row_num = b->has_row_num;
+    d.has_row_num = b->no_add_row_num;
   }
   return d;
 }
diff --git a/app/sheet/pivot.c b/app/sheet/pivot.c
index 82a1a6d8..cba3c875 100644
--- a/app/sheet/pivot.c
+++ b/app/sheet/pivot.c
@@ -165,7 +165,10 @@ static zsvsheet_status zsv_sqlite3_to_csv(zsvsheet_proc_context_t pctx, struct z
       fclose(writer_opts.stream);
 
     if (tmp_fn && zsv_file_exists(tmp_fn)) {
+      struct zsvsheet_opts zsvsheet_opts = {0};
+      zsvsheet_opts.hide_row_nums = 1;
       struct zsvsheet_ui_buffer_opts uibopts = {0};
+      uibopts.zsvsheet_opts = &zsvsheet_opts;
       uibopts.data_filename = tmp_fn;
       zst = zsvsheet_open_file_opts(pctx, &uibopts);
     } else {
@@ -205,11 +208,7 @@ zsvsheet_status pivot_drill_down(zsvsheet_proc_context_t ctx) {
     if (!zdb || !(sql_str = sqlite3_str_new(zdb->db)))
       zst = zsvsheet_status_memory;
     else if (zdb->rc == SQLITE_OK && zsv_sqlite3_add_csv(zdb, pd->data_filename, &pd->zopts, NULL) == SQLITE_OK) {
-      if (zsvsheet_buffer_info(buff).has_row_num)
-        sqlite3_str_appendf(sql_str, "select *");
-      else
-        sqlite3_str_appendf(sql_str, "select rowid as [Row #], *");
-      sqlite3_str_appendf(sql_str, " from data where %s = %Q", pd->value_sql, pr->value);
+      sqlite3_str_appendf(sql_str, "select rowid as [Row #], * from data where %s = %Q", pd->value_sql, pr->value);
       zst = zsv_sqlite3_to_csv(ctx, zdb, sqlite3_str_value(sql_str), NULL, NULL, NULL);
     }
 
diff --git a/app/sheet/read-data.c b/app/sheet/read-data.c
index f4878311..c8241d32 100644
--- a/app/sheet/read-data.c
+++ b/app/sheet/read-data.c
@@ -46,13 +46,13 @@ static void get_data_index_async(struct zsvsheet_ui_buffer *uibuffp, const char
 
 static int read_data(struct zsvsheet_ui_buffer **uibufferp,   // a new zsvsheet_ui_buffer will be allocated
                      struct zsvsheet_ui_buffer_opts *uibopts, // if *uibufferp == NULL and uibopts != NULL
-                     size_t start_row, size_t start_col, size_t header_span, struct zsvsheet_opts *zsvsheet_opts,
+                     size_t start_row, size_t start_col, size_t header_span,
                      struct zsv_prop_handler *custom_prop_handler) {
-  const char *filename = (uibufferp && *uibufferp) ? (*uibufferp)->filename : uibopts ? uibopts->filename : NULL;
+  struct zsvsheet_opts *zsvsheet_opts = uibopts->zsvsheet_opts;
   struct zsv_opts opts = {0};
   if (uibufferp && *uibufferp)
     opts = (*uibufferp)->zsv_opts;
-  else if (uibopts)
+  else
     opts = uibopts->zsv_opts;
 
   struct zsvsheet_ui_buffer *uibuff = uibufferp ? *uibufferp : NULL;
@@ -60,6 +60,7 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp,   // a new zsvsheet_
   size_t remaining_header_to_skip = header_span;
   size_t original_row_num = 0;
   FILE *fp;
+  const char *filename = NULL;
 
   if (uibuff) {
     if (uibuff->data_filename)
@@ -68,7 +69,7 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp,   // a new zsvsheet_
       filename = uibuff->filename;
   }
 
-  if (!filename && uibopts) {
+  if (!filename) {
     if (uibopts->data_filename)
       filename = uibopts->data_filename;
     else if (uibopts->filename)
@@ -116,17 +117,17 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp,   // a new zsvsheet_
   size_t find_len = zsvsheet_opts->find ? strlen(zsvsheet_opts->find) : 0;
   size_t rows_searched = 0;
   zsvsheet_screen_buffer_t buffer = uibuff ? uibuff->buffer : NULL;
-  if (uibuff && uibuff->has_row_num)
+  if (uibuff && uibuff->no_add_row_num)
     zsvsheet_opts->hide_row_nums = 1;
 
   while (zsv_next_row(parser) == zsv_status_row &&
          (rows_read == 0 || rows_read < zsvsheet_screen_buffer_rows(buffer))) { // for each row
 
     size_t col_count = zsv_cell_count(parser);
-    if (uibuff == NULL && uibufferp && uibopts && col_count > 0) {
+    if (uibuff == NULL && uibufferp && col_count > 0) {
       enum zsvsheet_priv_status stat;
       struct zsvsheet_ui_buffer *tmp_uibuff = NULL;
-      if (!(buffer = zsvsheet_screen_buffer_new(col_count, uibopts->buff_opts, &stat)) ||
+      if (!(buffer = zsvsheet_screen_buffer_new(col_count + 1, uibopts->buff_opts, &stat)) ||
           stat != zsvsheet_priv_status_ok || !(tmp_uibuff = zsvsheet_ui_buffer_new(buffer, uibopts))) {
         if (tmp_uibuff)
           zsvsheet_ui_buffer_delete(tmp_uibuff);
@@ -138,16 +139,16 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp,   // a new zsvsheet_
     }
 
     // row number
-    size_t rownum_column_offset = 0;
     if (rows_read == 0 && zsvsheet_opts->hide_row_nums == 0) {
       // Check if we already have Row #
       struct zsv_cell c = zsv_get_cell(parser, 0);
       if (c.len == ZSVSHEET_ROWNUM_HEADER_LEN && !memcmp(c.str, ZSVSHEET_ROWNUM_HEADER, c.len)) {
         zsvsheet_opts->hide_row_nums = 1;
         if (uibuff)
-          uibuff->has_row_num = 1;
+          uibuff->no_add_row_num = 1;
       }
     }
+    size_t rownum_column_offset = !zsvsheet_opts->hide_row_nums;
 
     original_row_num++;
     if (remaining_header_to_skip > 0) {
@@ -155,7 +156,7 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp,   // a new zsvsheet_
       continue;
     }
     if (uibuff) {
-      if (col_count + !buffer->opts.no_rownum_column > buffer->cols) {
+      if (col_count > buffer->cols) {
         if (zsvsheet_screen_buffer_grow(buffer, col_count) != zsvsheet_priv_status_ok)
           return -1;
       }
@@ -187,7 +188,6 @@ static int read_data(struct zsvsheet_ui_buffer **uibufferp,   // a new zsvsheet_
           sprintf(buff, "########");
         zsvsheet_screen_buffer_write_cell(buffer, rows_read, 0, (unsigned char *)buff);
       }
-      rownum_column_offset = 1;
     }
 
     for (size_t i = start_col; i < col_count && i + rownum_column_offset < zsvsheet_screen_buffer_cols(buffer); i++) {
@@ -262,14 +262,16 @@ static void *get_data_index(void *gdi) {
 static size_t zsvsheet_find_next(struct zsvsheet_ui_buffer *uib, const char *needle,
                                  struct zsvsheet_opts *zsvsheet_opts, size_t header_span,
                                  struct zsv_prop_handler *custom_prop_handler) {
+  struct zsvsheet_ui_buffer_opts uibopts = {0};
   struct zsvsheet_rowcol *input_offset = &uib->input_offset;
   struct zsvsheet_rowcol *buff_offset = &uib->buff_offset;
   size_t cursor_row = uib->cursor_row;
   zsvsheet_opts->find = needle;
   zsvsheet_opts->found_rownum = 0;
+  uibopts.zsvsheet_opts = zsvsheet_opts;
   // TO DO: check if it exists in current row, later column (and change 'cursor_row - 1' below to 'cursor_row')
-  read_data(&uib, NULL, input_offset->row + buff_offset->row + header_span + cursor_row - 1, 0, header_span,
-            zsvsheet_opts, custom_prop_handler);
+  read_data(&uib, &uibopts, input_offset->row + buff_offset->row + header_span + cursor_row - 1, 0, header_span,
+            custom_prop_handler);
   zsvsheet_opts->find = NULL;
   return zsvsheet_opts->found_rownum;
 }
diff --git a/app/sheet/screen_buffer.c b/app/sheet/screen_buffer.c
index 6008ed35..812adfc8 100644
--- a/app/sheet/screen_buffer.c
+++ b/app/sheet/screen_buffer.c
@@ -82,8 +82,6 @@ zsvsheet_screen_buffer_t zsvsheet_screen_buffer_new(size_t cols, struct zsvsheet
   if (opts->cell_buff_len < sizeof(void *) * 2)
     *stat = zsvsheet_priv_status_error;
   else {
-    if (!opts->no_rownum_column)
-      cols++;
     void *data = calloc(opts->rows, cols * opts->cell_buff_len);
     if (!data)
       *stat = zsvsheet_priv_status_memory;
@@ -110,9 +108,6 @@ enum zsvsheet_priv_status zsvsheet_screen_buffer_grow(zsvsheet_screen_buffer_t b
   size_t cell_buff_len = buff->opts.cell_buff_len;
   size_t old_row_len = old_cols * cell_buff_len;
 
-  if (!buff->opts.no_rownum_column)
-    cols++;
-
   size_t row_len = cols * cell_buff_len;
 
   assert(cols > old_cols);
diff --git a/app/sheet/screen_buffer.h b/app/sheet/screen_buffer.h
index cd1aa70a..a9012ed1 100644
--- a/app/sheet/screen_buffer.h
+++ b/app/sheet/screen_buffer.h
@@ -8,10 +8,9 @@
 typedef struct zsvsheet_screen_buffer *zsvsheet_screen_buffer_t;
 
 struct zsvsheet_screen_buffer_opts {
-  size_t cell_buff_len;  // default = 16. must be >= 2 * sizeof(void *)
-  size_t max_cell_len;   // length in bytes; defaults to 32767
-  size_t rows;           // rows to buffer. cannot be < 256
-  char no_rownum_column; // reserved. TO DO: if set, omit row num column
+  size_t cell_buff_len; // default = 16. must be >= 2 * sizeof(void *)
+  size_t max_cell_len;  // length in bytes; defaults to 32767
+  size_t rows;          // rows to buffer. cannot be < 256
 };
 
 zsvsheet_screen_buffer_t zsvsheet_screen_buffer_new(size_t cols, struct zsvsheet_screen_buffer_opts *opts,
diff --git a/app/sheet/transformation.c b/app/sheet/transformation.c
index 5d58f8ff..2b423b38 100644
--- a/app/sheet/transformation.c
+++ b/app/sheet/transformation.c
@@ -269,8 +269,10 @@ enum zsvsheet_status zsvsheet_push_transformation(zsvsheet_proc_context_t ctx,
     goto error;
   }
 
+  struct zsvsheet_opts zsvsheet_opts = {0};
   struct zsvsheet_ui_buffer_opts uibopts = {0};
 
+  uibopts.zsvsheet_opts = &zsvsheet_opts;
   uibopts.data_filename = zsvsheet_transformation_filename(trn);
   uibopts.write_after_open = 1;
 
diff --git a/app/sheet/ui_buffer.c b/app/sheet/ui_buffer.c
index 80314d59..b2bb770c 100644
--- a/app/sheet/ui_buffer.c
+++ b/app/sheet/ui_buffer.c
@@ -40,7 +40,7 @@ struct zsvsheet_ui_buffer {
   unsigned char index_ready : 1;
   unsigned char rownum_col_offset : 1;
   unsigned char index_started : 1;
-  unsigned char has_row_num : 1;
+  unsigned char no_add_row_num : 1;
   unsigned char mutex_inited : 1;
   unsigned char write_in_progress : 1;
   unsigned char write_done : 1;
@@ -92,11 +92,11 @@ void zsvsheet_ui_buffer_delete(struct zsvsheet_ui_buffer *ub) {
 }
 
 struct zsvsheet_ui_buffer_opts {
+  struct zsvsheet_opts *zsvsheet_opts;
   struct zsvsheet_screen_buffer_opts *buff_opts;
   const char *filename;
   const char *data_filename;
   struct zsv_opts zsv_opts; // options to use when opening this file
-  char no_rownum_col_offset;
   char write_after_open;
 };
 
@@ -108,9 +108,14 @@ struct zsvsheet_ui_buffer *zsvsheet_ui_buffer_new(zsvsheet_screen_buffer_t buffe
     uib->buffer = buffer;
     uib->mutex_inited = 1;
     memcpy(&uib->mutex, &init, sizeof(init));
-    if (!(uibopts && uibopts->no_rownum_col_offset))
-      uib->rownum_col_offset = 1;
+    uib->rownum_col_offset = !uibopts;
+
     if (uibopts) {
+      if (uibopts->zsvsheet_opts->hide_row_nums)
+        uib->no_add_row_num = 1;
+      else
+        uib->rownum_col_offset = 1;
+
       if (uibopts->filename && !(uib->filename = strdup(uibopts->filename))) {
         zsvsheet_ui_buffer_delete(uib);
         return NULL;
diff --git a/app/test/expected/test-sheet-pivot-1-groups.out b/app/test/expected/test-sheet-pivot-1-groups.out
index f173628f..05b370a3 100644
--- a/app/test/expected/test-sheet-pivot-1-groups.out
+++ b/app/test/expected/test-sheet-pivot-1-groups.out
@@ -1,25 +1,25 @@
-Row #                     value                     Count
-1                         ad                        27
-2                         ae                        147
-3                         af                        27822
-4                         ag                        55
-5                         ai                        14
-6                         al                        4730
-7                         am                        896
-8                         an                        75
-9                         ao                        6170
-10                        ar                        2817
-11                        at                        4740
-12                        au                        3403
-13                        aw                        37
-14                        az                        3522
-15                        ba                        5076
-16                        bb                        165
-17                        bd                        8280
-18                        be                        5056
-19                        bf                        3248
-20                        bg                        6258
-21                        bh                        102
-22                        bi                        646
-23                        bj                        1331
-? for help 1
+value                                   Count
+ad                                      27
+ae                                      147
+af                                      27822
+ag                                      55
+ai                                      14
+al                                      4730
+am                                      896
+an                                      75
+ao                                      6170
+ar                                      2817
+at                                      4740
+au                                      3403
+aw                                      37
+az                                      3522
+ba                                      5076
+bb                                      165
+bd                                      8280
+be                                      5056
+bf                                      3248
+bg                                      6258
+bh                                      102
+bi                                      646
+bj                                      1331
+? for help ad
diff --git a/app/test/expected/test-sheet-pivot-2-groups.out b/app/test/expected/test-sheet-pivot-2-groups.out
index 71316b11..d933c39e 100644
--- a/app/test/expected/test-sheet-pivot-2-groups.out
+++ b/app/test/expected/test-sheet-pivot-2-groups.out
@@ -1,25 +1,25 @@
-Row #                     value                     Count
-1                          C1                       1
-2                          C10                      1
-3                          C100                     1
-4                          C1000                    1
-5                          C1001                    1
-6                          C1002                    1
-7                          C1003                    1
-8                          C1004                    1
-9                          C1005                    1
-10                         C1006                    1
-11                         C1007                    1
-12                         C1008                    1
-13                         C1009                    1
-14                         C101                     1
-15                         C1010                    1
-16                         C1011                    1
-17                         C1012                    1
-18                         C1013                    1
-19                         C1014                    1
-20                         C1015                    1
-21                         C1016                    1
-22                         C1017                    1
-23                         C1018                    1
-? for help 1
+value                                   Count
+ C1                                     1
+ C10                                    1
+ C100                                   1
+ C1000                                  1
+ C1001                                  1
+ C1002                                  1
+ C1003                                  1
+ C1004                                  1
+ C1005                                  1
+ C1006                                  1
+ C1007                                  1
+ C1008                                  1
+ C1009                                  1
+ C101                                   1
+ C1010                                  1
+ C1011                                  1
+ C1012                                  1
+ C1013                                  1
+ C1014                                  1
+ C1015                                  1
+ C1016                                  1
+ C1017                                  1
+ C1018                                  1
+? for help  C1

From 5e606000f10e582fc275aee5fab6595928d19f3b Mon Sep 17 00:00:00 2001
From: Richard Palethorpe <io@richiejp.com>
Date: Thu, 19 Dec 2024 14:28:02 +0000
Subject: [PATCH 7/8] sheet: Allow extensions to disable row numbers

---
 app/cli.c                           |  1 +
 app/ext_example/mysheet_extension.c | 18 +++++++++---------
 app/sheet/file.c                    | 16 ++++++++++++++++
 app/sheet/handlers_internal.h       |  5 +++++
 include/zsv/ext.h                   |  5 +++++
 include/zsv/ext/sheet.h             |  9 +++++++++
 6 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/app/cli.c b/app/cli.c
index cc71173b..1bd187f6 100644
--- a/app/cli.c
+++ b/app/cli.c
@@ -419,6 +419,7 @@ static struct zsv_ext_callbacks *zsv_ext_callbacks_init(struct zsv_ext_callbacks
     e->ext_sheet_buffer_filename = zsvsheet_buffer_filename;
     e->ext_sheet_buffer_data_filename = zsvsheet_buffer_data_filename;
     e->ext_sheet_open_file = zsvsheet_open_file;
+    e->ext_sheet_open_file_opts = zsvsheet_ext_open_file_opts;
     e->ext_sheet_register_proc = zsvsheet_register_proc;
     e->ext_sheet_register_proc_key_binding = zsvsheet_register_proc_key_binding;
     e->ext_sheet_push_transformation = zsvsheet_push_transformation;
diff --git a/app/ext_example/mysheet_extension.c b/app/ext_example/mysheet_extension.c
index 2111572c..60d31d01 100644
--- a/app/ext_example/mysheet_extension.c
+++ b/app/ext_example/mysheet_extension.c
@@ -189,9 +189,12 @@ static zsvsheet_status zsv_sqlite3_to_csv(zsvsheet_proc_context_t pctx, struct z
     if (writer_opts.stream)
       fclose(writer_opts.stream);
 
-    if (tmp_fn && zsv_file_exists(tmp_fn))
-      zst = zsv_cb.ext_sheet_open_file(pctx, tmp_fn, NULL);
-    else {
+    if (tmp_fn && zsv_file_exists(tmp_fn)) {
+      struct zsvsheet_open_file_opts ofopts = {0};
+      ofopts.data_filename = tmp_fn;
+      ofopts.no_auto_row_num = 1;
+      zst = zsv_cb.ext_sheet_open_file_opts(pctx, &ofopts);
+    } else {
       if (zst == zsvsheet_status_ok) {
         zst = zsvsheet_status_error; // to do: make this more specific
         if (!err_msg && zdb && zdb->rc != SQLITE_OK)
@@ -229,12 +232,9 @@ zsvsheet_status pivot_drill_down(zsvsheet_proc_context_t ctx) {
     if (!zdb || !(sql_str = sqlite3_str_new(zdb->db)))
       zst = zsvsheet_status_memory;
     else if (zdb->rc == SQLITE_OK && zsv_cb.ext_sqlite3_add_csv(zdb, pd->data_filename, NULL, NULL) == SQLITE_OK) {
-      if (zsv_cb.ext_sheet_buffer_info(buff).has_row_num)
-        sqlite3_str_appendf(sql_str, "select *");
-      else
-        sqlite3_str_appendf(sql_str, "select rowid as [Row #], *");
+      sqlite3_str_appendf(sql_str, "select rowid as [Row #], *");
       sqlite3_str_appendf(sql_str, " from data where %s = %Q", pd->value_sql, pr->value);
-      fprintf(stderr, "SQL: %s\n", sqlite3_str_value(sql_str));
+      // fprintf(stderr, "SQL: %s\n", sqlite3_str_value(sql_str));
       zst = zsv_sqlite3_to_csv(ctx, zdb, sqlite3_str_value(sql_str), NULL, NULL, NULL);
     }
     if (sql_str)
@@ -327,7 +327,7 @@ enum zsv_ext_status zsv_ext_init(struct zsv_ext_callbacks *cb, zsv_execution_con
   int proc_id = zsv_cb.ext_sheet_register_proc("my-sheet-pivot", "my sheet pivot", my_pivot_table_command_handler);
   if (proc_id < 0)
     return zsv_ext_status_error;
-  zsv_cb.ext_sheet_register_proc_key_binding('v', proc_id);
+  zsv_cb.ext_sheet_register_proc_key_binding('s', proc_id);
   return zsv_ext_status_ok;
 }
 
diff --git a/app/sheet/file.c b/app/sheet/file.c
index 2ad41318..81aceadd 100644
--- a/app/sheet/file.c
+++ b/app/sheet/file.c
@@ -1,3 +1,5 @@
+#include <zsv/ext/sheet.h>
+
 int zsvsheet_ui_buffer_open_file_opts(struct zsvsheet_ui_buffer_opts *uibopts,
                                       struct zsv_prop_handler *custom_prop_handler,
                                       struct zsvsheet_ui_buffer **ui_buffer_stack_bottom,
@@ -65,3 +67,17 @@ zsvsheet_status zsvsheet_open_file_opts(struct zsvsheet_proc_context *ctx, struc
     return zsvsheet_status_error;
   return zsvsheet_status_ok;
 }
+
+zsvsheet_status zsvsheet_ext_open_file_opts(struct zsvsheet_proc_context *ctx, struct zsvsheet_open_file_opts *opts) {
+  struct zsvsheet_ui_buffer_opts uibopts = {0};
+  struct zsvsheet_opts zsvsheet_opts = {0};
+
+  zsvsheet_opts.hide_row_nums = opts->no_auto_row_num;
+  uibopts.zsvsheet_opts = &zsvsheet_opts;
+  uibopts.filename = opts->filename;
+  uibopts.data_filename = opts->data_filename;
+  if (opts->zsv_opts)
+    uibopts.zsv_opts = *opts->zsv_opts;
+
+  return zsvsheet_open_file_opts(ctx, &uibopts);
+}
diff --git a/app/sheet/handlers_internal.h b/app/sheet/handlers_internal.h
index b639fe45..8ea779ec 100644
--- a/app/sheet/handlers_internal.h
+++ b/app/sheet/handlers_internal.h
@@ -34,6 +34,11 @@ zsvsheet_status zsvsheet_open_file(struct zsvsheet_proc_context *ctx, const char
 
 /** extension support **/
 
+/**
+ * Open a tabular file with external facing options
+ */
+zsvsheet_status zsvsheet_ext_open_file_opts(struct zsvsheet_proc_context *ctx, struct zsvsheet_open_file_opts *opts);
+
 /**
  * Set the subcommand prompt
  */
diff --git a/include/zsv/ext.h b/include/zsv/ext.h
index 392399a2..44ca3a91 100644
--- a/include/zsv/ext.h
+++ b/include/zsv/ext.h
@@ -238,6 +238,11 @@ struct zsv_ext_callbacks {
    */
   zsvsheet_status (*ext_sheet_open_file)(zsvsheet_proc_context_t, const char *filepath, struct zsv_opts *zopts);
 
+  /**
+   * Open a tabular file with more available options
+   */
+  zsvsheet_status (*ext_sheet_open_file_opts)(struct zsvsheet_proc_context *ctx, struct zsvsheet_open_file_opts *opts);
+
   /**
    * Set custom context
    * @param on_close optional callback to invoke when the buffer is closed
diff --git a/include/zsv/ext/sheet.h b/include/zsv/ext/sheet.h
index f3e6b7e6..51c1e1c0 100644
--- a/include/zsv/ext/sheet.h
+++ b/include/zsv/ext/sheet.h
@@ -1,6 +1,8 @@
 #ifndef ZSVSHEET_H
 #define ZSVSHEET_H
 
+#include <zsv.h>
+
 /* custom sheet handler id */
 typedef int zsvsheet_proc_id_t;
 
@@ -22,6 +24,13 @@ typedef struct zsvsheet_subcommand_context *zsvsheet_subcommand_context_t;
 typedef void *zsvsheet_buffer_t;
 // int zsvsheet_ext_keypress(zsvsheet_proc_context_t);
 
+struct zsvsheet_open_file_opts {
+  const char *filename;
+  const char *data_filename;
+  struct zsv_opts *zsv_opts;
+  char no_auto_row_num;
+};
+
 typedef struct zsvsheet_transformation *zsvsheet_transformation;
 /**
  * Transformation options passed to zsvsheet_push_transformation

From 59e7add6638772220eb93a8275798917df030708 Mon Sep 17 00:00:00 2001
From: Richard Palethorpe <io@richiejp.com>
Date: Fri, 20 Dec 2024 12:25:31 +0000
Subject: [PATCH 8/8] sheet: Fix missing attrs type after file included in
 pivot.c

---
 app/sheet/handlers_internal.h | 2 ++
 app/sheet/pivot.c             | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/sheet/handlers_internal.h b/app/sheet/handlers_internal.h
index 8ea779ec..975007de 100644
--- a/app/sheet/handlers_internal.h
+++ b/app/sheet/handlers_internal.h
@@ -1,6 +1,8 @@
 #ifndef ZSVSHEET_HANDLER_INTERNAL_H
 #define ZSVSHEET_HANDLER_INTERNAL_H
 
+#include <zsv/ext.h>
+
 struct zsvsheet_context {
   const char *subcommand_value; // e.g. "/path/to/myfile.csv"
   int ch;                       // key press value from getch()
diff --git a/app/sheet/pivot.c b/app/sheet/pivot.c
index cba3c875..2819884d 100644
--- a/app/sheet/pivot.c
+++ b/app/sheet/pivot.c
@@ -85,7 +85,8 @@ static struct pivot_row *get_pivot_row_data(struct pivot_data *pd, size_t row_ix
 }
 
 // TO DO: return zsvsheet_status
-static enum zsv_ext_status get_cell_attrs(void *pdh, int *attrs, size_t start_row, size_t row_count, size_t cols) {
+static enum zsv_ext_status get_cell_attrs(void *pdh, zsvsheet_cell_attr_t *attrs, size_t start_row, size_t row_count,
+                                          size_t cols) {
   struct pivot_data *pd = pdh;
   size_t end_row = start_row + row_count;
   int attr = 0;