Skip to content

Commit

Permalink
prelim commit (#238)
Browse files Browse the repository at this point in the history 
* further modularize support for overwrites
  • Loading branch information
@liquidaty
liquidaty authored Oct 24, 2024
1 parent 9fcfe19 commit a851a46
Show file tree
Hide file tree
Showing 20 changed files with 2,019 additions and 306 deletions.
17 changes: 5 additions & 12 deletions app/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ CFLAGS+= -I${PREFIX}/include
THIS_LIB_BASE=$(shell cd .. && pwd)
INCLUDE_DIR=${THIS_LIB_BASE}/include
BUILD_DIR=${THIS_LIB_BASE}/build/${BUILD_SUBDIR}/${CCBN}
UTILS1=writer file err signal mem clock arg dl string dirs prop cache jq os
UTILS1=writer file err signal mem clock arg dl string dirs prop cache jq os overwrite

ZSV_EXTRAS ?=

Expand Down Expand Up @@ -241,7 +241,6 @@ YAJL_INCLUDE=-I${YAJL_SRC_DIR}/build/yajl-2.1.1/include

YAJL_HELPER_OBJ=${BUILD_DIR}-external/yajl_helper/yajl_helper.o
YAJL_HELPER_INCLUDE=-I${THIS_MAKEFILE_DIR}/external/yajl_helper
# YAJL_HELPER_INCLUDE=-I${THIS_MAKEFILE_DIR}/external
CFLAGS+=${YAJL_HELPER_INCLUDE} ${YAJL_INCLUDE}

# jq
Expand Down Expand Up @@ -275,9 +274,9 @@ SQLITE_SRC=${THIS_MAKEFILE_DIR}/external/sqlite3/sqlite3*.c
SQLITE_EXT=${BUILD_DIR}-external/sqlite3/sqlite3_and_csv_vtab.o
SQLITE_EXT_INCLUDE=-I${THIS_MAKEFILE_DIR}/external/sqlite3

# everything uses prop, which in turn uses yajl and jq and json
OBJECTS+= ${YAJL_OBJ} ${YAJL_HELPER_OBJ} ${BUILD_DIR}/objs/utils/json.o
MORE_SOURCE+= ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -I${JQ_INCLUDE_DIR}
# everything uses prop, which in turn uses yajl and jq and json and sqlite3
OBJECTS+= ${YAJL_OBJ} ${YAJL_HELPER_OBJ} ${BUILD_DIR}/objs/utils/json.o ${SQLITE_EXT}
MORE_SOURCE+= ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -I${JQ_INCLUDE_DIR} ${SQLITE_EXT_INCLUDE}
MORE_LIBS+=${JQ_LIB} ${LDFLAGS_JQ}

help:
Expand Down Expand Up @@ -425,15 +424,9 @@ ${JSONWRITER_OBJECT}: ${JSONWRITER_SRC}/jsonwriter.c
@mkdir -p `dirname "$@"`
${CC} ${CFLAGS} ${JSONWRITER_INCLUDE} -DINCLUDE_UTILS $< -c -o $@

# flatten stack desc use sglib
# ${STANDALONE_PFX}flatten${EXE} ${STANDALONE_PFX}stack${EXE} ${STANDALONE_PFX}desc${EXE}:
# sglib
MORE_SOURCE+=-I${THIS_MAKEFILE_DIR}/external/sglib

# sql, 2db, 2json, echo, compare use sqlite3
${CLI} ${STANDALONE_PFX}sql${EXE} ${STANDALONE_PFX}2db${EXE} ${STANDALONE_PFX}2json${EXE} ${STANDALONE_PFX}echo${EXE} ${STANDALONE_PFX}compare${EXE}: ${SQLITE_EXT}
${CLI} ${STANDALONE_PFX}sql${EXE} ${STANDALONE_PFX}2db${EXE} ${STANDALONE_PFX}2json${EXE} ${STANDALONE_PFX}echo${EXE} ${STANDALONE_PFX}compare${EXE}: MORE_OBJECTS+=${SQLITE_EXT}
${STANDALONE_PFX}sql${EXE} ${CLI_OBJ_PFX}sql.o ${STANDALONE_PFX}2db${EXE} ${CLI_OBJ_PFX}2db.o ${STANDALONE_PFX}2json${EXE} ${CLI_OBJ_PFX}2json.o ${STANDALONE_PFX}echo${EXE} ${CLI_OBJ_PFX}echo.o ${CLI_OBJ_PFX}compare.o: MORE_SOURCE+=${SQLITE_EXT_INCLUDE}

# 2json, desc, compare use jsonwriter
${CLI} ${STANDALONE_PFX}2json${EXE} ${STANDALONE_PFX}desc${EXE} ${STANDALONE_PFX}compare${EXE}: ${JSONWRITER_OBJECT}
${CLI} ${STANDALONE_PFX}2json${EXE} ${STANDALONE_PFX}desc${EXE} ${STANDALONE_PFX}compare${EXE}: MORE_OBJECTS+= ${JSONWRITER_OBJECT}
Expand Down
3 changes: 2 additions & 1 deletion app/cli.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include <zsv/utils/string.h>
#include <zsv/utils/dirs.h>
#include <zsv/utils/signal.h>
#include <zsv/utils/overwrite.h>
#include <zsv.h>
#include <zsv/ext.h>
#include "cli_internal.h"
Expand Down Expand Up @@ -431,12 +432,12 @@ static enum zsv_ext_status run_extension(int argc, const char *argv[], struct zs
}

struct zsv_execution_data ctx = {0};

if ((stat = execution_context_init(&ctx, argc, argv)) == zsv_ext_status_ok) {
struct zsv_opts opts;
zsv_args_to_opts(argc, argv, &argc, argv, &opts, ctx.opts_used);
zsv_set_default_opts(opts);
// need a corresponding zsv_set_default_custom_prop_handler?

stat = cmd->main(&ctx, ctx.argc - 1, &ctx.argv[1], &opts, ctx.opts_used);
}

Expand Down
210 changes: 41 additions & 169 deletions app/echo.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,7 @@
#include <zsv/utils/string.h>
#include <zsv/utils/mem.h>
#include <zsv/utils/arg.h>

enum zsv_echo_overwrite_input_type {
zsv_echo_overwrite_input_type_sqlite3 = 0
};
#include <zsv/utils/overwrite.h>

struct zsv_echo_data {
FILE *in;
Expand All @@ -34,24 +31,6 @@ struct zsv_echo_data {
zsv_parser parser;
size_t row_ix;

struct {
size_t row_ix;
size_t col_ix;
const unsigned char *str;
size_t len;
char eof;
} overwrite;

enum zsv_echo_overwrite_input_type overwrite_input_type;
struct {
struct {
char *filename;
sqlite3 *db;
sqlite3_stmt *stmt; // select row, column, overwrite
const char *sql;
} sqlite3;
} o;

unsigned char *skip_until_prefix;
size_t skip_until_prefix_len;

Expand All @@ -63,36 +42,6 @@ struct zsv_echo_data {
unsigned char _ : 5;
};

/**
* check if sqlite3 statement is valid
* return error
*/
static int zsv_echo_sqlite3_check_stmt(sqlite3_stmt *stmt) {
if (sqlite3_column_count(stmt) < 3)
return 1;
// TO DO: check that columns are row, column, value
return 0;
}

/**
* TO DO: verify original value
*/
void zsv_echo_get_next_overwrite(struct zsv_echo_data *data) {
if (!data->overwrite.eof) {
sqlite3_stmt *stmt = data->o.sqlite3.stmt;
if (stmt) {
if (sqlite3_step(stmt) == SQLITE_ROW) {
// row, column, value
data->overwrite.row_ix = sqlite3_column_int64(stmt, 0);
data->overwrite.col_ix = sqlite3_column_int64(stmt, 1);
data->overwrite.str = sqlite3_column_text(stmt, 2);
data->overwrite.len = sqlite3_column_bytes(stmt, 2);
} else
data->overwrite.eof = 1;
}
}
}

static void zsv_echo_get_max_nonempty_cols(void *hook) {
struct zsv_echo_data *data = hook;
unsigned row_nonempty_col_count = 0;
Expand Down Expand Up @@ -124,18 +73,11 @@ static void zsv_echo_row(void *hook) {
zsv_abort(data->parser);
} else {
for (size_t i = 0; i < j; i++) {
if (VERY_UNLIKELY(data->overwrite.row_ix == data->row_ix && data->overwrite.col_ix == i)) {
zsv_writer_cell(data->csv_writer, i == 0, data->overwrite.str, data->overwrite.len, 1);
zsv_echo_get_next_overwrite(data);
} else {
struct zsv_cell cell = zsv_get_cell(data->parser, i);
if (UNLIKELY(data->trim_white))
cell.str = (unsigned char *)zsv_strtrim(cell.str, &cell.len);
zsv_writer_cell(data->csv_writer, i == 0, cell.str, cell.len, cell.quoted);
}
struct zsv_cell cell = zsv_get_cell(data->parser, i);
if (UNLIKELY(data->trim_white))
cell.str = (unsigned char *)zsv_strtrim(cell.str, &cell.len);
zsv_writer_cell(data->csv_writer, i == 0, cell.str, cell.len, cell.quoted);
}
while (!data->overwrite.eof && data->overwrite.row_ix <= data->row_ix)
zsv_echo_get_next_overwrite(data);
}
data->row_ix++;
}
Expand Down Expand Up @@ -182,87 +124,29 @@ static int zsv_echo_usage() {

static void zsv_echo_cleanup(struct zsv_echo_data *data) {
zsv_writer_delete(data->csv_writer);
free(data->o.sqlite3.filename);
free(data->skip_until_prefix);
if (data->o.sqlite3.stmt)
sqlite3_finalize(data->o.sqlite3.stmt);
if (data->in && data->in != stdin)
fclose(data->in);
if (data->o.sqlite3.db)
sqlite3_close(data->o.sqlite3.db);

if (data->tmp_fn) {
remove(data->tmp_fn);
free(data->tmp_fn);
}
}

#define zsv_echo_sqlite3_prefix "sqlite3://"

static int zsv_echo_parse_overwrite_source(struct zsv_echo_data *data, const char *source, size_t len) {
size_t pfx_len;
if (len > (pfx_len = strlen(zsv_echo_sqlite3_prefix)) && !memcmp(source, zsv_echo_sqlite3_prefix, pfx_len)) {
data->o.sqlite3.filename = zsv_memdup(source + pfx_len, len - pfx_len);
char *q = memchr(data->o.sqlite3.filename, '?', len - pfx_len);
if (q) {
*q = '\0';
q++;
#define zsv_echo_sql_prefix "sql="
const char *sql = strstr(q, zsv_echo_sql_prefix);
if (sql)
data->o.sqlite3.sql = sql + strlen(zsv_echo_sql_prefix);
}
// open the sql connection
if (!(data->o.sqlite3.filename && *data->o.sqlite3.filename && data->o.sqlite3.sql && *data->o.sqlite3.sql)) {
free(data->o.sqlite3.filename);
fprintf(stderr, "Invalid query string\n");
return 1;
}

int rc = sqlite3_open_v2(data->o.sqlite3.filename, &data->o.sqlite3.db, SQLITE_OPEN_READONLY, NULL);
if (rc != SQLITE_OK || !data->o.sqlite3.db) {
fprintf(stderr, "%s: %s\n", sqlite3_errstr(rc), data->o.sqlite3.filename);
return 1;
}

if (!data->o.sqlite3.sql) {
// to do: detect it from the db
fprintf(stderr, "Missing sql select statement for sqlite3 echo data e.g.:\n"
" select row, column, value from overwrites order by row, column\n");
return 1;
}

rc = sqlite3_prepare_v2(data->o.sqlite3.db, data->o.sqlite3.sql, -1, &data->o.sqlite3.stmt, NULL);
if (rc != SQLITE_OK || !data->o.sqlite3.stmt) {
fprintf(stderr, "%s\n", sqlite3_errmsg(data->o.sqlite3.db));
return 1;
}

// successful sqlite3 connection
data->overwrite.eof = 0;
return 0;
}

fprintf(stderr, "Invalid overwrite source: %s\n", source);
return 1;
}

int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *opts,
int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *optsp,
struct zsv_prop_handler *custom_prop_handler, const char *opts_used) {
if (argc < 1 || (argc > 1 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")))) {
zsv_echo_usage();
return 0;
}

struct zsv_opts opts = *optsp;
struct zsv_csv_writer_options writer_opts = zsv_writer_get_default_opts();
struct zsv_echo_data data = {0};
data.overwrite_input_type = zsv_echo_overwrite_input_type_sqlite3;
struct zsv_overwrite_opts overwrite_opts = {0};

int err = 0;

const char *overwrites_csv = NULL;

data.overwrite.eof = 1;
for (int arg_i = 1; !err && arg_i < argc; arg_i++) {
const char *arg = argv[arg_i];
if (!strcmp(arg, "-b"))
Expand Down Expand Up @@ -291,20 +175,9 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
data.skip_until_prefix = (unsigned char *)strdup(argv[arg_i]);
data.skip_until_prefix_len = data.skip_until_prefix ? strlen((char *)data.skip_until_prefix) : 0;
}
} else if (!strcmp(arg, "--overwrite")) {
if (arg_i + 1 >= argc) {
fprintf(stderr, "Option %s requires a value\n", arg);
err = 1;
} else {
const char *src = argv[++arg_i];
if (strlen(src) > strlen(zsv_echo_sqlite3_prefix) &&
!memcmp(zsv_echo_sqlite3_prefix, src, strlen(zsv_echo_sqlite3_prefix)))
err = zsv_echo_parse_overwrite_source(&data, src, strlen(src));
else {
overwrites_csv = src;
}
}
} else if (!data.in) {
} else if (!strcmp(arg, "--overwrite"))
overwrite_opts.src = zsv_next_arg(++arg_i, argc, argv, &err);
else if (!data.in) {
#ifndef NO_STDIN
if (!strcmp(arg, "-"))
data.in = stdin;
Expand Down Expand Up @@ -338,7 +211,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op

unsigned char buff[4096];
if (data.skip_until_prefix)
opts->row_handler = zsv_echo_row_skip_until;
opts.row_handler = zsv_echo_row_skip_until;
else {
if (data.trim_columns) {
// first, save the file if it is stdin
Expand Down Expand Up @@ -366,7 +239,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
}
}
// next, determine the max number of columns from the left that contains data
struct zsv_opts tmp_opts = *opts;
struct zsv_opts tmp_opts = opts;
tmp_opts.row_handler = zsv_echo_get_max_nonempty_cols;
tmp_opts.stream = data.in;
tmp_opts.ctx = &data;
Expand All @@ -387,42 +260,41 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
data.in = fopen(data.tmp_fn ? data.tmp_fn : data.input_path, "rb");
}
}
opts->row_handler = zsv_echo_row;
opts.row_handler = zsv_echo_row;
}
opts->stream = data.in;
opts->ctx = &data;
opts.stream = data.in;
opts.ctx = &data;

data.csv_writer = zsv_writer_new(&writer_opts);

if (overwrites_csv) {
if (!(opts->overwrite.ctx = fopen(overwrites_csv, "rb"))) {
fprintf(stderr, "Unable to open for write: %s\n", overwrites_csv);
zsv_echo_cleanup(&data);
return 1;
if (overwrite_opts.src) {
if (!(opts.overwrite.ctx = zsv_overwrite_context_new(&overwrite_opts))) {
fprintf(stderr, "Out of memory!\n");
err = 1;
} else {
opts->overwrite.type = zsv_overwrite_type_csv;
opts->overwrite.close_ctx = (int (*)(void *))fclose;
opts.overwrite.open = zsv_overwrite_open;
opts.overwrite.next = zsv_overwrite_next;
opts.overwrite.close = zsv_overwrite_context_delete;
}
}

if (zsv_new_with_properties(opts, custom_prop_handler, data.input_path, opts_used, &data.parser) != zsv_status_ok ||
!data.csv_writer) {
zsv_echo_cleanup(&data);
return 1;
if (data.csv_writer && !err) {
if (zsv_new_with_properties(&opts, custom_prop_handler, data.input_path, opts_used, &data.parser) != zsv_status_ok)
err = 1;
else {
// create a local csv writer buff for faster performance
// unsigned char writer_buff[64];
zsv_writer_set_temp_buff(data.csv_writer, buff, sizeof(buff));

// process the input data
zsv_handle_ctrl_c_signal();
enum zsv_status status;
while (!zsv_signal_interrupted && (status = zsv_parse_more(data.parser)) == zsv_status_ok)
;

zsv_finish(data.parser);
zsv_delete(data.parser);
}
}

// create a local csv writer buff for faster performance
// unsigned char writer_buff[64];
zsv_writer_set_temp_buff(data.csv_writer, buff, sizeof(buff));

// process the input data
zsv_handle_ctrl_c_signal();
enum zsv_status status;
while (!zsv_signal_interrupted && (status = zsv_parse_more(data.parser)) == zsv_status_ok)
;

zsv_finish(data.parser);
zsv_delete(data.parser);
zsv_echo_cleanup(&data);
return 0;
return err;
}
2 changes: 1 addition & 1 deletion app/ext_example/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ CLI=${BUILD_DIR}/bin/cli
RUN_CLI=ZSV_CONFIG_DIR=/tmp ${CLI}

${BUILD_DIR}/bin/cli:
(cd .. && make CC=${CC} CONFIGFILE=${CONFIGFILEPATH} DEBUG=${DEBUG} ${BUILD_DIR}/bin/cli)
make -C .. build-cli CONFIGFILE=${CONFIGFILEPATH} DEBUG=${DEBUG}

${BUILD_DIR}/objs/utils/%.o:
(cd .. && make CONFIGFILE=${CONFIGFILEPATH} CC=${CC} DEBUG=${DEBUG} $@ )
Expand Down
Loading

0 comments on commit a851a46

Please sign in to comment.