Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

prelim commit #238

Merged
merged 11 commits into from
Oct 24, 2024
Merged

prelim commit #238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
94a8164
prelim commit
liquidaty Oct 23, 2024
57fa2b9
update Makefile to always add SQLITE_EXT dependency
liquidaty Oct 23, 2024
98d593f
further Makefile update for sqlite3.h include when building zsv.o
liquidaty Oct 23, 2024
8bf0261
fix prior commit
liquidaty Oct 23, 2024
e3a92b7
move overwrite handling to custom callbacks
liquidaty Oct 23, 2024
bcaa133
revert Makefile to undo prior commits in this branch
liquidaty Oct 23, 2024
6383b91
clang-format
liquidaty Oct 23, 2024
50beb8a
add general --overwrite-auto option
liquidaty Oct 24, 2024
4f3c7b1
clang-format
liquidaty Oct 24, 2024
8c4ebea
clang-format overwrite.h
liquidaty Oct 24, 2024
4a53b81
comment cleanup
liquidaty Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 5 additions & 12 deletions app/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ CFLAGS+= -I${PREFIX}/include
THIS_LIB_BASE=$(shell cd .. && pwd)
INCLUDE_DIR=${THIS_LIB_BASE}/include
BUILD_DIR=${THIS_LIB_BASE}/build/${BUILD_SUBDIR}/${CCBN}
UTILS1=writer file err signal mem clock arg dl string dirs prop cache jq os
UTILS1=writer file err signal mem clock arg dl string dirs prop cache jq os overwrite

ZSV_EXTRAS ?=

Expand Down Expand Up @@ -241,7 +241,6 @@ YAJL_INCLUDE=-I${YAJL_SRC_DIR}/build/yajl-2.1.1/include

YAJL_HELPER_OBJ=${BUILD_DIR}-external/yajl_helper/yajl_helper.o
YAJL_HELPER_INCLUDE=-I${THIS_MAKEFILE_DIR}/external/yajl_helper
# YAJL_HELPER_INCLUDE=-I${THIS_MAKEFILE_DIR}/external
CFLAGS+=${YAJL_HELPER_INCLUDE} ${YAJL_INCLUDE}

# jq
Expand Down Expand Up @@ -275,9 +274,9 @@ SQLITE_SRC=${THIS_MAKEFILE_DIR}/external/sqlite3/sqlite3*.c
SQLITE_EXT=${BUILD_DIR}-external/sqlite3/sqlite3_and_csv_vtab.o
SQLITE_EXT_INCLUDE=-I${THIS_MAKEFILE_DIR}/external/sqlite3

# everything uses prop, which in turn uses yajl and jq and json
OBJECTS+= ${YAJL_OBJ} ${YAJL_HELPER_OBJ} ${BUILD_DIR}/objs/utils/json.o
MORE_SOURCE+= ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -I${JQ_INCLUDE_DIR}
# everything uses prop, which in turn uses yajl and jq and json and sqlite3
OBJECTS+= ${YAJL_OBJ} ${YAJL_HELPER_OBJ} ${BUILD_DIR}/objs/utils/json.o ${SQLITE_EXT}
MORE_SOURCE+= ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -I${JQ_INCLUDE_DIR} ${SQLITE_EXT_INCLUDE}
MORE_LIBS+=${JQ_LIB} ${LDFLAGS_JQ}

help:
Expand Down Expand Up @@ -425,15 +424,9 @@ ${JSONWRITER_OBJECT}: ${JSONWRITER_SRC}/jsonwriter.c
@mkdir -p `dirname "$@"`
${CC} ${CFLAGS} ${JSONWRITER_INCLUDE} -DINCLUDE_UTILS $< -c -o $@

# flatten stack desc use sglib
# ${STANDALONE_PFX}flatten${EXE} ${STANDALONE_PFX}stack${EXE} ${STANDALONE_PFX}desc${EXE}:
# sglib
MORE_SOURCE+=-I${THIS_MAKEFILE_DIR}/external/sglib

# sql, 2db, 2json, echo, compare use sqlite3
${CLI} ${STANDALONE_PFX}sql${EXE} ${STANDALONE_PFX}2db${EXE} ${STANDALONE_PFX}2json${EXE} ${STANDALONE_PFX}echo${EXE} ${STANDALONE_PFX}compare${EXE}: ${SQLITE_EXT}
${CLI} ${STANDALONE_PFX}sql${EXE} ${STANDALONE_PFX}2db${EXE} ${STANDALONE_PFX}2json${EXE} ${STANDALONE_PFX}echo${EXE} ${STANDALONE_PFX}compare${EXE}: MORE_OBJECTS+=${SQLITE_EXT}
${STANDALONE_PFX}sql${EXE} ${CLI_OBJ_PFX}sql.o ${STANDALONE_PFX}2db${EXE} ${CLI_OBJ_PFX}2db.o ${STANDALONE_PFX}2json${EXE} ${CLI_OBJ_PFX}2json.o ${STANDALONE_PFX}echo${EXE} ${CLI_OBJ_PFX}echo.o ${CLI_OBJ_PFX}compare.o: MORE_SOURCE+=${SQLITE_EXT_INCLUDE}

# 2json, desc, compare use jsonwriter
${CLI} ${STANDALONE_PFX}2json${EXE} ${STANDALONE_PFX}desc${EXE} ${STANDALONE_PFX}compare${EXE}: ${JSONWRITER_OBJECT}
${CLI} ${STANDALONE_PFX}2json${EXE} ${STANDALONE_PFX}desc${EXE} ${STANDALONE_PFX}compare${EXE}: MORE_OBJECTS+= ${JSONWRITER_OBJECT}
Expand Down
3 changes: 2 additions & 1 deletion app/cli.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include <zsv/utils/string.h>
#include <zsv/utils/dirs.h>
#include <zsv/utils/signal.h>
#include <zsv/utils/overwrite.h>
#include <zsv.h>
#include <zsv/ext.h>
#include "cli_internal.h"
Expand Down Expand Up @@ -431,12 +432,12 @@ static enum zsv_ext_status run_extension(int argc, const char *argv[], struct zs
}

struct zsv_execution_data ctx = {0};

if ((stat = execution_context_init(&ctx, argc, argv)) == zsv_ext_status_ok) {
struct zsv_opts opts;
zsv_args_to_opts(argc, argv, &argc, argv, &opts, ctx.opts_used);
zsv_set_default_opts(opts);
// need a corresponding zsv_set_default_custom_prop_handler?

stat = cmd->main(&ctx, ctx.argc - 1, &ctx.argv[1], &opts, ctx.opts_used);
}

Expand Down
210 changes: 41 additions & 169 deletions app/echo.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,7 @@
#include <zsv/utils/string.h>
#include <zsv/utils/mem.h>
#include <zsv/utils/arg.h>

enum zsv_echo_overwrite_input_type {
zsv_echo_overwrite_input_type_sqlite3 = 0
};
#include <zsv/utils/overwrite.h>

struct zsv_echo_data {
FILE *in;
Expand All @@ -34,24 +31,6 @@ struct zsv_echo_data {
zsv_parser parser;
size_t row_ix;

struct {
size_t row_ix;
size_t col_ix;
const unsigned char *str;
size_t len;
char eof;
} overwrite;

enum zsv_echo_overwrite_input_type overwrite_input_type;
struct {
struct {
char *filename;
sqlite3 *db;
sqlite3_stmt *stmt; // select row, column, overwrite
const char *sql;
} sqlite3;
} o;

unsigned char *skip_until_prefix;
size_t skip_until_prefix_len;

Expand All @@ -63,36 +42,6 @@ struct zsv_echo_data {
unsigned char _ : 5;
};

/**
* check if sqlite3 statement is valid
* return error
*/
static int zsv_echo_sqlite3_check_stmt(sqlite3_stmt *stmt) {
if (sqlite3_column_count(stmt) < 3)
return 1;
// TO DO: check that columns are row, column, value
return 0;
}

/**
* TO DO: verify original value
*/
void zsv_echo_get_next_overwrite(struct zsv_echo_data *data) {
if (!data->overwrite.eof) {
sqlite3_stmt *stmt = data->o.sqlite3.stmt;
if (stmt) {
if (sqlite3_step(stmt) == SQLITE_ROW) {
// row, column, value
data->overwrite.row_ix = sqlite3_column_int64(stmt, 0);
data->overwrite.col_ix = sqlite3_column_int64(stmt, 1);
data->overwrite.str = sqlite3_column_text(stmt, 2);
data->overwrite.len = sqlite3_column_bytes(stmt, 2);
} else
data->overwrite.eof = 1;
}
}
}

static void zsv_echo_get_max_nonempty_cols(void *hook) {
struct zsv_echo_data *data = hook;
unsigned row_nonempty_col_count = 0;
Expand Down Expand Up @@ -124,18 +73,11 @@ static void zsv_echo_row(void *hook) {
zsv_abort(data->parser);
} else {
for (size_t i = 0; i < j; i++) {
if (VERY_UNLIKELY(data->overwrite.row_ix == data->row_ix && data->overwrite.col_ix == i)) {
zsv_writer_cell(data->csv_writer, i == 0, data->overwrite.str, data->overwrite.len, 1);
zsv_echo_get_next_overwrite(data);
} else {
struct zsv_cell cell = zsv_get_cell(data->parser, i);
if (UNLIKELY(data->trim_white))
cell.str = (unsigned char *)zsv_strtrim(cell.str, &cell.len);
zsv_writer_cell(data->csv_writer, i == 0, cell.str, cell.len, cell.quoted);
}
struct zsv_cell cell = zsv_get_cell(data->parser, i);
if (UNLIKELY(data->trim_white))
cell.str = (unsigned char *)zsv_strtrim(cell.str, &cell.len);
zsv_writer_cell(data->csv_writer, i == 0, cell.str, cell.len, cell.quoted);
}
while (!data->overwrite.eof && data->overwrite.row_ix <= data->row_ix)
zsv_echo_get_next_overwrite(data);
}
data->row_ix++;
}
Expand Down Expand Up @@ -182,87 +124,29 @@ static int zsv_echo_usage() {

static void zsv_echo_cleanup(struct zsv_echo_data *data) {
zsv_writer_delete(data->csv_writer);
free(data->o.sqlite3.filename);
free(data->skip_until_prefix);
if (data->o.sqlite3.stmt)
sqlite3_finalize(data->o.sqlite3.stmt);
if (data->in && data->in != stdin)
fclose(data->in);
if (data->o.sqlite3.db)
sqlite3_close(data->o.sqlite3.db);

if (data->tmp_fn) {
remove(data->tmp_fn);
free(data->tmp_fn);
}
}

#define zsv_echo_sqlite3_prefix "sqlite3://"

static int zsv_echo_parse_overwrite_source(struct zsv_echo_data *data, const char *source, size_t len) {
size_t pfx_len;
if (len > (pfx_len = strlen(zsv_echo_sqlite3_prefix)) && !memcmp(source, zsv_echo_sqlite3_prefix, pfx_len)) {
data->o.sqlite3.filename = zsv_memdup(source + pfx_len, len - pfx_len);
char *q = memchr(data->o.sqlite3.filename, '?', len - pfx_len);
if (q) {
*q = '\0';
q++;
#define zsv_echo_sql_prefix "sql="
const char *sql = strstr(q, zsv_echo_sql_prefix);
if (sql)
data->o.sqlite3.sql = sql + strlen(zsv_echo_sql_prefix);
}
// open the sql connection
if (!(data->o.sqlite3.filename && *data->o.sqlite3.filename && data->o.sqlite3.sql && *data->o.sqlite3.sql)) {
free(data->o.sqlite3.filename);
fprintf(stderr, "Invalid query string\n");
return 1;
}

int rc = sqlite3_open_v2(data->o.sqlite3.filename, &data->o.sqlite3.db, SQLITE_OPEN_READONLY, NULL);
if (rc != SQLITE_OK || !data->o.sqlite3.db) {
fprintf(stderr, "%s: %s\n", sqlite3_errstr(rc), data->o.sqlite3.filename);
return 1;
}

if (!data->o.sqlite3.sql) {
// to do: detect it from the db
fprintf(stderr, "Missing sql select statement for sqlite3 echo data e.g.:\n"
" select row, column, value from overwrites order by row, column\n");
return 1;
}

rc = sqlite3_prepare_v2(data->o.sqlite3.db, data->o.sqlite3.sql, -1, &data->o.sqlite3.stmt, NULL);
if (rc != SQLITE_OK || !data->o.sqlite3.stmt) {
fprintf(stderr, "%s\n", sqlite3_errmsg(data->o.sqlite3.db));
return 1;
}

// successful sqlite3 connection
data->overwrite.eof = 0;
return 0;
}

fprintf(stderr, "Invalid overwrite source: %s\n", source);
return 1;
}

int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *opts,
int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *optsp,
struct zsv_prop_handler *custom_prop_handler, const char *opts_used) {
if (argc < 1 || (argc > 1 && (!strcmp(argv[1], "-h") || !strcmp(argv[1], "--help")))) {
zsv_echo_usage();
return 0;
}

struct zsv_opts opts = *optsp;
struct zsv_csv_writer_options writer_opts = zsv_writer_get_default_opts();
struct zsv_echo_data data = {0};
data.overwrite_input_type = zsv_echo_overwrite_input_type_sqlite3;
struct zsv_overwrite_opts overwrite_opts = {0};

int err = 0;

const char *overwrites_csv = NULL;

data.overwrite.eof = 1;
for (int arg_i = 1; !err && arg_i < argc; arg_i++) {
const char *arg = argv[arg_i];
if (!strcmp(arg, "-b"))
Expand Down Expand Up @@ -291,20 +175,9 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
data.skip_until_prefix = (unsigned char *)strdup(argv[arg_i]);
data.skip_until_prefix_len = data.skip_until_prefix ? strlen((char *)data.skip_until_prefix) : 0;
}
} else if (!strcmp(arg, "--overwrite")) {
if (arg_i + 1 >= argc) {
fprintf(stderr, "Option %s requires a value\n", arg);
err = 1;
} else {
const char *src = argv[++arg_i];
if (strlen(src) > strlen(zsv_echo_sqlite3_prefix) &&
!memcmp(zsv_echo_sqlite3_prefix, src, strlen(zsv_echo_sqlite3_prefix)))
err = zsv_echo_parse_overwrite_source(&data, src, strlen(src));
else {
overwrites_csv = src;
}
}
} else if (!data.in) {
} else if (!strcmp(arg, "--overwrite"))
overwrite_opts.src = zsv_next_arg(++arg_i, argc, argv, &err);
else if (!data.in) {
#ifndef NO_STDIN
if (!strcmp(arg, "-"))
data.in = stdin;
Expand Down Expand Up @@ -338,7 +211,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op

unsigned char buff[4096];
if (data.skip_until_prefix)
opts->row_handler = zsv_echo_row_skip_until;
opts.row_handler = zsv_echo_row_skip_until;
else {
if (data.trim_columns) {
// first, save the file if it is stdin
Expand Down Expand Up @@ -366,7 +239,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
}
}
// next, determine the max number of columns from the left that contains data
struct zsv_opts tmp_opts = *opts;
struct zsv_opts tmp_opts = opts;
tmp_opts.row_handler = zsv_echo_get_max_nonempty_cols;
tmp_opts.stream = data.in;
tmp_opts.ctx = &data;
Expand All @@ -387,42 +260,41 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
data.in = fopen(data.tmp_fn ? data.tmp_fn : data.input_path, "rb");
}
}
opts->row_handler = zsv_echo_row;
opts.row_handler = zsv_echo_row;
}
opts->stream = data.in;
opts->ctx = &data;
opts.stream = data.in;
opts.ctx = &data;

data.csv_writer = zsv_writer_new(&writer_opts);

if (overwrites_csv) {
if (!(opts->overwrite.ctx = fopen(overwrites_csv, "rb"))) {
fprintf(stderr, "Unable to open for write: %s\n", overwrites_csv);
zsv_echo_cleanup(&data);
return 1;
if (overwrite_opts.src) {
if (!(opts.overwrite.ctx = zsv_overwrite_context_new(&overwrite_opts))) {
fprintf(stderr, "Out of memory!\n");
err = 1;
} else {
opts->overwrite.type = zsv_overwrite_type_csv;
opts->overwrite.close_ctx = (int (*)(void *))fclose;
opts.overwrite.open = zsv_overwrite_open;
opts.overwrite.next = zsv_overwrite_next;
opts.overwrite.close = zsv_overwrite_context_delete;
}
}

if (zsv_new_with_properties(opts, custom_prop_handler, data.input_path, opts_used, &data.parser) != zsv_status_ok ||
!data.csv_writer) {
zsv_echo_cleanup(&data);
return 1;
if (data.csv_writer && !err) {
if (zsv_new_with_properties(&opts, custom_prop_handler, data.input_path, opts_used, &data.parser) != zsv_status_ok)
err = 1;
else {
// create a local csv writer buff for faster performance
// unsigned char writer_buff[64];
zsv_writer_set_temp_buff(data.csv_writer, buff, sizeof(buff));

// process the input data
zsv_handle_ctrl_c_signal();
enum zsv_status status;
while (!zsv_signal_interrupted && (status = zsv_parse_more(data.parser)) == zsv_status_ok)
;

zsv_finish(data.parser);
zsv_delete(data.parser);
}
}

// create a local csv writer buff for faster performance
// unsigned char writer_buff[64];
zsv_writer_set_temp_buff(data.csv_writer, buff, sizeof(buff));

// process the input data
zsv_handle_ctrl_c_signal();
enum zsv_status status;
while (!zsv_signal_interrupted && (status = zsv_parse_more(data.parser)) == zsv_status_ok)
;

zsv_finish(data.parser);
zsv_delete(data.parser);
zsv_echo_cleanup(&data);
return 0;
return err;
}
2 changes: 1 addition & 1 deletion app/ext_example/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ CLI=${BUILD_DIR}/bin/cli
RUN_CLI=ZSV_CONFIG_DIR=/tmp ${CLI}

${BUILD_DIR}/bin/cli:
(cd .. && make CC=${CC} CONFIGFILE=${CONFIGFILEPATH} DEBUG=${DEBUG} ${BUILD_DIR}/bin/cli)
make -C .. build-cli CONFIGFILE=${CONFIGFILEPATH} DEBUG=${DEBUG}

${BUILD_DIR}/objs/utils/%.o:
(cd .. && make CONFIGFILE=${CONFIGFILEPATH} CC=${CC} DEBUG=${DEBUG} $@ )
Expand Down
Loading