From 67f21a7bec178466615c9f224bb212a1716819da Mon Sep 17 00:00:00 2001 From: liquidaty Date: Fri, 17 Mar 2023 22:53:55 -0700 Subject: [PATCH] prop command: add options for list copy export import clean * added features and tests --- app/2db.c | 40 +- app/Makefile | 7 + app/external/yajl_helper/yajl_helper.c | 52 +- app/external/yajl_helper/yajl_helper.h | 16 + app/prop.c | 780 +++++++++++++++++++++++-- app/sql.c | 2 +- app/test/prop/Makefile | 49 +- app/utils/dirs.c | 118 ++-- app/utils/file.c | 99 +++- include/zsv/utils/cache.h | 1 - include/zsv/utils/dirs.h | 36 ++ include/zsv/utils/file.h | 20 +- include/zsv/utils/prop.h | 27 + 13 files changed, 1112 insertions(+), 135 deletions(-) diff --git a/app/2db.c b/app/2db.c index fb6f06ee..3b2bd077 100644 --- a/app/2db.c +++ b/app/2db.c @@ -72,9 +72,9 @@ struct zsv_2db_data { char *connection_string; struct { - yajl_handle handle; +// yajl_handle handle; struct yajl_helper_parse_state st; - yajl_callbacks callbacks; +// yajl_callbacks callbacks; yajl_status yajl_stat; enum zsv_2db_state state; @@ -153,10 +153,9 @@ static void zsv_2db_delete(zsv_2db_handle data) { free(data->json_parser.row_values); - yajl_helper_parse_state_free(&data->json_parser.st); - if(data->json_parser.handle) - yajl_free(data->json_parser.handle); +// if(data->json_parser.handle) +// yajl_free(data->json_parser.handle); free(data); } @@ -165,11 +164,11 @@ static int zsv_2db_json_parse_err(struct zsv_2db_data *data, unsigned char *last_parsed_buff, size_t last_parsed_buff_len ) { - unsigned char *str = yajl_get_error(data->json_parser.handle, 1, + unsigned char *str = yajl_get_error(data->json_parser.st.yajl, 1, last_parsed_buff, last_parsed_buff_len); if(str) { fprintf(stderr, "Error parsing JSON: %s", (const char *)str); - yajl_free_error(data->json_parser.handle, str); + yajl_free_error(data->json_parser.st.yajl, str); } return 1; } @@ -626,18 +625,17 @@ static zsv_2db_handle zsv_2db_new(struct zsv_2db_options *opts) { sqlite3_exec(data->db, "PRAGMA journal_mode = OFF", NULL, NULL, NULL); // parse the input and create & populate the database table - yajl_helper_parse_state_init(&data->json_parser.st, 32, - json_start_map, json_end_map, json_map_key, - json_start_array, json_end_array, - json_process_value, - data); - yajl_helper_callbacks_init(&data->json_parser.callbacks, 1); - - data->json_parser.handle = yajl_alloc(&data->json_parser.callbacks, NULL, - &data->json_parser.st); - if(!data->json_parser.handle) { + if(yajl_helper_parse_state_init(&data->json_parser.st, 32, + json_start_map, json_end_map, json_map_key, + json_start_array, json_end_array, + json_process_value, + data) != yajl_status_ok) { fprintf(stderr, "Unable to get yajl parser\n"); err = 1; + } else { +// yajl_helper_callbacks_init(&data->json_parser.callbacks, 32); +// data->json_parser.handle = st->yajl; // yajl_alloc(&data->json_parser.callbacks, NULL, + // &data->json_parser.st); } } } @@ -683,7 +681,7 @@ static int zsv_2db_finish(zsv_2db_handle data) { // exportable static yajl_handle zsv_2db_yajl_handle(zsv_2db_handle data) { - return data->json_parser.handle; + return data->json_parser.st.yajl; } int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *zsv_opts, const char *opts_used) { @@ -770,12 +768,14 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *zs break; yajl_status stat = yajl_parse(zsv_2db_yajl_handle(data), buff, bytes_read); if(stat != yajl_status_ok) - err = zsv_2db_json_parse_err(data, buff, bytes_read); + // err = zsv_2db_json_parse_err(data, buff, bytes_read); + err = yajl_helper_print_err(data->json_parser.st.yajl, buff, bytes_read); } if(!err) { if(yajl_complete_parse(zsv_2db_yajl_handle(data)) != yajl_status_ok) - err = zsv_2db_json_parse_err(data, buff, bytes_read); + // err = zsv_2db_json_parse_err(data, buff, bytes_read); + err = yajl_helper_print_err(data->json_parser.st.yajl, buff, bytes_read); else if(zsv_2db_err(data) || zsv_2db_finish(data)) err = 1; } diff --git a/app/Makefile b/app/Makefile index a283fd13..c511a8d7 100644 --- a/app/Makefile +++ b/app/Makefile @@ -37,6 +37,13 @@ ifneq ($(ZSV_CACHE_PREFIX),) CFLAGS+= -DZSV_CACHE_PREFIX='${ZSV_CACHE_PREFIX}' endif +ifneq ($(ZSV_IS_PROP_FILE_HANDLER),) + CFLAGS+=-DZSV_IS_PROP_FILE_HANDLER=${ZSV_IS_PROP_FILE_HANDLER} +endif +ifneq ($(ZSV_IS_PROP_FILE_DEPTH),) + CFLAGS+=-DZSV_IS_PROP_FILE_DEPTH=${ZSV_IS_PROP_FILE_DEPTH} +endif + DEBUG=0 WIN= ifeq ($(WIN),) diff --git a/app/external/yajl_helper/yajl_helper.c b/app/external/yajl_helper/yajl_helper.c index aa833e45..b72ed19f 100644 --- a/app/external/yajl_helper/yajl_helper.c +++ b/app/external/yajl_helper/yajl_helper.c @@ -167,6 +167,24 @@ char json_value_truthy(struct json_value *value) { return 0; } +/** + * Print any error from the yajl parser + * Returns non-zero + */ +int yajl_helper_print_err(yajl_handle yajl, + unsigned char *last_parsed_buff, + size_t last_parsed_buff_len + ) { + unsigned char *str = yajl_get_error(yajl, 1, + last_parsed_buff, last_parsed_buff_len); + if(str) { + fprintf(stderr, "Error parsing JSON: %s", (const char *)str); + yajl_free_error(yajl, str); + } + return 1; +} + + const char *yajl_helper_get_map_key(struct yajl_helper_parse_state *st, unsigned int offset) { if(YAJL_HELPER_LEVEL(st) >= offset + 1) { unsigned int level = st->level - offset; @@ -176,12 +194,9 @@ const char *yajl_helper_get_map_key(struct yajl_helper_parse_state *st, unsigned return NULL; } -char yajl_helper_got_path(struct yajl_helper_parse_state *st, unsigned int level, const char *path) { - if(YAJL_HELPER_LEVEL(st) != level) - return 0; - - unsigned int this_level = st->level_offset + 1; - for(; *path && this_level <= st->level; path++) { +static char yajl_helper_got_path_aux(struct yajl_helper_parse_state *st, unsigned int level, const char *path) { + for(unsigned i = 1; *path && i <= level; path++, i++) { + unsigned this_level = st->level_offset + i; switch(*path) { case '{': case '[': @@ -199,8 +214,6 @@ char yajl_helper_got_path(struct yajl_helper_parse_state *st, unsigned int level path += len; } } - - this_level++; break; default: // map key start return 0; @@ -209,6 +222,18 @@ char yajl_helper_got_path(struct yajl_helper_parse_state *st, unsigned int level return 1; } +char yajl_helper_got_path(struct yajl_helper_parse_state *st, unsigned int level, const char *path) { + if(YAJL_HELPER_LEVEL(st) != level) + return 0; + return yajl_helper_got_path_aux(st, level, path); +} + +char yajl_helper_got_path_prefix(struct yajl_helper_parse_state *st, unsigned int level, const char *path) { + if(YAJL_HELPER_LEVEL(st) < level) + return 0; + return yajl_helper_got_path_aux(st, level, path); +} + char yajl_helper_path_is(struct yajl_helper_parse_state *st, const char *path) { unsigned int level = 0; for(int i = 0; path[i]; i++) @@ -318,10 +343,13 @@ static int yajl_helper_map_key(void *ctx, const unsigned char *stringVal, size_t static inline int process_value(struct yajl_helper_parse_state *st, void *ctx, struct json_value *v) { - int rc = st->value(ctx, v); - if(st->level && strchr("{[", st->stack[st->level-1]) && st->level <= st->max_level) - st->item_ind[st->level-1]++; - return rc; + if(st->value) { + int rc = st->value(ctx, v); + if(st->level && strchr("{[", st->stack[st->level-1]) && st->level <= st->max_level) + st->item_ind[st->level-1]++; + return rc; + } + return 1; } static int yajl_helper_number_str(void * ctx, const char * numberVal, diff --git a/app/external/yajl_helper/yajl_helper.h b/app/external/yajl_helper/yajl_helper.h index aa25ace2..5224cdc7 100644 --- a/app/external/yajl_helper/yajl_helper.h +++ b/app/external/yajl_helper/yajl_helper.h @@ -103,7 +103,14 @@ unsigned char *json_str_dup_if_len(struct json_value *value); // malloc() a new string and return that. returns buff, if buff was written to, else new malloc'd mem unsigned char *json_str_dup_if_len_buff(struct json_value *value, unsigned char *buff, size_t bufflen); +/* + * yajl_helper_got_path() and yajl_helper_got_path_prefix() are the same except that the former + * requires that the current level is equal to the level argument, and the latter only requires + * that the current level is greater than or equal to the level argument + */ char yajl_helper_got_path(struct yajl_helper_parse_state *st, unsigned int level, const char *path); +char yajl_helper_got_path_prefix(struct yajl_helper_parse_state *st, unsigned int level, const char *path); + char yajl_helper_path_is(struct yajl_helper_parse_state *st, const char *path); const char *yajl_helper_get_map_key(struct yajl_helper_parse_state *st, unsigned int offset); @@ -158,4 +165,13 @@ void int_list_free(struct int_list *e); */ void yajl_helper_dump_path(struct yajl_helper_parse_state *st, FILE *out); +/** + * Print any error from the yajl parser + * Returns non-zero + */ +int yajl_helper_print_err(yajl_handle yajl, + unsigned char *last_parsed_buff, + size_t last_parsed_buff_len + ); + #endif // ifdef YAJL_HELPER_H diff --git a/app/prop.c b/app/prop.c index a2772199..52245ac7 100644 --- a/app/prop.c +++ b/app/prop.c @@ -16,6 +16,7 @@ #include #include #include +#include // unlink, access #define ZSV_COMMAND_NO_OPTIONS #define ZSV_COMMAND prop @@ -24,6 +25,8 @@ #include #include #include +#include +#include #include #include @@ -32,20 +35,25 @@ const char *zsv_property_usage_msg[] = { " saved options will be applied by default when processing that file", "", "Usage: " APPNAME " [options]", - " where filepath is the path to the input CSV file (or when using --auto, - for stdin)", + " where filepath is the path to the input CSV file, or", + " when using --auto: input CSV file or - for stdin", + " when using --clean: directory to clean from (. for current directory)", " and options may be one or more of:", " -d,--header-row-span : set/unset/auto-detect header depth (see below)", " -R,--skip-head : set/unset/auto-detect initial rows to skip (see below)", - " --list-files : x", // output a list of all cache files - " --clear : delete all properties", - " --clear-orphans : delete properties of all orphaned files in the given file or directory path", + " --list-files : list all property sets associted with the given file", // output a list of all cache files + " --clear : delete all properties of given files", + " --clean : delete all files / dirs in the property cache of the given directory", + " that do not have a corresponding file in that directory", + " --dry : dry run, outputs files/dirs to remove. only for use with --clean", " --auto : guess the best property values. This is equivalent to:", " -d auto -R auto", " when using this option, a dash (-) can be used instead", " of a filepath to read from stdin", " --save [-f,--overwrite] : (only applicable with --auto) save the detected result", - " --export : export all properties to a single JSON file (- for stdout)", // to do: add option to check for valid JSON - " --import : import properties from a single JSON file (- for stdin)", // to do: add option to check for valid JSON + " --copy : copy properties to another file", // to do: opt to check valid JSON + " --export : export all properties to a single JSON file (- for stdout)", // to do: opt to check valid JSON + " --import : import properties from a single JSON file (- for stdin)", // to do: opt to check valid JSON " -f,--overwrite : overwrite any previously-saved properties", "", "For --header-row-span or --skip-head options, can be:", @@ -498,20 +506,693 @@ static int merge_and_save_properties(const unsigned char *filepath, return err; } +enum zsv_prop_mode { + zsv_prop_mode_default = 0, + zsv_prop_mode_list_files = 'l', + zsv_prop_mode_clean = 'K', + zsv_prop_mode_export = 'e', + zsv_prop_mode_import = 'i', + zsv_prop_mode_copy = 'c' +}; + +static enum zsv_prop_mode zsv_prop_get_mode(const char *opt) { + if(!strcmp(opt, "--clean")) return zsv_prop_mode_clean; + if(!strcmp(opt, "--list-files")) return zsv_prop_mode_list_files; + if(!strcmp(opt, "--copy")) return zsv_prop_mode_copy; + if(!strcmp(opt, "--export")) return zsv_prop_mode_export; + if(!strcmp(opt, "--import")) return zsv_prop_mode_import; + return zsv_prop_mode_default; +} + +struct prop_opts { + int64_t d; // ZSV_PROP_ARG_AUTO, ZSV_PROP_ARG_REMOVE or > 0 + int64_t R; // ZSV_PROP_ARG_AUTO, ZSV_PROP_ARG_REMOVE or > 0 + unsigned char clear:1; + unsigned char save:1; + unsigned char overwrite:1; + unsigned char _:3; +}; + +static int zsv_prop_execute_default(const unsigned char *filepath, struct zsv_opts zsv_opts, struct prop_opts opts) { + int err = 0; + struct zsv_file_properties fp = { 0 }; + if(opts.d >= 0 || opts.R >= 0 || opts.d == ZSV_PROP_ARG_REMOVE || opts.R == ZSV_PROP_ARG_REMOVE) + opts.overwrite = 1; + if(opts.d == ZSV_PROP_ARG_AUTO || opts.R == ZSV_PROP_ARG_AUTO) { + err = detect_properties(filepath, &fp, + opts.d == ZSV_PROP_ARG_AUTO, + opts.R == ZSV_PROP_ARG_AUTO, + &zsv_opts); + } + + if(!err) { + if(opts.d == ZSV_PROP_ARG_AUTO) + opts.d = fp.header_span; + + if(opts.R == ZSV_PROP_ARG_AUTO) + opts.R = fp.skip; + err = merge_and_save_properties(filepath, opts.save, opts.overwrite, opts.d, opts.R); + } + return err; +} + +int zsv_is_prop_file(struct zsv_foreach_dirent_handle *h, size_t depth) { + return depth == 1 && !strcmp(h->entry, "props.json"); +} + +struct is_property_ctx { + zsv_foreach_dirent_handler handler; + size_t max_depth; +}; + +#ifdef ZSV_IS_PROP_FILE_HANDLER +int ZSV_IS_PROP_FILE_HANDLER(struct zsv_foreach_dirent_handle *, size_t); +#endif + +struct is_property_ctx * +zsv_prop_get_or_set_is_prop_file( + int (*custom_is_prop_file)(struct zsv_foreach_dirent_handle *, size_t), + int max_depth, + char set + ) { + static struct is_property_ctx ctx = { +#ifndef ZSV_IS_PROP_FILE_HANDLER + .handler = zsv_is_prop_file, +#else + .handler = ZSV_IS_PROP_FILE_HANDLER, +#endif +#ifndef ZSV_IS_PROP_FILE_DEPTH + .max_depth = 1 +#else + .max_depth = ZSV_IS_PROP_FILE_DEPTH +#endif + }; + + if(set) { + if(!(ctx.handler = custom_is_prop_file)) { + ctx.handler = zsv_is_prop_file; + max_depth = 1; + } else + ctx.max_depth = max_depth; + } + return &ctx; +} + +static int zsv_prop_foreach_list(struct zsv_foreach_dirent_handle *h, size_t depth) { + if(!h->is_dir) { + struct is_property_ctx *ctx = (struct is_property_ctx *)h->ctx; + if(ctx->handler(h, depth)) + printf("%s\n", h->entry); + } + return 0; +} + +zsv_foreach_dirent_handler +zsv_prop_get_or_set_is_prop_dir( + int (*custom_is_prop_dir)(struct zsv_foreach_dirent_handle *, size_t), + char set + ) { + static int (*func)(struct zsv_foreach_dirent_handle *, size_t) = NULL; + if(set) + func = custom_is_prop_dir; + return func; +} + +static int zsv_prop_execute_list_files(const unsigned char *filepath, char verbose) { + int err = 0; + unsigned char *cache_path = zsv_cache_path(filepath, NULL, 0); + struct is_property_ctx ctx = *zsv_prop_get_or_set_is_prop_file(NULL, 0, 0); + if(cache_path) { + zsv_foreach_dirent((const char *)cache_path, ctx.max_depth, zsv_prop_foreach_list, + &ctx, verbose); + free(cache_path); + } + return err; +} + +struct zsv_prop_foreach_clean_ctx { + const char *dirpath; + unsigned char dry; +}; + +static int zsv_prop_foreach_clean(struct zsv_foreach_dirent_handle *h, size_t depth) { + int err = 0; + if(depth == 1) { + struct zsv_prop_foreach_clean_ctx *ctx = h->ctx; + if(h->is_dir) { + // h->entry is the name of the top-level file that this folder relates to + // make sure that the top-level file exists + h->no_recurse = 1; + + char *cache_owner_path; + asprintf(&cache_owner_path, "%s%c%s", ctx->dirpath, FILESLASH, h->entry); + if(!cache_owner_path) { + fprintf(stderr, "Out of memory!\n"); + return 1; + } + if(!zsv_file_exists(cache_owner_path)) { + if(ctx->dry) + printf("Orphaned: %s\n", h->parent_and_entry); + else + err = zsv_remove_dir_recursive((const unsigned char *)h->parent_and_entry); + } + free(cache_owner_path); + } else { + // there should be no files at depth 1, so just delete + if(ctx->dry) + printf("Unrecognized: %s\n", h->parent_and_entry); + else if(unlink(h->parent_and_entry)) { + perror(h->parent_and_entry); + err = 1; + } + } + } + return err; +} + +enum zsv_prop_foreach_copy_mode { + zsv_prop_foreach_copy_mode_check = 1, + zsv_prop_foreach_copy_mode_copy +}; + +struct zsv_prop_foreach_copy_ctx { + struct is_property_ctx is_property_ctx; + const unsigned char *src_cache_dir; + const unsigned char *dest_cache_dir; + enum zsv_prop_foreach_copy_mode mode; + int err; + unsigned char output_started:1; + unsigned char force:1; + unsigned char dry:1; + unsigned char _:5; +}; + +static int zsv_prop_foreach_copy(struct zsv_foreach_dirent_handle *h, size_t depth) { + if(!h->is_dir) { + struct zsv_prop_foreach_copy_ctx *ctx = h->ctx; + if(ctx->is_property_ctx.handler(h, depth)) { + char *dest_prop_filepath; + asprintf(&dest_prop_filepath, "%s%s", ctx->dest_cache_dir, h->parent_and_entry + strlen((const char *)ctx->src_cache_dir)); + if(!dest_prop_filepath) { + ctx->err = errno = ENOMEM; + perror(NULL); + } else { + switch(ctx->mode) { + case zsv_prop_foreach_copy_mode_check: + { + if(!zsv_file_readable(h->parent_and_entry, &ctx->err, NULL)) { // check if source is not readable + perror(h->parent_and_entry); + } else if(!ctx->force && access(dest_prop_filepath, F_OK) != -1) { // check if dest already exists + ctx->err = EEXIST; + if(!ctx->output_started) { + ctx->output_started = 1; + const char *msg = strerror(EEXIST); + fprintf(stderr, "%s:\n", msg ? msg : "File already exists"); + } + fprintf(stderr, " %s\n", dest_prop_filepath); + } else if(ctx->dry) + printf("%s => %s\n", h->parent_and_entry, dest_prop_filepath); + } + break; + case zsv_prop_foreach_copy_mode_copy: + if(!ctx->dry) { + char *dest_prop_filepath_tmp; + asprintf(&dest_prop_filepath_tmp, "%s.temp", dest_prop_filepath); + if(!dest_prop_filepath_tmp) { + ctx->err = errno = ENOMEM; + perror(NULL); + } else { + if(h->verbose) + fprintf(stderr, "Copying temp: %s => %s\n", h->parent_and_entry, dest_prop_filepath_tmp); + int err = zsv_copy_file(h->parent_and_entry, dest_prop_filepath_tmp); + if(err) + ctx->err = err; + else { + if(h->verbose) + fprintf(stderr, "Renaming: %s => %s\n", dest_prop_filepath_tmp, dest_prop_filepath); + if(rename(dest_prop_filepath_tmp, dest_prop_filepath)) { + const char *msg = strerror(errno); + fprintf(stderr, "Unable to rename %s -> %s: %s\n", dest_prop_filepath_tmp, dest_prop_filepath, msg ? msg : "Unknown error"); + ctx->err = errno; + } + } + free(dest_prop_filepath_tmp); + } + } + break; + } + free(dest_prop_filepath); + } + } + } + return 0; +} + +struct zsv_prop_foreach_export_ctx { + struct is_property_ctx is_property_ctx; + const unsigned char *src_cache_dir; + struct jv_to_json_ctx jctx; + zsv_jq_handle zjq; + unsigned count; // number of files exported so far + int err; +}; + +static int zsv_prop_foreach_export(struct zsv_foreach_dirent_handle *h, size_t depth) { + if(!h->is_dir) { + struct zsv_prop_foreach_export_ctx *ctx = h->ctx; + if(ctx->is_property_ctx.handler(h, depth) && !ctx->err) { + char suffix = 0; + if(strlen(h->parent_and_entry) > 5 && !zsv_stricmp((const unsigned char *)h->parent_and_entry + strlen(h->parent_and_entry) - 5, (const unsigned char *)".json")) + suffix = 'j'; // json + else if(strlen(h->parent_and_entry) > 4 && !zsv_stricmp((const unsigned char *)h->parent_and_entry + strlen(h->parent_and_entry) - 4, (const unsigned char *)".txt")) + suffix = 't'; // text + if(suffix) { + // for now, only handle json or txt + FILE *f = fopen(h->parent_and_entry, "rb"); + if(!f) + perror(h->parent_and_entry); + else { + // create an entry for this file. the map key is the file name; its value is the file contents + unsigned char *js = zsv_json_from_str((const unsigned char *)h->parent_and_entry + strlen((const char *)ctx->src_cache_dir) + 1); + if(!js) + errno = ENOMEM, perror(NULL); + else if(*js) { + if(ctx->count > 0) + if(zsv_jq_parse(ctx->zjq, ",", 1)) + ctx->err = 1; + if(!ctx->err) { + ctx->count++; + if(zsv_jq_parse(ctx->zjq, js, strlen((const char *)js)) || zsv_jq_parse(ctx->zjq, ":", 1)) + ctx->err = 1; + else { + switch(suffix) { + case 'j': // json + if(zsv_jq_parse_file(ctx->zjq, f)) + ctx->err = 1; + break; + case 't': // txt + // for now we are going to limit txt file values to 4096 chars and JSON-stringify it + { + unsigned char buff[4096]; + size_t n = fread(buff, 1, sizeof(buff), f); + unsigned char *txt_js = NULL; + if(n) { + txt_js = zsv_json_from_str_n(buff, n); + if(zsv_jq_parse(ctx->zjq, txt_js ? txt_js : (const unsigned char *)"null", txt_js ? strlen((const char *)txt_js) : 4)) + ctx->err = 1; + } + } + break; + } + } + } + } + free(js); + fclose(f); + } + } + } + } + return 0; +} + +static int zsv_prop_execute_copy(const char *src, const char *dest, unsigned char force, unsigned char dry, unsigned char verbose) { + int err = 0; + unsigned char *src_cache_dir = zsv_cache_path((const unsigned char *)src, NULL, 0); + unsigned char *dest_cache_dir = zsv_cache_path((const unsigned char *)dest, NULL, 0); + + if(!(src_cache_dir && dest_cache_dir)) + err = errno = ENOMEM, perror(NULL); + else { + // if !force, only proceed if: + // - src exists (file) + // - dest exists (file) + // - dest file property cache d.n. have conflicts + struct zsv_prop_foreach_copy_ctx ctx = { 0 }; + ctx.is_property_ctx = *zsv_prop_get_or_set_is_prop_file(NULL, 0, 0); + ctx.dest_cache_dir = dest_cache_dir; + ctx.src_cache_dir = src_cache_dir; + ctx.force = force; + ctx.dry = dry; + + if(!force) { + if(!zsv_file_exists(src)) + err = errno = ENOENT, perror(src); + if(!zsv_file_exists(dest)) + err = errno = ENOENT, perror(dest); + } + + if(!err) { + // for each property file, check if dest has same-named property file + ctx.mode = zsv_prop_foreach_copy_mode_check; + zsv_foreach_dirent((const char *)src_cache_dir, ctx.is_property_ctx.max_depth, zsv_prop_foreach_copy, + &ctx, verbose); + } + + if(!err && !(ctx.err && !force)) { + // copy the files + ctx.mode = zsv_prop_foreach_copy_mode_copy; + zsv_foreach_dirent((const char *)src_cache_dir, ctx.is_property_ctx.max_depth, zsv_prop_foreach_copy, + &ctx, verbose); + } + } + free(src_cache_dir); + free(dest_cache_dir); + return err; +} + +static int zsv_prop_execute_clean(const char *dirpath, unsigned char dry, unsigned char verbose) { + // TO DO: if ZSV_CACHE_DIR-tmp exists, delete it (file or dir) + int err = 0; + size_t dirpath_len = strlen(dirpath); + while(dirpath_len && memchr("/\\", dirpath[dirpath_len-1], 2) != NULL) + dirpath_len--; + if(!dirpath_len) + return 0; + + char *cache_parent; + if(!strcmp(dirpath, ".")) + cache_parent = strdup(ZSV_CACHE_DIR); + else + asprintf(&cache_parent, "%.*s%c%s", (int)dirpath_len, dirpath, FILESLASH, ZSV_CACHE_DIR); + if(!cache_parent) { + fprintf(stderr, "Out of memory!\n"); + return 1; + } + + struct zsv_prop_foreach_clean_ctx ctx = { 0 }; + ctx.dirpath = dirpath; + ctx.dry = dry; + + zsv_foreach_dirent(cache_parent, 0, zsv_prop_foreach_clean, &ctx, verbose); + free(cache_parent); + return err; +} + +static int zsv_prop_execute_export(const char *src, const char *dest, unsigned char verbose) { + int err = 0; + unsigned char *src_cache_dir = zsv_cache_path((const unsigned char *)src, NULL, 0); + if(!(src_cache_dir)) + err = errno = ENOMEM, perror(NULL); + else { + FILE *fdest = dest ? fopen(dest, "wb") : stdout; + if(!fdest) + err = errno, perror(dest); + else { + struct zsv_prop_foreach_export_ctx ctx = { 0 }; + ctx.is_property_ctx = *zsv_prop_get_or_set_is_prop_file(NULL, 0, 0); + ctx.src_cache_dir = src_cache_dir; + + // use a jq filter to pretty-print + ctx.jctx.write1 = zsv_jq_fwrite1; + ctx.jctx.ctx = fdest; + ctx.jctx.flags = JV_PRINT_PRETTY | JV_PRINT_SPACE1; + enum zsv_jq_status jqstat; + ctx.zjq = zsv_jq_new((const unsigned char *)".", jv_to_json_func, &ctx.jctx, &jqstat); + if(!ctx.zjq) + err = 1, fprintf(stderr, "zsv_jq_new\n"); + else { + if(jqstat == zsv_jq_status_ok && zsv_jq_parse(ctx.zjq, "{", 1) == zsv_jq_status_ok) { + // export each file + zsv_foreach_dirent((const char *)src_cache_dir, ctx.is_property_ctx.max_depth, zsv_prop_foreach_export, + &ctx, verbose); + if(!ctx.err && zsv_jq_parse(ctx.zjq, "}", 1)) + ctx.err = 1; + if(!ctx.err && zsv_jq_finish(ctx.zjq)) + ctx.err = 1; + zsv_jq_delete(ctx.zjq); + } + err = ctx.err; + } + fclose(fdest); + } + } + free(src_cache_dir); + return err; +} + +struct prop_import_ctx { + const char *filepath_prefix; + unsigned char buff[4096]; + size_t content_start; + FILE *out; + char *out_filepath; + struct jv_to_json_ctx jctx; + zsv_jq_handle zjq; + + int err; + unsigned char in_obj:1; + unsigned char do_check:1; + unsigned char dry:1; + unsigned char _:5; +}; + +static void prop_import_close_out(struct prop_import_ctx *ctx) { + if(ctx->zjq) { + zsv_jq_finish(ctx->zjq); + zsv_jq_delete(ctx->zjq); + ctx->zjq = NULL; + } + if(ctx->out) { + fclose(ctx->out); + ctx->out = NULL; + free(ctx->out_filepath); + ctx->out_filepath = NULL; + } +} + +static int prop_import_map_key(struct yajl_helper_parse_state *st, + const unsigned char *s, size_t len) { + if(yajl_helper_level(st) == 1 && len) { // new property file entry + struct prop_import_ctx *ctx = yajl_helper_data(st); + + char *fn = NULL; + if(ctx->filepath_prefix) + asprintf(&fn, "%s%c%.*s", ctx->filepath_prefix, FILESLASH, (int)len, s); + else + asprintf(&fn, "%.*s", (int)len, s); + if(!fn) { + errno = ENOMEM; + perror(NULL); + } else if(ctx->do_check) { + // we just want to check if the destination file exists + if(access(fn, F_OK) != -1) { // it exists + ctx->err = errno = EEXIST; + perror(fn); + } + } else if(ctx->dry) { // just output the name of the file + printf("%s\n", fn); + } else if(zsv_mkdirs(fn, 1)) { + fprintf(stderr, "Unable to create directories for %s\n", fn); + } else if(!((ctx->out = fopen(fn, "wb")))) { + perror(fn); + } else { + ctx->out_filepath = fn; + fn = NULL; + + // if it's a JSON file, use a jq filter to pretty-print + if(strlen(ctx->out_filepath) > 5 && !zsv_stricmp((const unsigned char *)ctx->out_filepath + strlen(ctx->out_filepath) - 5, (const unsigned char *)".json")) { + ctx->jctx.write1 = zsv_jq_fwrite1; + ctx->jctx.ctx = ctx->out; + ctx->jctx.flags = JV_PRINT_PRETTY | JV_PRINT_SPACE1; + enum zsv_jq_status jqstat; + ctx->zjq = zsv_jq_new((const unsigned char *)".", jv_to_json_func, &ctx->jctx, &jqstat); + if(!ctx->zjq) { + fprintf(stderr, "zsv_jq_new: unable to open for %s\n", ctx->out_filepath); + prop_import_close_out(ctx); + } + } + } + free(fn); + } + return 1; +} + +static int prop_import_start_obj(struct yajl_helper_parse_state *st) { + if(yajl_helper_level(st) == 2) { + struct prop_import_ctx *ctx = yajl_helper_data(st); + ctx->in_obj = 1; + ctx->content_start = yajl_get_bytes_consumed(st->yajl) - 1; + } + return 1; +} + +// prop_import_flush(): return err +static int prop_import_flush(yajl_handle yajl, struct prop_import_ctx *ctx) { + if(ctx->zjq) { + size_t current_position = yajl_get_bytes_consumed(yajl); + if(current_position <= ctx->content_start) + fprintf(stderr, "Error! prop_import_flush unexpected current position\n"); + else + zsv_jq_parse(ctx->zjq, ctx->buff + ctx->content_start, + current_position - ctx->content_start); + ctx->content_start = 0; + } + return 0; +} + +static int prop_import_end_obj(struct yajl_helper_parse_state *st) { + if(yajl_helper_level(st) == 1) { // just finished level 2 + struct prop_import_ctx *ctx = yajl_helper_data(st); + prop_import_flush(st->yajl, yajl_helper_data(st)); + prop_import_close_out(ctx); + ctx->in_obj = 0; + } + return 1; +} + +static int prop_import_process_value(struct yajl_helper_parse_state *st, + struct json_value *value) { + if(yajl_helper_level(st) == 1) { // just finished level 2 + struct prop_import_ctx *ctx = yajl_helper_data(st); + const unsigned char *jsstr; + size_t len; + json_value_default_string(value, &jsstr, &len); + if(ctx->zjq) { + unsigned char *js = len ? zsv_json_from_str_n(jsstr, len) : NULL; + if(js) + zsv_jq_parse(ctx->zjq, js, strlen((char *)js)); + else + zsv_jq_parse(ctx->zjq, "null", 4); + free(js); + } else if(len && ctx->out) + fwrite(jsstr, 1, len, ctx->out); + prop_import_close_out(ctx); + } + return 1; +} + + +static int zsv_prop_execute_import(const char *dest, const char *src, unsigned char force, unsigned char dry, unsigned char verbose) { + unsigned char *dest_cache_dir = NULL; + FILE *fsrc = NULL; + int err = 0; + + if(!force && !zsv_file_exists(dest)) { + err = errno = ENOENT; + perror(dest); + } else if(!(dest_cache_dir = zsv_cache_path((const unsigned char *)dest, NULL, 0))) { + err = errno = ENOMEM; + perror(NULL); + } else if(!(fsrc = src ? fopen(src, "rb") : stdin)) { + err = errno; + perror(src); + } else { + char *tmp_fn = NULL; + if(!force) { + // if input is stdin, we'll need to read it twice, so save it first + // this isn't the most efficient way to do it, as it reads it 3 times + // but it's easier and the diff is immaterial + if(fsrc == stdin) { + fsrc = NULL; + tmp_fn = zsv_get_temp_filename("zsv_prop_XXXXXXXX"); + src = (const char *)tmp_fn; + FILE *tmp_f; + if(!tmp_fn) { + err = errno = ENOMEM; + perror(NULL); + } else if(!(tmp_f = fopen(tmp_fn, "wb"))) { + err = errno; + perror(tmp_fn); + } else { + err = zsv_copy_file_ptr(stdin, tmp_f); + fclose(tmp_f); + if(!(fsrc = fopen(tmp_fn, "rb"))) { + err = errno; + perror(tmp_fn); + } + } + } + } + + if(!err) { + // we will run this loop either once (force) or twice (no force): + // 1. check before running (no force) + // 2. do the import + char do_check = !force; + + if(do_check && !zsv_dir_exists((const char *)dest_cache_dir)) + do_check = 0; + + for(int i = do_check ? 0 : 1; i < 2 && !err; i++) { + do_check = i == 0; + + size_t bytes_read; + struct yajl_helper_parse_state st; + struct prop_import_ctx ctx = { 0 }; + ctx.filepath_prefix = (const char *)dest_cache_dir; + + int (*start_obj)(struct yajl_helper_parse_state *st) = NULL; + int (*end_obj)(struct yajl_helper_parse_state *st) = NULL; + int (*process_value)(struct yajl_helper_parse_state *, struct json_value *) = NULL; + + if(do_check) + ctx.do_check = do_check; + else { + ctx.dry = dry; + if(!ctx.dry) { + start_obj = prop_import_start_obj; + end_obj = prop_import_end_obj; + process_value = prop_import_process_value; + } + } + + if(yajl_helper_parse_state_init(&st, 32, + start_obj, end_obj, // map start/end + prop_import_map_key, + start_obj, end_obj, // array start/end + process_value, + &ctx) != yajl_status_ok) { + err = errno = ENOMEM; + perror(NULL); + } else { + while((bytes_read = fread(ctx.buff, 1, sizeof(ctx.buff), fsrc)) > 0) { + if(yajl_parse(st.yajl, ctx.buff, bytes_read) != yajl_status_ok) + yajl_helper_print_err(st.yajl, ctx.buff, bytes_read); + if(ctx.in_obj) + prop_import_flush(st.yajl, &ctx); + } + if(yajl_complete_parse(st.yajl) != yajl_status_ok) + yajl_helper_print_err(st.yajl, ctx.buff, bytes_read); + + if(ctx.out) { // e.g. if bad JSON and parse failed + fclose(ctx.out); + free(ctx.out_filepath); + } + } + yajl_helper_parse_state_free(&st); + + if(ctx.err) + err = ctx.err; + if(i == 0) { + rewind(fsrc); + if(errno) { + err = errno; + perror(NULL); + } + } + } + } + if(tmp_fn) { + unlink(tmp_fn); + free(tmp_fn); + } + } + + if(fsrc && fsrc != stdin) + fclose(fsrc); + free(dest_cache_dir); + + return err; +} + int ZSV_MAIN_NO_OPTIONS_FUNC(ZSV_COMMAND)(int m_argc, const char *m_argv[]) { int err = 0; + char verbose = 0; if(m_argc < 2 || (m_argc > 1 && (!strcmp(m_argv[1], "-h") || !strcmp(m_argv[1], "--help")))) err = zsv_property_usage(stdout); else { - struct prop_opts { - int64_t d; // ZSV_PROP_ARG_AUTO, ZSV_PROP_ARG_REMOVE or > 0 - int64_t R; // ZSV_PROP_ARG_AUTO, ZSV_PROP_ARG_REMOVE or > 0 - unsigned char clear:1; - unsigned char save:1; - unsigned char overwrite:1; - unsigned char _:3; - }; struct prop_opts opts = { 0 }; opts.d = ZSV_PROP_ARG_NONE; opts.R = ZSV_PROP_ARG_NONE; @@ -523,13 +1204,32 @@ int ZSV_MAIN_NO_OPTIONS_FUNC(ZSV_COMMAND)(int m_argc, const char *m_argv[]) { if(m_argc == 3 && !strcmp("--clear", m_argv[2])) return zsv_cache_remove(filepath, zsv_cache_type_property); + enum zsv_prop_mode mode = zsv_prop_mode_default; + unsigned char dry = 0; + const char *mode_arg = NULL; // e.g. "--export" + const char *mode_value = NULL; // e.g. "saved_export.json" for(int i = 2; !err && i < m_argc; i++) { const char *opt = m_argv[i]; - if(!strcmp(opt, "-d") || !strcmp(opt, "--header-row-span")) + if(!strcmp(opt, "-v") || !strcmp(opt, "--verbose")) + verbose = 1; + else if(!strcmp(opt, "-d") || !strcmp(opt, "--header-row-span")) err = prop_arg_value(++i, m_argc, m_argv, &opts.d); else if(!strcmp(opt, "-R") || !strcmp(opt, "--skip-head")) err = prop_arg_value(++i, m_argc, m_argv, &opts.R); - else if(!strcmp(opt, "--clear")) + else if(zsv_prop_get_mode(opt)) { + if(mode_arg) + err = fprintf(stderr, "Option %s cannot be used together with %s\n", opt, mode_arg); + else { + mode = zsv_prop_get_mode(opt); + mode_arg = opt; + if(mode == zsv_prop_mode_export || mode == zsv_prop_mode_import || mode == zsv_prop_mode_copy) { + if(++i < m_argc) + mode_value = m_argv[i]; + else + err = fprintf(stderr, "Option %s requires a value\n", opt); + } + } + } else if(!strcmp(opt, "--clear")) err = fprintf(stderr, "--clear cannot be used in conjunction with any other options\n"); else if(!strcmp(opt, "--auto")) { if(opts.d != ZSV_PROP_ARG_NONE && opts.R != ZSV_PROP_ARG_NONE) @@ -544,6 +1244,8 @@ int ZSV_MAIN_NO_OPTIONS_FUNC(ZSV_COMMAND)(int m_argc, const char *m_argv[]) { opts.save = 1; else if(!strcmp(opt, "-f") || !strcmp(opt, "--overwrite")) opts.overwrite = 1; + else if(!strcmp(opt, "--dry")) + dry = 1; else { fprintf(stderr, "Unrecognized option: %s\n", opt); err = 1; @@ -559,7 +1261,9 @@ int ZSV_MAIN_NO_OPTIONS_FUNC(ZSV_COMMAND)(int m_argc, const char *m_argv[]) { if(have_auto && (have_specified || have_remove)) { fprintf(stderr, "Non-auto options may not be mixed with auto options\n"); err = 1; - } + } else if((have_auto || have_specified || have_remove || opts.save) + && mode != zsv_prop_mode_default) + err = fprintf(stderr, "Invalid options in combination with %s\n", mode_arg); if(have_specified || have_remove) { opts.save = 1; @@ -568,25 +1272,29 @@ int ZSV_MAIN_NO_OPTIONS_FUNC(ZSV_COMMAND)(int m_argc, const char *m_argv[]) { } if(!err) { - struct zsv_file_properties fp = { 0 }; - if(opts.d >= 0 || opts.R >= 0 || opts.d == ZSV_PROP_ARG_REMOVE || opts.R == ZSV_PROP_ARG_REMOVE) - opts.overwrite = 1; - if(opts.d == ZSV_PROP_ARG_AUTO || opts.R == ZSV_PROP_ARG_AUTO) { - struct zsv_opts zsv_opts; - zsv_args_to_opts(m_argc, m_argv, &m_argc, m_argv, &zsv_opts, NULL); - err = detect_properties(filepath, &fp, - opts.d == ZSV_PROP_ARG_AUTO, - opts.R == ZSV_PROP_ARG_AUTO, - &zsv_opts); - } - - if(!err) { - if(opts.d == ZSV_PROP_ARG_AUTO) - opts.d = fp.header_span; - - if(opts.R == ZSV_PROP_ARG_AUTO) - opts.R = fp.skip; - err = merge_and_save_properties(filepath, opts.save, opts.overwrite, opts.d, opts.R); + switch(mode) { + case zsv_prop_mode_list_files: + err = zsv_prop_execute_list_files(filepath, verbose); + break; + case zsv_prop_mode_clean: + err = zsv_prop_execute_clean((const char *)filepath, dry, verbose); + break; + case zsv_prop_mode_copy: + err = zsv_prop_execute_copy((const char *)filepath, mode_value, opts.overwrite, dry, verbose); + break; + case zsv_prop_mode_export: + err = zsv_prop_execute_export((const char *)filepath, mode_value && strcmp(mode_value, "-") ? mode_value : NULL, verbose); + break; + case zsv_prop_mode_import: + err = zsv_prop_execute_import((const char *)filepath, mode_value && strcmp(mode_value, "-") ? mode_value : NULL, opts.overwrite, dry, verbose); + break; + case zsv_prop_mode_default: + { + struct zsv_opts zsv_opts; + zsv_args_to_opts(m_argc, m_argv, &m_argc, m_argv, &zsv_opts, NULL); + err = zsv_prop_execute_default(filepath, zsv_opts, opts); + } + break; } } } diff --git a/app/sql.c b/app/sql.c index 317e4fd2..b648c3dc 100644 --- a/app/sql.c +++ b/app/sql.c @@ -307,7 +307,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op f = stdin; if(f == stdin) { - tmpfn = zsv_get_temp_filename("zsv_sql"); + tmpfn = zsv_get_temp_filename("zsv_sql_XXXXXXXX"); if(!tmpfn) { fprintf(stderr, "Unable to create temp file name\n"); } else { diff --git a/app/test/prop/Makefile b/app/test/prop/Makefile index 5f0d95ae..b99acf97 100644 --- a/app/test/prop/Makefile +++ b/app/test/prop/Makefile @@ -17,7 +17,7 @@ TMP_DIR=${THIS_MAKEFILE_DIR}/tmp TEST_PASS=echo "${COLOR_BLUE}$@: ${COLOR_GREEN}Passed${COLOR_NONE}" TEST_FAIL=(echo "${COLOR_BLUE}$@: ${COLOR_RED}Failed!${COLOR_NONE}" && exit 1) -TEST_INIT=mkdir -p ${TMP_DIR} && rm -f ${TMP_DIR}/test* && echo "${COLOR_PINK}$@: ${COLOR_NONE}" +TEST_INIT=mkdir -p ${TMP_DIR} && rm -rf ${TMP_DIR}/test* && echo "${COLOR_PINK}$@: ${COLOR_NONE}" LEAKS= ifneq ($(LEAKS),) @@ -36,7 +36,11 @@ help: @echo "# run all tests:" @echo " make test" -test: test-1 test-2 test-3 test-4 test-5 test-6 test-7 test-8 clean +ifeq ($(EXE),) + $(error EXE is not defined) +endif + +test: test-1 test-2 test-3 test-4 test-5 test-6 test-7 test-8 test-copy test-clean test-export test-import clean test-1: @${TEST_INIT} @@ -80,6 +84,47 @@ test-8: @${CHECK} [ "`${EXE} detect.csv|jq -c -S`" == '{"header-row-span":2,"skip-head":2}' ] && ${TEST_PASS} || ${TEST_FAIL} @${EXE} detect.csv --clear +test-copy: + @${TEST_INIT} + @rm -rf ${TMP_DIR}/$@ + @mkdir -p ${TMP_DIR}/$@/.zsv/data/abc.csv + @echo '{}' > ${TMP_DIR}/$@/.zsv/data/abc.csv/props.json + @touch ${TMP_DIR}/$@/abc.csv + @[ "`${EXE} ${TMP_DIR}/$@/abc.csv --copy ${TMP_DIR}/$@/def.csv 2>&1`" = "${TMP_DIR}/$@/def.csv: No such file or directory" ] && ${TEST_PASS} || ${TEST_FAIL} + @touch ${TMP_DIR}/$@/def.csv + @${EXE} ${TMP_DIR}/$@/abc.csv --copy ${TMP_DIR}/$@/def.csv + @cmp ${TMP_DIR}/$@/.zsv/data/abc.csv/props.json ${TMP_DIR}/$@/.zsv/data/def.csv/props.json && ${TEST_PASS} || ${TEST_FAIL} + +test-clean: + @${TEST_INIT} + @rm -rf ${TMP_DIR}/$@ + @mkdir -p ${TMP_DIR}/$@/.zsv/data/abc.csv + @echo '{}' > ${TMP_DIR}/$@/.zsv/data/abc.csv/props.json + @mkdir -p ${TMP_DIR}/$@/.zsv/data/def.csv + @echo '{}' > ${TMP_DIR}/$@/.zsv/data/def.csv/props.json + @touch ${TMP_DIR}/$@/abc.csv + @${EXE} ${TMP_DIR}/$@ --clean + @ [ ! -d ${TMP_DIR}/$@/.zsv/data/def.csv ] && [ -d ${TMP_DIR}/$@/.zsv/data/abc.csv ] && ${TEST_PASS} || ${TEST_FAIL} + +test-export: + @${TEST_INIT} + @rm -rf ${TMP_DIR}/$@ + @mkdir -p ${TMP_DIR}/$@/.zsv/data/abc.csv + @echo '{"a":1}' > ${TMP_DIR}/$@/.zsv/data/abc.csv/props.json + @touch ${TMP_DIR}/$@/abc.csv + @ [ "`${EXE} ${TMP_DIR}/$@/abc.csv --export - | jq -c`" = '{"props.json":{"a":1}}' ] && ${TEST_PASS} || ${TEST_FAIL} + +test-import: + @${TEST_INIT} + @rm -rf ${TMP_DIR}/$@ + @mkdir -p ${TMP_DIR}/$@/.zsv/data/abc.csv + @[ "`echo '{"props.json":{"b":1}}' | ${EXE} ${TMP_DIR}/$@/def.csv --import - 2>&1`" = "${TMP_DIR}/$@/def.csv: No such file or directory" ] && ${TEST_PASS} || ${TEST_FAIL} + @echo '{"props.json":{"b":1}}' | ${EXE} ${TMP_DIR}/$@/def.csv --import - -f + @[ "`jq -c < ${TMP_DIR}/$@/.zsv/data/def.csv/props.json`" = '{"b":1}' ] && ${TEST_PASS} || ${TEST_FAIL} + @touch ${TMP_DIR}/$@/def.csv + @[ "`echo '{"props.json":{"b":1}}' | ${EXE} ${TMP_DIR}/$@/def.csv --import - 2>&1`" = "${TMP_DIR}/$@/.zsv/data/def.csv/props.json: File exists" ] && ${TEST_PASS} || ${TEST_FAIL} + @[ "`echo '{"props.json":{"b":1}}' | ${EXE} ${TMP_DIR}/$@/def.csv --import - -f --dry`" = "${TMP_DIR}/$@/.zsv/data/def.csv/props.json" ] && ${TEST_PASS} || ${TEST_FAIL} + clean: @rm -rf ${TMP_DIR} diff --git a/app/utils/dirs.c b/app/utils/dirs.c index 5641344c..1484d4c7 100644 --- a/app/utils/dirs.c +++ b/app/utils/dirs.c @@ -181,11 +181,6 @@ to do: add support for this OS!; #endif /* end of: #if defined(_WIN32) */ -struct remove_dir_ctx { - int err; - struct dir_path *dirs; -}; - struct dir_path { struct dir_path *next; char *path; @@ -210,70 +205,105 @@ static int rmdir_w_msg(const char *path, int *err) { return *err; } -static void remove_files_collect_dirs(struct remove_dir_ctx *ctx, const char *path) { - // delete all files, collect dir names in reverse order +static int zsv_foreach_dirent_remove(struct zsv_foreach_dirent_handle *h, size_t depth) { + (void)(depth); + if(!h->is_dir) { // file + if(h->parent_and_entry) { + if(unlink(h->parent_and_entry)) { + perror(h->parent_and_entry); // "Unable to remove file"); + return 1; + } + } + } else { // dir + struct dir_path *dn = calloc(1, sizeof(*dn)); + if(!dn) { + fprintf(stderr, "Out of memory!\n"); + return 1; + } + if(h->parent_and_entry) { + dn->path = strdup(h->parent_and_entry); + dn->next = *((struct dir_path **)h->ctx); + *((struct dir_path **)h->ctx) = dn; + } + } + return 0; +} + +// return error +static +int zsv_foreach_dirent_aux(const char *dir_path, + size_t depth, + size_t max_depth, + zsv_foreach_dirent_handler handler, void *ctx, + char verbose + ) { + int err = 0; + if(!dir_path) + return 1; + + if(max_depth > 0 && depth > max_depth) + return 0; + DIR *dr; - struct dir_path *previous_dir = ctx->dirs; - struct dir_path *most_recent_dir = NULL; - if((dr = opendir(path))) { + if((dr = opendir(dir_path))) { struct dirent *de; while((de = readdir(dr)) != NULL) { if(!*de->d_name || !strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) continue; char *tmp; - asprintf(&tmp, "%s%c%s", path, FILESLASH, de->d_name); + asprintf(&tmp, "%s%c%s", dir_path, FILESLASH, de->d_name); if(!tmp) - fprintf(stderr, "Out of memory!\n"), ctx->err = 1; + fprintf(stderr, "Out of memory!\n"), err = 1; else { - struct stat s; - stat(tmp, &s); - if (s.st_mode & S_IFDIR) { // it's a dir. save for later - struct dir_path *dn = calloc(1, sizeof(*dn)); - if(!dn) - fprintf(stderr, "Out of memory!\n"), ctx->err = 1; - else { - most_recent_dir = dn; - dn->path = tmp; - dn->next = ctx->dirs; - ctx->dirs = dn; - } - } else { // not a dir. try to remove - if(unlink(tmp)) { - perror(tmp); // "Unable to remove file"); - // fprintf(stderr, "%s\n", tmp); - ctx->err = 1; - } - free(tmp); - } + struct zsv_foreach_dirent_handle h = { 0 }; + h.verbose = verbose; + stat(tmp, (struct stat *)&h.stat); + h.parent = dir_path; + h.entry = de->d_name; + h.parent_and_entry = tmp; + h.ctx = ctx; + char is_dir = h.stat.st_mode & S_IFDIR ? 1 : 0; + h.is_dir = is_dir; + if(handler) + err = handler(&h, depth + 1); + + if(is_dir && !h.no_recurse) + // recurse! + zsv_foreach_dirent_aux(tmp, depth + 1, max_depth, handler, ctx, verbose); + free(tmp); } } closedir(dr); } + return err; +} - // process all sub-dirs that we just collected - for(struct dir_path *dn = most_recent_dir; dn && dn != previous_dir; dn = dn->next) - remove_files_collect_dirs(ctx, dn->path); +int zsv_foreach_dirent(const char *dir_path, + size_t max_depth, + zsv_foreach_dirent_handler handler, void *ctx, + char verbose + ) { + return zsv_foreach_dirent_aux(dir_path, 0, max_depth, handler, ctx, verbose); } /** * Remove a directory and all of its contents */ int zsv_remove_dir_recursive(const unsigned char *path) { - const char *cpath = (void *)path; - struct remove_dir_ctx ctx = { 0 }; // we will delete all files first, then // delete directories in the reverse order we received them - remove_files_collect_dirs(&ctx, cpath); - + struct dir_path *reverse_dirs = NULL; + int err = zsv_foreach_dirent((const char *)path, 0, + zsv_foreach_dirent_remove, &reverse_dirs, 0); // unlink and free each dir - for(struct dir_path *next, *dn = ctx.dirs; dn; dn = next) { + for(struct dir_path *next, *dn = reverse_dirs; !err && dn; dn = next) { next = dn->next; - rmdir_w_msg(dn->path, &ctx.err); + rmdir_w_msg(dn->path, &err); free(dn->path); free(dn); } - if(!ctx.err) - rmdir_w_msg(cpath, &ctx.err); + if(!err) + rmdir_w_msg((const char *)path, &err); - return ctx.err; + return err; } diff --git a/app/utils/file.c b/app/utils/file.c index f09be0df..56d04f6b 100644 --- a/app/utils/file.c +++ b/app/utils/file.c @@ -13,6 +13,10 @@ #include // for close() #include // open +#include +#include + + #if defined(_WIN32) || defined(WIN32) || defined(WIN) #include @@ -33,16 +37,22 @@ char *zsv_get_temp_filename(const char *prefix) { } #else +/** + * Get a temp file name. The returned value, if any, will have been allocated + * on the heap, and the caller should `free()` + * + * @param prefix string with which the resulting file name will be prefixed + */ char *zsv_get_temp_filename(const char *prefix) { char *s = NULL; char *tmpdir = getenv("TMPDIR"); if(!tmpdir) tmpdir = "."; asprintf(&s, "%s/%s_XXXXXXXX", tmpdir, prefix); -#ifndef NDEBUG - fprintf(stderr, "creating temp file: %s\n", s ? s : "Out of memory!"); -#endif - if(s) { + if(!s) { + const char *msg = strerror(errno); + fprintf(stderr, "%s%c%s: %s\n", tmpdir, FILESLASH, prefix, msg ? msg : "Unknown error"); + } else { int fd = mkstemp(s); if(fd > 0) { close(fd); @@ -101,10 +111,74 @@ int zsv_file_exists(const char* filename) { int zsv_file_exists(const char* filename) { struct stat buffer; - return (stat(filename, &buffer) == 0); + if(stat(filename, &buffer) == 0) { + char is_dir = buffer.st_mode & S_IFDIR ? 1 : 0; + if(!is_dir) + return 1; + } + return 0; } #endif +/** + * Copy a file, given source and destination paths + * On error, output error message and return non-zero + */ +int zsv_copy_file(const char *src, const char *dest) { + // create one or more directories if needed + if(zsv_mkdirs(dest, 1)) { + fprintf(stderr, "Unable to create directories needed for %s\n", dest); + return -1; + } + + // copy the file + int err = 0; + FILE *fsrc = fopen(src, "rb"); + if(!fsrc) + err = errno ? errno : -1, perror(src); + else { + FILE *fdest = fopen(dest, "wb"); + if(!fdest) + err = errno ? errno : -1, perror(dest); + else { + err = zsv_copy_file_ptr(fsrc, fdest); + if(err) + perror(dest); + fclose(fdest); + } + fclose(fsrc); + } + return err; +} + +/** + * Copy a file, given source and destination FILE pointers + * Return error number per errno.h + */ +int zsv_copy_file_ptr(FILE *src, FILE *dest) { + int err = 0; + char buffer[4096]; + size_t bytes_read; + while((bytes_read = fread(buffer, 1, sizeof(buffer), src)) > 0) { + if(fwrite(buffer, 1, bytes_read, dest) != bytes_read) { + err = errno ? errno : -1; + break; + } + } + return err; +} + +size_t zsv_dir_len_basename(const char *filepath, const char **basename) { + for(size_t len = strlen(filepath); len; len--) { + if(filepath[len-1] == '/' || filepath[len-1] == '\\') { + *basename = filepath + len; + return len - 1; + } + } + + *basename = filepath; + return 0; +} int zsv_file_readable(const char *filename, int *err, FILE **f_out) { FILE *f; @@ -116,19 +190,8 @@ int zsv_file_readable(const char *filename, int *err, FILE **f_out) { rc = 0; if(err) *err = errno; - else switch(errno) { - case ENOENT: - fprintf(stderr, "File '%s' not found\n", filename); - break; - case EACCES: - fprintf(stderr, "No permissions to read '%s'\n", filename); - break; - case EISDIR: - fprintf(stderr, "File '%s' is a directory\n", filename); - break; - default: - fprintf(stderr, "Unknown error opening '%s'\n", filename); - } + else + perror(filename); } else { rc = 1; if(f_out) diff --git a/include/zsv/utils/cache.h b/include/zsv/utils/cache.h index 3980eba9..283bba1d 100644 --- a/include/zsv/utils/cache.h +++ b/include/zsv/utils/cache.h @@ -33,7 +33,6 @@ unsigned char *zsv_cache_path(const unsigned char *data_filepath, const unsigned char *cache_filename, char temp_file); - enum zsv_cache_type { zsv_cache_type_property = 1, zsv_cache_type_tag diff --git a/include/zsv/utils/dirs.h b/include/zsv/utils/dirs.h index 12428fc1..f79021ab 100644 --- a/include/zsv/utils/dirs.h +++ b/include/zsv/utils/dirs.h @@ -53,4 +53,40 @@ int zsv_mkdirs(const char *path, char path_is_filename); */ int zsv_remove_dir_recursive(const unsigned char *path); +#include + +struct zsv_foreach_dirent_handle { + const char *parent; /* name of the parent directory */ + const char *entry; /* file / dir name of current entry being processed */ + const char *parent_and_entry; /* parent + entry separated by file separator */ + const struct stat stat; /* stat of current entry */ + + void *ctx; /* caller-provided context to pass to handler */ + + unsigned char verbose:1; + unsigned char is_dir:1; /* non-zero if this entry is a directory */ + unsigned char no_recurse:1; /* set to 1 when handling a dir to prevent recursing into it */ + unsigned char _:5; +}; + +typedef int (*zsv_foreach_dirent_handler)(struct zsv_foreach_dirent_handle *h, size_t depth); + +/** + * Recursively process entries (files and folders) in a directory + * + * @param dir_path : path of directory to begin processing children of + * @param max_depth : maximum depth to recurse, or 0 for no maximum + * @param handler : caller-provided entry handler. return 0 on success, non-zero on error + * @param ctx : pointer passed to the handler + * @param verbose : non-zero for verbose output + * + * returns error + */ +int zsv_foreach_dirent(const char *dir_path, + size_t max_depth, + zsv_foreach_dirent_handler handler, + void *ctx, + char verbose + ); + #endif diff --git a/include/zsv/utils/file.h b/include/zsv/utils/file.h index 0afeb76e..127a259e 100644 --- a/include/zsv/utils/file.h +++ b/include/zsv/utils/file.h @@ -20,7 +20,7 @@ #endif // LINEEND /** * Get a temp file name. The returned value, if any, will have been allocated - * on the stack, and the caller should `free()` + * on the heap, and the caller should `free()` * * @param prefix string with which the resulting file name will be prefixed */ @@ -54,5 +54,23 @@ int zsv_file_readable(const char *filename, int *err, FILE **f_out); */ size_t zsv_filter_write(void *FILEp, unsigned char *buff, size_t bytes_read); +/** + * Get a file path's directory length and base name + * Returns the length of the directory portion of the path + * and the base name portion of the path + */ +size_t zsv_dir_len_basename(const char *filepath, const char **basename); + +/** + * Copy a file. Create any needed directories + * On error, prints error message and returns non-zero + */ +int zsv_copy_file(const char *src, const char *dest); + +/** + * Copy a file, given source and destination FILE pointers + * Return error number per errno.h + */ +int zsv_copy_file_ptr(FILE *src, FILE *dest); #endif diff --git a/include/zsv/utils/prop.h b/include/zsv/utils/prop.h index ba82ca42..7b733607 100644 --- a/include/zsv/utils/prop.h +++ b/include/zsv/utils/prop.h @@ -67,4 +67,31 @@ enum zsv_status zsv_new_with_properties(struct zsv_opts *opts, const char *opts_used, zsv_parser *handle_out ); + +/** + * If you are building your own CLI and incorporating zsv CLI commands into it, + * the `prop` command can be customized by providing your own function + * for determining whether a file in the property cache is a property file, + * which can be set using zsv_prop_get_or_set_is_prop_file() + * + * @param is_prop_file: your function, that returns non-zero if the given file entry + * is a property file. If NULL, is set to zsv_is_prop_file + * @param max_depth : maximum depth of any property file. if is_prop_file was NULL, + * max_depth is set to 1 + */ +#include "dirs.h" + +struct is_property_ctx; /* opaque structure for internal use */ +struct is_property_ctx * +zsv_prop_get_or_set_is_prop_file( + int (*is_prop_file)(struct zsv_foreach_dirent_handle *, size_t), + int max_depth, + char set + ); + +/** + * If you provide your own is_prop_file() function and you also want to include any + * zsv property file, your is_prop_file() can call zsv_is_prop_file() + */ +int zsv_is_prop_file(struct zsv_foreach_dirent_handle *h, size_t depth); #endif