Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

stack: remove superflous -T option #338

Merged
merged 3 commits into from
Dec 11, 2024
Merged

stack: remove superflous -T option #338

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
37f5ffa
stack: remove superflous -T option
liquidaty Dec 11, 2024
58d4238
stack: support duplicate column names (by default); add --unique option
liquidaty Dec 11, 2024
cd47869
stack: add tests for duplicate column names
liquidaty Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 46 additions & 44 deletions app/stack.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,11 @@ const char *zsv_stack_usage_msg[] = {
"Usage: " APPNAME " [options] filename [filename...]",
"",
"Options:",
" -o <filename>: output file",
" -b : output with BOM",
" -q : always add double-quotes",
" -T : input is tab-delimited, instead of comma-delimited",
" -o <filename> : output file",
" -b : output with BOM",
" -q : always add double-quotes",
" --unique : only output unique column names; in the case of duplicates,",
" only the last (right-most) column will be kept",
NULL,
};

Expand All @@ -40,6 +41,8 @@ typedef struct zsv_stack_colname {
struct zsv_stack_colname *next;
char unsigned *orig_name;
char unsigned *name;
unsigned occurrence;
unsigned dupes;
size_t global_position;
unsigned char color;
struct zsv_stack_colname *left;
Expand All @@ -48,12 +51,13 @@ typedef struct zsv_stack_colname {
} zsv_stack_colname;

static struct zsv_stack_colname *zsv_stack_colname_init(struct zsv_stack_colname *e, const unsigned char *name,
size_t len) {
size_t len, unsigned occurrence) {
memset(e, 0, sizeof(*e));
if (len)
e->name = zsv_strtolowercase(name, &len);
else
e->name = calloc(1, 2);
e->occurrence = occurrence;
return e;
}

Expand All @@ -70,7 +74,10 @@ static void zsv_stack_colname_delete(struct zsv_stack_colname *e) {
}

static int zsv_stack_colname_cmp(zsv_stack_colname *x, zsv_stack_colname *y) {
return strcmp((const char *)x->name, (const char *)y->name);
int i = strcmp((const char *)x->name, (const char *)y->name);
if (i == 0)
return x->occurrence == y->occurrence ? 0 : x->occurrence > y->occurrence ? 1 : -1;
return i;
}

SGLIB_DEFINE_RBTREE_PROTOTYPES(zsv_stack_colname, left, right, color, zsv_stack_colname_cmp);
Expand All @@ -94,6 +101,7 @@ struct zsv_stack_input_file {
zsv_parser parser;

// output_column_map[x] = n where x = output col ix, n = 0 if no map, else raw_column_ix
zsv_stack_colname *colnames;
size_t *output_column_map;
size_t output_column_map_size;
size_t header_row_end_offset; // location in buff at which the data row begins
Expand All @@ -115,6 +123,9 @@ struct zsv_stack_data {
int err;

zsv_csv_writer csv_writer;

unsigned char unique_column_names : 1;
unsigned char _ : 7;
};

static struct zsv_stack_input_file **zsv_stack_input_file_add(const char *filename,
Expand All @@ -139,6 +150,7 @@ static void zsv_stack_input_files_delete(struct zsv_stack_input_file *list) {
fclose(e->f);
if (e->output_column_map)
free(e->output_column_map);
zsv_stack_colname_tree_delete(&e->colnames);
zsv_delete(e->parser);
free(e);
}
Expand All @@ -150,14 +162,17 @@ static void zsv_stack_cleanup(struct zsv_stack_data *data) {
zsv_writer_delete(data->csv_writer);
}

static struct zsv_stack_colname *zsv_stack_colname_get_or_add(const unsigned char *name, size_t name_len,
static struct zsv_stack_colname *zsv_stack_colname_get_or_add(struct zsv_cell c, unsigned occurrence,
struct zsv_stack_colname **tree, int *added) {
const unsigned char *name = c.str;
size_t name_len = c.len;
struct zsv_stack_colname e;
zsv_stack_colname_init(&e, name, name_len);
zsv_stack_colname_init(&e, name, name_len, occurrence);
struct zsv_stack_colname *found = sglib_zsv_stack_colname_find_member(*tree, &e);
if (found)
if (found) {
found->dupes++;
zsv_stack_colname_free(&e);
else {
} else {
found = calloc(1, sizeof(*found));
if (found) {
*added = 1;
Expand All @@ -171,9 +186,9 @@ static struct zsv_stack_colname *zsv_stack_colname_get_or_add(const unsigned cha
}

// zsv_stack_consolidate_header(): return global position
static unsigned zsv_stack_consolidate_header(struct zsv_stack_data *d, const unsigned char *name, size_t name_len) {
static unsigned zsv_stack_consolidate_header(struct zsv_stack_data *d, struct zsv_cell cell, unsigned occurrence) {
int added = 0;
zsv_stack_colname *c = zsv_stack_colname_get_or_add(name, name_len, &d->colnames, &added);
zsv_stack_colname *c = zsv_stack_colname_get_or_add(cell, occurrence, &d->colnames, &added);
if (!c)
d->err = 1;
else {
Expand All @@ -190,14 +205,6 @@ static unsigned zsv_stack_consolidate_header(struct zsv_stack_data *d, const uns
return 0;
}

/*
static void zsv_stack_header_cell_wrapper(void *ctx, unsigned char *restrict utf8_value, size_t len) {
struct zsv_stack_input_file *input = ctx;
if(!input->headers_done)
zsv_stack_header_cell(ctx, utf8_value, len);
}
*/

static void zsv_stack_header_row(void *ctx) {
struct zsv_stack_input_file *input = ctx;
if (!input->headers_done && !zsv_row_is_blank(input->parser)) { // skip any blank leading rows
Expand Down Expand Up @@ -238,16 +245,16 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op

struct zsv_opts saved_opts = *opts;
struct zsv_stack_data data = {0};
char delimiter = 0; // defaults to csv
struct zsv_csv_writer_options writer_opts = zsv_writer_get_default_opts();
writer_opts.stream = stdout;

struct zsv_stack_input_file **next_input = &data.inputs;
for (int arg_i = 1; !data.err && arg_i < argc; arg_i++) {
const char *arg = argv[arg_i];
if (!strcmp(arg, "-b"))
writer_opts.with_bom = 1;
else if (!strcmp(arg, "-T"))
delimiter = '\t';
else if (!strcmp(arg, "--unique"))
data.unique_column_names = 1;
else if (!strcmp(arg, "-o")) {
arg_i++;
if (arg_i >= argc)
Expand All @@ -258,22 +265,10 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
fprintf(stderr, "Unable to open file for writing: %s\n", argv[arg_i]);
}
}
}
}

if (!(data.csv_writer = zsv_writer_new(&writer_opts)))
data.err = 1;

if (!data.err) {
struct zsv_stack_input_file **next_input = &data.inputs;
for (int arg_i = 1; !data.err && arg_i < argc; arg_i++) {
const char *arg = argv[arg_i];
if (*arg == '-') {
if (!strcmp(arg, "-o"))
arg_i++;
continue;
}

} else if (*arg == '-') {
fprintf(stderr, "Unrecognized option: %s\n", arg);
data.err = 1;
} else {
FILE *f = fopen(arg, "rb");
if (!f) {
fprintf(stderr, "Could not open file for reading: %s\n", arg);
Expand All @@ -283,13 +278,15 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
}
}

if (!(data.csv_writer = zsv_writer_new(&writer_opts)))
data.err = 1;

// collect all header names so we can line them up
unsigned i = 0;
for (struct zsv_stack_input_file *input = data.inputs; !data.err && input; input = input->next, i++) {
*opts = saved_opts;
opts->row_handler = zsv_stack_header_row;
opts->ctx = input;
opts->delimiter = delimiter;

// to do: max_cell_size
opts->stream = input->f;
Expand All @@ -311,7 +308,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
// of all inputs
size_t max_columns_count = 0;
for (struct zsv_stack_input_file *input = data.inputs; !data.err && input; input = input->next)
max_columns_count += zsv_cell_count(input->parser); // zsv_row_cells_count(input->row);
max_columns_count += zsv_cell_count(input->parser);

// next, for each input, align the input columns with the output columns
for (struct zsv_stack_input_file *input = data.inputs; input && !data.err; input = input->next) {
Expand All @@ -324,7 +321,14 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
size_t cols_used = zsv_cell_count(input->parser);
for (unsigned col_ix = 0; col_ix < cols_used; col_ix++) {
struct zsv_cell cell = zsv_get_cell(input->parser, col_ix);
size_t output_ix = zsv_stack_consolidate_header(&data, cell.str, cell.len);
unsigned occurrence = 0;
if (!data.unique_column_names) {
int added;
zsv_stack_colname *c = zsv_stack_colname_get_or_add(cell, 0, &input->colnames, &added);
if (c)
occurrence = c->dupes;
}
size_t output_ix = zsv_stack_consolidate_header(&data, cell, occurrence);
if (output_ix)
input->output_column_map[output_ix - 1] = col_ix + 1;
}
Expand Down Expand Up @@ -360,8 +364,6 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
*opts = saved_opts;
opts->row_handler = zsv_stack_data_row;
opts->ctx = input;
if (delimiter == '\t')
opts->delimiter = delimiter;

rewind(input->f);
input->headers_done = 0;
Expand Down
15 changes: 13 additions & 2 deletions app/test/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ test-blank-leading-rows-4: ${BUILD_DIR}/bin/zsv_select${EXE}
@${PREFIX} $< -R 2 ${TEST_DATA_DIR}/test/blank-leading-rows.csv ${REDIRECT} ${TMP_DIR}/[email protected] 2>&1
@${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL}

test-stack: test-stack1 test-stack2 test-stack3
test-stack: test-stack1 test-stack2 test-stack3 test-stack4

test-stack1: ${BUILD_DIR}/bin/zsv_stack${EXE}
@${TEST_INIT}
Expand All @@ -380,9 +380,20 @@ test-stack2: ${BUILD_DIR}/bin/zsv_stack${EXE}
test-stack3: ${BUILD_DIR}/bin/zsv_stack${EXE}
@${TEST_INIT}
@echo 'a,b,c' > /tmp/1.csv
@${PREFIX} $< /tmp/1.csv ${REDIRECT1} ${TMP_DIR}/[email protected]
@${PREFIX} $< /tmp/1.csv ${REDIRECT} ${TMP_DIR}/[email protected]
@${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL}

test-stack4: ${BUILD_DIR}/bin/zsv_stack${EXE}
@${TEST_INIT}
@echo 'a,b,c,a,b' > ${TMP_DIR}/[email protected]
@echo '1,2,3,4,5' >> ${TMP_DIR}/[email protected]
@echo 'a,b,c,a,b' > ${TMP_DIR}/[email protected]
@echo 'z1,z2,z3,z4,z5' >> ${TMP_DIR}/[email protected]
@${PREFIX} $< ${TMP_DIR}/[email protected] ${TMP_DIR}/[email protected] ${REDIRECT} ${TMP_DIR}/[email protected]
@${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL}
@${PREFIX} $< --unique ${TMP_DIR}/[email protected] ${TMP_DIR}/[email protected] ${REDIRECT} ${TMP_DIR}/[email protected]
@${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL}

test-sql test-flatten : test-%: ${BUILD_DIR}/bin/zsv_%${EXE}
@${TEST_INIT}
@( ( ! [ -s "${TEST_DATA_DIR}/test/$*.csv" ] ) && echo "No test input for $*") || \
Expand Down
3 changes: 3 additions & 0 deletions app/test/expected/test-stack4-2.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
a,b,c
4,5,3
z4,z5,z3
3 changes: 3 additions & 0 deletions app/test/expected/test-stack4.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
a,b,c,a,b
1,2,3,4,5
z1,z2,z3,z4,z5