zsv playground (#299)

* automatically generate zsv playground
* Update README

.github/workflows/ci.yml

@@ -34,6 +34,7 @@ env:
   AMD64_LINUX_CLANG: amd64-linux-clang
   AMD64_LINUX_MUSL: amd64-linux-musl
   AMD64_WINDOWS_MINGW: amd64-windows-mingw
+  AMD64_LINUX_WASM: amd64-linux-wasm
   AMD64_MACOSX_GCC: amd64-macosx-gcc
   ARM64_MACOSX_GCC: arm64-macosx-gcc
   AMD64_FREEBSD_GCC: amd64-freebsd-gcc
@@ -519,6 +520,9 @@ jobs:
     runs-on: ubuntu-latest
     container: alpine:latest
 
+    outputs:
+      TAG: ${{ needs.tag.outputs.TAG }}
+
     env:
       TAG: ${{ needs.tag.outputs.TAG }}
 
@@ -577,14 +581,14 @@ jobs:
       run: ./scripts/ci-upload-release-artifacts.sh
 
   ghcr:
-    needs: [tag, ci-musl]
+    needs: ci-musl
     runs-on: ubuntu-latest
 
     permissions:
       packages: write
 
     env:
-      TAG: ${{ needs.tag.outputs.TAG }}
+      TAG: ${{ needs.ci-musl.outputs.TAG }}
 
     steps:
     - name: Checkout
@@ -638,3 +642,57 @@ jobs:
         tags: |
           ghcr.io/liquidaty/zsv:${{ env.TAG }}
           ghcr.io/liquidaty/zsv:latest
+
+  ci-wasm:
+    needs: [tag, clang-format, cppcheck, shellcheck]
+    runs-on: ubuntu-latest
+
+    env:
+      TAG: ${{ needs.tag.outputs.TAG }}
+
+    steps:
+    - name: Set up emsdk
+      uses: mymindstorm/setup-emsdk@v14
+
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Build (${{ env.AMD64_LINUX_WASM }})
+      env:
+        PREFIX: ${{ env.AMD64_LINUX_WASM }}
+        CC: emcc
+        MAKE: make
+        RUN_TESTS: false
+        CONFIGFILE: "config.emcc"
+        CFLAGS: "-msse2 -msimd128"
+        CROSS_COMPILING: "yes"
+        NO_THREADING: "1"
+        STATIC_BUILD: "1"
+      run: |
+        emconfigure ./configure --enable-pic --disable-pie
+        emmake make install NO_STDIN=1 NO_PLAYGROUND=0
+        cp "$PREFIX"/bin/cli.em.{js,wasm} playground
+        sed "s|__VERSION__|$TAG|g" -i playground/index.html
+
+    - name: Upload GitHub Pages artifacts
+      uses: actions/upload-pages-artifact@v3
+      with:
+        path: playground
+
+  deploy-playground:
+    if: ${{ github.ref_name == 'main' }}
+    needs: ci-wasm
+    runs-on: ubuntu-latest
+
+    permissions:
+      pages: write
+      id-token: write
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+    - name: Deploy to GitHub Pages
+      id: deployment
+      uses: actions/deploy-pages@v4

.gitignore

@@ -69,8 +69,14 @@ cppcheck*
 tmp
 include/zsv.h
 app/test/worldcitiespop_mil.csv
+app/test/worldcitiespop_mil.tsv
 data/quoted5.csv
 data/loans_2.csv
 data/.zsv/data/loans_2.csv/overwrite.sqlite3
 compile_commands.json
 .cache
+zsvsheet_filter_*
+overwrite.sqlite3
+playground/*
+!playground/index.html
+playground/favicon.ico

README.md

@@ -16,6 +16,8 @@ lib only:
 [npm-url]: https://npmjs.org/package/zsv-lib
 [npm-version-image]: https://badgen.net/npm/v/zsv-lib
 
+Online playground: <https://liquidaty.github.io/zsv>
+
 zsv+lib is a fast CSV parser library and extensible command-line utility. It
 achieves high performance using SIMD operations, [efficient memory
 use](docs/memory.md) and other optimization techniques, and can also parse
@@ -413,7 +415,6 @@ helping, please post an issue.
 
 ### Possible enhancements and related developments
 
-- online "playground" (soon to be released)
 - optimize search; add search with hyperscan or re2 regex matching, possibly
   parallelize?
 - optional OpenMP or other multi-threading for row processing

app/Makefile

@@ -91,18 +91,25 @@ CONFIGURE_HOST=
 ifneq ($(HOST),)
   CONFIGURE_HOST=$(HOST)
 endif
+
 ifeq ($(WIN),0)
   BUILD_SUBDIR=$(shell uname)/${DBG_SUBDIR}
   EXE=
-
   ifneq ($(findstring emcc,$(CC)),) # emcc
     NO_THREADING ?= 1
+    NO_PLAYGROUND ?= 1
     EXE=.em.js
-    EXPORTED_FUNCTIONS='_free','_malloc'
-    CFLAGS+= -s EXPORTED_FUNCTIONS="[${EXPORTED_FUNCTIONS}]" -s EXPORTED_RUNTIME_METHODS="['setValue','allocateUTF8','stringToUTF8Array','lengthBytesUTF8','writeArrayToMemory']" -s RESERVED_FUNCTION_POINTERS=4 -s ALLOW_MEMORY_GROWTH=1
-    CFLAGS_EXE+= -s FORCE_FILESYSTEM=1 -lidbfs.js -s MAIN_MODULE
+    ifeq ($(NO_PLAYGROUND),0) # wasm playground
+      LDFLAGS += -sEXPORTED_RUNTIME_METHODS="['callMain']" -sINVOKE_RUN=0 -sALLOW_MEMORY_GROWTH=1
+    else
+      CFLAGS += -DNO_PLAYGROUND
+      LDFLAGS += -sEXPORTED_FUNCTIONS="['_free','_malloc']"
+      LDFLAGS += -sEXPORTED_RUNTIME_METHODS="['setValue','allocateUTF8','stringToUTF8Array','lengthBytesUTF8','writeArrayToMemory']"
+      LDFLAGS += -sRESERVED_FUNCTION_POINTERS=4 -sALLOW_MEMORY_GROWTH=1
+    endif
+    LDFLAGS += -sFORCE_FILESYSTEM=1 -lidbfs.js -sMAIN_MODULE
     ifeq ($(NO_THREADING),0)
-      CFLAGS_EXE+= -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=2 -s PTHREAD_POOL_SIZE=8
+      LDFLAGS += -sUSE_PTHREADS=1 -sPTHREAD_POOL_SIZE=2
     endif
   endif
 else
@@ -223,7 +230,7 @@ CLI_OBJ_PFX=${BUILD_DIR}/objs/cli_
 CLI_APP_OBJECT=${CLI_OBJ_PFX}cli.o
 CLI=${BUILD_DIR}/bin/cli${EXE}
 ifneq ($(findstring emcc,$(CC)),) # emcc
-    CLI_ADDITIONAL=${BUILD_DIR}/bin/cli.em.wasm
+  CLI_ADDITIONAL=${BUILD_DIR}/bin/cli.em.wasm
 endif
 CLI_OBJECTS=$(addprefix ${CLI_OBJ_PFX},$(addsuffix .o,${CLI_SOURCES}))
 
@@ -418,7 +425,7 @@ ${JQ_BUNDLE_LIB}: ${JQ_SRC} # -D_REENTRANT needed for clang to not break
 	# copy the directory because its timestamp gets updated after libjq is built
 	@cp -a ${JQ_SRC} ${JQ_SRC}-conf
 	cd ${JQ_SRC}-conf \
-	&& CC="${CC}" CFLAGS="${CFLAGS} -D_REENTRANT" ./configure \
+	&& CC="${CC}" CFLAGS="${CFLAGS} -w -D_REENTRANT" ./configure \
 	    --prefix="${JQ_PREFIX}" \
 	    --disable-maintainer-mode \
 	    --without-oniguruma \
@@ -458,11 +465,11 @@ ${STANDALONE_PFX}%${EXE}: %.c ${OBJECTS} ${MORE_OBJECTS} ${LIBZSV_INSTALL} ${UTF
 
 ${BUILD_DIR}-external/sqlite3/sqlite3_and_csv_vtab.o: ${BUILD_DIR}-external/%.o : external/%.c
 	@mkdir -p `dirname "$@"`
-	${CC} ${CFLAGS} -I${INCLUDE_DIR} ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -o $@ -c $<
+	${CC} ${CFLAGS} -w -I${INCLUDE_DIR} ${YAJL_INCLUDE} ${YAJL_HELPER_INCLUDE} -o $@ -c $<
 
 ${YAJL_OBJ}: ${BUILD_DIR}-external/yajl/%.o : external/yajl/src/%.c
 	@mkdir -p `dirname "$@"`
-	${CC} ${CFLAGS} ${YAJL_INCLUDE} -c external/yajl/src/$*.c -o $@
+	${CC} ${CFLAGS} -w ${YAJL_INCLUDE} -c external/yajl/src/$*.c -o $@
 
 ${YAJL_HELPER_OBJ}: external/yajl_helper/yajl_helper.c
 	@mkdir -p `dirname "$@"`

app/builtin/help.c

@@ -6,6 +6,8 @@
  * https://opensource.org/licenses/MIT
  */
 
+#include <stdio.h>
+
 static int main_help(int argc, const char *argv[]) {
   (void)(argc);
   (void)(argv);
@@ -29,9 +31,11 @@ static int main_help(int argc, const char *argv[]) {
 #endif
     "  zsv help [<command>]",
     "  zsv <command> <options> <arguments>  : run a command on data (see below for details)",
+#ifndef __EMSCRIPTEN__
     "  zsv <id>-<cmd> <options> <arguments> : invoke command 'cmd' of extension 'id'",
-    "  zsv thirdparty                       : view third-party licenses & acknowledgements",
     "  zsv license [<extension_id>]",
+#endif
+    "  zsv thirdparty                       : view third-party licenses & acknowledgements",
     "",
     "Options common to all commands except `prop`, `rm` and `jq`:",
 #ifdef ZSV_EXTRAS
@@ -83,6 +87,7 @@ static int main_help(int argc, const char *argv[]) {
   for (size_t i = 0; usage[i]; i++)
     fprintf(f, "%s\n", usage[i]);
 
+#ifndef __EMSCRIPTEN__
   char printed_init = 0;
   struct cli_config config;
   if (!config_init(&config, 1, 1, 0)) {
@@ -103,6 +108,7 @@ static int main_help(int argc, const char *argv[]) {
   }
   if (!printed_init)
     fprintf(f, "\n(No extended commands)\n");
+#endif
 
   return 0;
 }

app/builtin/thirdparty.c

@@ -23,6 +23,7 @@ static int main_thirdparty(int argc, const char *argv[]) {
   printf("Third-party licenses and acknowledgements");
   print_str_array("ZSV/lib third-party dependencies", "", zsv_thirdparty);
 
+#ifndef __EMSCRIPTEN__
   struct cli_config config;
   const char *ss[2];
   ss[1] = NULL;
@@ -34,6 +35,7 @@ static int main_thirdparty(int argc, const char *argv[]) {
       }
     }
   }
+#endif
   printf("\n");
   return 0;
 }

app/cli.c

@@ -46,7 +46,7 @@ zsvsheet_status zsvsheet_ext_prompt(struct zsvsheet_proc_context *ctx, char *buf
 typedef int(cmd_main)(int argc, const char *argv[]);
 typedef int(zsv_cmd)(int argc, const char *argv[], struct zsv_opts *opts, struct zsv_prop_handler *custom_prop_handler,
                      const char *opts_used);
-typedef int (*cmd_reserved)();
+typedef int (*cmd_reserved)(void);
 
 struct builtin_cmd {
   const char *name;
@@ -61,12 +61,15 @@ static int config_init(struct cli_config *c, char err_if_dl_not_found, char do_i
 
 #define CLI_BUILTIN_DECL_STATIC(x) static int main_##x(int argc, const char *argv[])
 
-CLI_BUILTIN_DECL_STATIC(license);
-CLI_BUILTIN_DECL_STATIC(thirdparty);
-CLI_BUILTIN_DECL_STATIC(help);
 CLI_BUILTIN_DECL_STATIC(version);
+CLI_BUILTIN_DECL_STATIC(help);
+CLI_BUILTIN_DECL_STATIC(thirdparty);
+
+#ifndef __EMSCRIPTEN__
 CLI_BUILTIN_DECL_STATIC(register);
 CLI_BUILTIN_DECL_STATIC(unregister);
+CLI_BUILTIN_DECL_STATIC(license);
+#endif
 
 ZSV_MAIN_DECL(select);
 ZSV_MAIN_DECL(count);
@@ -106,13 +109,14 @@ ZSV_MAINEXT_FUNC_DEFINE(sheet);
 #endif
 
 struct builtin_cmd builtin_cmds[] = {
-  CLI_BUILTIN_CMD(license),
-  CLI_BUILTIN_CMD(thirdparty),
-  CLI_BUILTIN_CMD(help),
   CLI_BUILTIN_CMD(version),
+  CLI_BUILTIN_CMD(help),
+  CLI_BUILTIN_CMD(thirdparty),
+#ifndef __EMSCRIPTEN__
   CLI_BUILTIN_CMD(register),
   CLI_BUILTIN_CMD(unregister),
-
+  CLI_BUILTIN_CMD(license),
+#endif
   CLI_BUILTIN_COMMAND(select),
   CLI_BUILTIN_COMMAND(count),
   CLI_BUILTIN_COMMAND(paste),
@@ -664,12 +668,12 @@ static char *dl_name_from_func(const void *func) {
   if (GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
                         (LPCTSTR)func, &hModule))
     return get_module_name(hModule);
-#endif
-
-#ifdef __APPLE__
+#elif defined __APPLE__
   Dl_info info;
   if (dladdr(func, &info))
     return strdup(info.dli_fname);
+#else
+  (void)(func);
 #endif
   return NULL;
 }

app/cli_export.h

@@ -1,10 +1,14 @@
 #ifndef ZSV_CLI_EXPORT_H
 #define ZSV_CLI_EXPORT_H
 
-#if defined(__EMSCRIPTEN__)
+#ifdef __EMSCRIPTEN__
 #include <emscripten.h>
 #define ZSV_CLI_EXPORT EMSCRIPTEN_KEEPALIVE
+#ifndef NO_PLAYGROUND
+#define ZSV_CLI_MAIN main
+#else
 #define ZSV_CLI_MAIN zsv
+#endif
 #else
 #define ZSV_CLI_EXPORT
 #endif

app/count-pull.c

@@ -13,7 +13,7 @@
 #define ZSV_COMMAND count_pull
 #include "zsv_command.h"
 
-static int count_usage() {
+static int count_usage(void) {
   static const char *usage = "Usage: count [options]\n"
                              "\n"
                              "Options:\n"

app/count.c

@@ -28,7 +28,7 @@ static void row(void *ctx) {
   ((struct data *)ctx)->rows++;
 }
 
-static int count_usage() {
+static int count_usage(void) {
   static const char *usage = "Usage: count [options]\n"
                              "\n"
                              "Options:\n"

app/desc.c

@@ -431,7 +431,7 @@ const char *zsv_desc_usage_msg[] = {
   NULL,
 };
 
-static int zsv_desc_usage() {
+static int zsv_desc_usage(void) {
   for (size_t i = 0; zsv_desc_usage_msg[i]; i++)
     fprintf(stdout, "%s\n", zsv_desc_usage_msg[i]);
   return 0;

app/echo.c

@@ -116,7 +116,7 @@ const char *zsv_echo_usage_msg[] = {
   NULL,
 };
 
-static int zsv_echo_usage() {
+static int zsv_echo_usage(void) {
   for (size_t i = 0; zsv_echo_usage_msg[i]; i++)
     fprintf(stdout, "%s\n", zsv_echo_usage_msg[i]);
   return 1;
@@ -199,7 +199,7 @@ int ZSV_MAIN_FUNC(ZSV_COMMAND)(int argc, const char *argv[], struct zsv_opts *op
 #ifndef NO_STDIN
     data.in = stdin;
 #else
-    fprintf(stderr, "No input\n");
+    fprintf(stderr, "Please specify an input file\n");
     err = 1;
 #endif
   }