diff --git a/.dockerignore b/.dockerignore index 3b1eb6cc..856cfe91 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,9 @@ /** !/app /app/cli_internal.*.in +!/data +!/docs/*.json +!examples !/include !/scripts !/src @@ -8,3 +11,4 @@ !AUTHORS !Makefile !LICENSE +!./ci/zsv diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d863f0b0..2a31624a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,11 @@ on: release: types: [published] +permissions: + contents: write + id-token: write + attestations: write + defaults: run: shell: bash @@ -109,11 +114,6 @@ jobs: runs-on: ${{ matrix.os }} - permissions: - contents: write - id-token: write - attestations: write - env: TAG: ${{ needs.tag.outputs.TAG }} @@ -125,7 +125,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt update - sudo apt install -y rpm alien musl-tools tmux + sudo apt install -y rpm alien tmux sudo apt remove -y jq - name: Set up macOS (AMD64 and ARM64) @@ -160,19 +160,6 @@ jobs: ./scripts/ci-create-debian-package.sh ./scripts/ci-create-rpm-package.sh - - name: Build on Linux (${{ env.AMD64_LINUX_MUSL }}) - if: runner.os == 'Linux' - env: - PREFIX: ${{ env.AMD64_LINUX_MUSL }} - CC: musl-gcc - LDFLAGS: -static - MAKE: make - RUN_TESTS: false - run: | - ./scripts/ci-build.sh - ./scripts/ci-create-debian-package.sh - ./scripts/ci-create-rpm-package.sh - - name: Build on macOS (${{ env.AMD64_MACOSX_GCC }}) if: matrix.os == 'macos-13' env: @@ -228,17 +215,6 @@ jobs: retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} if-no-files-found: error - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip) - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_GCC }}.deb) if: runner.os == 'Linux' uses: actions/upload-artifact@v4 @@ -261,17 +237,6 @@ jobs: retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} if-no-files-found: error - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.deb) - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.deb - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_GCC }}.rpm) if: runner.os == 'Linux' uses: actions/upload-artifact@v4 @@ -294,17 +259,6 @@ jobs: retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} if-no-files-found: error - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.rpm) - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.rpm - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_MACOSX_GCC }}.zip) if: matrix.os == 'macos-13' uses: actions/upload-artifact@v4 @@ -349,17 +303,6 @@ jobs: retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} if-no-files-found: error - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz) - if: runner.os == 'Linux' - uses: actions/upload-artifact@v4 - env: - ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz - with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - if-no-files-found: error - - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_MACOSX_GCC }}.tar.gz) if: matrix.os == 'macos-13' uses: actions/upload-artifact@v4 @@ -398,11 +341,6 @@ jobs: needs: [tag, clang-format, cppcheck, shellcheck] runs-on: ubuntu-latest - permissions: - contents: write - id-token: write - attestations: write - env: TAG: ${{ needs.tag.outputs.TAG }} @@ -467,11 +405,6 @@ jobs: needs: [tag, clang-format, cppcheck, shellcheck] runs-on: ubuntu-latest - permissions: - contents: write - id-token: write - attestations: write - env: TAG: ${{ needs.tag.outputs.TAG }} @@ -501,7 +434,7 @@ jobs: sed 's|--enable-pc-files|--enable-pc-files --enable-widec|' -i "$NCURSES_PORTFILE" fi ./vcpkg install ncurses:x64-mingw-static - tree -h ./installed/x64-mingw-static + tree ./installed/x64-mingw-static - name: Checkout uses: actions/checkout@v4 @@ -571,8 +504,70 @@ jobs: if: startsWith(github.ref, 'refs/tags/v') run: ./scripts/ci-upload-release-artifacts.sh + ci-musl: + needs: [tag, clang-format, cppcheck, shellcheck] + runs-on: ubuntu-latest + container: alpine:latest + + env: + TAG: ${{ needs.tag.outputs.TAG }} + + steps: + - name: Set up dependencies + shell: sh + run: apk add bash gcc make musl-dev perl ncurses-dev ncurses-static tmux file sqlite curl zip + + - name: Checkout + uses: actions/checkout@v4 + + - name: Build (${{ env.AMD64_LINUX_MUSL }}) + env: + PREFIX: ${{ env.AMD64_LINUX_MUSL }} + CC: gcc + MAKE: make + RUN_TESTS: true + STATIC_BUILD: "1" + run: ./scripts/ci-build.sh + + - name: Prepare build artifacts for upload + run: ./scripts/ci-prepare-artifacts-for-upload.sh + + - name: Attest build artifacts for release + if: startsWith(github.ref, 'refs/tags/v') + uses: actions/attest-build-provenance@v1 + with: + subject-path: ${{ env.ARTIFACT_DIR }}/* + + - name: Verify attestations of release artifacts + if: startsWith(github.ref, 'refs/tags/v') + run: ./scripts/ci-verify-attestations.sh + + - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip) + uses: actions/upload-artifact@v4 + env: + ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip + with: + name: ${{ env.ARTIFACT_NAME }} + path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + if-no-files-found: error + + - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz) + uses: actions/upload-artifact@v4 + env: + ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz + with: + name: ${{ env.ARTIFACT_NAME }} + path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }} + retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + if-no-files-found: error + + - name: Upload release artifacts + if: startsWith(github.ref, 'refs/tags/v') + run: ./scripts/ci-upload-release-artifacts.sh + ghcr: - needs: [tag] + needs: [tag, ci-musl] runs-on: ubuntu-latest permissions: @@ -584,6 +579,27 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + sparse-checkout: | + Dockerfile.ci + + - name: Download (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip) + uses: actions/download-artifact@v4 + with: + name: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip + path: ${{ env.AMD64_LINUX_MUSL }} + + - name: Unzip + env: + ZIP: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip + DIR: ${{ env.AMD64_LINUX_MUSL }} + run: | + cd "$DIR" + unzip -o "$ZIP" + cd .. + mkdir -p ./ci + mv ./"$DIR"/bin/zsv ./ci/ + rm -rf ./"$DIR" - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -604,6 +620,9 @@ jobs: env: DOCKER_BUILD_RECORD_UPLOAD: false with: + no-cache: true + context: . + file: Dockerfile.ci platforms: linux/amd64 push: ${{ startsWith(github.ref, 'refs/tags/v') }} tags: | diff --git a/.gitignore b/.gitignore index e9c5fcbd..db592ce0 100644 --- a/.gitignore +++ b/.gitignore @@ -63,5 +63,7 @@ tmp include/zsv.h app/test/worldcitiespop_mil.csv data/quoted5.csv +data/loans_2.csv +data/.zsv/data/loans_2.csv/overwrite.sqlite3 compile_commands.json .cache diff --git a/Dockerfile b/Dockerfile index f490f07d..6dbbba0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ FROM alpine:latest AS build LABEL maintainer="Liquidaty" -LABEL description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser" LABEL url="https://github.com/liquidaty/zsv" +LABEL org.opencontainers.image.description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser" -RUN apk add --no-cache gcc make musl-dev perl +RUN apk add bash gcc make musl-dev perl ncurses-dev ncurses-static tmux file sqlite curl zip WORKDIR /zsv COPY . . @@ -14,7 +14,7 @@ RUN \ CC=gcc \ MAKE=make \ ARTIFACT_DIR=artifacts \ - RUN_TESTS=false \ + RUN_TESTS=true \ STATIC_BUILD=1 \ SKIP_ZIP_ARCHIVE=true \ SKIP_TAR_ARCHIVE=true \ diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 00000000..8fd9a527 --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,10 @@ +FROM scratch + +LABEL maintainer="Liquidaty" +LABEL url="https://github.com/liquidaty/zsv" +LABEL org.opencontainers.image.description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser" + +WORKDIR /zsv +COPY ci/zsv . + +ENTRYPOINT [ "./zsv" ] diff --git a/app/test/Makefile b/app/test/Makefile index becfa625..8d26afcf 100644 --- a/app/test/Makefile +++ b/app/test/Makefile @@ -84,7 +84,6 @@ else CMP=cmp endif - help: @echo "To run all tests: make test [LEAKS=1]" @echo "To run individual test: make test-xxx" @@ -331,7 +330,6 @@ test-mv: ${BUILD_DIR}/bin/zsv_prop${EXE} ${BUILD_DIR}/bin/zsv_mv${EXE} @${PREFIX} ${BUILD_DIR}/bin/zsv_mv${EXE} ${TMP_DIR}/$@.csv ${TMP_DIR}/$@-moved.csv ${REDIRECT} /dev/null @find ${TMP_DIR}/.zsv/data/$@-moved.csv/props.json -type f >/dev/null && ${TEST_PASS} || ${TEST_FAIL} - test-blank-leading-rows: test-blank-leading-rows-1 test-blank-leading-rows-2 test-blank-leading-rows-3 test-blank-leading-rows-4 test-blank-leading-rows-1: ${BUILD_DIR}/bin/zsv_select${EXE} @@ -424,7 +422,6 @@ test-serialize-position-id: ${BUILD_DIR}/bin/zsv_serialize${EXE} ${TEST_DATA_DIR @(${PREFIX} $< --id-column 3 < ${TEST_DATA_DIR}/test/serialize.csv -a 'Interest Paid Through Date' -a 'original INTEREST Only Term' ${REDIRECT1} ${TMP_DIR}/$@.out && \ ${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL}) - test-serialize : test-%: ${BUILD_DIR}/bin/zsv_%${EXE} test-serialize-quoted test-serialize-additional test-serialize-position-id @${TEST_INIT} @( ( ! [ -s "${TEST_DATA_DIR}/test/$*.csv" ] ) && echo "No test input for $*") || \ @@ -460,7 +457,6 @@ test-sql5: ${BUILD_DIR}/bin/zsv_sql${EXE} # test blank rows @(${PREFIX} $< /tmp/1.csv 'select * from data' ${REDIRECT1} ${TMP_DIR}/$@.out) @${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL} - ${BUILD_DIR}/bin/zsv_%${EXE}: make -C .. $@ CONFIGFILE=${CONFIGFILEPATH} DEBUG=${DEBUG} @@ -540,7 +536,6 @@ test-compare-tolerance: ${BUILD_DIR}/bin/zsv_compare${EXE} @(${PREFIX} $< --tolerance 0.00001 ../../data/compare/tolerance1.csv ../../data/compare/tolerance2.csv ${REDIRECT1} ${TMP_DIR}/$@.out4 && \ ${CMP} ${TMP_DIR}/$@.out4 expected/$@.out4 && ${TEST_PASS} || ${TEST_FAIL}) - test-compare: test-%: ${BUILD_DIR}/bin/zsv_%${EXE} @${TEST_INIT} @(${PREFIX} $< compare/t1.csv compare/t2.csv compare/t3.csv ${REDIRECT1} ${TMP_DIR}/$@.out && \ diff --git a/configure b/configure index a77da4db..822539c1 100755 --- a/configure +++ b/configure @@ -35,7 +35,7 @@ Optional features: --force-sse2 force compile with (no CPU check) or without -msse2 [auto] --enable-sheet compile with 'sheet' feature (requires ncurses) [yes] --enable-shared build/install zsv shared library - --ncurses-dynamic compile without NCURSES_STATIC or NCURSESW_STATIC defines + --ncurses-dynamic compile without NCURSES_STATIC --enable-lto compile with LTO (works with some but not all platforms/compilers) [no] --enable-whole-program compile without -fwhole-program even if no -flto [yes] --enable-pie build with position independent executables [auto] @@ -78,7 +78,7 @@ tryranlib () { test -z "$RANLIB" && cmdexists "$1" && RANLIB=$1 ; } tryawk () { test -z "$AWK" && "$1" 'function works () {return 0} BEGIN{exit works()}' && AWK=$1 ; } trymake() { if test -z "$MAKE" && cmdexists "$1" ; then - v=$($1 --version | grep 'GNU Make' | sed -e 's/.*Make //') + v=$($1 --version | grep 'GNU Make' | sed -e 's/.*Make //') if [ "$v" = "" ] ; then printf "Found but not using non-GNU make (%s)..." "$1" else @@ -86,7 +86,7 @@ trymake() { maj=$(echo "$v" | cut -d. -f 1) min=$(echo "$v" | cut -d. -f 2) if ! test "$maj" -gt 3 -o '(' "$maj" -eq 3 -a "$min" -ge 81 ')'; then - printf "Warning: using make, but version %s < 3.81 has not been tested. " "$v" + printf "WARNING: using make, but version %s < 3.81 has not been tested. " "$v" fi fi fi @@ -101,14 +101,14 @@ trycchdr () { printf "checking whether there is a header called %s... " "$2" dn=$(dirname "$2") if [ "$dn" != "" ]; then - dn="/$dn" - bn=$(basename "$2") - for x in $CCSEARCHPATH ; do - fnd=$fnd"$x$dn " - done + dn="/$dn" + bn=$(basename "$2") + for x in $CCSEARCHPATH ; do + fnd=$fnd"$x$dn " + done else - bn="$2" - fnd=$CCSEARCHPATH + bn="$2" + fnd=$CCSEARCHPATH fi upper2=$(echo "$2" | tr '[:lower:]' '[:upper:]' | tr . _ | tr / _) if find "$fnd" -name "$bn" 2>/dev/null | grep "/$2$" >/dev/null 2>&1 ; then @@ -179,7 +179,7 @@ NO_$flag = 1" echo "Failed: $CC -o $tmpo tmp.c" cat "$tmpc" echo "------" - }>> "$CONFIGFILE".log + } >>"$CONFIGFILE".log return 1 fi } @@ -344,7 +344,9 @@ for arg ; do --docdir=*) DOCDIR=${arg#*=} ;; --mandir=*) MANDIR=${arg#*=} ;; --minimal|--minimal=yes) MINIMAL=yes;; + --try-avx512|--try-avx512=yes) TRY_AVX512=yes;; + --try-avx512=no) TRY_AVX512=no;; --force-avx2|--force-avx2=yes) FORCE_AVX2=yes;; --force-avx2=no) FORCE_AVX2=no;; @@ -564,7 +566,7 @@ if [ "$TRY_SHEET" != "no" ]; then exit 1 fi if [ "$LDFLAGS_NCURSES" = "" ] ; then - echo "Warning: could not find either ncurses or ncursesw library; cannot build 'sheet' feature" + echo "WARNING: could not find either ncurses or ncursesw library; cannot build 'sheet' feature" ZSVSHEET_BUILD=0 fi fi @@ -600,19 +602,18 @@ if [ "$FORCE_AVX2" = "no" ]; then elif [ "$FORCE_AVX2" = "yes" ] ; then CFLAGS_AVX=-mavx2 if [ "$CROSS_COMPILING" = "no" ] ; then - trycpusupport avx2 || echo "warning: avx2 forced but not supported on native CPU" + trycpusupport avx2 || echo "WARNING: avx2 forced but not supported on native CPU" fi elif [ "$FORCE_AVX2" = "auto" ] && [ "$CROSS_COMPILING" = "no" ] ; then trycpusupport avx2 && CFLAGS_AVX=-mavx2 fi -if [ "$FORCE_AVX" = "yes" ] ; then - CFLAGS_AVX=-mavx || echo "warning: avx forced but not supported on native CPU" -elif [ "$FORCE_AVX" = "auto" ] && [ "$CFLAGS_AVX" = "" ] && [ "$CROSS_COMPILING" = "no" ] ; then - trycpusupport avx && CFLAGS_AVX=-mavx -fi if [ "$FORCE_AVX" = "no" ]; then tryflag CFLAGS -mno-avx +elif [ "$FORCE_AVX" = "yes" ] ; then + CFLAGS_AVX=-mavx || echo "WARNING: avx forced but not supported on native CPU" +elif [ "$FORCE_AVX" = "auto" ] && [ "$CFLAGS_AVX" = "" ] && [ "$CROSS_COMPILING" = "no" ] ; then + trycpusupport avx && CFLAGS_AVX=-mavx fi if [ "$FORCE_SSE2" = "no" ]; then @@ -623,7 +624,7 @@ elif [ "$FORCE_SSE2" = "yes" ] ; then CFLAGS_SSE="$CFLAGS_SSE -msimd128 -experimental-wasm-simd" fi if [ "$CROSS_COMPILING" = "no" ] ; then - trycpusupport sse2 || echo "warning: sse2 forced but not supported on native CPU" + trycpusupport sse2 || echo "WARNING: sse2 forced but not supported on native CPU" fi elif [ "$FORCE_SSE2" = "auto" ] && [ "$CROSS_COMPILING" = "no" ] ; then if [ "$CFLAGS_SSE" = "" ] && [ "$CROSS_COMPILING" = "no" ] ; then @@ -689,7 +690,7 @@ fi # Check function availability if [ "$TRY_AVX512" = "yes" ]; then - printf "checking whether avx512 instructions are available..." + printf "checking whether avx512 instructions are available... " HAVE_AVX512= tryccfn CFLAGS_AVX_512 "_mm512_movepi8_mask" "immintrin.h" && \ trycchdr CFLAGS_AVX_512 "immintrin.h" && \ @@ -699,10 +700,10 @@ if [ "$TRY_AVX512" = "yes" ]; then CFLAGS_AVX_512="-mbmi -mavx512bw -DHAVE_AVX512=1" && \ HAVE_AVX512=1 if [ "$HAVE_AVX512" = "1" ]; then - echo "yes" + echo "yes" else - echo "no" - echo "WARNING: --try-avx512 option enabled, but no avx512 instruction set available" + echo "no" + echo "WARNING: --try-avx512 option enabled, but no avx512 instruction set available" fi fi @@ -857,7 +858,11 @@ fi if [ "$ZSVSHEET_BUILD" = "1" ]; then echo "* - build sheet feature: yes *" - echo "* ncurses library: $NCLIB *" + if [ "$NCLIB" = "ncurses" ]; then + echo "* ncurses library: ncurses *" + elif [ "$NCLIB" = "ncursesw" ]; then + echo "* ncurses library: ncursesw *" + fi else echo "* - build sheet feature: no *" fi diff --git a/examples/lib/Makefile b/examples/lib/Makefile index f18a2e73..d3a5b271 100644 --- a/examples/lib/Makefile +++ b/examples/lib/Makefile @@ -79,11 +79,11 @@ test-tiny: build/simple${EXE} test-eol: test-eol-1 test-eol-2 test-eol-3 test-eol-4 -test-eol-%: build/simple${EXE} build/pull${EXE} - @$< ${TEST_DATA_DIR}/test/no-eol-$*.csv > ${TMP_DIR}/$@.out +test-eol-%: ${BUILD_DIR}/simple${EXE} ${BUILD_DIR}/pull${EXE} + @${BUILD_DIR}/simple${EXE} ${TEST_DATA_DIR}/test/no-eol-$*.csv > ${TMP_DIR}/$@.out @cmp ${TMP_DIR}/$@.out test/expected/$@.out && ${TEST_PASS} || ${TEST_FAIL} - @build/pull${EXE} ${TEST_DATA_DIR}/test/no-eol-$*.csv > ${TMP_DIR}/$@.out + @${BUILD_DIR}/pull${EXE} ${TEST_DATA_DIR}/test/no-eol-$*.csv > ${TMP_DIR}/$@.out @cmp ${TMP_DIR}/$@.out test/expected/$@.out && ${TEST_PASS} || ${TEST_FAIL} simple print_my_column parse_by_chunk pull: % : ${BUILD_DIR}/%${EXE} diff --git a/scripts/ci-build.sh b/scripts/ci-build.sh index 4bcd6dd9..d5f39851 100755 --- a/scripts/ci-build.sh +++ b/scripts/ci-build.sh @@ -27,6 +27,8 @@ if [ "$SKIP_TAR_ARCHIVE" != true ]; then SKIP_TAR_ARCHIVE=false fi +WITHOUT_SIMD=${WITHOUT_SIMD:-false} + #JQ_DIR="$PWD/jq" #JQ_PREFIX="$JQ_DIR/build" #JQ_INCLUDE_DIR="$JQ_PREFIX/include" @@ -42,6 +44,7 @@ echo "[INF] LDFLAGS: $LDFLAGS" echo "[INF] MAKE: $MAKE" echo "[INF] RUN_TESTS: $RUN_TESTS" echo "[INF] ARTIFACT_DIR: $ARTIFACT_DIR" +echo "[INF] WITHOUT_SIMD: $WITHOUT_SIMD" echo "[INF] SKIP_ZIP_ARCHIVE: $SKIP_ZIP_ARCHIVE" echo "[INF] SKIP_TAR_ARCHIVE: $SKIP_TAR_ARCHIVE" #echo "[INF] JQ_DIR: $JQ_DIR" @@ -56,10 +59,22 @@ echo "[INF] Listing compiler version [$CC]" echo "[INF] Configuring zsv" # CFLAGS="-I$JQ_INCLUDE_DIR" LDFLAGS="-L$JQ_LIB_DIR" -./configure \ - --prefix="$PREFIX" \ - --disable-termcap -# --enable-jq + +if [ "$WITHOUT_SIMD" = true ]; then + ./configure \ + --prefix="$PREFIX" \ + --disable-termcap \ + --arch=none \ + --try-avx512=no \ + --force-avx2=no \ + --force-avx=no \ + --force-sse2=no +else + ./configure \ + --prefix="$PREFIX" \ + --disable-termcap + # --enable-jq +fi if [ "$RUN_TESTS" = true ]; then echo "[INF] Running tests" @@ -67,8 +82,8 @@ if [ "$RUN_TESTS" = true ]; then "$MAKE" test echo "[INF] Tests completed successfully!" - if [ "$CC" = "musl-gcc" ] && [ "$(echo "$LDFLAGS" | grep -- "-static")" != "" ]; then - echo "[WRN] Dynamic extensions are not supported with static musl build! Skipping tests..." + if [ "$(echo "$LDFLAGS" | grep -- "-static")" != "" ] || [ "$STATIC_BUILD" = "1" ]; then + echo "[WRN] Dynamic extensions are not supported with static builds! Skipping tests..." else echo "[INF] Configuring example extension and running example extension tests" echo "[INF] (cd app/ext_example && $MAKE CONFIGFILE=../../config.mk test)" @@ -80,7 +95,7 @@ fi echo "[INF] Building" rm -rf build "$PREFIX" /usr/local/etc/zsv.ini "$MAKE" install -tree -h "$PREFIX" +tree "$PREFIX" echo "[INF] Built successfully!" mkdir -p "$ARTIFACT_DIR" @@ -90,7 +105,7 @@ if [ "$SKIP_ZIP_ARCHIVE" = false ]; then echo "[INF] Compressing [$ZIP]" cd "$PREFIX" zip -r "$ZIP" . - ls -Gghl "$ZIP" + ls -hl "$ZIP" cd .. mv "$PREFIX/$ZIP" "$ARTIFACT_DIR" echo "[INF] Compressed! [$ZIP]" @@ -100,7 +115,7 @@ if [ "$SKIP_TAR_ARCHIVE" = false ]; then TAR="$PREFIX.tar.gz" echo "[INF] Compressing [$TAR]" tar -czvf "$TAR" "$PREFIX" - ls -Gghl "$TAR" + ls -hl "$TAR" mv "$TAR" "$ARTIFACT_DIR" echo "[INF] Compressed! [$TAR]" fi diff --git a/scripts/ci-create-debian-package.sh b/scripts/ci-create-debian-package.sh index f3767df8..edf600aa 100755 --- a/scripts/ci-create-debian-package.sh +++ b/scripts/ci-create-debian-package.sh @@ -65,7 +65,7 @@ Homepage: https://github.com/liquidaty/zsv Installed-Size: $INSTALLED_SIZE EOF -ls -Gghl "$DEBIAN_CONTROL_FILE" +ls -hl "$DEBIAN_CONTROL_FILE" echo "[INF] Dumping [$DEBIAN_CONTROL_FILE]" echo "[INF] --- [$DEBIAN_CONTROL_FILE] ---" @@ -81,7 +81,7 @@ rm -rf /usr/bin/zsv EOF chmod +x "$DEBIAN_PREINST_SCRIPT" -ls -Gghl "$DEBIAN_PREINST_SCRIPT" +ls -hl "$DEBIAN_PREINST_SCRIPT" echo "[INF] Dumping [$DEBIAN_PREINST_SCRIPT]" echo "[INF] --- [$DEBIAN_PREINST_SCRIPT] ---" @@ -91,7 +91,7 @@ echo "[INF] --- [$DEBIAN_PREINST_SCRIPT] ---" echo "[INF] Building debian package" dpkg-deb --root-owner-group --build "$PREFIX" dpkg --contents "$DEBIAN_PKG" -ls -Gghl "$DEBIAN_PKG" +ls -hl "$DEBIAN_PKG" mv -f "$DEBIAN_PKG" "$ARTIFACT_DIR/" mv -f "./$PREFIX/usr/bin" "./$PREFIX/" diff --git a/scripts/ci-create-rpm-package.sh b/scripts/ci-create-rpm-package.sh index b318f638..7e94f0a6 100755 --- a/scripts/ci-create-rpm-package.sh +++ b/scripts/ci-create-rpm-package.sh @@ -94,7 +94,7 @@ rpm -qlp "$RPM_PKG_PATH" mv "$RPM_PKG_PATH" "$ARTIFACT_DIR/" rm -rf "$RPM_DIR" -ls -Gghl "$ARTIFACT_DIR/$RPM_PKG" +ls -hl "$ARTIFACT_DIR/$RPM_PKG" echo "[INF] Verifying RPM package [$ARTIFACT_DIR/$RPM_PKG]" diff --git a/scripts/ci-prepare-artifacts-for-upload.sh b/scripts/ci-prepare-artifacts-for-upload.sh index 708b10a9..aad40607 100755 --- a/scripts/ci-prepare-artifacts-for-upload.sh +++ b/scripts/ci-prepare-artifacts-for-upload.sh @@ -47,6 +47,6 @@ prepare 'nupkg' cd .. echo "[INF] Listing" -ls -Gghl "$ARTIFACT_DIR" +ls -hl "$ARTIFACT_DIR" echo "[INF] --- [DONE] ---" diff --git a/scripts/ci-run-cppcheck.sh b/scripts/ci-run-cppcheck.sh index d8048c92..333447f5 100755 --- a/scripts/ci-run-cppcheck.sh +++ b/scripts/ci-run-cppcheck.sh @@ -31,7 +31,7 @@ cppcheck \ --project="$CPPCHECK_PROJECT_FILE" \ --xml 2>"$CPPCHECK_XML_OUTPUT_FILE" -ls -Gghl "$CPPCHECK_XML_OUTPUT_FILE" +ls -hl "$CPPCHECK_XML_OUTPUT_FILE" echo "[INF] Generating HTML report..." cppcheck-htmlreport \ diff --git a/scripts/ci-update-homebrew-tap.sh b/scripts/ci-update-homebrew-tap.sh index a9bd2e7c..d0ab0a36 100755 --- a/scripts/ci-update-homebrew-tap.sh +++ b/scripts/ci-update-homebrew-tap.sh @@ -29,7 +29,7 @@ echo "[INF] HOMEBREW_TAP_FORMULA: $HOMEBREW_TAP_FORMULA" echo "[INF] Downloading release tar file [$TAR_URL]" wget -q "$TAR_URL" -ls -Gghl "$TAR" +ls -hl "$TAR" echo "[INF] Calculating SHA256 of $TAR" SHA256=$(openssl dgst -sha256 "$TAR" | cut -d ' ' -f2 | tr -d '\n') @@ -37,7 +37,7 @@ echo "[INF] SHA256: $SHA256" rm -f "$TAR" echo "[INF] Setting up GitHub credentials" -echo "$HOMEBREW_TAP_DEPLOY_KEY" > $HOMEBREW_TAP_DEPLOY_KEY_FILE +echo "$HOMEBREW_TAP_DEPLOY_KEY" >$HOMEBREW_TAP_DEPLOY_KEY_FILE chmod 400 $HOMEBREW_TAP_DEPLOY_KEY_FILE export GIT_SSH_COMMAND="ssh -i $PWD/$HOMEBREW_TAP_DEPLOY_KEY_FILE -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" git config --global user.name "zsv-ci" diff --git a/src/Makefile b/src/Makefile index 664bc4c5..1adaa31d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -27,7 +27,9 @@ ifeq ($(WIN),) endif endif -include ${THIS_MAKEFILE_DIR}/../make/sharedlib.mk # sets SHAREDLIB_EXT +ifeq ($(BUILD_SHAREDLIB),1) + include ${THIS_MAKEFILE_DIR}/../make/sharedlib.mk # sets SHAREDLIB_EXT +endif CFLAGS+= ${CFLAG_O} ${CFLAGS_OPT} CFLAGS+= ${CFLAGS_AUTO}