[CI] musl build updates

.dockerignore

@@ -1,10 +1,14 @@
 /**
 !/app
 /app/cli_internal.*.in
+!/data
+!/docs/*.json
+!examples
 !/include
 !/scripts
 !/src
 !configure
 !AUTHORS
 !Makefile
 !LICENSE
+!./ci/zsv

.github/workflows/ci.yml

@@ -10,6 +10,11 @@ on:
   release:
     types: [published]
 
+permissions:
+  contents: write
+  id-token: write
+  attestations: write
+
 defaults:
   run:
     shell: bash
@@ -109,11 +114,6 @@ jobs:
 
     runs-on: ${{ matrix.os }}
 
-    permissions:
-      contents: write
-      id-token: write
-      attestations: write
-
     env:
       TAG: ${{ needs.tag.outputs.TAG }}
 
@@ -125,7 +125,7 @@ jobs:
       if: runner.os == 'Linux'
       run: |
         sudo apt update
-        sudo apt install -y rpm alien musl-tools tmux
+        sudo apt install -y rpm alien tmux
         sudo apt remove -y jq
 
     - name: Set up macOS (AMD64 and ARM64)
@@ -160,19 +160,6 @@ jobs:
         ./scripts/ci-create-debian-package.sh
         ./scripts/ci-create-rpm-package.sh
 
-    - name: Build on Linux (${{ env.AMD64_LINUX_MUSL }})
-      if: runner.os == 'Linux'
-      env:
-        PREFIX: ${{ env.AMD64_LINUX_MUSL }}
-        CC: musl-gcc
-        LDFLAGS: -static
-        MAKE: make
-        RUN_TESTS: false
-      run: |
-        ./scripts/ci-build.sh
-        ./scripts/ci-create-debian-package.sh
-        ./scripts/ci-create-rpm-package.sh
-
     - name: Build on macOS (${{ env.AMD64_MACOSX_GCC }})
       if: matrix.os == 'macos-13'
       env:
@@ -228,17 +215,6 @@ jobs:
         retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
         if-no-files-found: error
 
-    - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip)
-      if: runner.os == 'Linux'
-      uses: actions/upload-artifact@v4
-      env:
-        ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip
-      with:
-        name: ${{ env.ARTIFACT_NAME }}
-        path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
-        retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
-        if-no-files-found: error
-
     - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_GCC }}.deb)
       if: runner.os == 'Linux'
       uses: actions/upload-artifact@v4
@@ -261,17 +237,6 @@ jobs:
         retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
         if-no-files-found: error
 
-    - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.deb)
-      if: runner.os == 'Linux'
-      uses: actions/upload-artifact@v4
-      env:
-        ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.deb
-      with:
-        name: ${{ env.ARTIFACT_NAME }}
-        path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
-        retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
-        if-no-files-found: error
-
     - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_GCC }}.rpm)
       if: runner.os == 'Linux'
       uses: actions/upload-artifact@v4
@@ -294,17 +259,6 @@ jobs:
         retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
         if-no-files-found: error
 
-    - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.rpm)
-      if: runner.os == 'Linux'
-      uses: actions/upload-artifact@v4
-      env:
-        ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.rpm
-      with:
-        name: ${{ env.ARTIFACT_NAME }}
-        path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
-        retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
-        if-no-files-found: error
-
     - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_MACOSX_GCC }}.zip)
       if: matrix.os == 'macos-13'
       uses: actions/upload-artifact@v4
@@ -349,17 +303,6 @@ jobs:
         retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
         if-no-files-found: error
 
-    - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz)
-      if: runner.os == 'Linux'
-      uses: actions/upload-artifact@v4
-      env:
-        ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz
-      with:
-        name: ${{ env.ARTIFACT_NAME }}
-        path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
-        retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
-        if-no-files-found: error
-
     - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_MACOSX_GCC }}.tar.gz)
       if: matrix.os == 'macos-13'
       uses: actions/upload-artifact@v4
@@ -398,11 +341,6 @@ jobs:
     needs: [tag, clang-format, cppcheck, shellcheck]
     runs-on: ubuntu-latest
 
-    permissions:
-      contents: write
-      id-token: write
-      attestations: write
-
     env:
       TAG: ${{ needs.tag.outputs.TAG }}
 
@@ -467,11 +405,6 @@ jobs:
     needs: [tag, clang-format, cppcheck, shellcheck]
     runs-on: ubuntu-latest
 
-    permissions:
-      contents: write
-      id-token: write
-      attestations: write
-
     env:
       TAG: ${{ needs.tag.outputs.TAG }}
 
@@ -501,7 +434,7 @@ jobs:
           sed 's|--enable-pc-files|--enable-pc-files --enable-widec|' -i "$NCURSES_PORTFILE"
         fi
         ./vcpkg install ncurses:x64-mingw-static
-        tree -h ./installed/x64-mingw-static
+        tree ./installed/x64-mingw-static
 
     - name: Checkout
       uses: actions/checkout@v4
@@ -571,8 +504,70 @@ jobs:
       if: startsWith(github.ref, 'refs/tags/v')
       run: ./scripts/ci-upload-release-artifacts.sh
 
+  ci-musl:
+    needs: [tag, clang-format, cppcheck, shellcheck]
+    runs-on: ubuntu-latest
+    container: alpine:latest
+
+    env:
+      TAG: ${{ needs.tag.outputs.TAG }}
+
+    steps:
+    - name: Set up dependencies
+      shell: sh
+      run: apk add bash gcc make musl-dev perl ncurses-dev ncurses-static tmux file sqlite curl zip
+
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Build (${{ env.AMD64_LINUX_MUSL }})
+      env:
+        PREFIX: ${{ env.AMD64_LINUX_MUSL }}
+        CC: gcc
+        MAKE: make
+        RUN_TESTS: true
+        STATIC_BUILD: "1"
+      run: ./scripts/ci-build.sh
+
+    - name: Prepare build artifacts for upload
+      run: ./scripts/ci-prepare-artifacts-for-upload.sh
+
+    - name: Attest build artifacts for release
+      if: startsWith(github.ref, 'refs/tags/v')
+      uses: actions/attest-build-provenance@v1
+      with:
+        subject-path: ${{ env.ARTIFACT_DIR }}/*
+
+    - name: Verify attestations of release artifacts
+      if: startsWith(github.ref, 'refs/tags/v')
+      run: ./scripts/ci-verify-attestations.sh
+
+    - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip)
+      uses: actions/upload-artifact@v4
+      env:
+        ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip
+      with:
+        name: ${{ env.ARTIFACT_NAME }}
+        path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
+        retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
+        if-no-files-found: error
+
+    - name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz)
+      uses: actions/upload-artifact@v4
+      env:
+        ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz
+      with:
+        name: ${{ env.ARTIFACT_NAME }}
+        path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
+        retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
+        if-no-files-found: error
+
+    - name: Upload release artifacts
+      if: startsWith(github.ref, 'refs/tags/v')
+      run: ./scripts/ci-upload-release-artifacts.sh
+
   ghcr:
-    needs: [tag]
+    needs: [tag, ci-musl]
     runs-on: ubuntu-latest
 
     permissions:
@@ -584,6 +579,27 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+      with:
+        sparse-checkout: |
+          Dockerfile.ci
+
+    - name: Download (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip)
+      uses: actions/download-artifact@v4
+      with:
+        name: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip
+        path: ${{ env.AMD64_LINUX_MUSL }}
+
+    - name: Unzip
+      env:
+        ZIP: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip
+        DIR: ${{ env.AMD64_LINUX_MUSL }}
+      run: |
+        cd "$DIR"
+        unzip -o "$ZIP"
+        cd ..
+        mkdir -p ./ci
+        mv ./"$DIR"/bin/zsv ./ci/
+        rm -rf ./"$DIR"
 
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
@@ -604,6 +620,9 @@ jobs:
       env:
         DOCKER_BUILD_RECORD_UPLOAD: false
       with:
+        no-cache: true
+        context: .
+        file: Dockerfile.ci
         platforms: linux/amd64
         push: ${{ startsWith(github.ref, 'refs/tags/v') }}
         tags: |

.gitignore

@@ -63,5 +63,7 @@ tmp
 include/zsv.h
 app/test/worldcitiespop_mil.csv
 data/quoted5.csv
+data/loans_2.csv
+data/.zsv/data/loans_2.csv/overwrite.sqlite3
 compile_commands.json
 .cache

Dockerfile

@@ -1,10 +1,10 @@
 FROM alpine:latest AS build
 
 LABEL maintainer="Liquidaty"
-LABEL description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser"
 LABEL url="https://github.com/liquidaty/zsv"
+LABEL org.opencontainers.image.description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser"
 
-RUN apk add --no-cache gcc make musl-dev perl
+RUN apk add bash gcc make musl-dev perl ncurses-dev ncurses-static tmux file sqlite curl zip
 
 WORKDIR /zsv
 COPY . .
@@ -14,7 +14,7 @@ RUN \
     CC=gcc \
     MAKE=make \
     ARTIFACT_DIR=artifacts \
-    RUN_TESTS=false \
+    RUN_TESTS=true \
     STATIC_BUILD=1 \
     SKIP_ZIP_ARCHIVE=true \
     SKIP_TAR_ARCHIVE=true \

Dockerfile.ci

@@ -0,0 +1,10 @@
+FROM scratch
+
+LABEL maintainer="Liquidaty"
+LABEL url="https://github.com/liquidaty/zsv"
+LABEL org.opencontainers.image.description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser"
+
+WORKDIR /zsv
+COPY ci/zsv .
+
+ENTRYPOINT [ "./zsv" ]

app/test/Makefile

@@ -84,7 +84,6 @@ else
   CMP=cmp
 endif
 
-
 help:
 	@echo "To run all tests: make test [LEAKS=1]"
 	@echo "To run individual test: make test-xxx"
@@ -331,7 +330,6 @@ test-mv: ${BUILD_DIR}/bin/zsv_prop${EXE} ${BUILD_DIR}/bin/zsv_mv${EXE}
 	@${PREFIX} ${BUILD_DIR}/bin/zsv_mv${EXE} ${TMP_DIR}/[email protected] ${TMP_DIR}/[email protected] ${REDIRECT} /dev/null
 	@find ${TMP_DIR}/.zsv/data/[email protected]/props.json -type f >/dev/null && ${TEST_PASS} || ${TEST_FAIL}
 
-
 test-blank-leading-rows: test-blank-leading-rows-1 test-blank-leading-rows-2 test-blank-leading-rows-3 test-blank-leading-rows-4
 
 test-blank-leading-rows-1: ${BUILD_DIR}/bin/zsv_select${EXE}
@@ -424,7 +422,6 @@ test-serialize-position-id: ${BUILD_DIR}/bin/zsv_serialize${EXE} ${TEST_DATA_DIR
 	@(${PREFIX} $< --id-column 3 < ${TEST_DATA_DIR}/test/serialize.csv -a 'Interest Paid Through Date' -a 'original INTEREST Only Term' ${REDIRECT1} ${TMP_DIR}/[email protected] && \
 	${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL})
 
-
 test-serialize : test-%: ${BUILD_DIR}/bin/zsv_%${EXE} test-serialize-quoted test-serialize-additional test-serialize-position-id
 	@${TEST_INIT}
 	@( ( ! [ -s "${TEST_DATA_DIR}/test/$*.csv" ] ) && echo "No test input for $*") || \
@@ -460,7 +457,6 @@ test-sql5: ${BUILD_DIR}/bin/zsv_sql${EXE} # test blank rows
 	@(${PREFIX} $< /tmp/1.csv 'select * from data' ${REDIRECT1} ${TMP_DIR}/[email protected])
 	@${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL}
 
-
 ${BUILD_DIR}/bin/zsv_%${EXE}:
 	make -C .. $@ CONFIGFILE=${CONFIGFILEPATH} DEBUG=${DEBUG}
 
@@ -540,7 +536,6 @@ test-compare-tolerance: ${BUILD_DIR}/bin/zsv_compare${EXE}
 	@(${PREFIX} $< --tolerance 0.00001 ../../data/compare/tolerance1.csv ../../data/compare/tolerance2.csv ${REDIRECT1} ${TMP_DIR}/[email protected] && \
 	${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL})
 
-
 test-compare: test-%: ${BUILD_DIR}/bin/zsv_%${EXE}
 	@${TEST_INIT}
 	@(${PREFIX} $< compare/t1.csv compare/t2.csv compare/t3.csv ${REDIRECT1} ${TMP_DIR}/[email protected] && \