Skip to content

Commit

Permalink
[CI] musl build updates (#256)
Browse files Browse the repository at this point in the history
  • Loading branch information
iamazeem authored Nov 6, 2024
1 parent db51ac5 commit 522e5d4
Show file tree
Hide file tree
Showing 15 changed files with 180 additions and 128 deletions.
4 changes: 4 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
/**
!/app
/app/cli_internal.*.in
!/data
!/docs/*.json
!examples
!/include
!/scripts
!/src
!configure
!AUTHORS
!Makefile
!LICENSE
!./ci/zsv
169 changes: 94 additions & 75 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ on:
release:
types: [published]

permissions:
contents: write
id-token: write
attestations: write

defaults:
run:
shell: bash
Expand Down Expand Up @@ -119,11 +124,6 @@ jobs:

runs-on: ${{ matrix.os }}

permissions:
contents: write
id-token: write
attestations: write

env:
TAG: ${{ needs.tag.outputs.TAG }}

Expand All @@ -135,7 +135,7 @@ jobs:
if: runner.os == 'Linux'
run: |
sudo apt update
sudo apt install -y rpm alien musl-tools tmux
sudo apt install -y rpm alien tmux
sudo apt remove -y jq
- name: Set up macOS (AMD64 and ARM64)
Expand Down Expand Up @@ -170,19 +170,6 @@ jobs:
./scripts/ci-create-debian-package.sh
./scripts/ci-create-rpm-package.sh
- name: Build on Linux (${{ env.AMD64_LINUX_MUSL }})
if: runner.os == 'Linux'
env:
PREFIX: ${{ env.AMD64_LINUX_MUSL }}
CC: musl-gcc
LDFLAGS: -static
MAKE: make
RUN_TESTS: false
run: |
./scripts/ci-build.sh
./scripts/ci-create-debian-package.sh
./scripts/ci-create-rpm-package.sh
- name: Build on macOS (${{ env.AMD64_MACOSX_GCC }})
if: matrix.os == 'macos-13'
env:
Expand Down Expand Up @@ -238,17 +225,6 @@ jobs:
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip)
if: runner.os == 'Linux'
uses: actions/upload-artifact@v4
env:
ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip
with:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_GCC }}.deb)
if: runner.os == 'Linux'
uses: actions/upload-artifact@v4
Expand All @@ -271,17 +247,6 @@ jobs:
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.deb)
if: runner.os == 'Linux'
uses: actions/upload-artifact@v4
env:
ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.deb
with:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_GCC }}.rpm)
if: runner.os == 'Linux'
uses: actions/upload-artifact@v4
Expand All @@ -304,17 +269,6 @@ jobs:
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.rpm)
if: runner.os == 'Linux'
uses: actions/upload-artifact@v4
env:
ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.rpm
with:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_MACOSX_GCC }}.zip)
if: matrix.os == 'macos-13'
uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -359,17 +313,6 @@ jobs:
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz)
if: runner.os == 'Linux'
uses: actions/upload-artifact@v4
env:
ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz
with:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_MACOSX_GCC }}.tar.gz)
if: matrix.os == 'macos-13'
uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -408,11 +351,6 @@ jobs:
needs: [tag, clang-format, cppcheck, shellcheck]
runs-on: ubuntu-latest

permissions:
contents: write
id-token: write
attestations: write

env:
TAG: ${{ needs.tag.outputs.TAG }}

Expand Down Expand Up @@ -477,11 +415,6 @@ jobs:
needs: [tag, clang-format, cppcheck, shellcheck]
runs-on: ubuntu-latest

permissions:
contents: write
id-token: write
attestations: write

env:
TAG: ${{ needs.tag.outputs.TAG }}

Expand Down Expand Up @@ -511,7 +444,7 @@ jobs:
sed 's|--enable-pc-files|--enable-pc-files --enable-widec|' -i "$NCURSES_PORTFILE"
fi
./vcpkg install ncurses:x64-mingw-static
tree -h ./installed/x64-mingw-static
tree ./installed/x64-mingw-static
- name: Checkout
uses: actions/checkout@v4
Expand Down Expand Up @@ -581,8 +514,70 @@ jobs:
if: startsWith(github.ref, 'refs/tags/v')
run: ./scripts/ci-upload-release-artifacts.sh

ci-musl:
needs: [tag, clang-format, cppcheck, shellcheck]
runs-on: ubuntu-latest
container: alpine:latest

env:
TAG: ${{ needs.tag.outputs.TAG }}

steps:
- name: Set up dependencies
shell: sh
run: apk add bash gcc make musl-dev perl ncurses-dev ncurses-static tmux file sqlite curl zip

- name: Checkout
uses: actions/checkout@v4

- name: Build (${{ env.AMD64_LINUX_MUSL }})
env:
PREFIX: ${{ env.AMD64_LINUX_MUSL }}
CC: gcc
MAKE: make
RUN_TESTS: true
STATIC_BUILD: "1"
run: ./scripts/ci-build.sh

- name: Prepare build artifacts for upload
run: ./scripts/ci-prepare-artifacts-for-upload.sh

- name: Attest build artifacts for release
if: startsWith(github.ref, 'refs/tags/v')
uses: actions/attest-build-provenance@v1
with:
subject-path: ${{ env.ARTIFACT_DIR }}/*

- name: Verify attestations of release artifacts
if: startsWith(github.ref, 'refs/tags/v')
run: ./scripts/ci-verify-attestations.sh

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip)
uses: actions/upload-artifact@v4
env:
ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip
with:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz)
uses: actions/upload-artifact@v4
env:
ARTIFACT_NAME: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.tar.gz
with:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ env.ARTIFACT_DIR }}/${{ env.ARTIFACT_NAME }}
retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
if-no-files-found: error

- name: Upload release artifacts
if: startsWith(github.ref, 'refs/tags/v')
run: ./scripts/ci-upload-release-artifacts.sh

ghcr:
needs: [tag]
needs: [tag, ci-musl]
runs-on: ubuntu-latest

permissions:
Expand All @@ -594,6 +589,27 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v4
with:
sparse-checkout: |
Dockerfile.ci
- name: Download (zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip)
uses: actions/download-artifact@v4
with:
name: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip
path: ${{ env.AMD64_LINUX_MUSL }}

- name: Unzip
env:
ZIP: zsv-${{ env.TAG }}-${{ env.AMD64_LINUX_MUSL }}.zip
DIR: ${{ env.AMD64_LINUX_MUSL }}
run: |
cd "$DIR"
unzip -o "$ZIP"
cd ..
mkdir -p ./ci
mv ./"$DIR"/bin/zsv ./ci/
rm -rf ./"$DIR"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
Expand All @@ -614,6 +630,9 @@ jobs:
env:
DOCKER_BUILD_RECORD_UPLOAD: false
with:
no-cache: true
context: .
file: Dockerfile.ci
platforms: linux/amd64
push: ${{ startsWith(github.ref, 'refs/tags/v') }}
tags: |
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -63,5 +63,7 @@ tmp
include/zsv.h
app/test/worldcitiespop_mil.csv
data/quoted5.csv
data/loans_2.csv
data/.zsv/data/loans_2.csv/overwrite.sqlite3
compile_commands.json
.cache
6 changes: 3 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
FROM alpine:latest AS build

LABEL maintainer="Liquidaty"
LABEL description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser"
LABEL url="https://github.com/liquidaty/zsv"
LABEL org.opencontainers.image.description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser"

RUN apk add --no-cache gcc make musl-dev perl
RUN apk add bash gcc make musl-dev perl ncurses-dev ncurses-static tmux file sqlite curl zip

WORKDIR /zsv
COPY . .
Expand All @@ -14,7 +14,7 @@ RUN \
CC=gcc \
MAKE=make \
ARTIFACT_DIR=artifacts \
RUN_TESTS=false \
RUN_TESTS=true \
STATIC_BUILD=1 \
SKIP_ZIP_ARCHIVE=true \
SKIP_TAR_ARCHIVE=true \
Expand Down
10 changes: 10 additions & 0 deletions Dockerfile.ci
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
FROM scratch

LABEL maintainer="Liquidaty"
LABEL url="https://github.com/liquidaty/zsv"
LABEL org.opencontainers.image.description="zsv: tabular data swiss-army knife CLI + world's fastest (simd) CSV parser"

WORKDIR /zsv
COPY ci/zsv .

ENTRYPOINT [ "./zsv" ]
5 changes: 0 additions & 5 deletions app/test/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ else
CMP=cmp
endif


help:
@echo "To run all tests: make test [LEAKS=1]"
@echo "To run individual test: make test-xxx"
Expand Down Expand Up @@ -337,7 +336,6 @@ test-mv: ${BUILD_DIR}/bin/zsv_prop${EXE} ${BUILD_DIR}/bin/zsv_mv${EXE}
@${PREFIX} ${BUILD_DIR}/bin/zsv_mv${EXE} ${TMP_DIR}/$@.csv ${TMP_DIR}/$@-moved.csv ${REDIRECT} /dev/null
@find ${TMP_DIR}/.zsv/data/$@-moved.csv/props.json -type f >/dev/null && ${TEST_PASS} || ${TEST_FAIL}


test-blank-leading-rows: test-blank-leading-rows-1 test-blank-leading-rows-2 test-blank-leading-rows-3 test-blank-leading-rows-4

test-blank-leading-rows-1: ${BUILD_DIR}/bin/zsv_select${EXE}
Expand Down Expand Up @@ -430,7 +428,6 @@ test-serialize-position-id: ${BUILD_DIR}/bin/zsv_serialize${EXE} ${TEST_DATA_DIR
@(${PREFIX} $< --id-column 3 < ${TEST_DATA_DIR}/test/serialize.csv -a 'Interest Paid Through Date' -a 'original INTEREST Only Term' ${REDIRECT1} ${TMP_DIR}/$@.out && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL})


test-serialize : test-%: ${BUILD_DIR}/bin/zsv_%${EXE} test-serialize-quoted test-serialize-additional test-serialize-position-id
@${TEST_INIT}
@( ( ! [ -s "${TEST_DATA_DIR}/test/$*.csv" ] ) && echo "No test input for $*") || \
Expand Down Expand Up @@ -466,7 +463,6 @@ test-sql5: ${BUILD_DIR}/bin/zsv_sql${EXE} # test blank rows
@(${PREFIX} $< /tmp/1.csv 'select * from data' ${REDIRECT1} ${TMP_DIR}/$@.out)
@${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL}


${BUILD_DIR}/bin/zsv_%${EXE}:
make -C .. $@ CONFIGFILE=${CONFIGFILEPATH} DEBUG=${DEBUG}

Expand Down Expand Up @@ -546,7 +542,6 @@ test-compare-tolerance: ${BUILD_DIR}/bin/zsv_compare${EXE}
@(${PREFIX} $< --tolerance 0.00001 ../../data/compare/tolerance1.csv ../../data/compare/tolerance2.csv ${REDIRECT1} ${TMP_DIR}/[email protected] && \
${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || ${TEST_FAIL})


test-compare: test-%: ${BUILD_DIR}/bin/zsv_%${EXE}
@${TEST_INIT}
@(${PREFIX} $< compare/t1.csv compare/t2.csv compare/t3.csv ${REDIRECT1} ${TMP_DIR}/$@.out && \
Expand Down
Loading

0 comments on commit 522e5d4

Please sign in to comment.