CI: embedings

.github/workflows/build_template.yml

@@ -31,14 +31,17 @@ on:
         required: false
         type: string
         default: boost_nov22
+      embeddings_artifact:
+        required: true
+        type: string
       artifact_list:
         required: false
         type: string
         default: "build/columnar/lib_manticore_columnar.so build/secondary/lib_manticore_secondary.so build/_deps/manticore-build/src/searchd build/_deps/manticore-build/src/indexer build/_deps/manticore-build/src/indextool build/_deps/manticore-build/config/*.c build/_deps/manticore-build/config/*.h"
       HOMEBREW_PREFIX:
         required: false
         type: string
-        default:
+        default: ""
       MANTICORE_LOCATOR:
         required: false
         type: string
@@ -74,10 +77,20 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           set-safe-directory: true
 
+      - name: Download embeddings lib 
+        uses: manticoresoftware/download_artifact_with_retries@v2
+        continue-on-error: true
+        with:
+          name: ${{ inputs.embeddings_artifact }}
+          path: ./embeddings-lib/
+
       - name: Initialization
         run: | # without adding the safe.directory cmake fails to do git log -1 --date=short --format=%cd
           bash /sysroot/root/entry_point.sh
           git config --global --add safe.directory /__w/columnar/columnar
+          mkdir -p embeddings/target/release
+          mv ./embeddings-lib/build/* embeddings/target/release/
+          rm -fr ./embeddings-lib
 
       - name: Check out main cache before building
         uses: actions/[email protected]

.github/workflows/embedding_build_template.yml

@@ -0,0 +1,109 @@
+name: Embeddings Build Template
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        required: false
+        type: string
+        default: "ubuntu-22.04"
+      distr:
+        required: true
+        type: string
+      arch:
+        required: true
+        type: string
+      artifact_list:
+        required: false
+        type: string
+        default: "target/release/lib*.so"
+      setup:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  build:
+    runs-on: ${{ inputs.runner }}
+    defaults:
+      run:
+        shell: bash
+    timeout-minutes: 30
+    steps:
+      - name: Setup
+        if: ${{ inputs.setup }}
+        run: |
+          ${{ inputs.setup }}
+
+      - name: Check distribution type
+        id: vars
+        run: |
+          declare -A target_map=(
+            [linux]="unknown-linux-musl"
+            [macos]="apple-darwin"
+            [windows]="pc-windows-msvc"
+          )
+
+          if [[ -n "${target_map[${{ inputs.distr }}]}" ]]; then
+            target="${{ inputs.arch }}-${target_map[${{ inputs.distr }}]}"
+            echo "target=${target}" >> $GITHUB_OUTPUT
+          else
+            echo "Unsupported distribution type: ${{ inputs.distr }}"
+            exit 1
+          fi
+
+          lib_dir="./embeddings/target/${target}/release"
+          echo "lib_dir=${lib_dir}" >> $GITHUB_OUTPUT
+
+      - uses: actions/checkout@v3
+        with:
+          toolchain: stable
+          target: ${{ steps.vars.outputs.target }}
+          override: true
+      - name: Prepare build args
+        id: params
+        run: |
+          args=(
+            "--target" 
+            "${{ steps.vars.outputs.target }}" 
+            "--lib"
+            "--release" 
+            "--manifest-path" 
+            "./embeddings/Cargo.toml"
+          )
+          echo "args=${args[@]}" >> $GITHUB_OUTPUT
+
+      - name: Build
+        if: ${{ inputs.distr == 'linux' || inputs.distr == 'macos' }}
+        run: |
+          if [[ "${{ inputs.distr }}" == "macos" ]]; then
+            export CC=o64-clang
+            export CXX=o64-clang++
+          fi
+          docker run --rm \
+            -e CC -e CXX \
+            -v $(pwd):/src \
+            -w /src \
+            joseluisq/rust-linux-darwin-builder:1.83.0 \
+            sh -c "cargo build ${{ steps.params.outputs.args }}"
+
+          sudo chown -hR $(id -u):$(id -g) ${{ steps.vars.outputs.lib_dir }}
+
+      - name: Build
+        if: ${{ inputs.distr == 'windows' }}
+        run: |
+          rustup target add ${{ steps.vars.outputs.target }}
+          cargo build ${{ steps.params.outputs.args }}
+
+      - run: | 
+          mkdir build
+          lib_dir="${{ steps.vars.outputs.lib_dir }}"
+          ls -la $lib_dir/*
+          find $lib_dir -type f \( -name "libmanticoresearch*.a" -o -name "manticoresearch*.lib" \) -ls -exec mv {} ./build \;
+
+      - name: Upload build artifacts
+        if: success()
+        uses: manticoresoftware/upload_artifact_with_retries@v2
+        with:
+          name: embeddings_${{ inputs.distr }}_${{ inputs.arch }}
+          path: ./build/*

.github/workflows/pack_publish.yml

@@ -31,9 +31,37 @@ jobs:
     steps:
       - run: echo "All set to build packages"
 
+  embedding_linux_osx_pack:
+    needs: pack
+    uses: ./.github/workflows/embedding_build_template.yml
+    strategy:
+      fail-fast: false
+      matrix:
+        distr: [linux, macos]
+        arch: [x86_64, aarch64]
+    name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing
+    with:
+      runner: ubuntu-22.04
+      distr: ${{ matrix.distr }}
+      arch: ${{ matrix.arch }}
+
+  embedding_windows_pack:
+    needs: pack
+    uses: ./.github/workflows/embedding_build_template.yml
+    strategy:
+      fail-fast: false
+      matrix:
+        distr: [windows]
+        arch: [x86_64, aarch64]
+    name: ${{ matrix.distr }} ${{ matrix.arch }} embedding packing
+    with:
+      runner: windows-2022
+      distr: ${{ matrix.distr }}
+      arch: ${{ matrix.arch }}
+
   pack_debian_ubuntu:
     uses: ./.github/workflows/build_template.yml
-    needs: pack
+    needs: embedding_linux_osx_pack
     strategy:
       fail-fast: false
       matrix:
@@ -50,10 +78,11 @@ jobs:
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*deb"
+      embeddings_artifact: embeddings_linux_${{ matrix.arch }}
 
   pack_rhel:
     uses: ./.github/workflows/build_template.yml
-    needs: pack
+    needs: embedding_linux_osx_pack
     strategy:
       fail-fast: false
       matrix:
@@ -73,10 +102,12 @@ jobs:
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*rpm"
+      embeddings_artifact: embeddings_linux_${{ matrix.arch }}
+
 
   pack_macos:
     uses: ./.github/workflows/build_template.yml
-    needs: pack
+    needs: embedding_linux_osx_pack
     strategy:
       fail-fast: false
       matrix:
@@ -94,11 +125,12 @@ jobs:
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*tar.gz"
+      embeddings_artifact: embeddings_macos_${{ matrix.arch == 'arm64' && 'aarch64' || matrix.arch }}
 
   pack_windows:
     name: Windows x64 package
     uses: ./.github/workflows/build_template.yml
-    needs: pack
+    needs: embedding_windows_pack
     with:
       DISTR: windows
       arch: x64
@@ -109,6 +141,7 @@ jobs:
         cmake --build . --target package
       cache_key: pack_${{ matrix.DISTR }}_${{ matrix.arch }}
       artifact_list: "build/manticore*exe build/manticore*zip"
+      embeddings_artifact: embeddings_windows_x86_64
 
   # virtual job to simplify the CI
   # This job depends on all the package preparation jobs that have to pass before we can start publishing packages

embeddings/.editorconfig

@@ -0,0 +1,25 @@
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+visual_wrap = true
+max_line_length = 120
+
+[*]
+indent_style = tab
+indent_size = 2
+
+[**.yaml]
+indent_style = space
+indent_size = 2
+
+[**.yml]
+indent_style = space
+indent_size = 2
+
+[**.md]
+indent_style = space
+indent_size = 2

embeddings/.gitignore

@@ -0,0 +1,3 @@
+target/**
+.cache
+