cli v2

.bazelrc

@@ -1,2 +1,5 @@
 build --java_language_version=11
-build --java_runtime_version=11
+build --java_runtime_version=11
+build --remote_cache=https://storage.googleapis.com/zipline-bazel-cache
+build --google_default_credentials
+test --test_output=errors

.github/image/Dockerfile

@@ -1,15 +1,9 @@
-FROM alpine:latest
+FROM ubuntu:latest
 
 # build using command: docker build --progress=plain -t chronon-base .
 
 # Install necessary tools and Python
-RUN apk add --no-cache wget curl bash python3 py3-pip
-
-# Add Amazon Corretto repository
-RUN wget -O /etc/apk/keys/amazoncorretto.rsa.pub https://apk.corretto.aws/amazoncorretto.rsa.pub && \
-    echo "https://apk.corretto.aws/" >> /etc/apk/repositories && \
-    apk update
-RUN apk add --no-cache amazon-corretto-17
+RUN apt update && apt install -y wget curl bash python3 python3-pip openjdk-17-jdk python3.12-venv
 
 # java
 ENV JAVA_HOME=/usr/lib/jvm/default-jvm
@@ -21,15 +15,16 @@ ENV PATH="/usr/local/sbt/bin:${PATH}"
 
 # bazel
 RUN curl -fsSL "https://github.com/bazelbuild/bazelisk/releases/download/v1.18.0/bazelisk-linux-amd64" -o /usr/local/bin/bazel
-ENV PATH="/usr/local/bin/bazel:${PATH}"
+RUN chmod +x /usr/local/bin/bazel
+ENV PATH="/usr/local/bin:${PATH}"
 
 # thrift
 ARG THRIFT_VERSION=0.21.0
-RUN apk add --no-cache \
-        build-base \
+RUN apt install -y \
+        build-essential \
         cmake \
-        boost-dev \
-        openssl-dev \
+        libboost-dev \
+        libssl-dev \
         libevent-dev \
         bison \
         flex \
@@ -45,32 +40,34 @@ RUN apk add --no-cache \
     make install && \
     cd .. && \
     rm -rf thrift-${THRIFT_VERSION} thrift-${THRIFT_VERSION}.tar.gz && \
-    apk del \
-        build-base \
+    apt purge -y \
+        build-essential \
         cmake \
-        boost-dev \
-        openssl-dev \
+        libboost-dev \
+        libssl-dev \
         libevent-dev \
         bison \
         flex \
         autoconf \
         automake \
         libtool \
         curl && \
-    rm -rf /var/cache/apk/*
+    apt autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
 
 # Upgrade pip and install some common Python packages
-RUN python3 -m pip install --break-system-packages --upgrade pip && \
-    pip3 install --break-system-packages pytest tox flake8
+RUN pip3 install --break-system-packages pytest tox flake8
 
-# spark uses snappy to write files, snappy needs gcompat
-RUN apk add gcompat
+RUN apt update && apt install -y build-essential git
+RUN mkdir -p /usr/lib/jvm && ln -s /usr/lib/jvm/java-17-openjdk-amd64/ /usr/lib/jvm/default-jvm
 
 # Verify installations
 RUN java -version && \
     thrift -version && \
     python3 --version  && \
-    pip3 --version
+    pip3 --version && \
+    bazel --version && \
+    git --version
 
 # Set working directory
 WORKDIR /app

.github/workflows/test_python.yaml

@@ -33,20 +33,28 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Python virtual environment
+      - name: Configure Git
         run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "GitHub Actions"
+
+      - name: Set up Python virtual environment
+        shell: bash
+        run: |          
           python3 -m venv chronon_py_env
           source chronon_py_env/bin/activate
 
       - name: Run Chronon Python lint
+        shell: bash
         run: |
           source chronon_py_env/bin/activate
           cd api/py/ai/chronon
           pip install importlib-metadata==4.11.4
           pip install flake8
-          flake8 --extend-ignore=W605,Q000,F631
+          flake8 --extend-ignore=W605,Q000,F631 --exclude=cli
 
       - name: Run Chronon Python tests
+        shell: bash
         run: |
           source chronon_py_env/bin/activate
           thrift --gen py -out api/py/ api/thrift/common.thrift

.github/workflows/test_scala_fmt.yaml

@@ -33,6 +33,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Set up locale
+        run: |
+          export LANG=en_US.UTF-8
+          export LC_ALL=en_US.UTF-8
+
+      - name: Run Sbt Clean
+        run: sbt clean
+
       - name: Check Scalafmt
         run: sbt +scalafmtCheck
 

.github/workflows/test_scala_no_spark.yaml

@@ -42,48 +42,55 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set SBT resources
+      - name: Setup Bazel cache credentials
         run: |
-          echo "SBT_OPTS=-Xmx8G -Xms2G" >> $GITHUB_ENV
+          echo "${{ secrets.BAZEL_CACHE_CREDENTIALS }}" | base64 -d > bazel-cache-key.json
 
       - name: Run Flink tests
         run: |
-          export SBT_OPTS="-Xmx24G -Xms4G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
-          sbt "++ 2.12.18 flink/test"
-
-      - name: Run Async KV Store writer test
-        run: |
-          sbt "++ 2.12.18 flink/testOnly -- -n asyncKVStoreWriterTest"
+          bazel test \
+            --remote_cache=https://storage.googleapis.com/zipline-bazel-cache \
+            --google_credentials=bazel-cache-key.json \
+            //flink:test
 
       - name: Run Aggregator tests
         run: |
-          sbt "++ 2.12.18 aggregator/test"
+          bazel test \
+            --remote_cache=https://storage.googleapis.com/zipline-bazel-cache \
+            --google_credentials=bazel-cache-key.json \
+            //aggregator:test
 
       - name: Run Online tests
         run: |
-          sbt "++ 2.12.18 online/test"
-
-      - name: Run Catalyst Util tests
-        run: |
-          sbt "++ 2.12.18 online/testOnly -- -n catalystUtilHiveUdfTest"      
+          bazel test \
+            --remote_cache=https://storage.googleapis.com/zipline-bazel-cache \
+            --google_credentials=bazel-cache-key.json \
+            //online:test
 
       - name: Run api tests
         run: |
-          sbt "++ 2.12.18 api/test"
+          bazel test \
+            --remote_cache=https://storage.googleapis.com/zipline-bazel-cache \
+            --google_credentials=bazel-cache-key.json \
+            //api:test
 
       - name: Run hub tests
         run: |
-          export SBT_OPTS="-Xmx8G -Xms2G"
-          sbt "++ 2.12.18 hub/test"
+          bazel test \
+            --remote_cache=https://storage.googleapis.com/zipline-bazel-cache \
+            --google_credentials=bazel-cache-key.json \
+            //hub:test
 
       - name: Run service tests
         run: |
-          export SBT_OPTS="-Xmx8G -Xms2G"
-          sbt "++ 2.12.18 service/test"
+          bazel test \
+            --remote_cache=https://storage.googleapis.com/zipline-bazel-cache \
+            --google_credentials=bazel-cache-key.json \
+            //service:test
 
       - name: Run orchestrator tests
         run: |
-          export SBT_OPTS="-Xmx8G -Xms2G"
-          sbt "++ 2.12.18 orchestration/test"
-
-
+          bazel test \
+            --remote_cache=https://storage.googleapis.com/zipline-bazel-cache \
+            --google_credentials=bazel-cache-key.json \
+            //orchestration:test

.github/workflows/test_scala_spark.yaml

@@ -35,85 +35,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Run other spark tests
+      - name: Setup Bazel cache credentials
         run: |
-          export SBT_OPTS="-Xmx24G -Xms4G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
-          sbt "spark/testOnly"
+          echo "${{ secrets.BAZEL_CACHE_CREDENTIALS }}" | base64 -d > bazel-cache-key.json
 
-  join_spark_tests:
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/${{ github.repository }}-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Run spark join tests
-        run: |
-          export SBT_OPTS="-Xmx8G -Xms2G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
-          sbt "spark/testOnly -- -n jointest"
-
-  table_utils_delta_format_spark_tests:
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/${{ github.repository }}-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Run table utils format test for Delta Lake
-        run: |
-          export SBT_OPTS="-Xmx8G -Xms2G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
-          sbt "spark/testOnly ai.chronon.spark.test.TableUtilsFormatTest"
-        env:
-          format_test: deltalake
-
-  mutation_spark_tests:
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/${{ github.repository }}-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Run spark mutations tests
-        run: |
-          export SBT_OPTS="-Xmx8G -Xms2G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
-          sbt "spark/testOnly -- -n mutationstest"
-
-  fetcher_spark_tests:
-    runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/${{ github.repository }}-ci:latest
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-    defaults:
-      run:
-        working-directory: ${{ github.workspace }}
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Run spark fetcher tests
+      - name: Run Spark tests
         run: |
-          export SBT_OPTS="-Xmx8G -Xms2G --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
-          sbt "spark/testOnly -- -n fetchertest"
+          bazel test \
+            --remote_cache=https://storage.googleapis.com/zipline-bazel-cache \
+            --google_credentials=bazel-cache-key.json \
+            //spark:test