Bazel migration phase 2

.bazelrc

@@ -27,3 +27,7 @@ build --incompatible_default_to_explicit_init_py
 
 # https://github.com/bazelbuild/bazel/issues/2377
 test --spawn_strategy=standalone
+test --test_timeout=6000
+
+# Use Java 11 as the JVM for Bazel builds
+build --java_language_version=8 --java_runtime_version=remotejdk_8

.circleci/Dockerfile

@@ -1,41 +1,67 @@
-ARG base_image
+# Set a default base image version (e.g., ubuntu:25.04)
+ARG base_image=ubuntu:25.04
+
+# Use the argument in the FROM statement
 FROM ${base_image}
-USER root
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-ENV PATH /opt/conda/bin:$PATH
-ENV THRIFT_VERSION 0.11.0
 
-WORKDIR /img-build
+ARG DEBIAN_FRONTEND=noninteractive
+ENV LC_ALL=C.utf-8
+ENV THRIFT_VERSION 0.13.0
+
+RUN apt-get update && apt-get install -y software-properties-common && \
+    apt-add-repository ppa:deadsnakes/ppa \
+    && apt-get install -y \
+  automake \
+  autotools-dev \
+  bison \
+  byacc \
+  build-essential \
+  ca-certificates \
+  chromium-browser \
+  clang \
+  cmake \
+  curl \
+  docker.io \
+  gettext-base \
+  git \
+  git-lfs \
+  jq \
+  libbz2-dev \
+  libc++-dev \
+  libc++abi-dev \
+  libcurl4-openssl-dev \
+  libffi-dev \
+  libgdbm-dev \
+  liblzma-dev \
+  libmysqlclient-dev \
+  libncurses5-dev \
+  libnss3-dev \
+  libpcre3 \
+  libpcre3-dev \
+  libreadline-dev \
+  libsqlite3-dev \
+  libssl-dev \
+  libtinfo-dev \
+  locales \
+  pkg-config \
+  python3.8 \
+  python3.8-dev \
+  python3.8-distutils \
+  zlib1g-dev \
+  openjdk-11-jdk \
+  zip && \
+  rm -rf /var/lib/apt/lists/*
 
+# Set JAVA_HOME environment variable
+ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+ENV PATH="$JAVA_HOME/bin:$PATH"
 
-# Add sbt repo to sources list
-RUN echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list
-RUN echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee /etc/apt/sources.list.d/sbt_old.list
-RUN curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo apt-key add
+RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100 && \
+    update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100
 
-# Install prereqs
-RUN apt-get update && apt-get -y -q install \
-    automake \
-    bison \
-    cmake \
-    curl \
-    flex \
-    g++ \
-    git \
-    libboost-dev \
-    libboost-filesystem-dev \
-    libboost-program-options-dev \
-    libboost-system-dev \
-    libboost-test-dev \
-    libevent-dev \
-    libssl-dev \
-    libtool \
-    make \
-    openjdk-8-jdk \
-    pkg-config \
-    sbt \
-    bazelisk \
-    && apt-get clean
+# Install Bazel
+ADD https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 /usr/local/bin/bazel
+RUN chmod 755 /usr/local/bin/bazel
 
 # Install thrift
 RUN curl -sSL "http://archive.apache.org/dist/thrift/$THRIFT_VERSION/thrift-$THRIFT_VERSION.tar.gz" -o thrift.tar.gz \
@@ -49,35 +75,14 @@ RUN curl -sSL "http://archive.apache.org/dist/thrift/$THRIFT_VERSION/thrift-$THR
 	&& cd / \
 	&& rm -rf /usr/src/thrift
 
-# Install Scala
-ENV SCALA_VERSION 2.11.7
-ENV SCALA_DEB http://www.scala-lang.org/files/archive/scala-$SCALA_VERSION.deb
-
-RUN wget --quiet --output-document=scala.deb $SCALA_DEB && \
-    dpkg -i scala.deb && \
-    rm -f *.deb
-
-# Install sbt
-RUN apt-get update && apt-get -y -q install \
-    sbt \
-    && apt-get clean
-
-# Install conda
-RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure && \
-    wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
-    bash ~/miniconda.sh -b -p /opt/conda && \
-    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-    # clean up
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-    rm ~/*.sh
 
-RUN conda create -y -n chronon_py python=3.7
-RUN conda install -y -q -n chronon_py --no-deps virtualenv
-RUN  /opt/conda/envs/chronon_py/bin/pip install \
-   flake8==5.0.4 flake8-quotes==3.3.1 thrift==0.11.0 click==7.0 thrift_json==0.1.0 nose>=1.3.7
+# Create a non-root user
+RUN useradd -m -s /bin/bash chronon
 
-# Install python build dependencies
-RUN  /opt/conda/envs/chronon_py/bin/pip install build
+# Set working directory
+WORKDIR /home/chronon
+RUN chown -R chronon:chronon /home/chronon/
+USER chronon
 
-WORKDIR /
-RUN rm -rf /img-build
+# Cmd to run when starting the container
+CMD ["/bin/bash"]

.circleci/config.yml

@@ -2,14 +2,14 @@ version: 2.1
 
 base_defaults: &base_defaults
     resource_class: xlarge
-    working_directory: /chronon
+    working_directory: /home/chronon/workspace
 
 executors:
     docker_baseimg_executor:
         resource_class: xlarge
-        working_directory: /chronon
+        working_directory: /home/chronon/workspace
         docker:
-            - image: houpy0829/chronon-ci:base--f87f50dc520f7a73894ae024eb78bd305d5b08e2
+            - image: krisnaru/chronon:krish-test-6
 
 jobs:
     "Pull Docker Image":
@@ -24,7 +24,7 @@ jobs:
                 name: Pull existing docker image
                 command: |
                     set +o pipefail
-                    docker pull houpy0829/chronon-ci:base--f87f50dc520f7a73894ae024eb78bd305d5b08e2 || true
+                    docker pull krisnaru/chronon:krish-test-6 || true
 
     "Scala 12 -- Spark 3 Tests":
         executor: docker_baseimg_executor
@@ -34,23 +34,7 @@ jobs:
                   name: Run Spark 3.1.1 tests
                   shell: /bin/bash -leuxo pipefail
                   command: |
-                      conda activate chronon_py
-                      # Increase if we see OOM.
-                      export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
-                      sbt "++ 2.12.12 test"
-            - store_test_results:
-                  path: /chronon/spark/target/test-reports
-            - store_test_results:
-                  path: /chronon/aggregator/target/test-reports
-            - run:
-                  name: Compress spark-warehouse
-                  command: |
-                    cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
-                  when: on_fail
-            - store_artifacts:
-                  path: /tmp/spark-warehouse.tar.gz
-                  destination: spark_warehouse.tar.gz
-                  when: on_fail
+                      bazel test --config scala_2.12 --config spark_3.1 //spark:test
 
     "Scala 13 -- Tests":
         executor: docker_baseimg_executor
@@ -60,23 +44,7 @@ jobs:
                   name: Run Scala 13 tests
                   shell: /bin/bash -leuxo pipefail
                   command: |
-                      conda activate chronon_py
-                      # Increase if we see OOM.
-                      export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
-                      sbt "++ 2.13.6 test"
-            - store_test_results:
-                  path: /chronon/spark/target/test-reports
-            - store_test_results:
-                  path: /chronon/aggregator/target/test-reports
-            - run:
-                  name: Compress spark-warehouse
-                  command: |
-                    cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
-                  when: on_fail
-            - store_artifacts:
-                  path: /tmp/spark-warehouse.tar.gz
-                  destination: spark_warehouse.tar.gz
-                  when: on_fail
+                      bazel test --config scala_2.13 --config spark_3.2 //spark:test
 
     # run these separately as we need a isolated JVM to not have Spark session settings interfere with other runs
     # long term goal is to refactor the current testing spark session builder and avoid adding new single test to CI
@@ -90,49 +58,20 @@ jobs:
               format_test: deltalake
             shell: /bin/bash -leuxo pipefail
             command: |
-              conda activate chronon_py
-              # Increase if we see OOM.
-              export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
-              sbt '++ 2.13.6' "testOnly ai.chronon.spark.test.TableUtilsFormatTest"
-        - store_test_results:
-            path: /chronon/spark/target/test-reports
-        - store_test_results:
-            path: /chronon/aggregator/target/test-reports
-        - run:
-            name: Compress spark-warehouse
-            command: |
-              cd /tmp/ && tar -czvf spark-warehouse.tar.gz chronon/spark-warehouse
-            when: on_fail
-        - store_artifacts:
-            path: /tmp/spark-warehouse.tar.gz
-            destination: spark_warehouse.tar.gz
-            when: on_fail
+              bazel test --config scala_2.13 --config spark_3.1 //spark:TableUtilsFormatTest
 
-    "Scala 11 -- Compile":
-      executor: docker_baseimg_executor
-      steps:
-        - checkout
-        - run:
-            name: Compile Scala 11
-            shell: /bin/bash -leuxo pipefail
-            command: |
-              conda activate chronon_py
-              # Increase if we see OOM.
-              export SBT_OPTS="-XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=4G -Xmx4G -Xms2G"
-              sbt "++ 2.11.12 compile"
-
-    "Chronon Python Lint":
-        executor: docker_baseimg_executor
-        steps:
-            - checkout
-            - run:
-                  name: Run Chronon Python lint
-                  shell: /bin/bash -leuxo pipefail
-                  command: |
-                      conda activate chronon_py
-                      cd /chronon/api/py/ai/chronon
-                      pip install importlib-metadata==4.11.4 #Install importlib-metadata < 5
-                      flake8 --extend-ignore=W605,Q000,F631,E203
+#    "Chronon Python Lint":
+#        executor: docker_baseimg_executor
+#        steps:
+#            - checkout
+#            - run:
+#                  name: Run Chronon Python lint
+#                  shell: /bin/bash -leuxo pipefail
+#                  command: |
+#                      conda activate chronon_py
+#                      cd /chronon/api/py/ai/chronon
+#                      pip install importlib-metadata==4.11.4 #Install importlib-metadata < 5
+#                      flake8 --extend-ignore=W605,Q000,F631,E203
 
     "Chronon Python Tests":
         executor: docker_baseimg_executor
@@ -142,35 +81,24 @@ jobs:
                   name: Run Chronon Python tests
                   shell: /bin/bash -leuxo pipefail
                   command: |
-                    conda activate chronon_py
-                    pushd /chronon/api/
-                    thrift --gen py -out /chronon/api/py/ai/chronon\
-                       /chronon/api/thrift/api.thrift                 # Generate thrift files
-                    cd /chronon/                                      # Go to Python module
-                    pip install -r api/py/requirements/dev.txt        # Install latest requirements
-                    tox                                               # Run tests
-                    popd
-            - store_artifacts:
-                  path: /chronon/api/py/htmlcov
+                    bazel test //api/py:api_test
 
-    "Scalafmt Check":
-        executor: docker_baseimg_executor
-        steps:
-            - checkout
-            - run:
-                  name: Run ScalafmtCheck
-                  shell: /bin/bash -leuxo pipefail
-                  command: |
-                      conda activate chronon_py
-                      sbt +scalafmtCheck
+
+#    "Scalafmt Check":
+#        executor: docker_baseimg_executor
+#        steps:
+#            - checkout
+#            - run:
+#                  name: Run ScalafmtCheck
+#                  shell: /bin/bash -leuxo pipefail
+#                  command: |
+#                      conda activate chronon_py
+#                      sbt +scalafmtCheck
 
 workflows:
     build_test_deploy:
         jobs:
             - "Pull Docker Image"
-            - "Scala 11 -- Compile":
-                requires:
-                  - "Pull Docker Image"
             - "Scala 12 -- Spark 3 Tests":
                   requires:
                       - "Pull Docker Image"
@@ -180,12 +108,12 @@ workflows:
             - "Scala 13 -- Delta Lake Format Tests":
                   requires:
                     - "Pull Docker Image"
-            - "Scalafmt Check":
-                  requires:
-                      - "Pull Docker Image"
+#            - "Scalafmt Check":
+#                  requires:
+#                      - "Pull Docker Image"
             - "Chronon Python Tests":
                   requires:
                       - "Pull Docker Image"
-            - "Chronon Python Lint":
-                  requires:
-                      - "Pull Docker Image"
+#            - "Chronon Python Lint":
+#                  requires:
+#                      - "Pull Docker Image"

WORKSPACE

@@ -101,6 +101,25 @@ load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
 
 rules_jvm_external_setup()
 
+http_archive(
+    name = "contrib_rules_jvm",
+    sha256 = "e6cd8f54b7491fb3caea1e78c2c740b88c73c7a43150ec8a826ae347cc332fc7",
+    strip_prefix = "rules_jvm-0.27.0",
+    url = "https://github.com/bazel-contrib/rules_jvm/releases/download/v0.27.0/rules_jvm-v0.27.0.tar.gz",
+)
+
+# Fetches the contrib_rules_jvm dependencies.
+# If you want to have a different version of some dependency,
+# you should fetch it *before* calling this.
+load("@contrib_rules_jvm//:repositories.bzl", "contrib_rules_jvm_deps")
+
+contrib_rules_jvm_deps()
+
+# Now ensure that the downloaded deps are properly configured
+load("@contrib_rules_jvm//:setup.bzl", "contrib_rules_jvm_setup")
+
+contrib_rules_jvm_setup()
+
 ## Scala support
 
 # See https://github.com/bazelbuild/rules_scala/releases for up to date version information.