Introduce Auto-scaling Based on CPU Load for Triangle Count Operation

.github/workflows/build.yml

@@ -6,39 +6,33 @@ on:
   pull_request:
     branches: [master]
 
-env:
-  # Customize the build type here (Release, Debug, RelWithDebInfo, etc.)
-  BUILD_TYPE: Release
-
 jobs:
   unit-tests:
     runs-on: ubuntu-latest
+    if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip unit tests')}}
 
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{github.head_ref}}
-          fetch-depth: 0
           repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+
+      - name: Build unit test
+        run: |
+          docker build -t jasminegraph .
+          docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit .
+
       - uses: jupyterhub/action-k3s-helm@v4
         with:
           docker-enabled: true
 
       - name: Grant permissions for default user
         run: kubectl apply -f ./k8s/rbac.yaml
 
-      - name: Docker Build
-        run: docker build -t jasminegraph .
-
-      - name: Build unit test
-        run: docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit .
-
       - name: Run unit test
         run: |
-          mkdir coverage
-          kubectl apply -f ./.github/workflows/resources/unit-test-conf.yaml
-          sleep 30
-          kubectl logs -f jasminegraph-unit-test-pod
+          chmod +x ./.github/workflows/resources/unit-test.sh
+          ./.github/workflows/resources/unit-test.sh          
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4
@@ -50,16 +44,15 @@ jobs:
 
   docker-integration-tests:
     runs-on: ubuntu-latest
+    if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip docker integration')}}
 
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{github.head_ref}}
-        fetch-depth: 0
         repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
 
     - name: Docker Build
-      # Build JasmineGraph docker image
       run: docker build -t jasminegraph .
 
     - name: Run Integration Tests
@@ -69,13 +62,14 @@ jobs:
 
   k8s-integration-tests:
     runs-on: ubuntu-latest
+    if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip k8s integration')}}
 
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{github.head_ref}}
-          fetch-depth: 0
           repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+
       - uses: jupyterhub/action-k3s-helm@v4
         with:
           docker-enabled: true

.github/workflows/code-lint.yml

@@ -1,35 +1,37 @@
 name: Style Check
 
 on:
-    push:
-      branches: [master]
-    pull_request:
-      branches: [master]
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 
 jobs:
-    style-check:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v3
-
-            - name: Install tools
-              run: |
-                sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt
-                sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint
-                pip install pylint
-
-            - name: hadolint check
-              if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}}
-              run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 --ignore DL3013 -t warning
-
-            - name: shfmt check
-              if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}}
-              run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci
-
-            - name: cpplint check
-              if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}}
-              run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --recursive .
-
-            - name: pylint check
-              if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}}
-              run: pylint -d C0103,E0401,R0801,R0902,R0913,R0914,R0915,W0603,W0621,W0718 --recursive=y .
+  style-check:
+    runs-on: ubuntu-latest
+    if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip style check')}}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install tools
+        run: |
+          sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt
+          sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint
+          pip install pylint
+
+      - name: hadolint check
+        if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}}
+        run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 --ignore DL3013 -t warning
+
+      - name: shfmt check
+        if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}}
+        run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci
+
+      - name: cpplint check
+        if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}}
+        run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --recursive .
+
+      - name: pylint check
+        if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}}
+        run: pylint -d C0103,E0401,R0801,R0902,R0913,R0914,R0915,W0603,W0621,W0718 --recursive=y .

.github/workflows/release.yml

@@ -0,0 +1,42 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+env:
+  IMAGE: "${{ secrets.DOCKER_USER }}/jasminegraph:${{  github.ref_name }}"
+
+jobs:
+    release:
+      runs-on: ubuntu-latest
+
+      steps:
+        - uses: actions/checkout@v4
+          with:
+            ref: ${{github.ref}}
+            repository: ${{ github.repository }}
+
+        - name: Docker Build
+          run: |
+            sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${IMAGE}#" ./conf/jasminegraph-server.properties
+            docker build -t "jasminegraph:temp" .
+            CONTAINER_ID=$(docker create "jasminegraph:temp")
+            docker export "${CONTAINER_ID}" | docker import - "jasminegraph:pre-build"
+            docker build -t "${IMAGE}" -f .github/workflows/resources/Dockerfile .
+
+        - name: Docker Minimal Image Build
+          run: |
+            sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${IMAGE}-minimal#" ./conf/jasminegraph-server.properties
+            chmod +x ./minimal/build.sh
+            ./minimal/build.sh
+            docker tag "jasminegraph:minimal" "${IMAGE}-minimal"
+
+        - name: Docker Login
+          run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}
+
+        - name: Docker Push
+          run: |
+            docker push "${IMAGE}"
+            docker push "${IMAGE}-minimal"

.github/workflows/resources/Dockerfile

@@ -0,0 +1,9 @@
+FROM jasminegraph:pre-build
+
+ENV HOME="/home/ubuntu"
+ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph"
+
+WORKDIR "${JASMINEGRAPH_HOME}"
+
+ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"]
+CMD ["bash"]

.github/workflows/resources/unit-test.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Function to check if pod is in Pending/Running state
+pod_status=""
+check_pod_status() {
+    pod_status=$(kubectl get pod jasminegraph-unit-test-pod --no-headers -o custom-columns=":status.phase")
+
+    if [[ $pod_status == "Pending" || $pod_status == "ContainerCreating" ]]; then
+        echo 0
+    else
+        echo 1
+    fi
+}
+
+mkdir coverage
+kubectl apply -f ./k8s/configs.yaml
+kubectl apply -f ./.github/workflows/resources/unit-test-conf.yaml
+
+timeout=300 # Set the timeout in seconds (adjust as needed)
+start_time=$(date +%s)
+
+while [[ $(check_pod_status) == "0" ]]; do
+    sleep 10
+
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+
+    if [[ $elapsed_time -ge $timeout ]]; then
+        echo "Timeout reached. Exiting loop."
+        break
+    fi
+done
+
+echo "----------------------------- logs -----------------------------"
+kubectl logs -f jasminegraph-unit-test-pod
+
+check_pod_status
+if [[ $pod_status != "Running" && $pod_status != "Completed" ]]; then
+    echo "Pod jasminegraph-unit-test-pod is in $pod_status state"
+    echo "Unit tests failed"
+    echo "----------------------------- details --------------------------"
+    kubectl describe pod jasminegraph-unit-test-pod
+    exit 1
+fi

.gitignore

@@ -84,4 +84,5 @@ __pycache__
 *.db
 *.db-*
 
-coverage.xml
+coverage.xml
+*.tmp

CMakeLists.txt

@@ -5,7 +5,8 @@ set(CMAKE_CXX_STANDARD 11)
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 
-set(HEADERS src/backend/JasmineGraphBackend.h
+set(HEADERS globals.h
+        src/backend/JasmineGraphBackend.h
         src/backend/JasmineGraphBackendProtocol.h
         src/centralstore/JasmineGraphHashMapCentralStore.h
         src/centralstore/JasmineGraphHashMapDuplicateCentralStore.h
@@ -45,7 +46,6 @@ set(HEADERS src/backend/JasmineGraphBackend.h
         src/server/JasmineGraphInstanceService.h
         src/server/JasmineGraphServer.h
         src/util/Conts.h
-        src/util/PlacesToNodeMapper.h
         src/util/Utils.h
         src/util/dbutil/attributestore_generated.h
         src/util/dbutil/edgestore_generated.h
@@ -113,7 +113,6 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp
         src/server/JasmineGraphInstanceService.cpp
         src/server/JasmineGraphServer.cpp
         src/util/Conts.cpp
-        src/util/PlacesToNodeMapper.cpp
         src/util/Utils.cpp
         src/util/kafka/KafkaCC.cpp
         src/util/kafka/StreamHandler.cpp
@@ -134,6 +133,10 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp
         src/util/dbinterface/DBInterface.cpp
 )
 
+if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+add_compile_options(-DUNIT_TEST)
+endif ()
+
 add_library(JasmineGraphLib ${HEADERS} ${SOURCES})
 add_executable(JasmineGraph main.h main.cpp)
 
@@ -144,6 +147,8 @@ if (CMAKE_ENABLE_DEBUG)
     target_compile_options(JasmineGraph PRIVATE -g)
 endif ()
 
+SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-s,--gc-sections")
+
 #TODO: Need to install the library separately
 target_link_libraries(JasmineGraphLib PRIVATE sqlite3)
 target_link_libraries(JasmineGraphLib PRIVATE pthread)
@@ -155,19 +160,13 @@ target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libflatb
 target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libjsoncpp.so)
 target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libcppkafka.so)
 target_link_libraries(JasmineGraph JasmineGraphLib)
-target_include_directories(JasmineGraph PRIVATE /usr/include/python3)
-
-set(PYTHON_EXECUTABLE "/usr/bin/python3")
-set(PYTHON_INCLUDE_DIR "/usr/include/python3")
-set(PYTHON_LIBRARIES "/usr/lib/x86_64-linux-gnu/libpython3.so")
+target_link_libraries(JasmineGraph curl)
 
-include_directories(${PYTHON_INCLUDE_DIRS})
-include_directories(${PYTHON_DIRECTORIES})
 include_directories(/usr/local/include/yaml-cpp)
-target_link_libraries(JasmineGraphLib PRIVATE ${PYTHON_LIBRARIES})
 target_link_libraries(JasmineGraphLib PRIVATE m)
 target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libkubernetes.so)
 target_link_libraries(JasmineGraphLib PRIVATE yaml-cpp)
+target_link_libraries(JasmineGraphLib PRIVATE curl)
 
 if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
     # Include google test

Dockerfile

@@ -1,10 +1,13 @@
 FROM miyurud/jasminegraph-prerequisites:20240101T095619
+
+RUN apt-get update && apt-get install -y libcurl4-openssl-dev sysstat nmon
+RUN rm -r /usr/lib/python3.8/distutils
+RUN apt-get purge -y libpython3.8-dev python3.8-dev python3.8-distutils libpython3.8 python3.8
+
 ENV HOME="/home/ubuntu"
 ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph"
 
 RUN ln -sf /usr/bin/python3.8 /usr/bin/python3
-RUN ln -sf /usr/include/python3.8 /usr/include/python3
-RUN ln -sf /usr/lib/x86_64-linux-gnu/libpython3.8.so /usr/lib/x86_64-linux-gnu/libpython3.so
 
 WORKDIR "${JASMINEGRAPH_HOME}"
 
@@ -17,6 +20,7 @@ COPY ./build.sh ./build.sh
 COPY ./CMakeLists.txt ./CMakeLists.txt
 COPY ./main.h ./main.h
 COPY ./main.cpp ./main.cpp
+COPY ./globals.h ./globals.h
 COPY ./src ./src
 
 RUN if [ "$DEBUG" = "true" ]; then echo "building in DEBUG mode" && sh build.sh --debug; else sh build.sh; fi
@@ -25,6 +29,7 @@ COPY ./run-docker.sh ./run-docker.sh
 COPY ./src_python ./src_python
 COPY ./conf ./conf
 COPY ./k8s ./k8s
+COPY ./ddl ./ddl
 
 ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"]
 CMD ["bash"]

README.md

@@ -25,10 +25,17 @@ Run JasmineGraph in Kubernetes environment by providing appropriate parameters:
         --PERFORMANCE_DB_PATH "$(pwd)/performancedb" \
         --DATA_PATH "$(pwd)/data" \
         --LOG_PATH "$(pwd)/logs" \
+        --AGGREGATE_PATH "$(pwd)/aggregate" \
         --NO_OF_WORKERS 2 \
-        --ENABLE_NMON false
+        --ENABLE_NMON false \
+        --MAX_COUNT 4
 
->Note: `NO_OF_WORKERS` and `ENABLE_NMON` are optional arguments, and it defaults to `2` and `false` respectively.
+>Note: `NO_OF_WORKERS`, `MAX_COUNT`, and `ENABLE_NMON` are optional arguments, and it defaults to `2`, `4` and `false` respectively.
+
+
+Remove all resources created by JasmineGraph in Kubernetes environment:
+
+    ./start-k8s.sh clean
 
 
 ## 3. Contact Information