Skip to content

Commit

Permalink
Merge branch 'master' into dynamic_triangles_2
Browse files Browse the repository at this point in the history
  • Loading branch information
Ashok12011234 authored Apr 27, 2024
2 parents efeb54e + ea22010 commit 23e3bfe
Show file tree
Hide file tree
Showing 97 changed files with 3,811 additions and 3,559 deletions.
6 changes: 0 additions & 6 deletions .github/actions/jasminegraph-action/action.yml

This file was deleted.

30 changes: 12 additions & 18 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,39 +6,33 @@ on:
pull_request:
branches: [master]

env:
# Customize the build type here (Release, Debug, RelWithDebInfo, etc.)
BUILD_TYPE: Release

jobs:
unit-tests:
runs-on: ubuntu-latest
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip unit tests')}}

steps:
- uses: actions/checkout@v4
with:
ref: ${{github.head_ref}}
fetch-depth: 0
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

- name: Build unit test
run: |
docker build -t jasminegraph .
docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit .
- uses: jupyterhub/action-k3s-helm@v4
with:
docker-enabled: true

- name: Grant permissions for default user
run: kubectl apply -f ./k8s/rbac.yaml

- name: Docker Build
run: docker build -t jasminegraph .

- name: Build unit test
run: docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit .

- name: Run unit test
run: |
mkdir coverage
kubectl apply -f ./.github/workflows/resources/unit-test-conf.yaml
sleep 30
kubectl logs -f jasminegraph-unit-test-pod
chmod +x ./.github/workflows/resources/unit-test.sh
./.github/workflows/resources/unit-test.sh
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v4
Expand All @@ -50,16 +44,15 @@ jobs:

docker-integration-tests:
runs-on: ubuntu-latest
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip docker integration')}}

steps:
- uses: actions/checkout@v4
with:
ref: ${{github.head_ref}}
fetch-depth: 0
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

- name: Docker Build
# Build JasmineGraph docker image
run: docker build -t jasminegraph .

- name: Run Integration Tests
Expand All @@ -69,13 +62,14 @@ jobs:
k8s-integration-tests:
runs-on: ubuntu-latest
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip k8s integration')}}

steps:
- uses: actions/checkout@v4
with:
ref: ${{github.head_ref}}
fetch-depth: 0
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

- uses: jupyterhub/action-k3s-helm@v4
with:
docker-enabled: true
Expand Down
62 changes: 32 additions & 30 deletions .github/workflows/code-lint.yml
Original file line number Diff line number Diff line change
@@ -1,35 +1,37 @@
name: Style Check

on:
push:
branches: [master]
pull_request:
branches: [master]
push:
branches: [master]
pull_request:
branches: [master]

jobs:
style-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- name: Install tools
run: |
sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt
sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint
pip install pylint
- name: hadolint check
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}}
run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 --ignore DL3013 -t warning

- name: shfmt check
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}}
run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci

- name: cpplint check
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}}
run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --recursive .

- name: pylint check
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}}
run: pylint -d C0103,E0401,R0801,R0902,R0913,R0914,R0915,W0603,W0621,W0718 --recursive=y .
style-check:
runs-on: ubuntu-latest
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip style check')}}

steps:
- uses: actions/checkout@v3

- name: Install tools
run: |
sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt
sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint
pip install pylint
- name: hadolint check
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}}
run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 --ignore DL3013 -t warning

- name: shfmt check
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}}
run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci

- name: cpplint check
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}}
run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --recursive .

- name: pylint check
if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}}
run: pylint -d C0103,E0401,R0801,R0902,R0913,R0914,R0915,W0603,W0621,W0718 --recursive=y .
42 changes: 42 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Release

on:
push:
tags:
- 'v*'

env:
IMAGE: "${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}"

jobs:
release:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
with:
ref: ${{github.ref}}
repository: ${{ github.repository }}

- name: Docker Build
run: |
sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${IMAGE}#" ./conf/jasminegraph-server.properties
docker build -t "jasminegraph:temp" .
CONTAINER_ID=$(docker create "jasminegraph:temp")
docker export "${CONTAINER_ID}" | docker import - "jasminegraph:pre-build"
docker build -t "${IMAGE}" -f .github/workflows/resources/Dockerfile .
- name: Docker Minimal Image Build
run: |
sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${IMAGE}-minimal#" ./conf/jasminegraph-server.properties
chmod +x ./minimal/build.sh
./minimal/build.sh
docker tag "jasminegraph:minimal" "${IMAGE}-minimal"
- name: Docker Login
run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }}

- name: Docker Push
run: |
docker push "${IMAGE}"
docker push "${IMAGE}-minimal"
9 changes: 9 additions & 0 deletions .github/workflows/resources/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
FROM jasminegraph:pre-build

ENV HOME="/home/ubuntu"
ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph"

WORKDIR "${JASMINEGRAPH_HOME}"

ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"]
CMD ["bash"]
44 changes: 44 additions & 0 deletions .github/workflows/resources/unit-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#!/bin/bash

# Function to check if pod is in Pending/Running state
pod_status=""
check_pod_status() {
pod_status=$(kubectl get pod jasminegraph-unit-test-pod --no-headers -o custom-columns=":status.phase")

if [[ $pod_status == "Pending" || $pod_status == "ContainerCreating" ]]; then
echo 0
else
echo 1
fi
}

mkdir coverage
kubectl apply -f ./k8s/configs.yaml
kubectl apply -f ./.github/workflows/resources/unit-test-conf.yaml

timeout=300 # Set the timeout in seconds (adjust as needed)
start_time=$(date +%s)

while [[ $(check_pod_status) == "0" ]]; do
sleep 10

current_time=$(date +%s)
elapsed_time=$((current_time - start_time))

if [[ $elapsed_time -ge $timeout ]]; then
echo "Timeout reached. Exiting loop."
break
fi
done

echo "----------------------------- logs -----------------------------"
kubectl logs -f jasminegraph-unit-test-pod

check_pod_status
if [[ $pod_status != "Running" && $pod_status != "Completed" ]]; then
echo "Pod jasminegraph-unit-test-pod is in $pod_status state"
echo "Unit tests failed"
echo "----------------------------- details --------------------------"
kubectl describe pod jasminegraph-unit-test-pod
exit 1
fi
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -84,4 +84,5 @@ __pycache__
*.db
*.db-*

coverage.xml
coverage.xml
*.tmp
21 changes: 10 additions & 11 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ set(CMAKE_CXX_STANDARD 11)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)

set(HEADERS src/backend/JasmineGraphBackend.h
set(HEADERS globals.h
src/backend/JasmineGraphBackend.h
src/backend/JasmineGraphBackendProtocol.h
src/centralstore/JasmineGraphHashMapCentralStore.h
src/centralstore/JasmineGraphHashMapDuplicateCentralStore.h
Expand Down Expand Up @@ -44,7 +45,6 @@ set(HEADERS src/backend/JasmineGraphBackend.h
src/server/JasmineGraphInstanceService.h
src/server/JasmineGraphServer.h
src/util/Conts.h
src/util/PlacesToNodeMapper.h
src/util/Utils.h
src/util/dbutil/attributestore_generated.h
src/util/dbutil/edgestore_generated.h
Expand Down Expand Up @@ -111,7 +111,6 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp
src/server/JasmineGraphInstanceService.cpp
src/server/JasmineGraphServer.cpp
src/util/Conts.cpp
src/util/PlacesToNodeMapper.cpp
src/util/Utils.cpp
src/util/kafka/KafkaCC.cpp
src/util/kafka/StreamHandler.cpp
Expand All @@ -132,6 +131,10 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp
src/util/dbinterface/DBInterface.cpp
)

if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
add_compile_options(-DUNIT_TEST)
endif ()

add_library(JasmineGraphLib ${HEADERS} ${SOURCES})
add_executable(JasmineGraph main.h main.cpp)

Expand All @@ -142,6 +145,8 @@ if (CMAKE_ENABLE_DEBUG)
target_compile_options(JasmineGraph PRIVATE -g)
endif ()

SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-s,--gc-sections")

#TODO: Need to install the library separately
target_link_libraries(JasmineGraphLib PRIVATE sqlite3)
target_link_libraries(JasmineGraphLib PRIVATE pthread)
Expand All @@ -153,19 +158,13 @@ target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libflatb
target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libjsoncpp.so)
target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libcppkafka.so)
target_link_libraries(JasmineGraph JasmineGraphLib)
target_include_directories(JasmineGraph PRIVATE /usr/include/python3)

set(PYTHON_EXECUTABLE "/usr/bin/python3")
set(PYTHON_INCLUDE_DIR "/usr/include/python3")
set(PYTHON_LIBRARIES "/usr/lib/x86_64-linux-gnu/libpython3.so")
target_link_libraries(JasmineGraph curl)

include_directories(${PYTHON_INCLUDE_DIRS})
include_directories(${PYTHON_DIRECTORIES})
include_directories(/usr/local/include/yaml-cpp)
target_link_libraries(JasmineGraphLib PRIVATE ${PYTHON_LIBRARIES})
target_link_libraries(JasmineGraphLib PRIVATE m)
target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libkubernetes.so)
target_link_libraries(JasmineGraphLib PRIVATE yaml-cpp)
target_link_libraries(JasmineGraphLib PRIVATE curl)

if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
# Include google test
Expand Down
9 changes: 7 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
FROM miyurud/jasminegraph-prerequisites:20240101T095619

RUN apt-get update && apt-get install -y libcurl4-openssl-dev sysstat nmon
RUN rm -r /usr/lib/python3.8/distutils
RUN apt-get purge -y libpython3.8-dev python3.8-dev python3.8-distutils libpython3.8 python3.8

ENV HOME="/home/ubuntu"
ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph"

RUN ln -sf /usr/bin/python3.8 /usr/bin/python3
RUN ln -sf /usr/include/python3.8 /usr/include/python3
RUN ln -sf /usr/lib/x86_64-linux-gnu/libpython3.8.so /usr/lib/x86_64-linux-gnu/libpython3.so

WORKDIR "${JASMINEGRAPH_HOME}"

Expand All @@ -17,6 +20,7 @@ COPY ./build.sh ./build.sh
COPY ./CMakeLists.txt ./CMakeLists.txt
COPY ./main.h ./main.h
COPY ./main.cpp ./main.cpp
COPY ./globals.h ./globals.h
COPY ./src ./src

RUN if [ "$DEBUG" = "true" ]; then echo "building in DEBUG mode" && sh build.sh --debug; else sh build.sh; fi
Expand All @@ -25,6 +29,7 @@ COPY ./run-docker.sh ./run-docker.sh
COPY ./src_python ./src_python
COPY ./conf ./conf
COPY ./k8s ./k8s
COPY ./ddl ./ddl

ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"]
CMD ["bash"]
11 changes: 9 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,17 @@ Run JasmineGraph in Kubernetes environment by providing appropriate parameters:
--PERFORMANCE_DB_PATH "$(pwd)/performancedb" \
--DATA_PATH "$(pwd)/data" \
--LOG_PATH "$(pwd)/logs" \
--AGGREGATE_PATH "$(pwd)/aggregate" \
--NO_OF_WORKERS 2 \
--ENABLE_NMON false
--ENABLE_NMON false \
--MAX_COUNT 4

>Note: `NO_OF_WORKERS` and `ENABLE_NMON` are optional arguments, and it defaults to `2` and `false` respectively.
>Note: `NO_OF_WORKERS`, `MAX_COUNT`, and `ENABLE_NMON` are optional arguments, and it defaults to `2`, `4` and `false` respectively.

Remove all resources created by JasmineGraph in Kubernetes environment:

./start-k8s.sh clean


## 3. Contact Information
Expand Down
8 changes: 0 additions & 8 deletions conf/hosts.txt

This file was deleted.

Loading

0 comments on commit 23e3bfe

Please sign in to comment.