diff --git a/.github/actions/jasminegraph-action/action.yml b/.github/actions/jasminegraph-action/action.yml deleted file mode 100644 index d7716e4c..00000000 --- a/.github/actions/jasminegraph-action/action.yml +++ /dev/null @@ -1,6 +0,0 @@ -# action.yml -name: 'Build' -description: 'Build a JasmineGraph docker image' -runs: - using: 'docker' - image: 'docker/Dockerfile' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5822d693..70396db6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,20 +6,22 @@ on: pull_request: branches: [master] -env: - # Customize the build type here (Release, Debug, RelWithDebInfo, etc.) - BUILD_TYPE: Release - jobs: unit-tests: runs-on: ubuntu-latest + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip unit tests')}} steps: - uses: actions/checkout@v4 with: ref: ${{github.head_ref}} - fetch-depth: 0 repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + + - name: Build unit test + run: | + docker build -t jasminegraph . + docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit . + - uses: jupyterhub/action-k3s-helm@v4 with: docker-enabled: true @@ -27,18 +29,10 @@ jobs: - name: Grant permissions for default user run: kubectl apply -f ./k8s/rbac.yaml - - name: Docker Build - run: docker build -t jasminegraph . - - - name: Build unit test - run: docker build -f ./tests/unit/Dockerfile -t jasminegraph:unit . - - name: Run unit test run: | - mkdir coverage - kubectl apply -f ./.github/workflows/resources/unit-test-conf.yaml - sleep 30 - kubectl logs -f jasminegraph-unit-test-pod + chmod +x ./.github/workflows/resources/unit-test.sh + ./.github/workflows/resources/unit-test.sh - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4 @@ -50,16 +44,15 @@ jobs: docker-integration-tests: runs-on: ubuntu-latest + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip docker integration')}} steps: - uses: actions/checkout@v4 with: ref: ${{github.head_ref}} - fetch-depth: 0 repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - name: Docker Build - # Build JasmineGraph docker image run: docker build -t jasminegraph . - name: Run Integration Tests @@ -69,13 +62,14 @@ jobs: k8s-integration-tests: runs-on: ubuntu-latest + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip k8s integration')}} steps: - uses: actions/checkout@v4 with: ref: ${{github.head_ref}} - fetch-depth: 0 repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + - uses: jupyterhub/action-k3s-helm@v4 with: docker-enabled: true diff --git a/.github/workflows/code-lint.yml b/.github/workflows/code-lint.yml index baa0768c..ba796421 100644 --- a/.github/workflows/code-lint.yml +++ b/.github/workflows/code-lint.yml @@ -1,35 +1,37 @@ name: Style Check on: - push: - branches: [master] - pull_request: - branches: [master] + push: + branches: [master] + pull_request: + branches: [master] jobs: - style-check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Install tools - run: | - sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt - sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint - pip install pylint - - - name: hadolint check - if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}} - run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 --ignore DL3013 -t warning - - - name: shfmt check - if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}} - run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci - - - name: cpplint check - if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}} - run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --recursive . - - - name: pylint check - if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}} - run: pylint -d C0103,E0401,R0801,R0902,R0913,R0914,R0915,W0603,W0621,W0718 --recursive=y . + style-check: + runs-on: ubuntu-latest + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip style check')}} + + steps: + - uses: actions/checkout@v3 + + - name: Install tools + run: | + sudo apt-get update && sudo apt-get install -y --no-install-recommends cpplint shfmt + sudo wget https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 -O /bin/hadolint && sudo chmod +x /bin/hadolint + pip install pylint + + - name: hadolint check + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip hadolint Check')}} + run: find . -type f -name '*Dockerfile*' -print0 | xargs -0 hadolint --ignore DL3008 --ignore DL3013 -t warning + + - name: shfmt check + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip shfmt Check')}} + run: find . -type f -name '*.sh' -print0 | xargs -0 shfmt -d -s -i 4 -ci + + - name: cpplint check + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip cpplint Check')}} + run: cpplint --linelength=120 --filter=-build,-runtime,-readability/todo,-whitespace/todo,-readability/casting,-readability/braces,-readability/fn_size,-legal/copyright --recursive . + + - name: pylint check + if: ${{!contains(github.event.pull_request.labels.*.name, 'Skip pylint Check')}} + run: pylint -d C0103,E0401,R0801,R0902,R0913,R0914,R0915,W0603,W0621,W0718 --recursive=y . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..8571c021 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,42 @@ +name: Release + +on: + push: + tags: + - 'v*' + +env: + IMAGE: "${{ secrets.DOCKER_USER }}/jasminegraph:${{ github.ref_name }}" + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{github.ref}} + repository: ${{ github.repository }} + + - name: Docker Build + run: | + sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${IMAGE}#" ./conf/jasminegraph-server.properties + docker build -t "jasminegraph:temp" . + CONTAINER_ID=$(docker create "jasminegraph:temp") + docker export "${CONTAINER_ID}" | docker import - "jasminegraph:pre-build" + docker build -t "${IMAGE}" -f .github/workflows/resources/Dockerfile . + + - name: Docker Minimal Image Build + run: | + sed -i "s#org.jasminegraph.k8s.image=.*#org.jasminegraph.k8s.image=${IMAGE}-minimal#" ./conf/jasminegraph-server.properties + chmod +x ./minimal/build.sh + ./minimal/build.sh + docker tag "jasminegraph:minimal" "${IMAGE}-minimal" + + - name: Docker Login + run: docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_PASS }} + + - name: Docker Push + run: | + docker push "${IMAGE}" + docker push "${IMAGE}-minimal" diff --git a/.github/workflows/resources/Dockerfile b/.github/workflows/resources/Dockerfile new file mode 100644 index 00000000..57878a19 --- /dev/null +++ b/.github/workflows/resources/Dockerfile @@ -0,0 +1,9 @@ +FROM jasminegraph:pre-build + +ENV HOME="/home/ubuntu" +ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph" + +WORKDIR "${JASMINEGRAPH_HOME}" + +ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"] +CMD ["bash"] \ No newline at end of file diff --git a/.github/workflows/resources/unit-test.sh b/.github/workflows/resources/unit-test.sh new file mode 100755 index 00000000..4846ea9b --- /dev/null +++ b/.github/workflows/resources/unit-test.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Function to check if pod is in Pending/Running state +pod_status="" +check_pod_status() { + pod_status=$(kubectl get pod jasminegraph-unit-test-pod --no-headers -o custom-columns=":status.phase") + + if [[ $pod_status == "Pending" || $pod_status == "ContainerCreating" ]]; then + echo 0 + else + echo 1 + fi +} + +mkdir coverage +kubectl apply -f ./k8s/configs.yaml +kubectl apply -f ./.github/workflows/resources/unit-test-conf.yaml + +timeout=300 # Set the timeout in seconds (adjust as needed) +start_time=$(date +%s) + +while [[ $(check_pod_status) == "0" ]]; do + sleep 10 + + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + + if [[ $elapsed_time -ge $timeout ]]; then + echo "Timeout reached. Exiting loop." + break + fi +done + +echo "----------------------------- logs -----------------------------" +kubectl logs -f jasminegraph-unit-test-pod + +check_pod_status +if [[ $pod_status != "Running" && $pod_status != "Completed" ]]; then + echo "Pod jasminegraph-unit-test-pod is in $pod_status state" + echo "Unit tests failed" + echo "----------------------------- details --------------------------" + kubectl describe pod jasminegraph-unit-test-pod + exit 1 +fi diff --git a/.gitignore b/.gitignore index 295a8661..eb4c9980 100644 --- a/.gitignore +++ b/.gitignore @@ -84,4 +84,5 @@ __pycache__ *.db *.db-* -coverage.xml \ No newline at end of file +coverage.xml +*.tmp diff --git a/CMakeLists.txt b/CMakeLists.txt index 95b8804d..3fb89fed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,8 @@ set(CMAKE_CXX_STANDARD 11) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -set(HEADERS src/backend/JasmineGraphBackend.h +set(HEADERS globals.h + src/backend/JasmineGraphBackend.h src/backend/JasmineGraphBackendProtocol.h src/centralstore/JasmineGraphHashMapCentralStore.h src/centralstore/JasmineGraphHashMapDuplicateCentralStore.h @@ -45,7 +46,6 @@ set(HEADERS src/backend/JasmineGraphBackend.h src/server/JasmineGraphInstanceService.h src/server/JasmineGraphServer.h src/util/Conts.h - src/util/PlacesToNodeMapper.h src/util/Utils.h src/util/dbutil/attributestore_generated.h src/util/dbutil/edgestore_generated.h @@ -113,7 +113,6 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp src/server/JasmineGraphInstanceService.cpp src/server/JasmineGraphServer.cpp src/util/Conts.cpp - src/util/PlacesToNodeMapper.cpp src/util/Utils.cpp src/util/kafka/KafkaCC.cpp src/util/kafka/StreamHandler.cpp @@ -134,6 +133,10 @@ set(SOURCES src/backend/JasmineGraphBackend.cpp src/util/dbinterface/DBInterface.cpp ) +if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") +add_compile_options(-DUNIT_TEST) +endif () + add_library(JasmineGraphLib ${HEADERS} ${SOURCES}) add_executable(JasmineGraph main.h main.cpp) @@ -144,6 +147,8 @@ if (CMAKE_ENABLE_DEBUG) target_compile_options(JasmineGraph PRIVATE -g) endif () +SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-s,--gc-sections") + #TODO: Need to install the library separately target_link_libraries(JasmineGraphLib PRIVATE sqlite3) target_link_libraries(JasmineGraphLib PRIVATE pthread) @@ -155,19 +160,13 @@ target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libflatb target_link_libraries(JasmineGraphLib PRIVATE /usr/lib/x86_64-linux-gnu/libjsoncpp.so) target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libcppkafka.so) target_link_libraries(JasmineGraph JasmineGraphLib) -target_include_directories(JasmineGraph PRIVATE /usr/include/python3) - -set(PYTHON_EXECUTABLE "/usr/bin/python3") -set(PYTHON_INCLUDE_DIR "/usr/include/python3") -set(PYTHON_LIBRARIES "/usr/lib/x86_64-linux-gnu/libpython3.so") +target_link_libraries(JasmineGraph curl) -include_directories(${PYTHON_INCLUDE_DIRS}) -include_directories(${PYTHON_DIRECTORIES}) include_directories(/usr/local/include/yaml-cpp) -target_link_libraries(JasmineGraphLib PRIVATE ${PYTHON_LIBRARIES}) target_link_libraries(JasmineGraphLib PRIVATE m) target_link_libraries(JasmineGraphLib PRIVATE /usr/local/lib/libkubernetes.so) target_link_libraries(JasmineGraphLib PRIVATE yaml-cpp) +target_link_libraries(JasmineGraphLib PRIVATE curl) if (CMAKE_BUILD_TYPE STREQUAL "DEBUG") # Include google test diff --git a/Dockerfile b/Dockerfile index 00145b7a..5d29c35e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,13 @@ FROM miyurud/jasminegraph-prerequisites:20240101T095619 + +RUN apt-get update && apt-get install -y libcurl4-openssl-dev sysstat nmon +RUN rm -r /usr/lib/python3.8/distutils +RUN apt-get purge -y libpython3.8-dev python3.8-dev python3.8-distutils libpython3.8 python3.8 + ENV HOME="/home/ubuntu" ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph" RUN ln -sf /usr/bin/python3.8 /usr/bin/python3 -RUN ln -sf /usr/include/python3.8 /usr/include/python3 -RUN ln -sf /usr/lib/x86_64-linux-gnu/libpython3.8.so /usr/lib/x86_64-linux-gnu/libpython3.so WORKDIR "${JASMINEGRAPH_HOME}" @@ -17,6 +20,7 @@ COPY ./build.sh ./build.sh COPY ./CMakeLists.txt ./CMakeLists.txt COPY ./main.h ./main.h COPY ./main.cpp ./main.cpp +COPY ./globals.h ./globals.h COPY ./src ./src RUN if [ "$DEBUG" = "true" ]; then echo "building in DEBUG mode" && sh build.sh --debug; else sh build.sh; fi @@ -25,6 +29,7 @@ COPY ./run-docker.sh ./run-docker.sh COPY ./src_python ./src_python COPY ./conf ./conf COPY ./k8s ./k8s +COPY ./ddl ./ddl ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"] CMD ["bash"] diff --git a/README.md b/README.md index a107ad5f..892cd8bc 100755 --- a/README.md +++ b/README.md @@ -25,10 +25,17 @@ Run JasmineGraph in Kubernetes environment by providing appropriate parameters: --PERFORMANCE_DB_PATH "$(pwd)/performancedb" \ --DATA_PATH "$(pwd)/data" \ --LOG_PATH "$(pwd)/logs" \ + --AGGREGATE_PATH "$(pwd)/aggregate" \ --NO_OF_WORKERS 2 \ - --ENABLE_NMON false + --ENABLE_NMON false \ + --MAX_COUNT 4 ->Note: `NO_OF_WORKERS` and `ENABLE_NMON` are optional arguments, and it defaults to `2` and `false` respectively. +>Note: `NO_OF_WORKERS`, `MAX_COUNT`, and `ENABLE_NMON` are optional arguments, and it defaults to `2`, `4` and `false` respectively. + + +Remove all resources created by JasmineGraph in Kubernetes environment: + + ./start-k8s.sh clean ## 3. Contact Information diff --git a/conf/hosts.txt b/conf/hosts.txt deleted file mode 100644 index 964b511a..00000000 --- a/conf/hosts.txt +++ /dev/null @@ -1,8 +0,0 @@ -#This is the hosts file for Jasminegraph. All the host names on which Jasminegraph workers should get started needs -#to be listed here -localhost -#anuradhak@10.8.100.245 -#192.168.1.8 -#192.168.1.7 -#ubuntu@18.219.24.222 -#anuradha@10.10.28.181 \ No newline at end of file diff --git a/conf/jasminegraph-server.properties b/conf/jasminegraph-server.properties index 454c5ba2..c03de76e 100644 --- a/conf/jasminegraph-server.properties +++ b/conf/jasminegraph-server.properties @@ -24,15 +24,11 @@ org.jasminegraph.server.nworkers=2 #org.jasminegraph.server.npartitions is the number of partitions into which the graph should be partitioned org.jasminegraph.server.npartitions=2 org.jasminegraph.server.streaming.kafka.host=127.0.0.1:9092 -#org.jasminegraph.server.runtime.location is the location where temporary files of JasmineGraph master is kept -org.jasminegraph.server.runtime.location=./tmp org.jasminegraph.worker.path=/var/tmp/jasminegraph/ #Path to keep jasminegraph artifacts in order to copy them to remote locations. If this is not set then the artifacts in the #JASMINEGRAPH_HOME location will be copied instead. #if it is empty please leave a space, else it will cause an issue org.jasminegraph.artifact.path= -#org.jasminegraph.partitioner.metis.bin is the location where the METIS graph partitioner's gpmetis executable is installed -org.jasminegraph.partitioner.metis.bin=/usr/local/bin #The following folder is the location where workers keep their data. #This is the location where the actual data storage takes place in JasmineGraph. org.jasminegraph.server.instance.datafolder=/var/tmp/jasminegraph-localstore @@ -42,9 +38,8 @@ org.jasminegraph.server.instance.trainedmodelfolder=/var/tmp/jasminegraph-locals org.jasminegraph.server.instance.local=/var/tmp org.jasminegraph.server.instance=/var/tmp - #This parameter controls the nmon stat collection enable and disable -org.jasminegraph.server.enable.nmon=false; +org.jasminegraph.server.enable.nmon=true; #This parameter holds the path to nmon stat file org.jasminegraph.server.nmon.file.location=/var/tmp/nmon/ #This parameter holds the number of snapshots to be collected @@ -57,7 +52,6 @@ org.jasminegraph.server.modelDir=/var/tmp/jasminegraph-localstore/models/ #-------------------------------------------------------------------------------- #Federated Learning Parameters #-------------------------------------------------------------------------------- - org.jasminegraph.fl.location=/home/ubuntu/software/jasminegraph/src_python/ org.jasminegraph.fl.dataDir=/home/ubuntu/software/jasminegraph/src_python/data/ org.jasminegraph.fl.weights=/home/ubuntu/software/jasminegraph/src_python/weights/ @@ -76,9 +70,15 @@ org.jasminegraph.fl.organization.file=/home/ubuntu/software/jasminegraph/conf/ho #-------------------------------------------------------------------------------- #Scheduler Specific Parameters -org.jasminegraph.scheduler.enabled=false +org.jasminegraph.scheduler.enabled=true #PerformanceCollector Scheduler Timing. Run once every 120 seconds -org.jasminegraph.scheduler.performancecollector.timing=120 +org.jasminegraph.scheduler.performancecollector.timing=30 + +#-------------------------------------------------------------------------------- +#PerformanceCollector +#-------------------------------------------------------------------------------- +org.jasminegraph.collector.pushgateway=http://192.168.43.135:9091/ +org.jasminegraph.collector.prometheus=http://192.168.43.135:9090/ #-------------------------------------------------------------------------------- #MetaDB information @@ -87,12 +87,6 @@ org.jasminegraph.centralstore.location=/home/tmp/centralstore org.jasminegraph.db.location=./metadb/jasminegraph_meta.db org.jasminegraph.performance.db.location=./performancedb/jasminegraph_performance.db org.jasminegraph.streaming.db.location=/var/tmp/jasminegraph/jasminegraph_streaming.db -#-------------------------------------------------------------------------------- -#Jetty specific parameters - Note that this is for Java 7 -#-------------------------------------------------------------------------------- - -#Java 1.6 Jetty -org.acacia.ui.jetty.home=/home/ubuntu/software/jetty-distribution-8.1.9.v20130131 #-------------------------------------------------------------------------------- #K8S information @@ -101,9 +95,14 @@ org.jasminegraph.k8s.config=/etc/rancher/k3s/k3s.yaml org.jasminegraph.k8s.namespace=default org.jasminegraph.k8s.image=jasminegraph +#-------------------------------------------------------------------------------- +# Auto-scaling +#-------------------------------------------------------------------------------- +org.jasminegraph.autopartition.enabled=false + #-------------------------------------------------------------------------------- #Native store information #-------------------------------------------------------------------------------- #This parameter holds the maximum label size of Node Block -org.jasminegraph.nativestore.max.label.size=43 \ No newline at end of file +org.jasminegraph.nativestore.max.label.size=43 diff --git a/src/metadb/ddl.sql b/ddl/metadb.sql similarity index 100% rename from src/metadb/ddl.sql rename to ddl/metadb.sql diff --git a/src/performancedb/ddl.sql b/ddl/perfdb.sql similarity index 100% rename from src/performancedb/ddl.sql rename to ddl/perfdb.sql diff --git a/src/streamingdb/ddl.sql b/ddl/streamingdb.sql similarity index 100% rename from src/streamingdb/ddl.sql rename to ddl/streamingdb.sql diff --git a/src/util/PlacesToNodeMapper.h b/globals.h similarity index 52% rename from src/util/PlacesToNodeMapper.h rename to globals.h index f51bf887..6adda481 100644 --- a/src/util/PlacesToNodeMapper.h +++ b/globals.h @@ -1,5 +1,5 @@ /** -Copyright 2018 JasmineGraph Team +Copyright 2024 JasmineGraph Team Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -11,22 +11,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -#ifndef JASMINEGRAPH_PLACESTONODEMAPPER_H -#define JASMINEGRAPH_PLACESTONODEMAPPER_H - #include -#include "Utils.h" - -class PlacesToNodeMapper { - private: - static int numberOfWorkers; +#ifndef JASMINEGRAPH__GLOBALS_H_ +#define JASMINEGRAPH__GLOBALS_H_ - public: - static std::string getHost(long placeId); - static std::vector getInstancePorts(long placeId); - static std::vector getInstancePortsList(long placeId); - static std::vector getFileTransferServicePort(long placeId); -}; +#define PROFILE_NATIVE 0 +#define PROFILE_DOCKER 1 +#define PROFILE_K8S 2 +extern int jasminegraph_profile; -#endif // JASMINEGRAPH_PLACESTONODEMAPPER_H +#endif // JASMINEGRAPH__GLOBALS_H_ diff --git a/k8s/configs.yaml b/k8s/configs.yaml new file mode 100644 index 00000000..08e5adfa --- /dev/null +++ b/k8s/configs.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: jasminegraph-config + labels: + application: jasminegraph +data: + pushgateway_address: "${pushgateway_address}" + prometheus_address: "${prometheus_address}" + max_worker_count: "${max_worker_count}" + auto_scaling_enabled: "true" + scale_on_adgr: "false" diff --git a/k8s/master-deployment.yaml b/k8s/master-deployment.yaml index 81e4e020..01b766fc 100644 --- a/k8s/master-deployment.yaml +++ b/k8s/master-deployment.yaml @@ -2,6 +2,8 @@ apiVersion: apps/v1 kind: Deployment metadata: name: jasminegraph-master-deployment + labels: + application: jasminegraph spec: replicas: 1 selector: @@ -27,6 +29,9 @@ spec: - name: jasminegraph-logs persistentVolumeClaim: claimName: jasminegraph-logs-claim + - name: jasminegraph-aggregate + persistentVolumeClaim: + claimName: jasminegraph-aggregate-claim containers: - name: jasminegraph-master image: jasminegraph @@ -53,4 +58,6 @@ spec: - mountPath: /var/tmp/data name: jasminegraph-data - mountPath: /tmp/jasminegraph - name: jasminegraph-logs \ No newline at end of file + name: jasminegraph-logs + - mountPath: /var/tmp/jasminegraph-aggregate + name: jasminegraph-aggregate diff --git a/k8s/master-service.yaml b/k8s/master-service.yaml index b5e3dd31..0cbc8fee 100644 --- a/k8s/master-service.yaml +++ b/k8s/master-service.yaml @@ -3,6 +3,7 @@ kind: Service metadata: name: jasminegraph-master-service labels: + application: jasminegraph service: jasminegraph-master spec: ports: diff --git a/k8s/prometheus.yaml b/k8s/prometheus.yaml new file mode 100644 index 00000000..6e8dca66 --- /dev/null +++ b/k8s/prometheus.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server-conf + labels: + application: jasminegraph + app: prometheus +data: + prometheus.yml: | + global: + scrape_interval: 15s + external_labels: + monitor: 'codelab-monitor' + scrape_configs: + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'pushgateway' + scrape_interval: 2s + static_configs: + - targets: ['${pushgateway_address}'] + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + labels: + application: jasminegraph + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + name: prometheus + labels: + app: prometheus + spec: + containers: + - name: prometheus + image: prom/prometheus + ports: + - containerPort: 9090 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: prometheus-config + mountPath: /etc/prometheus + volumes: + - name: prometheus-config + configMap: + name: prometheus-server-conf + restartPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + labels: + application: jasminegraph + app: prometheus +spec: + ports: + - port: 9090 + targetPort: 9090 + nodePort: 30003 + selector: + app: prometheus + type: NodePort diff --git a/k8s/pushgateway.yaml b/k8s/pushgateway.yaml new file mode 100644 index 00000000..3da961b5 --- /dev/null +++ b/k8s/pushgateway.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pushgateway + labels: + application: jasminegraph + app: pushgateway +spec: + replicas: 1 + selector: + matchLabels: + app: pushgateway + template: + metadata: + name: pushgateway + labels: + app: pushgateway + spec: + containers: + - name: pushgateway + image: prom/pushgateway + ports: + - containerPort: 9091 + imagePullPolicy: IfNotPresent + restartPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: pushgateway + labels: + application: jasminegraph + app: pushgateway +spec: + ports: + - port: 9091 + targetPort: 9091 + selector: + app: pushgateway + \ No newline at end of file diff --git a/k8s/volumes.yaml b/k8s/volumes.yaml index 3faede3c..36e74e76 100644 --- a/k8s/volumes.yaml +++ b/k8s/volumes.yaml @@ -3,6 +3,7 @@ kind: PersistentVolume metadata: name: jasminegraph-metadb labels: + application: jasminegraph type: local spec: storageClassName: manual @@ -21,6 +22,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jasminegraph-metadb-claim + labels: + application: jasminegraph spec: storageClassName: manual accessModes: @@ -35,6 +38,7 @@ kind: PersistentVolume metadata: name: jasminegraph-performancedb labels: + application: jasminegraph type: local spec: storageClassName: manual @@ -53,6 +57,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jasminegraph-performancedb-claim + labels: + application: jasminegraph spec: storageClassName: manual accessModes: @@ -67,6 +73,7 @@ kind: PersistentVolume metadata: name: jasminegraph-data labels: + application: jasminegraph type: local spec: storageClassName: manual @@ -82,6 +89,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jasminegraph-data-claim + labels: + application: jasminegraph spec: storageClassName: manual accessModes: @@ -96,6 +105,7 @@ kind: PersistentVolume metadata: name: jasminegraph-logs labels: + application: jasminegraph type: local spec: storageClassName: manual @@ -111,6 +121,40 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: jasminegraph-logs-claim + labels: + application: jasminegraph +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: jasminegraph-aggregate + labels: + application: jasminegraph + type: local +spec: + storageClassName: manual + capacity: + storage: 1Gi + accessModes: + - ReadWriteMany + hostPath: + path: "${aggregate_path}" + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: jasminegraph-aggregate-claim + labels: + application: jasminegraph spec: storageClassName: manual accessModes: diff --git a/k8s/worker-deployment.yaml b/k8s/worker-deployment.yaml index d4b39e82..ae7a5dca 100644 --- a/k8s/worker-deployment.yaml +++ b/k8s/worker-deployment.yaml @@ -3,6 +3,7 @@ kind: Deployment metadata: name: "jasminegraph-worker-deployment" labels: + application: "jasminegraph" deployment: "jasminegraph-worker" workerId: "" spec: @@ -19,7 +20,10 @@ spec: workerId: "" service: "jasminegraph" spec: - nodeName: "" + volumes: + - name: "jasminegraph-worker-data" + persistentVolumeClaim: + claimName: "jasminegraph-worker-data-claim" containers: - name: "jasminegraph-worker" image: "" @@ -28,7 +32,7 @@ spec: - "--MODE" - "2" - "--HOST_NAME" - - "jasminegraph-worker" + - "" - "--MASTERIP" - "" - "--SERVER_PORT" @@ -44,4 +48,7 @@ spec: value: "" ports: - containerPort: 7777 - - containerPort: 7778 \ No newline at end of file + - containerPort: 7778 + volumeMounts: + - mountPath: "/var/tmp/" + name: "jasminegraph-worker-data" diff --git a/k8s/worker-service.yaml b/k8s/worker-service.yaml index bab2501e..3f897318 100644 --- a/k8s/worker-service.yaml +++ b/k8s/worker-service.yaml @@ -3,6 +3,7 @@ kind: Service metadata: name: jasminegraph-worker-service labels: + application: jasminegraph service: jasminegraph-worker workerId: spec: diff --git a/k8s/worker-volume-claim.yaml b/k8s/worker-volume-claim.yaml new file mode 100644 index 00000000..99edf8fd --- /dev/null +++ b/k8s/worker-volume-claim.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "jasminegraph-worker-data-claim" + labels: + application: jasminegraph +spec: + storageClassName: manual + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/k8s/worker-volume.yaml b/k8s/worker-volume.yaml new file mode 100644 index 00000000..a895d859 --- /dev/null +++ b/k8s/worker-volume.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: "jasminegraph-worker-data" + labels: + application: jasminegraph + type: local +spec: + storageClassName: manual + capacity: + storage: 1Gi + accessModes: + - ReadWriteMany + hostPath: + path: "/var/tmp/worker" \ No newline at end of file diff --git a/main.cpp b/main.cpp index a966ccc9..44a00d3c 100644 --- a/main.cpp +++ b/main.cpp @@ -18,6 +18,8 @@ limitations under the License. #include #include +#include "globals.h" +#include "src/k8s/K8sWorkerController.h" #include "src/server/JasmineGraphInstance.h" #include "src/util/logger/Logger.h" #include "src/util/scheduler/SchedulerService.h" @@ -28,77 +30,92 @@ JasmineGraphInstance *instance; SchedulerService schedulerService; Logger main_logger; -void fnExit3(void) { - delete (server); - puts("Shutting down the server."); -} +#ifndef UNIT_TEST +int jasminegraph_profile = PROFILE_DOCKER; +#endif + +enum args { + PROFILE = 1, + MODE = 2 +}; + +enum master_mode_args { + MASTER_IP = 3, + NUMBER_OF_WORKERS = 4, + WORKER_IPS = 5, + ENABLE_NMON = 6 +}; + +enum worker_mode_args { + HOST_NAME = 3, + MASTER_HOST = 4, + SERVER_PORT = 5, + SERVER_DATA_PORT = 6, + WORKER_ENABLE_NMON = 7 +}; + +void fnExit3(void) { delete (server); } int main(int argc, char *argv[]) { atexit(fnExit3); if (argc <= 1) { - main_logger.log( + main_logger.error( "\"Use argument 1 to start JasmineGraph in Master mode. Use 2 " - " to start as worker", - "error"); + " to start as worker"); return -1; } std::cout << argc << std::endl; - int mode = atoi(argv[2]); + int mode = atoi(argv[args::MODE]); std::string JASMINEGRAPH_HOME = Utils::getJasmineGraphHome(); - std::string profile = argv[1]; // This can be either "docker" or "native" + jasminegraph_profile = strcmp(argv[args::PROFILE], "docker") == 0 ? PROFILE_DOCKER : PROFILE_K8S; std::string enableNmon = "false"; + main_logger.info("Using JASMINE_GRAPH_HOME=" + JASMINEGRAPH_HOME); - main_logger.log("Using JASMINE_GRAPH_HOME", "info"); - std::cout << JASMINEGRAPH_HOME << std::endl; + StatisticCollector::init(); + thread schedulerThread(SchedulerService::startScheduler); if (mode == Conts::JASMINEGRAPH_RUNTIME_PROFILE_MASTER) { - std::string masterIp = argv[3]; - int numberOfWorkers = atoi(argv[4]); - std::string workerIps = argv[5]; - enableNmon = argv[6]; + std::string masterIp = argv[master_mode_args::MASTER_IP]; + int numberOfWorkers = atoi(argv[master_mode_args::NUMBER_OF_WORKERS]); + std::string workerIps = argv[master_mode_args::WORKER_IPS]; + enableNmon = argv[master_mode_args::ENABLE_NMON]; server = JasmineGraphServer::getInstance(); - thread schedulerThread(SchedulerService::startScheduler); - if (profile == Conts::PROFILE_K8S) { - K8sInterface* interface = new K8sInterface(); - masterIp = interface->getMasterIp(); + if (jasminegraph_profile == PROFILE_K8S) { + std::unique_ptr k8sInterface(new K8sInterface()); + masterIp = k8sInterface->getMasterIp(); if (masterIp.empty()) { - masterIp = interface->createJasmineGraphMasterService()->spec->cluster_ip; + masterIp = k8sInterface->createJasmineGraphMasterService()->spec->cluster_ip; } } - server->run(profile, masterIp, numberOfWorkers, workerIps, enableNmon); - - while (server->isRunning()) { - usleep(microseconds); - } + server->run(masterIp, numberOfWorkers, workerIps, enableNmon); schedulerThread.join(); delete server; } else if (mode == Conts::JASMINEGRAPH_RUNTIME_PROFILE_WORKER) { - main_logger.log(to_string(argc), "info"); + main_logger.info(to_string(argc)); - if (argc < 5) { - main_logger.log("Need Four arguments. Use 2 to start as worker", - "info"); + if (argc < 8) { + main_logger.info( + "Need 7 arguments. Use 2 to " + "start as worker"); return -1; } string hostName; - hostName = argv[3]; - std::string masterHost = argv[4]; - int serverPort = atoi(argv[5]); - int serverDataPort = atoi(argv[6]); - enableNmon = argv[7]; + hostName = argv[worker_mode_args::HOST_NAME]; + setenv("HOST_NAME", argv[worker_mode_args::HOST_NAME], 1); + std::string masterHost = argv[worker_mode_args::MASTER_HOST]; + int serverPort = atoi(argv[worker_mode_args::SERVER_PORT]); + setenv("PORT", argv[worker_mode_args::SERVER_PORT], 1); + int serverDataPort = atoi(argv[worker_mode_args::SERVER_DATA_PORT]); + enableNmon = argv[worker_mode_args::WORKER_ENABLE_NMON]; std::cout << "In worker mode" << std::endl; instance = new JasmineGraphInstance(); - instance->start_running(profile, hostName, masterHost, serverPort, serverDataPort, enableNmon); - - while (instance->isRunning()) { - usleep(microseconds); - } + instance->start_running(hostName, masterHost, serverPort, serverDataPort, enableNmon); delete instance; } diff --git a/minimal/Dockerfile.fs b/minimal/Dockerfile.fs new file mode 100644 index 00000000..4849d226 --- /dev/null +++ b/minimal/Dockerfile.fs @@ -0,0 +1,16 @@ +FROM busybox:stable-glibc + +COPY --chmod=755 files.tmp/lib/* /lib/x86_64-linux-gnu/ +COPY --chmod=755 files.tmp/bin/* /bin/ + +RUN mkdir -p /home/ubuntu/software/jasminegraph/conf +RUN mkdir -p /home/ubuntu/software/jasminegraph/k8s +RUN mkdir -p /home/ubuntu/software/jasminegraph/metadb +RUN mkdir -p /home/ubuntu/software/jasminegraph/performancedb +RUN mkdir -p /home/ubuntu/software/jasminegraph/ddl + +COPY --chmod=755 files.tmp/JasmineGraph /home/ubuntu/software/jasminegraph/ +COPY --chmod=755 minimal/run-docker.sh /home/ubuntu/software/jasminegraph/ +COPY --chmod=644 conf/jasminegraph-server.properties /home/ubuntu/software/jasminegraph/conf/ +COPY --chmod=644 k8s/*.yaml /home/ubuntu/software/jasminegraph/k8s/ +COPY --chmod=644 ddl/*.sql /home/ubuntu/software/jasminegraph/ddl/ diff --git a/minimal/Dockerfile.minimal b/minimal/Dockerfile.minimal new file mode 100644 index 00000000..7ddd81d4 --- /dev/null +++ b/minimal/Dockerfile.minimal @@ -0,0 +1,7 @@ +FROM jasminegraph:squash + +ENV HOME="/home/ubuntu" +ENV JASMINEGRAPH_HOME="${HOME}/software/jasminegraph" +WORKDIR "${JASMINEGRAPH_HOME}" +ENTRYPOINT ["/home/ubuntu/software/jasminegraph/run-docker.sh"] +CMD ["ash"] diff --git a/minimal/build.sh b/minimal/build.sh new file mode 100755 index 00000000..1a3aa77f --- /dev/null +++ b/minimal/build.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +set -e + +libs=( + /lib/x86_64-linux-gnu/libcurl.so.4 + /lib/x86_64-linux-gnu/libsqlite3.so.0 + /lib/x86_64-linux-gnu/libfmt.so.8 + /lib/x86_64-linux-gnu/libxerces-c-3.2.so + /lib/x86_64-linux-gnu/libjsoncpp.so.25 + /usr/local/lib/libcppkafka.so.0.4.0 + /usr/local/lib/libkubernetes.so + /lib/x86_64-linux-gnu/libyaml-cpp.so.0.7 + /lib/x86_64-linux-gnu/libstdc++.so.6 + /lib/x86_64-linux-gnu/libgcc_s.so.1 + /lib/x86_64-linux-gnu/libnghttp2.so.14 + /lib/x86_64-linux-gnu/libidn2.so.0 + /lib/x86_64-linux-gnu/librtmp.so.1 + /lib/x86_64-linux-gnu/libssh.so.4 + /lib/x86_64-linux-gnu/libpsl.so.5 + /lib/x86_64-linux-gnu/libssl.so.3 + /lib/x86_64-linux-gnu/libcrypto.so.3 + /lib/x86_64-linux-gnu/libgssapi_krb5.so.2 + /lib/x86_64-linux-gnu/libldap-2.5.so.0 + /lib/x86_64-linux-gnu/liblber-2.5.so.0 + /lib/x86_64-linux-gnu/libzstd.so.1 + /lib/x86_64-linux-gnu/libbrotlidec.so.1 + /lib/x86_64-linux-gnu/libz.so.1 + /lib/x86_64-linux-gnu/libcurl-gnutls.so.4 + /lib/x86_64-linux-gnu/libicuuc.so.70 + /lib/x86_64-linux-gnu/librdkafka.so.1 + /usr/local/lib/libyaml.so + /usr/local/lib/libwebsockets.so.18 + /lib/x86_64-linux-gnu/libunistring.so.2 + /lib/x86_64-linux-gnu/libgnutls.so.30 + /lib/x86_64-linux-gnu/libhogweed.so.6 + /lib/x86_64-linux-gnu/libnettle.so.8 + /lib/x86_64-linux-gnu/libgmp.so.10 + /lib/x86_64-linux-gnu/libkrb5.so.3 + /lib/x86_64-linux-gnu/libk5crypto.so.3 + /lib/x86_64-linux-gnu/libcom_err.so.2 + /lib/x86_64-linux-gnu/libkrb5support.so.0 + /lib/x86_64-linux-gnu/libsasl2.so.2 + /lib/x86_64-linux-gnu/libbrotlicommon.so.1 + /lib/x86_64-linux-gnu/libicudata.so.70 + /lib/x86_64-linux-gnu/liblz4.so.1 + /lib/x86_64-linux-gnu/libp11-kit.so.0 + /lib/x86_64-linux-gnu/libtasn1.so.6 + /lib/x86_64-linux-gnu/libkeyutils.so.1 + /lib/x86_64-linux-gnu/libresolv.so.2 + /lib/x86_64-linux-gnu/libffi.so.8 + /lib/x86_64-linux-gnu/libncurses.so.6 + /usr/local/lib/libmetis.so +) + +bins=( + /usr/bin/nmon + /usr/local/bin/gpmetis +) + +if [ ! -f Dockerfile ]; then + echo 'Run the script from JasmineGraph root' + exit 1 +fi + +docker build -t jasminegraph . + +TEMP_DIR="files.tmp" +mkdir -p "$TEMP_DIR" +cd "$TEMP_DIR" +rm -r * || true +mkdir -p bin +mkdir -p lib + +cont_jasmine="$(docker create jasminegraph)" +for file in "${libs[@]}"; do + echo "Copying $file" + docker cp -L "$cont_jasmine":"$file" lib/ | sed '/Copying from container/d' +done +for file in "${bins[@]}"; do + echo "Copying $file" + docker cp -L "$cont_jasmine":"$file" bin/ | sed '/Copying from container/d' +done +echo 'Copying JasmineGraph' +docker cp "$cont_jasmine":/home/ubuntu/software/jasminegraph/JasmineGraph ./ | sed '/Copying from container/d' +docker rm "$cont_jasmine" + +docker build -t jasminegraph:fs -f ../minimal/Dockerfile.fs .. +cont_fs="$(docker create jasminegraph:fs)" +docker export "$cont_fs" | docker import - jasminegraph:squash +docker rm "$cont_fs" +docker rmi jasminegraph:fs + +docker build -t jasminegraph:minimal -f ../minimal/Dockerfile.minimal . + +docker rmi jasminegraph:squash +cd .. +rm -r "$TEMP_DIR" + +echo 'Created docker image jasminegraph:minimal' diff --git a/minimal/run-docker.sh b/minimal/run-docker.sh new file mode 100755 index 00000000..a3a7715f --- /dev/null +++ b/minimal/run-docker.sh @@ -0,0 +1,71 @@ +#!/bin/ash + +MODE=${MODE} +MASTERIP=${MASTERIP} +WORKERS=${WORKERS} +WORKERIP=${WORKERIP} +HOST_NAME=${HOST_NAME} +SERVER_PORT=${SERVER_PORT} +SERVER_DATA_PORT=${SERVER_DATA_PORT} +ENABLE_NMON=${ENABLE_NMON} +DEBUG=${DEBUG} +PROFILE=${PROFILE} + +if [ -z "$PROFILE" ]; then + PROFILE="k8s" +fi + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + export $param="$2" + echo $1 $2 + fi + + shift +done + +if [ -z "$MODE" ]; then + exit 1 +fi + +if [ $MODE -eq 1 ]; then + if [ "$PROFILE" = "k8s" ]; then + MASTERIP="x" + WORKERIP="x" + fi + if [ -z "$MASTERIP" ]; then + exit 1 + fi + if [ -z "$WORKERS" ]; then + exit 1 + fi + if [ -z "$WORKERIP" ]; then + exit 1 + fi +else + if [ -z "$MASTERIP" ]; then + exit 1 + fi + if [ -z "$SERVER_PORT" ]; then + exit 1 + fi + if [ -z "$SERVER_DATA_PORT" ]; then + exit 1 + fi +fi + +if [ ! -d /tmp/jasminegraph ]; then + mkdir /tmp/jasminegraph +fi + +if [ $MODE -eq 1 ]; then + ./JasmineGraph $PROFILE $MODE $MASTERIP $WORKERS $WORKERIP $ENABLE_NMON +else + ./JasmineGraph $PROFILE $MODE $HOST_NAME $MASTERIP $SERVER_PORT $SERVER_DATA_PORT $ENABLE_NMON +fi + +if [ "$TESTING" = "true" ]; then + chmod -R go+w /tmp/jasminegraph +fi diff --git a/python/.keep b/python/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/samples/ml/single-cluster-fl-unsupervised/.keep b/samples/ml/single-cluster-fl-unsupervised/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/samples/ml/single-cluster-fl/.keep b/samples/ml/single-cluster-fl/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/samples/ml/single-cluster-unsupervised/.keep b/samples/ml/single-cluster-unsupervised/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/src/backend/JasmineGraphBackend.cpp b/src/backend/JasmineGraphBackend.cpp index b7816443..4e526b73 100644 --- a/src/backend/JasmineGraphBackend.cpp +++ b/src/backend/JasmineGraphBackend.cpp @@ -27,17 +27,12 @@ void *backendservicesesion(void *dummyPt) { int connFd = sessionargs->connFd; SQLiteDBInterface *sqLiteDbInterface = sessionargs->sqlite; delete sessionargs; - backend_logger.log("Thread No: " + to_string(pthread_self()), "info"); - char data[301]; - bzero(data, 301); + backend_logger.info("Thread No: " + to_string(pthread_self())); + char data[BACKEND_DATA_LENGTH + 1]; bool loop = false; while (!loop) { - bzero(data, 301); - read(connFd, data, 300); - - string line(data); - backend_logger.log("Command received: " + line, "info"); - line = Utils::trim_copy(line); + string line = Utils::read_str_trim_wrapper(connFd, data, BACKEND_DATA_LENGTH); + backend_logger.info("Command received: " + line); if (line.compare(EXIT_BACKEND) == 0) { write(connFd, EXIT_ACK.c_str(), EXIT_ACK.size()); @@ -47,32 +42,22 @@ void *backendservicesesion(void *dummyPt) { write(connFd, HANDSHAKE_OK.c_str(), HANDSHAKE_OK.size()); write(connFd, "\r\n", 2); - char host[301]; - bzero(host, 301); - read(connFd, host, 300); - string hostname(host); - hostname = Utils::trim_copy(hostname); + string hostname = Utils::read_str_trim_wrapper(connFd, data, BACKEND_DATA_LENGTH); write(connFd, HOST_OK.c_str(), HOST_OK.size()); - backend_logger.log("Hostname of the worker: " + hostname, "info"); + backend_logger.info("Hostname of the worker: " + hostname); } else if (line.compare(ACKNOWLEGE_MASTER) == 0) { int result_wr = write(connFd, WORKER_INFO_SEND.c_str(), WORKER_INFO_SEND.size()); if (result_wr < 0) { - backend_logger.log("Error writing to socket", "error"); + backend_logger.error("Error writing to socket"); } result_wr = write(connFd, "\r\n", 2); if (result_wr < 0) { - backend_logger.log("Error writing to socket", "error"); + backend_logger.error("Error writing to socket"); } // We get the name and the path to graph as a pair separated by |. - char worker_info_data[301]; - bzero(worker_info_data, 301); - string name = ""; - - read(connFd, worker_info_data, 300); - - string worker_info(worker_info_data); + string worker_info = Utils::read_str_trim_wrapper(connFd, data, BACKEND_DATA_LENGTH); worker_info.erase(std::remove(worker_info.begin(), worker_info.end(), '\n'), worker_info.end()); worker_info.erase(std::remove(worker_info.begin(), worker_info.end(), '\r'), worker_info.end()); @@ -85,10 +70,11 @@ void *backendservicesesion(void *dummyPt) { break; } else { - backend_logger.log("Message format not recognized", "error"); + backend_logger.error("Message format not recognized"); + sleep(1); } } - backend_logger.log("Closing thread " + to_string(pthread_self()) + " and connection", "info"); + backend_logger.info("Closing thread " + to_string(pthread_self()) + " and connection"); close(connFd); return NULL; } @@ -111,7 +97,7 @@ int JasmineGraphBackend::run() { listenFd = socket(AF_INET, SOCK_STREAM, 0); if (listenFd < 0) { - backend_logger.log("Cannot open socket", "error"); + backend_logger.error("Cannot open socket"); return 0; } @@ -128,7 +114,7 @@ int JasmineGraphBackend::run() { // bind socket if (bind(listenFd, (struct sockaddr *)&svrAdd, sizeof(svrAdd)) < 0) { - backend_logger.log("Cannot bind on port " + portNo, "error"); + backend_logger.error("Cannot bind on port " + portNo); return 0; } @@ -137,21 +123,22 @@ int JasmineGraphBackend::run() { len = sizeof(clntAdd); while (true) { - backend_logger.log("Backend Listening", "info"); + backend_logger.info("Backend Listening"); // this is where client connects. svr will hang in this mode until client conn int connFd = accept(listenFd, (struct sockaddr *)&clntAdd, &len); if (connFd < 0) { - backend_logger.log("Cannot accept connection", "error"); + backend_logger.error("Cannot accept connection"); continue; } - backend_logger.log("Connection successful", "info"); + backend_logger.info("Connection successful"); backendservicesessionargs *sessionargs = new backendservicesessionargs; sessionargs->sqlite = this->sqlite; sessionargs->connFd = connFd; pthread_t pt; pthread_create(&pt, NULL, backendservicesesion, sessionargs); + pthread_detach(pt); } return 1; diff --git a/src/centralstore/JasmineGraphHashMapCentralStore.cpp b/src/centralstore/JasmineGraphHashMapCentralStore.cpp index 047bd62f..7dc1f634 100644 --- a/src/centralstore/JasmineGraphHashMapCentralStore.cpp +++ b/src/centralstore/JasmineGraphHashMapCentralStore.cpp @@ -58,13 +58,13 @@ bool JasmineGraphHashMapCentralStore::loadGraph() { vertexCount = centralSubgraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } -bool JasmineGraphHashMapCentralStore::loadGraph(std::string fileName) { +bool JasmineGraphHashMapCentralStore::loadGraph(const std::string &edgeStorePath) { bool result = false; - std::string edgeStorePath = fileName; std::ifstream dbFile; dbFile.open(edgeStorePath, std::ios::binary | std::ios::in); @@ -88,6 +88,7 @@ bool JasmineGraphHashMapCentralStore::loadGraph(std::string fileName) { vertexCount = centralSubgraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } @@ -134,10 +135,8 @@ map JasmineGraphHashMapCentralStore::getOutDegreeDistributionHashMap for (map>::iterator it = centralSubgraphMap.begin(); it != centralSubgraphMap.end(); ++it) { long distribution = (it->second).size(); - auto key = it->first; - auto nodes = it->second; - unordered_set neighboursOfNeighbour = nodes; - distributionHashMap.insert(std::make_pair(it->first, distribution)); + unordered_set neighboursOfNeighbour = it->second; + distributionHashMap[it->first] = distribution; } return distributionHashMap; } @@ -145,17 +144,15 @@ map JasmineGraphHashMapCentralStore::getOutDegreeDistributionHashMap map JasmineGraphHashMapCentralStore::getInDegreeDistributionHashMap() { map distributionHashMap; - for (map>::iterator it = centralSubgraphMap.begin(); it != centralSubgraphMap.end(); - ++it) { - unordered_set distribution = it->second; - + for (auto it = centralSubgraphMap.begin(); it != centralSubgraphMap.end(); ++it) { + const auto &distribution = it->second; for (auto itr = distribution.begin(); itr != distribution.end(); ++itr) { - std::map::iterator distMapItr = distributionHashMap.find(*itr); + auto distMapItr = distributionHashMap.find(*itr); if (distMapItr != distributionHashMap.end()) { long previousValue = distMapItr->second; distMapItr->second = previousValue + 1; } else { - distributionHashMap.insert(std::make_pair(*itr, 1)); + distributionHashMap[*itr] = 1; } } } @@ -183,9 +180,8 @@ long JasmineGraphHashMapCentralStore::getVertexCount() { long JasmineGraphHashMapCentralStore::getEdgeCount() { if (edgeCount == 0) { - std::map>::iterator localSubGraphMapIterator; long mapSize = centralSubgraphMap.size(); - for (localSubGraphMapIterator = centralSubgraphMap.begin(); + for (auto localSubGraphMapIterator = centralSubgraphMap.begin(); localSubGraphMapIterator != centralSubgraphMap.end(); localSubGraphMapIterator++) { edgeCount = edgeCount + localSubGraphMapIterator->second.size(); } @@ -212,35 +208,27 @@ void JasmineGraphHashMapCentralStore::toLocalSubGraphMap(const PartEdgeMapStore auto value = entry->value(); const flatbuffers::Vector &vector = *value; unordered_set valueSet(vector.begin(), vector.end()); - centralSubgraphMap.insert(std::make_pair(key, valueSet)); + centralSubgraphMap[key] = valueSet; } } -bool JasmineGraphHashMapCentralStore::storePartEdgeMap(std::map> edgeMap, - const std::string savePath) { - bool result = false; +bool JasmineGraphHashMapCentralStore::storePartEdgeMap(const std::map> &edgeMap, + const std::string &savePath) { flatbuffers::FlatBufferBuilder builder; std::vector> edgeStoreEntriesVector; - std::map>::iterator mapIterator; - for (mapIterator = edgeMap.begin(); mapIterator != edgeMap.end(); mapIterator++) { + for (auto mapIterator = edgeMap.begin(); mapIterator != edgeMap.end(); mapIterator++) { int key = mapIterator->first; - std::vector value = mapIterator->second; - std::vector valueVector(value.begin(), value.end()); - auto flatbufferVector = builder.CreateVector(valueVector); + auto flatbufferVector = builder.CreateVector(mapIterator->second); auto edgeStoreEntry = CreatePartEdgeMapStoreEntry(builder, key, flatbufferVector); edgeStoreEntriesVector.push_back(edgeStoreEntry); } auto flatBuffersEdgeStoreEntriesVector = builder.CreateVectorOfSortedTables(&edgeStoreEntriesVector); - auto edgeStore = CreatePartEdgeMapStore(builder, flatBuffersEdgeStoreEntriesVector); - builder.Finish(edgeStore); flatbuffers::SaveFile(savePath.c_str(), (const char *)builder.GetBufferPointer(), (size_t)builder.GetSize(), true); - result = true; - - return result; + return true; } diff --git a/src/centralstore/JasmineGraphHashMapCentralStore.h b/src/centralstore/JasmineGraphHashMapCentralStore.h index e51a25eb..1c266b83 100644 --- a/src/centralstore/JasmineGraphHashMapCentralStore.h +++ b/src/centralstore/JasmineGraphHashMapCentralStore.h @@ -60,7 +60,7 @@ class JasmineGraphHashMapCentralStore : public JasmineGraphLocalStore { bool loadGraph(); - bool loadGraph(std::string fileName); + bool loadGraph(const std::string &fileName); bool storeGraph(); @@ -80,7 +80,7 @@ class JasmineGraphHashMapCentralStore : public JasmineGraphLocalStore { long getEdgeCount(); - bool storePartEdgeMap(std::map> edgeMap, const std::string savePath); + static bool storePartEdgeMap(const std::map> &edgeMap, const std::string &savePath); }; #endif // JASMINEGRAPH_JASMINEGRAPHHASHMAPCENTRALSTORE_H diff --git a/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp b/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp index fc5a5e78..78605831 100644 --- a/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp +++ b/src/centralstore/JasmineGraphHashMapDuplicateCentralStore.cpp @@ -53,6 +53,7 @@ bool JasmineGraphHashMapDuplicateCentralStore::loadGraph() { vertexCount = centralDuplicateStoreSubgraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } @@ -83,6 +84,7 @@ bool JasmineGraphHashMapDuplicateCentralStore::loadGraph(std::string fileName) { vertexCount = centralDuplicateStoreSubgraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } @@ -131,7 +133,7 @@ map JasmineGraphHashMapDuplicateCentralStore::getOutDegreeDistributi for (map>::iterator it = centralDuplicateStoreSubgraphMap.begin(); it != centralDuplicateStoreSubgraphMap.end(); ++it) { long distribution = (it->second).size(); - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } @@ -186,7 +188,7 @@ void JasmineGraphHashMapDuplicateCentralStore::toLocalSubGraphMap(const PartEdge auto value = entry->value(); const flatbuffers::Vector &vector = *value; unordered_set valueSet(vector.begin(), vector.end()); - centralDuplicateStoreSubgraphMap.insert(std::make_pair(key, valueSet)); + centralDuplicateStoreSubgraphMap[key] = valueSet; } } diff --git a/src/frontend/JasmineGraphFrontEnd.cpp b/src/frontend/JasmineGraphFrontEnd.cpp index 53e640b5..a398b25a 100644 --- a/src/frontend/JasmineGraphFrontEnd.cpp +++ b/src/frontend/JasmineGraphFrontEnd.cpp @@ -15,12 +15,12 @@ limitations under the License. #include +#include #include #include #include -#include // C++ header file for file access +#include #include -#include // C++ header file for printing #include #include #include @@ -55,14 +55,13 @@ using namespace std::chrono; std::atomic highPriorityTaskCount; static int connFd; -static int currentFESession; +static volatile int currentFESession; static bool canCalibrate = true; Logger frontend_logger; std::set processData; -std::mutex aggregateWeightMutex; -std::mutex triangleTreeMutex; std::string stream_topic_name; +static std::string getPartitionCount(std::string path); static void list_command(int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p); static void add_rdf_command(std::string masterIP, int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p); static void add_graph_command(std::string masterIP, int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p); @@ -93,6 +92,19 @@ static void start_remote_worker_command(int connFd, bool *loop_exit_p); static void sla_command(int connFd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, bool *loop_exit_p); +static vector getWorkerClients(SQLiteDBInterface *sqlite) { + const vector &workerList = Utils::getWorkerList(sqlite); + vector workerClients; + for (int i = 0; i < workerList.size(); i++) { + Utils::worker currentWorker = workerList.at(i); + string workerHost = currentWorker.hostname; + int workerPort = atoi(string(currentWorker.port).c_str()); + DataPublisher *workerClient = new DataPublisher(workerPort, workerHost); + workerClients.push_back(workerClient); + } + return workerClients; +} + void *frontendservicesesion(void *dummyPt) { frontendservicesessionargs *sessionargs = (frontendservicesessionargs *)dummyPt; std::string masterIP = sessionargs->masterIP; @@ -101,13 +113,16 @@ void *frontendservicesesion(void *dummyPt) { PerformanceSQLiteDBInterface *perfSqlite = sessionargs->perfSqlite; JobScheduler *jobScheduler = sessionargs->jobScheduler; delete sessionargs; - frontend_logger.info("Thread No: " + to_string(pthread_self())); - frontend_logger.info("Master IP: " + masterIP); - char data[FRONTEND_DATA_LENGTH + 1]; - bzero(data, FRONTEND_DATA_LENGTH + 1); - vector workerList = Utils::getWorkerList(sqlite); - vector workerClients; + if (currentFESession++ > Conts::MAX_FE_SESSIONS) { + if (!Utils::send_str_wrapper(connFd, "JasmineGraph server is busy. Please try again later.")) { + frontend_logger.error("Error writing to socket"); + } + close(connFd); + currentFESession--; + return NULL; + } + char data[FRONTEND_DATA_LENGTH + 1]; // Initiate Thread thread input_stream_handler; // Initiate kafka consumer parameters @@ -118,42 +133,28 @@ void *frontendservicesesion(void *dummyPt) { KafkaConnector *kstream; Partitioner graphPartitioner(numberOfPartitions, 1, spt::Algorithms::HASH); - for (int i = 0; i < workerList.size(); i++) { - Utils::worker currentWorker = workerList.at(i); - string workerHost = currentWorker.hostname; - string workerID = currentWorker.workerID; - int workerPort = atoi(string(currentWorker.port).c_str()); - DataPublisher *workerClient = new DataPublisher(workerPort, workerHost); - workerClients.push_back(workerClient); - } + vector workerClients; + bool workerClientsInitialized = false; + bool loop_exit = false; + int failCnt = 0; while (!loop_exit) { - if (currentFESession == Conts::MAX_FE_SESSIONS + 1) { - currentFESession--; - std::string errorResponse = "Jasminegraph Server is Busy. Please try again later."; - int result_wr = write(connFd, errorResponse.c_str(), errorResponse.length()); - if (result_wr < 0) { - frontend_logger.error("Error writing to socket"); + string line = Utils::read_str_wrapper(connFd, data, FRONTEND_DATA_LENGTH, true); + if (line.empty()) { + failCnt++; + if (failCnt > 4) { + break; } - break; - } - - bzero(data, FRONTEND_DATA_LENGTH + 1); - read(connFd, data, FRONTEND_DATA_LENGTH); - - string line(data); - if (line.compare("\r\n") == 0) { + sleep(1); continue; } + failCnt = 0; + line = Utils::trim_copy(line); frontend_logger.info("Command received: " + line); - if (line.empty()) { - currentFESession--; - break; + continue; } - line = Utils::trim_copy(line); - if (currentFESession > 1) { canCalibrate = false; } else { @@ -162,7 +163,6 @@ void *frontendservicesesion(void *dummyPt) { } if (line.compare(EXIT) == 0) { - currentFESession--; break; } else if (line.compare(LIST) == 0) { list_command(connFd, sqlite, &loop_exit); @@ -179,6 +179,10 @@ void *frontendservicesesion(void *dummyPt) { } else if (line.compare(ADGR_CUST) == 0) { add_graph_cust_command(masterIP, connFd, sqlite, &loop_exit); } else if (line.compare(ADD_STREAM_KAFKA) == 0) { + if (!workerClientsInitialized) { + workerClients = getWorkerClients(sqlite); + workerClientsInitialized = true; + } add_stream_kafka_command(connFd, kafka_server_IP, configs, kstream, input_stream_handler, workerClients, numberOfPartitions, &loop_exit); } else if (line.compare(STOP_STREAM_KAFKA) == 0) { @@ -229,6 +233,7 @@ void *frontendservicesesion(void *dummyPt) { } frontend_logger.info("Closing thread " + to_string(pthread_self()) + " and connection"); close(connFd); + currentFESession--; return NULL; } @@ -302,6 +307,7 @@ int JasmineGraphFrontEnd::run() { sessionargs->jobScheduler = this->jobScheduler; pthread_t pt; pthread_create(&pt, NULL, frontendservicesesion, sessionargs); + pthread_detach(pt); } } @@ -434,20 +440,12 @@ void JasmineGraphFrontEnd::getAndUpdateUploadTime(std::string graphID, SQLiteDBI frontend_logger.info("Upload time updated in the database"); } -JasmineGraphHashMapCentralStore JasmineGraphFrontEnd::loadCentralStore(std::string centralStoreFileName) { - frontend_logger.info("Loading Central Store File : Started " + centralStoreFileName); - JasmineGraphHashMapCentralStore *jasmineGraphHashMapCentralStore = new JasmineGraphHashMapCentralStore(); - jasmineGraphHashMapCentralStore->loadGraph(centralStoreFileName); - frontend_logger.info("Loading Central Store File : Completed"); - return *jasmineGraphHashMapCentralStore; -} - map JasmineGraphFrontEnd::getOutDegreeDistributionHashMap(map> graphMap) { map distributionHashMap; for (map>::iterator it = graphMap.begin(); it != graphMap.end(); ++it) { long distribution = (it->second).size(); - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } @@ -563,6 +561,18 @@ bool JasmineGraphFrontEnd::modelExistsByID(string id, SQLiteDBInterface *sqlite) return result; } +static std::string getPartitionCount(std::string path) { + if (Utils::getJasmineGraphProperty("org.jasminegraph.autopartition.enabled") != "true") { + return ""; + } + ifstream dataFile(path); + size_t edges = std::count(std::istreambuf_iterator(dataFile), std::istreambuf_iterator(), '\n'); + dataFile.close(); + int partCnt = (int)round(pow(edges, 0.2) / 6); + if (partCnt < 2) partCnt = 2; + return to_string(partCnt); +} + static void list_command(int connFd, SQLiteDBInterface *sqlite, bool *loop_exit_p) { std::stringstream ss; std::vector>> v = @@ -679,14 +689,14 @@ static void add_rdf_command(std::string masterIP, int connFd, SQLiteDBInterface GetConfig appConfig; appConfig.readConfigFile(path, newGraphID); - MetisPartitioner *metisPartitioner = new MetisPartitioner(sqlite); + MetisPartitioner metisPartitioner(sqlite); vector> fullFileList; string input_file_path = Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(newGraphID) + "/" + to_string(newGraphID); - metisPartitioner->loadDataSet(input_file_path, newGraphID); + metisPartitioner.loadDataSet(input_file_path, newGraphID); - metisPartitioner->constructMetisFormat(Conts::GRAPH_TYPE_RDF); - fullFileList = metisPartitioner->partitioneWithGPMetis(""); + metisPartitioner.constructMetisFormat(Conts::GRAPH_TYPE_RDF); + fullFileList = metisPartitioner.partitioneWithGPMetis(""); JasmineGraphServer *server = JasmineGraphServer::getInstance(); server->uploadGraphLocally(newGraphID, Conts::GRAPH_WITH_ATTRIBUTES, fullFileList, masterIP); Utils::deleteDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(newGraphID)); @@ -750,9 +760,7 @@ static void add_graph_command(std::string masterIP, int connFd, SQLiteDBInterfac name = strArr[0]; path = strArr[1]; - if (strArr.size() == 3) { - partitionCount = strArr[2]; - } + partitionCount = getPartitionCount(path); if (JasmineGraphFrontEnd::graphExists(path, sqlite)) { frontend_logger.error("Graph exists"); @@ -769,27 +777,23 @@ static void add_graph_command(std::string masterIP, int connFd, SQLiteDBInterfac name + "\", \"" + path + "\", \"" + uploadStartTime + "\", \"\",\"" + to_string(Conts::GRAPH_STATUS::LOADING) + "\", \"\", \"\", \"\")"; int newGraphID = sqlite->runInsert(sqlStatement); - MetisPartitioner *partitioner = new MetisPartitioner(sqlite); + MetisPartitioner partitioner(sqlite); vector> fullFileList; - partitioner->loadDataSet(path, newGraphID); - int result = partitioner->constructMetisFormat(Conts::GRAPH_TYPE_NORMAL); + partitioner.loadDataSet(path, newGraphID); + int result = partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL); if (result == 0) { - string reformattedFilePath = partitioner->reformatDataSet(path, newGraphID); - partitioner->loadDataSet(reformattedFilePath, newGraphID); - partitioner->constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); - fullFileList = partitioner->partitioneWithGPMetis(partitionCount); + string reformattedFilePath = partitioner.reformatDataSet(path, newGraphID); + partitioner.loadDataSet(reformattedFilePath, newGraphID); + partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); + fullFileList = partitioner.partitioneWithGPMetis(partitionCount); } else { - fullFileList = partitioner->partitioneWithGPMetis(partitionCount); + fullFileList = partitioner.partitioneWithGPMetis(partitionCount); } frontend_logger.info("Upload done"); JasmineGraphServer *server = JasmineGraphServer::getInstance(); server->uploadGraphLocally(newGraphID, Conts::GRAPH_TYPE_NORMAL, fullFileList, masterIP); Utils::deleteDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(newGraphID)); - string workerCountQuery = "select count(*) from worker"; - std::vector>> results = sqlite->runSelect(workerCountQuery); - string workerCount = results[0][0].second; - int nWorkers = atoi(workerCount.c_str()); JasmineGraphFrontEnd::getAndUpdateUploadTime(to_string(newGraphID), sqlite); int result_wr = write(connFd, DONE.c_str(), DONE.size()); if (result_wr < 0) { @@ -943,19 +947,18 @@ static void add_graph_cust_command(std::string masterIP, int connFd, SQLiteDBInt name + "\", \"" + edgeListPath + "\", \"" + uploadStartTime + "\", \"\",\"" + to_string(Conts::GRAPH_STATUS::LOADING) + "\", \"\", \"\", \"\")"; int newGraphID = sqlite->runInsert(sqlStatement); - MetisPartitioner *partitioner = new MetisPartitioner(sqlite); + MetisPartitioner partitioner(sqlite); vector> fullFileList; - partitioner->loadContentData(attributeListPath, graphAttributeType, newGraphID, attrDataType); - partitioner->loadDataSet(edgeListPath, newGraphID); - int result = partitioner->constructMetisFormat(Conts::GRAPH_TYPE_NORMAL); + partitioner.loadContentData(attributeListPath, graphAttributeType, newGraphID, attrDataType); + partitioner.loadDataSet(edgeListPath, newGraphID); + int result = partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL); if (result == 0) { - string reformattedFilePath = partitioner->reformatDataSet(edgeListPath, newGraphID); - partitioner->loadDataSet(reformattedFilePath, newGraphID); - partitioner->constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); - fullFileList = partitioner->partitioneWithGPMetis(""); - } else { - fullFileList = partitioner->partitioneWithGPMetis(""); + string reformattedFilePath = partitioner.reformatDataSet(edgeListPath, newGraphID); + partitioner.loadDataSet(reformattedFilePath, newGraphID); + partitioner.constructMetisFormat(Conts::GRAPH_TYPE_NORMAL_REFORMATTED); } + fullFileList = partitioner.partitioneWithGPMetis(""); + // Graph type should be changed to identify graphs with attributes // because this graph type has additional attribute files to be uploaded JasmineGraphServer *server = JasmineGraphServer::getInstance(); @@ -1171,7 +1174,6 @@ static void add_stream_kafka_command(int connFd, std::string &kafka_server_IP, c // reading kafka_server IP from the given file. std::vector::iterator it; vector vec = Utils::getFileContent(file_path_s); - it = vec.begin(); for (it = vec.begin(); it < vec.end(); it++) { std::string item = *it; if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { @@ -1219,14 +1221,14 @@ static void add_stream_kafka_command(int connFd, std::string &kafka_server_IP, c } // create kafka consumer and graph partitioner kstream = new KafkaConnector(configs); -// Create the Partitioner object. + // Create the Partitioner object. Partitioner graphPartitioner(numberOfPartitions, 0, spt::Algorithms::FENNEL); -// Create the KafkaConnector object. + // Create the KafkaConnector object. kstream = new KafkaConnector(configs); // Subscribe to the Kafka topic. kstream->Subscribe(topic_name_s); -// Create the StreamHandler object. - StreamHandler* stream_handler = new StreamHandler(kstream, numberOfPartitions, workerClients); + // Create the StreamHandler object. + StreamHandler *stream_handler = new StreamHandler(kstream, numberOfPartitions, workerClients); frontend_logger.info("Start listening to " + topic_name_s); input_stream_handler_thread = thread(&StreamHandler::listen_to_kafka_topic, stream_handler); @@ -1381,14 +1383,17 @@ static void triangles_command(std::string masterIP, int connFd, SQLiteDBInterfac int threadPriority = std::atoi(priority.c_str()); + static volatile int reqCounter = 0; + string reqId = to_string(reqCounter++); + frontend_logger.info("Started processing request " + reqId); auto begin = chrono::high_resolution_clock::now(); JobRequest jobDetails; jobDetails.setJobId(std::to_string(uniqueId)); jobDetails.setJobType(TRIANGLES); - long graphSLA; - // All high priority threads will be set the same high priority level + long graphSLA = -1; // This prevents auto calibration for priority=1 (=default priority) if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { + // All high priority threads will be set the same high priority level threadPriority = Conts::HIGH_PRIORITY_DEFAULT_VALUE; graphSLA = JasmineGraphFrontEnd::getSLAForGraphId(sqlite, perfSqlite, graph_id, TRIANGLES, Conts::SLA_CATEGORY::LATENCY); @@ -1448,8 +1453,8 @@ static void triangles_command(std::string masterIP, int connFd, SQLiteDBInterfac auto end = chrono::high_resolution_clock::now(); auto dur = end - begin; auto msDuration = std::chrono::duration_cast(dur).count(); - frontend_logger.info("Triangle Count: " + triangleCount + " Time Taken: " + to_string(msDuration) + - " milliseconds"); + frontend_logger.info("Req: " + reqId + " Triangle Count: " + triangleCount + + " Time Taken: " + to_string(msDuration) + " milliseconds"); result_wr = write(connFd, triangleCount.c_str(), triangleCount.length()); if (result_wr < 0) { frontend_logger.error("Error writing to socket"); @@ -2047,9 +2052,9 @@ static void page_rank_command(std::string masterIP, int connFd, SQLiteDBInterfac jobDetails.setJobId(std::to_string(uniqueId)); jobDetails.setJobType(PAGE_RANK); - long graphSLA; - // All high priority threads will be set the same high priority level + long graphSLA = -1; // This prevents auto calibration for priority=1 (=default priority) if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { + // All high priority threads will be set the same high priority level threadPriority = Conts::HIGH_PRIORITY_DEFAULT_VALUE; graphSLA = JasmineGraphFrontEnd::getSLAForGraphId(sqlite, perfSqlite, graphID, PAGE_RANK, Conts::SLA_CATEGORY::LATENCY); @@ -2110,8 +2115,7 @@ static void page_rank_command(std::string masterIP, int connFd, SQLiteDBInterfac auto end = chrono::high_resolution_clock::now(); auto dur = end - begin; auto msDuration = std::chrono::duration_cast(dur).count(); - frontend_logger.info("PageRank Time Taken : " + to_string(msDuration) + - " milliseconds"); + frontend_logger.info("PageRank Time Taken : " + to_string(msDuration) + " milliseconds"); result_wr = write(connFd, DONE.c_str(), FRONTEND_COMMAND_LENGTH); if (result_wr < 0) { @@ -2329,7 +2333,6 @@ static void start_remote_worker_command(int connFd, bool *loop_exit_p) { string host = ""; string port = ""; string dataPort = ""; - string profile = ""; string masterHost = ""; string enableNmon = ""; @@ -2348,11 +2351,10 @@ static void start_remote_worker_command(int connFd, bool *loop_exit_p) { host = strArr[0]; port = strArr[1]; dataPort = strArr[2]; - profile = strArr[3]; masterHost = strArr[4]; enableNmon = strArr[5]; - JasmineGraphServer::spawnNewWorker(host, port, dataPort, profile, masterHost, enableNmon); + JasmineGraphServer::spawnNewWorker(host, port, dataPort, masterHost, enableNmon); } static void sla_command(int connFd, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfSqlite, diff --git a/src/frontend/JasmineGraphFrontEnd.h b/src/frontend/JasmineGraphFrontEnd.h index 5d318c1e..ba542be7 100644 --- a/src/frontend/JasmineGraphFrontEnd.h +++ b/src/frontend/JasmineGraphFrontEnd.h @@ -38,7 +38,6 @@ limitations under the License. #include "../metadb/SQLiteDBInterface.h" #include "../performancedb/PerformanceSQLiteDBInterface.h" #include "../query/algorithms/triangles/Triangles.h" -#include "../util/PlacesToNodeMapper.h" #include "core/scheduler/JobScheduler.h" class JasmineGraphHashMapCentralStore; @@ -67,8 +66,6 @@ class JasmineGraphFrontEnd { static bool isGraphActiveAndTrained(std::string graphID, SQLiteDBInterface *sqlite); - static JasmineGraphHashMapCentralStore loadCentralStore(std::string centralStoreFileName); - static map getOutDegreeDistributionHashMap(map> graphMap); static bool isGraphActive(string graphID, SQLiteDBInterface *sqlite); @@ -80,9 +77,6 @@ class JasmineGraphFrontEnd { static int getRunningHighPriorityTaskCount(); static bool areRunningJobsForSameGraph(); - static std::vector> fileCombinations; - static std::map combinationWorkerMap; - static std::map>> triangleTree; std::map> *streamsState; std::map streamingThreads; diff --git a/src/frontend/core/executor/AbstractExecutor.cpp b/src/frontend/core/executor/AbstractExecutor.cpp index 33eec575..28d5eb3f 100644 --- a/src/frontend/core/executor/AbstractExecutor.cpp +++ b/src/frontend/core/executor/AbstractExecutor.cpp @@ -12,6 +12,7 @@ limitations under the License. */ #include "AbstractExecutor.h" + #include "../../../performance/metrics/PerformanceUtil.h" AbstractExecutor::AbstractExecutor(JobRequest jobRequest) { this->request = jobRequest; } @@ -87,8 +88,7 @@ int AbstractExecutor::collectPerformaceData(PerformanceSQLiteDBInterface *perDB, if (!(isMaster.find("true") != std::string::npos || host == "localhost" || host.compare(masterIP) == 0)) { PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization( - host, atoi(serverPort.c_str()), isHostReporter, "false", - placeId, elapsedTime, masterIP); + host, atoi(serverPort.c_str()), isHostReporter, "false", placeId, elapsedTime, masterIP); } } @@ -108,8 +108,8 @@ int AbstractExecutor::collectPerformaceData(PerformanceSQLiteDBInterface *perDB, } } - PerformanceUtil::updateRemoteResourceConsumption(perDB, graphId, partitionCount, placeList, - slaCategoryId, masterIP); + PerformanceUtil::updateRemoteResourceConsumption(perDB, graphId, partitionCount, placeList, slaCategoryId, + masterIP); PerformanceUtil::updateResourceConsumption(perDB, graphId, partitionCount, placeList, slaCategoryId); return 0; diff --git a/src/frontend/core/executor/impl/PageRankExecutor.cpp b/src/frontend/core/executor/impl/PageRankExecutor.cpp index d4d40670..3e1524d1 100644 --- a/src/frontend/core/executor/impl/PageRankExecutor.cpp +++ b/src/frontend/core/executor/impl/PageRankExecutor.cpp @@ -20,8 +20,7 @@ Logger pageRank_logger; PageRankExecutor::PageRankExecutor() {} -PageRankExecutor::PageRankExecutor(SQLiteDBInterface *db, PerformanceSQLiteDBInterface *perfDb, - JobRequest jobRequest) { +PageRankExecutor::PageRankExecutor(SQLiteDBInterface *db, PerformanceSQLiteDBInterface *perfDb, JobRequest jobRequest) { this->sqlite = db; this->perfDB = perfDb; this->request = jobRequest; @@ -72,8 +71,7 @@ void PageRankExecutor::execute() { processStatusMutex.unlock(); } - pageRank_logger.log( - "###PAGERANK-EXECUTOR### Started with graph ID : " + graphId + " Master IP : " + masterIP, "info"); + pageRank_logger.info("###PAGERANK-EXECUTOR### Started with graph ID : " + graphId + " Master IP : " + masterIP); int partitionCount = 0; std::vector> intermRes; @@ -81,52 +79,51 @@ void PageRankExecutor::execute() { auto begin = chrono::high_resolution_clock::now(); - std::map graphPartitionedHosts = - JasmineGraphServer::getWorkerPartitions(graphId); - string partition; + std::map graphPartitionedHosts = + JasmineGraphServer::getGraphPartitionedHosts(graphId); string host; int port; int dataPort; std::string workerList; - std::map::iterator workerIter; - for (workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { - JasmineGraphServer::workerPartition workerPartition = workerIter->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + for (auto workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { + JasmineGraphServer::workerPartitions workerPartition = workerIter->second; + host = workerIter->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; + for (auto partitionIterator = workerPartition.partitionID.begin(); + partitionIterator != workerPartition.partitionID.end(); partitionIterator++) { + std::string partition = *partitionIterator; + workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } - - workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } workerList.pop_back(); pageRank_logger.info("Worker list " + workerList); - for (workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { - JasmineGraphServer::workerPartition workerPartition = workerIter->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; + for (auto workerIter = graphPartitionedHosts.begin(); workerIter != graphPartitionedHosts.end(); workerIter++) { + JasmineGraphServer::workerPartitions workerPartition = workerIter->second; + host = workerIter->first; port = workerPartition.port; dataPort = workerPartition.dataPort; - intermRes.push_back(std::async( - std::launch::async, PageRankExecutor::doPageRank, graphId, alpha, - iterations, partition, host, port, dataPort, workerList)); + for (auto partitionIterator = workerPartition.partitionID.begin(); + partitionIterator != workerPartition.partitionID.end(); partitionIterator++) { + std::string partition = *partitionIterator; + intermRes.push_back(std::async(std::launch::async, PageRankExecutor::doPageRank, graphId, alpha, iterations, + partition, host, port, dataPort, workerList)); + } } PerformanceUtil::init(); std::string query = - "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " - "graph_sla.graph_id='" + - graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + - "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + - "' and sla_category.command='" + PAGE_RANK + "';"; + "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " + "graph_sla.graph_id='" + + graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + + "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + PAGE_RANK + + "';"; std::vector>> queryResults = perfDB->runSelect(query); @@ -138,9 +135,9 @@ void PageRankExecutor::execute() { canCalibrate = false; } } else { - pageRank_logger.log("###PAGERANK-EXECUTOR### Inserting initial record for SLA ", "info"); + pageRank_logger.info("###PAGERANK-EXECUTOR### Inserting initial record for SLA "); Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, PAGE_RANK, Conts::SLA_CATEGORY::LATENCY); - statResponse.push_back(std::async(std::launch::async, collectPerformaceData, perfDB, + statResponse.push_back(std::async(std::launch::async, AbstractExecutor::collectPerformaceData, perfDB, graphId.c_str(), PAGE_RANK, Conts::SLA_CATEGORY::LATENCY, partitionCount, masterIP, autoCalibrate)); isStatCollect = true; @@ -150,8 +147,7 @@ void PageRankExecutor::execute() { futureCall.get(); } - pageRank_logger.info( - "###PAGERANK-EXECUTOR### Getting PageRank : Completed"); + pageRank_logger.info("###PAGERANK-EXECUTOR### Getting PageRank : Completed"); workerResponded = true; JobResponse jobResponse; @@ -175,8 +171,7 @@ void PageRankExecutor::execute() { } processStatusMutex.lock(); - std::set::iterator processCompleteIterator; - for (processCompleteIterator = processData.begin(); processCompleteIterator != processData.end(); + for (auto processCompleteIterator = processData.begin(); processCompleteIterator != processData.end(); ++processCompleteIterator) { ProcessInfo processInformation = *processCompleteIterator; @@ -193,133 +188,133 @@ int PageRankExecutor::getUid() { return ++uid; } -void PageRankExecutor::doPageRank(std::string graphID, double alpha, int iterations, string partition, - string host, int port, int dataPort, std::string workerList) { - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } +void PageRankExecutor::doPageRank(std::string graphID, double alpha, int iterations, string partition, string host, + int port, int dataPort, std::string workerList) { + if (host.find('@') != std::string::npos) { + host = Utils::split(host, '@')[1]; + } - int sockfd; - char data[DATA_BUFFER_SIZE]; - struct sockaddr_in serv_addr; - struct hostent *server; + int sockfd; + char data[DATA_BUFFER_SIZE]; + struct sockaddr_in serv_addr; + struct hostent *server; - sockfd = socket(AF_INET, SOCK_STREAM, 0); + sockfd = socket(AF_INET, SOCK_STREAM, 0); - if (sockfd < 0) { - pageRank_logger.error("Cannot create socket"); - return; - } - server = gethostbyname(host.c_str()); - if (server == NULL) { - pageRank_logger.error("ERROR, no host named " + host); - return; - } + if (sockfd < 0) { + pageRank_logger.error("Cannot create socket"); + return; + } + server = gethostbyname(host.c_str()); + if (server == NULL) { + pageRank_logger.error("ERROR, no host named " + host); + return; + } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - pageRank_logger.error("Error connecting to socket"); - return; - } + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + pageRank_logger.error("Error connecting to socket"); + return; + } - if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::PAGE_RANK)) { - pageRank_logger.error("Error writing to socket"); - return; - } - pageRank_logger.info("Sent : " + JasmineGraphInstanceProtocol::PAGE_RANK); - - string response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::PAGE_RANK)) { + pageRank_logger.error("Error writing to socket"); + return; + } + pageRank_logger.info("Sent : " + JasmineGraphInstanceProtocol::PAGE_RANK); - if (!Utils::send_str_wrapper(sockfd, graphID)) { - pageRank_logger.error("Error writing to socket"); - return; - } - pageRank_logger.info("Sent : Graph ID " + graphID); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + string response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } - if (!Utils::send_str_wrapper(sockfd, partition)) { - pageRank_logger.error("Error writing to socket"); - return; - } - pageRank_logger.info("Sent : Partition ID " + partition); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + if (!Utils::send_str_wrapper(sockfd, graphID)) { + pageRank_logger.error("Error writing to socket"); + return; + } + pageRank_logger.info("Sent : Graph ID " + graphID); - if (!Utils::send_str_wrapper(sockfd, workerList)) { - pageRank_logger.error("Error writing to socket"); - } - pageRank_logger.info("Sent : Host List "); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } - long graphVertexCount = JasmineGraphServer::getGraphVertexCount(graphID); - if (!Utils::send_str_wrapper(sockfd, std::to_string(graphVertexCount))) { - pageRank_logger.error("Error writing to socket"); - } - pageRank_logger.info("graph vertex count: " + std::to_string(graphVertexCount)); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + if (!Utils::send_str_wrapper(sockfd, partition)) { + pageRank_logger.error("Error writing to socket"); + return; + } + pageRank_logger.info("Sent : Partition ID " + partition); - if (!Utils::send_str_wrapper(sockfd, std::to_string(alpha))) { - pageRank_logger.error("Error writing to socket"); - return; - } - pageRank_logger.info("PageRank alpha value sent : " + std::to_string(alpha)); - - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } - if (!Utils::send_str_wrapper(sockfd, std::to_string(iterations))) { - pageRank_logger.error("Error writing to socket"); - return; - } + if (!Utils::send_str_wrapper(sockfd, workerList)) { + pageRank_logger.error("Error writing to socket"); + } + pageRank_logger.info("Sent : Host List "); - response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); - } else { - pageRank_logger.error("Error reading from socket"); - return; - } + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } + + long graphVertexCount = JasmineGraphServer::getGraphVertexCount(graphID); + if (!Utils::send_str_wrapper(sockfd, std::to_string(graphVertexCount))) { + pageRank_logger.error("Error writing to socket"); + } + pageRank_logger.info("graph vertex count: " + std::to_string(graphVertexCount)); + + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } + + if (!Utils::send_str_wrapper(sockfd, std::to_string(alpha))) { + pageRank_logger.error("Error writing to socket"); + return; + } + pageRank_logger.info("PageRank alpha value sent : " + std::to_string(alpha)); + + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } + + if (!Utils::send_str_wrapper(sockfd, std::to_string(iterations))) { + pageRank_logger.error("Error writing to socket"); + return; + } + + response = Utils::read_str_trim_wrapper(sockfd, data, FRONTEND_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { + pageRank_logger.info("Received : " + JasmineGraphInstanceProtocol::OK); + } else { + pageRank_logger.error("Error reading from socket"); + return; + } return; } diff --git a/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp b/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp index 269d37ab..a4b70de9 100644 --- a/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/StreamingTriangleCountExecutor.cpp @@ -567,7 +567,7 @@ std::vector> StreamingTriangleCountExecutor::getWorkerCombin std::vector workerIdVector(workerIdSet.begin(), workerIdSet.end()); - std::vector> workerIdCombination = getCombinations(workerIdVector); + std::vector> workerIdCombination = AbstractExecutor::getCombinations(workerIdVector); return workerIdCombination; } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp index f2abe8e0..2c182d54 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.cpp +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.cpp @@ -13,17 +13,33 @@ limitations under the License. #include "TriangleCountExecutor.h" +#include +#include + +#include "../../../../../globals.h" +#include "../../../../k8s/K8sWorkerController.h" + using namespace std::chrono; Logger triangleCount_logger; -std::vector> TriangleCountExecutor::fileCombinations; -std::map TriangleCountExecutor::combinationWorkerMap; -std::map>> TriangleCountExecutor::triangleTree; bool isStatCollect = false; -std::mutex fileCombinationMutex; std::mutex processStatusMutex; std::mutex responseVectorMutex; +static std::mutex fileCombinationMutex; +static std::mutex aggregateWeightMutex; +static std::mutex schedulerMutex; + +static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, + std::string aggregatorPort, std::string masterIP, std::string fileType, + std::string fileName); +static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, + int threadPriority, + const std::map> &partitionMap); +static int updateTriangleTreeAndGetTriangleCount( + const std::vector &triangles, + std::unordered_map>> *triangleTree_p, + std::mutex *triangleTreeMutex_p); TriangleCountExecutor::TriangleCountExecutor() {} @@ -34,13 +50,391 @@ TriangleCountExecutor::TriangleCountExecutor(SQLiteDBInterface *db, PerformanceS this->request = jobRequest; } +static void allocate(int p, string w, std::map &alloc, std::set &remain, + std::map> &p_avail, std::map &loads) { + alloc[p] = w; + remain.erase(p); + p_avail.erase(p); + loads[w]++; + if (loads[w] >= 3) { + for (auto it = p_avail.begin(); it != p_avail.end(); it++) { + auto &ws = it->second; + auto itr = std::find(ws.begin(), ws.end(), w); + if (itr != ws.end()) { + ws.erase(itr); + } + } + } +} + +static int get_min_partition(std::set &remain, std::map> &p_avail) { + int p0 = *remain.begin(); + size_t minimum = 1000000000; + for (auto it = remain.begin(); it != remain.end(); it++) { + int partition = *it; + auto &workers = p_avail[partition]; + if (workers.size() > 0 && workers.size() < minimum) { + minimum = workers.size(); + p0 = partition; + } + } + return p0; +} + +static const std::vector LOAD_PREFERENCE = {2, 3, 1, 0}; + +static int alloc_plan(std::map &alloc, std::set &remain, std::map> &p_avail, + std::map &loads) { + for (bool done = false; !done;) { + string w = ""; + done = true; + int p; + for (auto it = remain.begin(); it != remain.end(); it++) { + p = *it; + if (p_avail[p].size() == 1) { + w = p_avail[p][0]; + done = false; + break; + } + } + if (!w.empty()) allocate(p, w, alloc, remain, p_avail, loads); + } + if (remain.empty()) return 0; + int p0 = get_min_partition(remain, p_avail); + auto &ws = p_avail[p0]; + if (ws.empty()) return (int)remain.size(); + sort(ws.begin(), ws.end(), [&loads](string &w1, string &w2) { + return LOAD_PREFERENCE[loads[w1]] > LOAD_PREFERENCE[loads[w2]]; + }); // load=1 goes first and load=3 goes last. The order is 1,0,2,3 for 4 cores. + struct best_alloc { + std::map alloc; + std::set remain; + std::map> p_avail; + std::map loads; + }; + int best_rem = remain.size(); + struct best_alloc best = {.alloc = alloc, .remain = remain, .p_avail = p_avail, .loads = loads}; + for (auto it = ws.begin(); it != ws.end(); it++) { + string w = *it; + auto alloc2 = alloc; // need copy => do not copy reference + auto remain2 = remain; // need copy => do not copy reference + auto p_avail2 = p_avail; // need copy => do not copy reference + auto loads2 = loads; // need copy => do not copy reference + allocate(p0, w, alloc2, remain2, p_avail2, loads2); + int rem = alloc_plan(alloc2, remain2, p_avail2, loads2); + if (rem == 0) { + remain.clear(); + p_avail.clear(); + alloc.insert(alloc2.begin(), alloc2.end()); + loads.insert(loads2.begin(), loads2.end()); + return 0; + } + if (rem < best_rem) { + best_rem = rem; + best = {.alloc = alloc2, .remain = remain2, .p_avail = p_avail2, .loads = loads2}; + } + } + alloc.insert(best.alloc.begin(), best.alloc.end()); + remain.clear(); + remain.insert(best.remain.begin(), best.remain.end()); + p_avail.clear(); + p_avail.insert(best.p_avail.begin(), best.p_avail.end()); + loads.clear(); + loads.insert(best.loads.begin(), best.loads.end()); + return best_rem; +} + +static std::vector reallocate_parts(std::map &alloc, std::set &remain, + const std::map> &P_AVAIL) { + map P_COUNT; + for (auto it = P_AVAIL.begin(); it != P_AVAIL.end(); it++) { + P_COUNT[it->first] = it->second.size(); + } + vector remain_l(remain.begin(), remain.end()); + sort(remain_l.begin(), remain_l.end(), + [&P_COUNT](int &p1, int &p2) { return P_COUNT[p1] > P_COUNT[p2]; }); // partitions with more copies goes first + vector PARTITIONS; + for (auto it = P_COUNT.begin(); it != P_COUNT.end(); it++) { + PARTITIONS.push_back(it->first); + } + sort(PARTITIONS.begin(), PARTITIONS.end(), [&P_COUNT](int &p1, int &p2) { + return P_COUNT[p1] < P_COUNT[p2]; + }); // partitions with fewer copies goes first + vector copying; + while (!remain_l.empty()) { + int p0 = remain_l.back(); + remain_l.pop_back(); + int w_cnt = P_COUNT[p0]; + if (w_cnt == 1) { + copying.push_back(p0); + continue; + } + const auto &ws = P_AVAIL.find(p0)->second; + bool need_pushing = true; + for (auto it = PARTITIONS.begin(); it != PARTITIONS.end(); it++) { + int p = *it; + if (w_cnt <= P_COUNT[p]) { + copying.push_back(p0); // assuming PARTITIONS are in sorted order of copy count + need_pushing = false; + break; + } + if (alloc.find(p) == alloc.end()) { + continue; + } + auto w = alloc[p]; + if (std::find(ws.begin(), ws.end(), w) != ws.end()) { + alloc.erase(p); + alloc[p0] = w; + remain_l.push_back(p); + need_pushing = false; + break; + } + } + if (need_pushing) copying.push_back(p0); + } + return copying; +} + +static void scale_up(std::map &loads, map &workers, int copy_cnt) { + int curr_load = 0; + for (auto it = loads.begin(); it != loads.end(); it++) { + curr_load += it->second; + } + int n_cores = copy_cnt + curr_load - 3 * loads.size(); + if (n_cores < 0) { + return; + } + int n_workers = n_cores / 2 + 1; // allocate a little more to prevent saturation + if (n_cores % 2 > 0) n_workers++; + if (n_workers == 0) return; + + K8sWorkerController *k8sController = K8sWorkerController::getInstance(); + map w_new = k8sController->scaleUp(n_workers); + + for (auto it = w_new.begin(); it != w_new.end(); it++) { + loads[it->first] = 0.1; + workers[it->first] = it->second; + } +} + +static int alloc_net_plan(std::map &alloc, std::vector &parts, + std::map> &transfer, std::map &net_loads, + std::map &loads, const std::map> &p_avail, + int curr_best) { + int curr_load = std::max_element(net_loads.begin(), net_loads.end(), + [](const std::map::value_type &p1, + const std::map::value_type &p2) { return p1.second < p2.second; }) + ->second; + if (curr_load >= curr_best) { + return curr_load; + } + if (parts.empty()) { + if (net_loads.empty()) return 0; + return curr_load; + } + struct best_net_alloc { + std::map alloc; + std::map> transfer; + std::map net_loads; + std::map loads; + }; + int best = curr_best; + struct best_net_alloc best_plan = {.transfer = transfer, .net_loads = net_loads, .loads = loads}; + int p = parts.back(); + parts.pop_back(); + vector wts; + for (auto it = loads.begin(); it != loads.end(); it++) { + wts.push_back(it->first); + } + sort(wts.begin(), wts.end(), [&loads](string &w1, string &w2) { + int l1 = loads[w1]; + int l2 = loads[w2]; + if (l1 < 3 || l2 < 3) return l1 < l2; + return l1 <= l2; + }); // load=1 goes first and load=3 goes last. The order is 1,0,2,3 for 4 cores. + const auto &ws = p_avail.find(p)->second; + int minLoad = 100000000; + for (auto it = loads.begin(); it != loads.end(); it++) { + int load = it->second; + if (minLoad > load) { + minLoad = load; + } + } + for (auto itf = ws.begin(); itf != ws.end(); itf++) { + auto wf = *itf; + for (auto itt = wts.begin(); itt != wts.end(); itt++) { + auto wt = *itt; + int currLoad = loads[wt]; + if (currLoad > minLoad) continue; + auto alloc2 = alloc; // need copy => do not copy reference + auto parts2 = parts; // need copy => do not copy reference + auto transfer2 = transfer; // need copy => do not copy reference + auto net_loads2 = net_loads; // need copy => do not copy reference + auto loads2 = loads; // need copy => do not copy reference + if (wf != wt) { + transfer2[p] = {wf, wt}; // assume + net_loads2[wf]++; + net_loads2[wt]++; + } + alloc2[p] = wt; + loads2[wt]++; + int new_net_load = alloc_net_plan(alloc2, parts2, transfer2, net_loads2, loads2, p_avail, best); + if (new_net_load < best) { + best = new_net_load; + best_plan = {.alloc = alloc2, .transfer = transfer2, .net_loads = net_loads2, .loads = loads2}; + } + } + } + alloc.clear(); + alloc.insert(best_plan.alloc.begin(), best_plan.alloc.end()); + auto &b_transfer = best_plan.transfer; + for (auto it = b_transfer.begin(); it != b_transfer.end(); it++) { + transfer[it->first] = it->second; + } + net_loads.clear(); + net_loads.insert(best_plan.net_loads.begin(), best_plan.net_loads.end()); + loads.clear(); + loads.insert(best_plan.loads.begin(), best_plan.loads.end()); + return best; +} + +static void filter_partitions(std::map> &partitionMap, SQLiteDBInterface *sqlite, + string graphId) { + map workers; // id => "ip:port" + const std::vector>> &results = + sqlite->runSelect("SELECT DISTINCT idworker,ip,server_port FROM worker;"); + for (int i = 0; i < results.size(); i++) { + string workerId = results[i][0].second; + string ip = results[i][1].second; + string port = results[i][2].second; + workers[workerId] = ip + ":" + port; + } + + map loads; + const map &cpu_map = Utils::getMetricMap("cpu_usage"); + for (auto it = workers.begin(); it != workers.end(); it++) { + auto workerId = it->first; + auto worker = it->second; + const auto workerLoadIt = cpu_map.find(worker); + if (workerLoadIt != cpu_map.end()) { + double load = stod(workerLoadIt->second.c_str()); + if (load < 0) load = 0; + loads[workerId] = (int)(4 * load); + } else { + loads[workerId] = 0; + } + } + + std::map> p_avail; + std::set remain; + for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { + auto worker = it->first; + auto &partitions = it->second; + for (auto partitionIt = partitions.begin(); partitionIt != partitions.end(); partitionIt++) { + auto partition = stoi(*partitionIt); + p_avail[partition].push_back(worker); + remain.insert(partition); + } + } + const std::map> P_AVAIL = p_avail; // get a copy and make it const + + for (auto loadIt = loads.begin(); loadIt != loads.end(); loadIt++) { + if (loadIt->second < 3) continue; + auto w = loadIt->first; + for (auto it = p_avail.begin(); it != p_avail.end(); it++) { + auto &partitionWorkers = it->second; + for (auto workerIt = partitionWorkers.begin(); workerIt != partitionWorkers.end();) { + if (*workerIt == w) { + partitionWorkers.erase(workerIt); + } else { + workerIt++; + } + } + } + } + + std::map alloc; + int unallocated = alloc_plan(alloc, remain, p_avail, loads); + if (unallocated > 0) { + triangleCount_logger.info(to_string(unallocated) + " partitions remaining after alloc_plan"); + auto copying = reallocate_parts(alloc, remain, P_AVAIL); + scale_up(loads, workers, copying.size()); + triangleCount_logger.info("Scale up completed"); + + map net_loads; + for (auto it = loads.begin(); it != loads.end(); it++) { + net_loads[it->first] = 0; + } + for (auto it = workers.begin(); it != workers.end(); it++) { + if (loads.find(it->first) == loads.end()) { + loads[it->first] = 3; + } + } + + std::map> transfer; + int net_load = alloc_net_plan(alloc, copying, transfer, net_loads, loads, P_AVAIL, 100000000); + for (auto it = transfer.begin(); it != transfer.end(); it++) { + auto p = it->first; + auto w_to = it->second.second; + alloc[p] = w_to; + } + + if (!transfer.empty()) { + const std::vector>> &workerData = + sqlite->runSelect("SELECT DISTINCT ip,server_port,server_data_port FROM worker;"); + map dataPortMap; // "ip:port" => data_port + for (int i = 0; i < workerData.size(); i++) { + string ip = workerData[i][0].second; + string port = workerData[i][1].second; + string dport = workerData[i][2].second; + dataPortMap[ip + ":" + port] = dport; + } + map workers_r; // "ip:port" => id + for (auto it = workers.begin(); it != workers.end(); it++) { + workers_r[it->second] = it->first; + } + thread transferThreads[transfer.size()]; + int threadCnt = 0; + for (auto it = transfer.begin(); it != transfer.end(); it++) { + auto partition = it->first; + auto from_worker = it->second.first; + auto to_worker = it->second.second; + auto w_from = workers[from_worker]; + auto w_to = workers[to_worker]; + const auto &ip_port_from = Utils::split(w_from, ':'); + auto ip_from = ip_port_from[0]; + auto port_from = stoi(ip_port_from[1]); + const auto &ip_port_to = Utils::split(w_to, ':'); + auto ip_to = ip_port_to[0]; + auto dport_to = stoi(dataPortMap[w_to]); + transferThreads[threadCnt++] = std::thread(&Utils::transferPartition, ip_from, port_from, ip_to, + dport_to, graphId, to_string(partition), to_worker, sqlite); + } + for (int i = 0; i < threadCnt; i++) { + transferThreads[i].join(); + } + } + } + partitionMap.clear(); + for (auto it = alloc.begin(); it != alloc.end(); it++) { + auto partition = it->first; + auto worker = it->second; + partitionMap[worker].push_back(to_string(partition)); + } +} + void TriangleCountExecutor::execute() { + static time_t last_exec = 0; + schedulerMutex.lock(); + time_t curr_time = time(NULL); + if (curr_time < last_exec + 8) { + sleep(last_exec + 9 - curr_time); + } int uniqueId = getUid(); std::string masterIP = request.getMasterIP(); std::string graphId = request.getParameter(Conts::PARAM_KEYS::GRAPH_ID); std::string canCalibrateString = request.getParameter(Conts::PARAM_KEYS::CAN_CALIBRATE); std::string queueTime = request.getParameter(Conts::PARAM_KEYS::QUEUE_TIME); - std::string graphSLAString = request.getParameter(Conts::PARAM_KEYS::GRAPH_SLA); bool canCalibrate = Utils::parseBoolean(canCalibrateString); int threadPriority = request.getPriority(); @@ -54,7 +448,6 @@ void TriangleCountExecutor::execute() { // Below code is used to update the process details processStatusMutex.lock(); - std::set::iterator processIterator; bool processInfoExists = false; std::chrono::milliseconds startTime = duration_cast(system_clock::now().time_since_epoch()); @@ -82,39 +475,58 @@ void TriangleCountExecutor::execute() { long result = 0; bool isCompositeAggregation = false; Utils::worker aggregatorWorker; - vector workerList = Utils::getWorkerList(sqlite); - int workerListSize = workerList.size(); - int partitionCount = 0; std::vector> intermRes; std::vector> statResponse; - std::vector> remoteCopyRes; - PlacesToNodeMapper placesToNodeMapper; std::vector compositeCentralStoreFiles; - int slaStatCount = 0; auto begin = chrono::high_resolution_clock::now(); string sqlStatement = - "SELECT worker_idworker, name,ip,user,server_port,server_data_port,partition_idpartition " + "SELECT DISTINCT worker_idworker,partition_idpartition " "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " "WHERE partition_graph_idgraph=" + graphId + ";"; - std::vector>> results = sqlite->runSelect(sqlStatement); + const std::vector>> &results = sqlite->runSelect(sqlStatement); + + std::map> partitionMap; + + for (auto i = results.begin(); i != results.end(); ++i) { + const std::vector> &rowData = *i; + + string workerID = rowData.at(0).second; + string partitionId = rowData.at(1).second; + + if (partitionMap.find(workerID) == partitionMap.end()) { + std::vector partitionVec; + partitionVec.push_back(partitionId); + partitionMap[workerID] = partitionVec; + } else { + partitionMap[workerID].push_back(partitionId); + } + + triangleCount_logger.info("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : PartitionId " + partitionId); + } if (results.size() > Conts::COMPOSITE_CENTRAL_STORE_WORKER_THRESHOLD) { isCompositeAggregation = true; } + std::unique_ptr k8sInterface(new K8sInterface()); + if (jasminegraph_profile == PROFILE_K8S && k8sInterface->getJasmineGraphConfig("auto_scaling_enabled") == "true") { + filter_partitions(partitionMap, sqlite, graphId); + } + + std::vector> fileCombinations; if (isCompositeAggregation) { std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::vector graphFiles = Utils::getListOfFilesInDirectory(aggregatorFilePath); - std::vector::iterator graphFilesIterator; std::string compositeFileNameFormat = graphId + "_compositecentralstore_"; - for (graphFilesIterator = graphFiles.begin(); graphFilesIterator != graphFiles.end(); ++graphFilesIterator) { + for (auto graphFilesIterator = graphFiles.begin(); graphFilesIterator != graphFiles.end(); + ++graphFilesIterator) { std::string graphFileName = *graphFilesIterator; if ((graphFileName.find(compositeFileNameFormat) == 0) && @@ -122,59 +534,34 @@ void TriangleCountExecutor::execute() { compositeCentralStoreFiles.push_back(graphFileName); } } - fileCombinations = getCombinations(compositeCentralStoreFiles); - } - - std::map> partitionMap; - - for (std::vector>>::iterator i = results.begin(); i != results.end(); ++i) { - std::vector> rowData = *i; - - string workerID = rowData.at(0).second; - string name = rowData.at(1).second; - string host = rowData.at(2).second; - string user = rowData.at(3).second; - string serverPort = rowData.at(4).second; - string serverDataPort = rowData.at(5).second; - string partitionId = rowData.at(6).second; - - if (partitionMap.find(workerID) == partitionMap.end()) { - std::vector partitionVec; - partitionVec.push_back(partitionId); - partitionMap.insert(std::pair>(workerID, partitionVec)); - } else { - std::vector partitionVec = partitionMap.find(workerID)->second; - partitionVec.push_back(partitionId); - } - - triangleCount_logger.log("###TRIANGLE-COUNT-EXECUTOR### Getting Triangle Count : Host " + host + - " Server Port " + serverPort + " PartitionId " + partitionId, - "info"); - } - - for (auto &&futureCall : remoteCopyRes) { - futureCall.wait(); + fileCombinations = AbstractExecutor::getCombinations(compositeCentralStoreFiles); } + std::map combinationWorkerMap; + std::unordered_map>> triangleTree; + std::mutex triangleTreeMutex; + int partitionCount = 0; + vector workerList = Utils::getWorkerList(sqlite); + int workerListSize = workerList.size(); for (int i = 0; i < workerListSize; i++) { Utils::worker currentWorker = workerList.at(i); string host = currentWorker.hostname; string workerID = currentWorker.workerID; string partitionId; - - std::vector partitionList = partitionMap[workerID]; - - std::vector::iterator partitionIterator; - - for (partitionIterator = partitionList.begin(); partitionIterator != partitionList.end(); ++partitionIterator) { + int workerPort = atoi(string(currentWorker.port).c_str()); + int workerDataPort = atoi(string(currentWorker.dataPort).c_str()); + triangleCount_logger.info("worker_" + workerID + " host=" + host + ":" + to_string(workerPort) + ":" + + to_string(workerDataPort)); + const std::vector &partitionList = partitionMap[workerID]; + for (auto partitionIterator = partitionList.begin(); partitionIterator != partitionList.end(); + ++partitionIterator) { partitionCount++; - int workerPort = atoi(string(currentWorker.port).c_str()); - int workerDataPort = atoi(string(currentWorker.dataPort).c_str()); - partitionId = *partitionIterator; + triangleCount_logger.info("> partition" + partitionId); intermRes.push_back(std::async( std::launch::async, TriangleCountExecutor::getTriangleCount, atoi(graphId.c_str()), host, workerPort, - workerDataPort, atoi(partitionId.c_str()), masterIP, uniqueId, isCompositeAggregation, threadPriority)); + workerDataPort, atoi(partitionId.c_str()), masterIP, uniqueId, isCompositeAggregation, threadPriority, + fileCombinations, &combinationWorkerMap, &triangleTree, &triangleTreeMutex)); } } @@ -184,10 +571,10 @@ void TriangleCountExecutor::execute() { "SELECT attempt from graph_sla INNER JOIN sla_category where graph_sla.id_sla_category=sla_category.id and " "graph_sla.graph_id='" + graphId + "' and graph_sla.partition_count='" + std::to_string(partitionCount) + - "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + - "' and sla_category.command='" + TRIANGLES + "';"; + "' and sla_category.category='" + Conts::SLA_CATEGORY::LATENCY + "' and sla_category.command='" + TRIANGLES + + "';"; - std::vector>> queryResults = perfDB->runSelect(query); + const std::vector>> &queryResults = perfDB->runSelect(query); if (queryResults.size() > 0) { std::string attemptString = queryResults[0][0].second; @@ -199,19 +586,25 @@ void TriangleCountExecutor::execute() { } else { triangleCount_logger.log("###TRIANGLE-COUNT-EXECUTOR### Inserting initial record for SLA ", "info"); Utils::updateSLAInformation(perfDB, graphId, partitionCount, 0, TRIANGLES, Conts::SLA_CATEGORY::LATENCY); - statResponse.push_back(std::async(std::launch::async, collectPerformaceData, perfDB, + statResponse.push_back(std::async(std::launch::async, AbstractExecutor::collectPerformaceData, perfDB, graphId.c_str(), TRIANGLES, Conts::SLA_CATEGORY::LATENCY, partitionCount, masterIP, autoCalibrate)); isStatCollect = true; } + last_exec = time(NULL); + schedulerMutex.unlock(); + for (auto &&futureCall : intermRes) { + triangleCount_logger.info("Waiting for result. uuid=" + to_string(uniqueId)); result += futureCall.get(); } + triangleTree.clear(); + combinationWorkerMap.clear(); if (!isCompositeAggregation) { long aggregatedTriangleCount = - TriangleCountExecutor::aggregateCentralStoreTriangles(sqlite, graphId, masterIP, threadPriority); + aggregateCentralStoreTriangles(sqlite, graphId, masterIP, threadPriority, partitionMap); result += aggregatedTriangleCount; workerResponded = true; triangleCount_logger.log( @@ -242,8 +635,7 @@ void TriangleCountExecutor::execute() { } processStatusMutex.lock(); - std::set::iterator processCompleteIterator; - for (processCompleteIterator = processData.begin(); processCompleteIterator != processData.end(); + for (auto processCompleteIterator = processData.begin(); processCompleteIterator != processData.end(); ++processCompleteIterator) { ProcessInfo processInformation = *processCompleteIterator; @@ -253,16 +645,16 @@ void TriangleCountExecutor::execute() { } } processStatusMutex.unlock(); - - triangleTree.clear(); - combinationWorkerMap.clear(); } -long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int port, int dataPort, int partitionId, - std::string masterIP, int uniqueId, bool isCompositeAggregation, - int threadPriority) { +long TriangleCountExecutor::getTriangleCount( + int graphId, std::string host, int port, int dataPort, int partitionId, std::string masterIP, int uniqueId, + bool isCompositeAggregation, int threadPriority, std::vector> fileCombinations, + std::map *combinationWorkerMap_p, + std::unordered_map>> *triangleTree_p, + std::mutex *triangleTreeMutex_p) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -272,7 +664,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return 0; } @@ -293,11 +685,10 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; + triangleCount_logger.error("ERROR connecting"); return 0; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); @@ -306,11 +697,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int } triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); @@ -319,12 +706,9 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -336,13 +720,9 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::TRIANGLES, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); result_wr = write(sockfd, std::to_string(graphId).c_str(), std::to_string(graphId).size()); @@ -350,13 +730,9 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Graph ID " + std::to_string(graphId), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -369,10 +745,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int triangleCount_logger.log("Sent : Partition ID " + std::to_string(partitionId), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -382,45 +755,36 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Thread Priority " + std::to_string(threadPriority), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); triangleCount_logger.log("Got response : |" + response + "|", "info"); - response = Utils::trim_copy(response); triangleCount = atol(response.c_str()); } if (isCompositeAggregation) { triangleCount_logger.log("###COMPOSITE### Started Composite aggregation ", "info"); - static std::vector>::iterator combinationsIterator; - for (int combinationIndex = 0; combinationIndex < fileCombinations.size(); ++combinationIndex) { - std::vector fileList = fileCombinations.at(combinationIndex); - std::vector::iterator fileListIterator; - std::vector::iterator listIterator; + const std::vector &fileList = fileCombinations.at(combinationIndex); std::set partitionIdSet; std::set partitionSet; std::map tempWeightMap; - std::set::iterator partitionSetIterator; std::set transferRequireFiles; std::string combinationKey = ""; std::string availableFiles = ""; std::string transferredFiles = ""; bool isAggregateValid = false; - for (listIterator = fileList.begin(); listIterator != fileList.end(); ++listIterator) { + for (auto listIterator = fileList.begin(); listIterator != fileList.end(); ++listIterator) { std::string fileName = *listIterator; size_t lastIndex = fileName.find_last_of("."); string rawFileName = fileName.substr(0, lastIndex); - std::vector fileNameParts = Utils::split(rawFileName, '_'); + const std::vector &fileNameParts = Utils::split(rawFileName, '_'); - /*Partition numbers are extracted from the file name. The starting index of partition number is 2. - * Therefore the loop starts with 2*/ + /*Partition numbers are extracted from the file name. The starting index of partition number + * is 2. Therefore the loop starts with 2*/ for (int index = 2; index < fileNameParts.size(); ++index) { partitionSet.insert(fileNameParts[index]); } @@ -430,12 +794,11 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int continue; } - if (proceedOrNot(partitionSet, partitionId)) { - } else { + if (!proceedOrNot(partitionSet, partitionId)) { continue; } - for (fileListIterator = fileList.begin(); fileListIterator != fileList.end(); ++fileListIterator) { + for (auto fileListIterator = fileList.begin(); fileListIterator != fileList.end(); ++fileListIterator) { std::string fileName = *fileListIterator; bool isTransferRequired = true; @@ -466,6 +829,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int std::string adjustedTransferredFile = transferredFiles.substr(0, transferredFiles.size() - 1); fileCombinationMutex.lock(); + std::map &combinationWorkerMap = *combinationWorkerMap_p; if (combinationWorkerMap.find(combinationKey) == combinationWorkerMap.end()) { if (partitionIdSet.find(std::to_string(partitionId)) != partitionIdSet.end()) { combinationWorkerMap[combinationKey] = std::to_string(partitionId); @@ -475,9 +839,7 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int fileCombinationMutex.unlock(); if (isAggregateValid) { - std::set::iterator transferRequireFileIterator; - - for (transferRequireFileIterator = transferRequireFiles.begin(); + for (auto transferRequireFileIterator = transferRequireFiles.begin(); transferRequireFileIterator != transferRequireFiles.end(); ++transferRequireFileIterator) { std::string transferFileName = *transferRequireFileIterator; std::string fileAccessible = isFileAccessibleToWorker( @@ -490,16 +852,14 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int } } - std::string compositeTriangles = - countCompositeCentralStoreTriangles(host, std::to_string(port), adjustedTransferredFile, - masterIP, adjustedAvailableFiles, threadPriority); - triangleCount_logger.log("###COMPOSITE### Retrieved Composite triangle list ", "debug"); - std::vector triangles = Utils::split(compositeTriangles, ':'); - + const auto &triangles = + countCompositeCentralStoreTriangles(host, std::to_string(port), adjustedTransferredFile, + masterIP, adjustedAvailableFiles, threadPriority); if (triangles.size() > 0) { - triangleCount += updateTriangleTreeAndGetTriangleCount(triangles); + triangleCount += + updateTriangleTreeAndGetTriangleCount(triangles, triangleTree_p, triangleTreeMutex_p); } } updateMap(partitionId); @@ -507,22 +867,24 @@ long TriangleCountExecutor::getTriangleCount(int graphId, std::string host, int } triangleCount_logger.info("###COMPOSITE### Returning Total Triangles from executer "); - + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return triangleCount; } else { triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return 0; } bool TriangleCountExecutor::proceedOrNot(std::set partitionSet, int partitionId) { const std::lock_guard lock(aggregateWeightMutex); - std::set::iterator partitionSetIterator; std::map tempWeightMap; - for (partitionSetIterator = partitionSet.begin(); partitionSetIterator != partitionSet.end(); + for (auto partitionSetIterator = partitionSet.begin(); partitionSetIterator != partitionSet.end(); ++partitionSetIterator) { std::string partitionIdString = *partitionSetIterator; int currentPartitionId = atoi(partitionIdString.c_str()); @@ -532,9 +894,7 @@ bool TriangleCountExecutor::proceedOrNot(std::set partitionSet, int part int currentWorkerWeight = tempWeightMap[partitionId]; pair entryWithMinValue = make_pair(partitionId, currentWorkerWeight); - map::iterator currentEntry; - - for (currentEntry = aggregateWeightMap.begin(); currentEntry != aggregateWeightMap.end(); ++currentEntry) { + for (auto currentEntry = aggregateWeightMap.begin(); currentEntry != aggregateWeightMap.end(); ++currentEntry) { if (entryWithMinValue.second > currentEntry->second) { entryWithMinValue = make_pair(currentEntry->first, currentEntry->second); } @@ -551,7 +911,6 @@ bool TriangleCountExecutor::proceedOrNot(std::set partitionSet, int part result = true; } - aggregateWeightMutex.unlock(); return result; } @@ -566,14 +925,18 @@ void TriangleCountExecutor::updateMap(int partitionId) { "info"); } -int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(std::vector triangles) { +static int updateTriangleTreeAndGetTriangleCount( + const std::vector &triangles, + std::unordered_map>> *triangleTree_p, + std::mutex *triangleTreeMutex_p) { + std::mutex &triangleTreeMutex = *triangleTreeMutex_p; const std::lock_guard lock1(triangleTreeMutex); - std::vector::iterator triangleIterator; int aggregateCount = 0; + auto &triangleTree = *triangleTree_p; triangleCount_logger.log("###COMPOSITE### Triangle Tree locked ", "debug"); - for (triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { + for (auto triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { std::string triangle = *triangleIterator; if (!triangle.empty() && triangle != "NILL") { @@ -583,19 +946,17 @@ int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(std::vector> itemRes = triangleTree[vertexOne]; - - std::map>::iterator itemResIterator = itemRes.find(vertexTwo); - + auto &itemRes = triangleTree[vertexOne]; + auto itemResIterator = itemRes.find(vertexTwo); if (itemResIterator != itemRes.end()) { - std::vector list = itemRes[vertexTwo]; - - if (std::find(list.begin(), list.end(), vertexThree) == list.end()) { - triangleTree[vertexOne][vertexTwo].push_back(vertexThree); + auto &set2 = itemResIterator->second; + auto set2Iter = set2.find(vertexThree); + if (set2Iter == set2.end()) { + set2.insert(vertexThree); aggregateCount++; } } else { - triangleTree[vertexOne][vertexTwo].push_back(vertexThree); + triangleTree[vertexOne][vertexTwo].insert(vertexThree); aggregateCount++; } } @@ -604,109 +965,93 @@ int TriangleCountExecutor::updateTriangleTreeAndGetTriangleCount(std::vector> workerCombinations = getWorkerCombination(sqlite, graphId); +static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, + int threadPriority, + const std::map> &partitionMap) { + vector partitionsVector; + std::map partitionWorkerMap; // partition_id => worker_id + for (auto it = partitionMap.begin(); it != partitionMap.end(); it++) { + const auto &parts = it->second; + string worker = it->first; + for (auto partsIt = parts.begin(); partsIt != parts.end(); partsIt++) { + string partition = *partsIt; + partitionWorkerMap[partition] = worker; + partitionsVector.push_back(partition); + } + } + + const std::vector> &partitionCombinations = AbstractExecutor::getCombinations(partitionsVector); std::map workerWeightMap; - std::vector>::iterator workerCombinationsIterator; std::vector> triangleCountResponse; std::string result = ""; long aggregatedTriangleCount = 0; - for (workerCombinationsIterator = workerCombinations.begin(); - workerCombinationsIterator != workerCombinations.end(); ++workerCombinationsIterator) { - std::vector workerCombination = *workerCombinationsIterator; - std::map::iterator workerWeightMapIterator; + const std::vector>> &workerDataResult = + sqlite->runSelect("SELECT DISTINCT idworker,ip,server_port,server_data_port FROM worker;"); + map> workerDataMap; // worker_id => [ip,port,data_port] + for (auto it = workerDataResult.begin(); it != workerDataResult.end(); it++) { + const auto &ipPortDport = *it; + string id = ipPortDport[0].second; + string ip = ipPortDport[1].second; + string port = ipPortDport[2].second; + string dport = ipPortDport[3].second; + workerDataMap[id] = {ip, port, dport}; + } + + for (auto partitonCombinationsIterator = partitionCombinations.begin(); + partitonCombinationsIterator != partitionCombinations.end(); partitonCombinationsIterator++) { + const std::vector &partitionCombination = *partitonCombinationsIterator; std::vector> remoteGraphCopyResponse; int minimumWeight = 0; std::string minWeightWorker; - string aggregatorHost = ""; - std::string partitionIdList = ""; - - std::vector::iterator workerCombinationIterator; - std::vector::iterator aggregatorCopyCombinationIterator; - - for (workerCombinationIterator = workerCombination.begin(); - workerCombinationIterator != workerCombination.end(); ++workerCombinationIterator) { - std::string workerId = *workerCombinationIterator; - - workerWeightMapIterator = workerWeightMap.find(workerId); + std::string minWeightWorkerPartition; + for (auto partCombinationIterator = partitionCombination.begin(); + partCombinationIterator != partitionCombination.end(); partCombinationIterator++) { + string part = *partCombinationIterator; + string workerId = partitionWorkerMap[part]; + auto workerWeightMapIterator = workerWeightMap.find(workerId); if (workerWeightMapIterator != workerWeightMap.end()) { - int weight = workerWeightMap.at(workerId); + int weight = workerWeightMapIterator->second; if (minimumWeight == 0 || minimumWeight > weight) { minimumWeight = weight + 1; minWeightWorker = workerId; + minWeightWorkerPartition = part; } } else { minimumWeight = 1; minWeightWorker = workerId; + minWeightWorkerPartition = part; } } + workerWeightMap[minWeightWorker] = minimumWeight; - string aggregatorSqlStatement = - "SELECT ip,user,server_port,server_data_port,partition_idpartition " - "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " - "WHERE partition_graph_idgraph=" + - graphId + " and idworker=" + minWeightWorker + ";"; - - std::vector>> result = sqlite->runSelect(aggregatorSqlStatement); - - vector> aggregatorData = result.at(0); - - std::string aggregatorIp = aggregatorData.at(0).second; - std::string aggregatorUser = aggregatorData.at(1).second; - std::string aggregatorPort = aggregatorData.at(2).second; - std::string aggregatorDataPort = aggregatorData.at(3).second; - std::string aggregatorPartitionId = aggregatorData.at(4).second; + const auto &workerData = workerDataMap[minWeightWorker]; + std::string aggregatorIp = workerData[0]; + std::string aggregatorPort = workerData[1]; + std::string aggregatorDataPort = workerData[2]; - if ((aggregatorIp.find("localhost") != std::string::npos) || aggregatorIp == masterIP) { - aggregatorHost = aggregatorIp; - } else { - aggregatorHost = aggregatorUser + "@" + aggregatorIp; - } + std::string aggregatorPartitionId = minWeightWorkerPartition; - for (aggregatorCopyCombinationIterator = workerCombination.begin(); - aggregatorCopyCombinationIterator != workerCombination.end(); ++aggregatorCopyCombinationIterator) { - std::string workerId = *aggregatorCopyCombinationIterator; - string host = ""; + std::string partitionIdList = ""; + for (auto partitionCombinationIterator = partitionCombination.begin(); + partitionCombinationIterator != partitionCombination.end(); ++partitionCombinationIterator) { + string part = *partitionCombinationIterator; + string workerId = partitionWorkerMap[part]; + if (part != minWeightWorkerPartition) { + partitionIdList += part + ","; + } if (workerId != minWeightWorker) { - string sqlStatement = - "SELECT ip,user,server_port,server_data_port,partition_idpartition " - "FROM worker_has_partition INNER JOIN worker ON " - "worker_has_partition.worker_idworker=worker.idworker " - "WHERE partition_graph_idgraph=" + - graphId + " and idworker=" + workerId + ";"; - - std::vector>> result = sqlite->runSelect(sqlStatement); - - vector> workerData = result.at(0); - - std::string workerIp = workerData.at(0).second; - std::string workerUser = workerData.at(1).second; - std::string workerPort = workerData.at(2).second; - std::string workerDataPort = workerData.at(3).second; - std::string partitionId = workerData.at(4).second; - - if ((workerIp.find("localhost") != std::string::npos) || workerIp == masterIP) { - host = workerIp; - } else { - host = workerUser + "@" + workerIp; - } - - partitionIdList += partitionId + ","; - std::string centralStoreAvailable = isFileAccessibleToWorker( - graphId, partitionId, aggregatorHost, aggregatorPort, masterIP, + graphId, part, aggregatorIp, aggregatorPort, masterIP, JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE, std::string()); if (centralStoreAvailable.compare("false") == 0) { - remoteGraphCopyResponse.push_back( - std::async(std::launch::async, TriangleCountExecutor::copyCentralStoreToAggregator, - aggregatorHost, aggregatorPort, aggregatorDataPort, atoi(graphId.c_str()), - atoi(partitionId.c_str()), masterIP)); + remoteGraphCopyResponse.push_back(std::async( + std::launch::async, TriangleCountExecutor::copyCentralStoreToAggregator, aggregatorIp, + aggregatorPort, aggregatorDataPort, atoi(graphId.c_str()), atoi(part.c_str()), masterIP)); } } } @@ -716,22 +1061,19 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq } std::string adjustedPartitionIdList = partitionIdList.substr(0, partitionIdList.size() - 1); - workerWeightMap[minWeightWorker] = minimumWeight; triangleCountResponse.push_back(std::async( - std::launch::async, TriangleCountExecutor::countCentralStoreTriangles, aggregatorHost, aggregatorPort, - aggregatorHost, aggregatorPartitionId, adjustedPartitionIdList, graphId, masterIP, threadPriority)); + std::launch::async, TriangleCountExecutor::countCentralStoreTriangles, aggregatorPort, aggregatorIp, + aggregatorPartitionId, adjustedPartitionIdList, graphId, masterIP, threadPriority)); } for (auto &&futureCall : triangleCountResponse) { result = result + ":" + futureCall.get(); } - std::vector triangles = Utils::split(result, ':'); - std::vector::iterator triangleIterator; + const std::vector &triangles = Utils::split(result, ':'); std::set uniqueTriangleSet; - - for (triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { + for (auto triangleIterator = triangles.begin(); triangleIterator != triangles.end(); ++triangleIterator) { std::string triangle = *triangleIterator; if (!triangle.empty() && triangle != "NILL") { @@ -740,16 +1082,16 @@ long TriangleCountExecutor::aggregateCentralStoreTriangles(SQLiteDBInterface *sq } aggregatedTriangleCount = uniqueTriangleSet.size(); + uniqueTriangleSet.clear(); return aggregatedTriangleCount; } -string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std::string partitionId, - std::string aggregatorHostName, std::string aggregatorPort, - std::string masterIP, std::string fileType, - std::string fileName) { +static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, + std::string aggregatorPort, std::string masterIP, std::string fileType, + std::string fileName) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -759,14 +1101,10 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return 0; } - if (aggregatorHostName.find('@') != std::string::npos) { - aggregatorHostName = Utils::split(aggregatorHostName, '@')[1]; - } - server = gethostbyname(aggregatorHostName.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + aggregatorHostName); @@ -778,26 +1116,19 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - // TODO::exit + triangleCount_logger.error("ERROR connecting"); return 0; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -805,12 +1136,9 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -823,11 +1151,7 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_TYPE) == 0) { result_wr = write(sockfd, fileType.c_str(), fileType.size()); @@ -835,11 +1159,7 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (fileType.compare(JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE) == 0) { if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { result_wr = write(sockfd, graphId.c_str(), graphId.size()); @@ -848,11 +1168,7 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { result_wr = write(sockfd, partitionId.c_str(), partitionId.size()); @@ -860,10 +1176,7 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - isFileAccessible = Utils::trim_copy(response); + isFileAccessible = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } } } else if (fileType.compare(JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_COMPOSITE) == 0) { @@ -876,15 +1189,13 @@ string TriangleCountExecutor::isFileAccessibleToWorker(std::string graphId, std: triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - isFileAccessible = Utils::trim_copy(response); + isFileAccessible = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } } } } - + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return isFileAccessible; } @@ -893,7 +1204,7 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st std::string aggregatorDataPort, std::string fileName, std::string masterIP) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -907,7 +1218,7 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return 0; } @@ -926,26 +1237,19 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - // TODO::exit + triangleCount_logger.error("ERROR connecting"); return 0; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -953,12 +1257,9 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -970,14 +1271,10 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::SEND_COMPOSITE_CENTRALSTORE_TO_AGGREGATOR, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_NAME) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_NAME, "info"); result_wr = write(sockfd, fileName.c_str(), fileName.size()); @@ -985,14 +1282,9 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : File Name " + fileName, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_LEN) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_LEN, "info"); result_wr = write(sockfd, fileLength.c_str(), fileLength.size()); @@ -1000,20 +1292,14 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : File Length: " + fileLength, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT, "info"); triangleCount_logger.log("Going to send file through service", "info"); - JasmineGraphServer::sendFileThroughService(aggregatorHostName, - std::atoi(aggregatorDataPort.c_str()), fileName, - aggregateStoreFile, masterIP); + Utils::sendFileThroughService(aggregatorHostName, std::atoi(aggregatorDataPort.c_str()), fileName, + aggregateStoreFile); } } } @@ -1030,10 +1316,8 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::FILE_RECV_CHK, "info"); triangleCount_logger.log("Checking if file is received", "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT, "info"); triangleCount_logger.log("Checking file status : " + to_string(count), "info"); @@ -1044,6 +1328,8 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_ACK, "info"); triangleCount_logger.log("File transfer completed for file : " + aggregateStoreFile, "info"); break; + } else { + triangleCount_logger.error("Invalid response " + response); } } @@ -1055,12 +1341,9 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT, "info"); sleep(1); @@ -1075,16 +1358,16 @@ std::string TriangleCountExecutor::copyCompositeCentralStoreToAggregator(std::st triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return response; } -string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string aggregatorHostName, - std::string aggregatorPort, - std::string compositeCentralStoreFileList, - std::string masterIP, std::string availableFileList, - int threadPriority) { +std::vector TriangleCountExecutor::countCompositeCentralStoreTriangles( + std::string aggregatorHostName, std::string aggregatorPort, std::string compositeCentralStoreFileList, + std::string masterIP, std::string availableFileList, int threadPriority) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -1093,14 +1376,14 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return 0; + triangleCount_logger.error("Cannot create socket"); + return {}; } server = gethostbyname(aggregatorHostName.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + aggregatorHostName); - return 0; + return {}; } bzero((char *)&serv_addr, sizeof(serv_addr)); @@ -1108,26 +1391,19 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - // TODO::exit - return 0; + triangleCount_logger.error("ERROR connecting"); + return {}; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -1135,13 +1411,10 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); triangleCount_logger.log("Port : " + aggregatorPort, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -1153,14 +1426,10 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::AGGREGATE_COMPOSITE_CENTRALSTORE_TRIANGLES, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); result_wr = write(sockfd, availableFileList.c_str(), availableFileList.size()); @@ -1171,10 +1440,7 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag triangleCount_logger.log("Sent : Available File List " + availableFileList, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1197,18 +1463,13 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag std::string chunk = chunksVector.at(loopCount); write(sockfd, chunk.c_str(), chunk.size()); } else { - bzero(data, 301); - read(sockfd, data, 300); - string chunkStatus = (data); + string chunkStatus = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); std::string chunk = chunksVector.at(loopCount); write(sockfd, chunk.c_str(), chunk.size()); } } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1221,28 +1482,21 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag triangleCount_logger.log("Sent : Thread Priority " + std::to_string(threadPriority), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); string status = response.substr(response.size() - 5); - std::string result = response.substr(0, response.size() - 5); - + std::basic_ostringstream resultStream; + resultStream << response.substr(0, response.size() - 5); while (status.compare("/SEND") == 0) { result_wr = write(sockfd, status.c_str(), status.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); status = response.substr(response.size() - 5); - std::string triangleResponse = response.substr(0, response.size() - 5); - result = result + triangleResponse; + resultStream << response.substr(0, response.size() - 5); } - response = result; + response = resultStream.str(); } triangleCount_logger.log("Aggregate Response Received", "info"); @@ -1251,34 +1505,9 @@ string TriangleCountExecutor::countCompositeCentralStoreTriangles(std::string ag triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } - return response; -} - -std::vector> TriangleCountExecutor::getWorkerCombination(SQLiteDBInterface *sqlite, - std::string graphId) { - std::set workerIdSet; - - string sqlStatement = - "SELECT worker_idworker " - "FROM worker_has_partition INNER JOIN worker ON worker_has_partition.worker_idworker=worker.idworker " - "WHERE partition_graph_idgraph=" + - graphId + ";"; - - std::vector>> results = sqlite->runSelect(sqlStatement); - - for (std::vector>>::iterator i = results.begin(); i != results.end(); ++i) { - std::vector> rowData = *i; - - string workerId = rowData.at(0).second; - - workerIdSet.insert(workerId); - } - - std::vector workerIdVector(workerIdSet.begin(), workerIdSet.end()); - - std::vector> workerIdCombination = getCombinations(workerIdVector); - - return workerIdCombination; + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return Utils::split(response, ':'); } std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggregatorHostName, @@ -1286,14 +1515,14 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr std::string aggregatorDataPort, int graphId, int partitionId, std::string masterIP) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; struct hostent *server; - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); + std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::string fileName = std::to_string(graphId) + "_centralstore_" + std::to_string(partitionId) + ".gz"; - std::string centralStoreFile = aggregatorFilePath + "/" + fileName; + std::string centralStoreFile = aggregatorDirPath + "/" + fileName; int fileSize = Utils::getFileSize(centralStoreFile); std::string fileLength = to_string(fileSize); @@ -1301,18 +1530,14 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return 0; - } - - if (aggregatorHostName.find('@') != std::string::npos) { - aggregatorHostName = Utils::split(aggregatorHostName, '@')[1]; + triangleCount_logger.error("Cannot create socket"); + return ""; } server = gethostbyname(aggregatorHostName.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + aggregatorHostName); - return 0; + return ""; } bzero((char *)&serv_addr, sizeof(serv_addr)); @@ -1320,26 +1545,19 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - // TODO::exit - return 0; + triangleCount_logger.error("ERROR connecting"); + return ""; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -1347,12 +1565,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -1364,13 +1579,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::SEND_CENTRALSTORE_TO_AGGREGATOR, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_NAME) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_NAME, "info"); result_wr = write(sockfd, fileName.c_str(), fileName.size()); @@ -1378,14 +1589,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : File Name " + fileName, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_LEN) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_LEN, "info"); result_wr = write(sockfd, fileLength.c_str(), fileLength.size()); @@ -1393,20 +1599,14 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : File Length: " + fileLength, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT, "info"); triangleCount_logger.log("Going to send file through service", "info"); - JasmineGraphServer::sendFileThroughService(aggregatorHostName, - std::atoi(aggregatorDataPort.c_str()), fileName, - centralStoreFile, masterIP); + Utils::sendFileThroughService(aggregatorHostName, std::atoi(aggregatorDataPort.c_str()), fileName, + centralStoreFile); } } } @@ -1423,10 +1623,8 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::FILE_RECV_CHK, "info"); triangleCount_logger.log("Checking if file is received", "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT, "info"); triangleCount_logger.log("Checking file status : " + to_string(count), "info"); @@ -1437,6 +1635,8 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::FILE_ACK, "info"); triangleCount_logger.log("File transfer completed for file : " + centralStoreFile, "info"); break; + } else { + triangleCount_logger.error("Invalid response " + response); } } @@ -1448,12 +1648,9 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT, "info"); sleep(1); @@ -1468,15 +1665,17 @@ std::string TriangleCountExecutor::copyCentralStoreToAggregator(std::string aggr triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return response; } -string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, - std::string host, std::string partitionId, - std::string partitionIdList, std::string graphId, - std::string masterIP, int threadPriority) { +string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorPort, std::string host, + std::string partitionId, std::string partitionIdList, + std::string graphId, std::string masterIP, + int threadPriority) { int sockfd; - char data[301]; + char data[INSTANCE_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -1485,14 +1684,10 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; + triangleCount_logger.error("Cannot create socket"); return 0; } - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - server = gethostbyname(host.c_str()); if (server == NULL) { triangleCount_logger.error("ERROR, no host named " + host); @@ -1504,26 +1699,19 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(atoi(aggregatorPort.c_str())); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - // TODO::exit + triangleCount_logger.error("ERROR connecting"); return 0; } - bzero(data, 301); int result_wr = write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); + string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); result_wr = write(sockfd, masterIP.c_str(), masterIP.size()); @@ -1531,12 +1719,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + masterIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); + response = Utils::read_str_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); } else { @@ -1548,13 +1733,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : " + JasmineGraphInstanceProtocol::AGGREGATE_CENTRALSTORE_TRIANGLES, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { triangleCount_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); result_wr = write(sockfd, graphId.c_str(), graphId.size()); @@ -1562,13 +1743,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Graph ID " + graphId, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1578,13 +1755,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - triangleCount_logger.log("Sent : Partition ID " + partitionId, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1594,13 +1767,9 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } + triangleCount_logger.log("Sent : Partition ID List : " + partitionIdList, "info"); - triangleCount_logger.log("Sent : Partition ID List : " + partitionId, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); } if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { @@ -1613,12 +1782,10 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH triangleCount_logger.log("Sent : Thread Priority " + std::to_string(threadPriority), "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); string status = response.substr(response.size() - 5); - std::string result = response.substr(0, response.size() - 5); + std::basic_ostringstream resultStream; + resultStream << response.substr(0, response.size() - 5); while (status == "/SEND") { result_wr = write(sockfd, status.c_str(), status.size()); @@ -1626,21 +1793,19 @@ string TriangleCountExecutor::countCentralStoreTriangles(std::string aggregatorH if (result_wr < 0) { triangleCount_logger.log("Error writing to socket", "error"); } - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); + response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); status = response.substr(response.size() - 5); - std::string triangleResponse = response.substr(0, response.size() - 5); - result = result + triangleResponse; + resultStream << response.substr(0, response.size() - 5); } - response = result; + response = resultStream.str(); } } else { triangleCount_logger.log("There was an error in the upload process and the response is :: " + response, "error"); } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); return response; } diff --git a/src/frontend/core/executor/impl/TriangleCountExecutor.h b/src/frontend/core/executor/impl/TriangleCountExecutor.h index e294b779..717348e3 100644 --- a/src/frontend/core/executor/impl/TriangleCountExecutor.h +++ b/src/frontend/core/executor/impl/TriangleCountExecutor.h @@ -15,6 +15,8 @@ limitations under the License. #define JASMINEGRAPH_TRIANGLECOUNTEXECUTOR_H #include +#include +#include #include "../../../../metadb/SQLiteDBInterface.h" #include "../../../../performance/metrics/PerformanceUtil.h" @@ -35,44 +37,35 @@ class TriangleCountExecutor : public AbstractExecutor { int getUid(); - static long getTriangleCount(int graphId, std::string host, int port, int dataPort, int partitionId, - std::string masterIP, int uniqueId, bool isCompositeAggregation, int threadPriority); - - static long aggregateCentralStoreTriangles(SQLiteDBInterface *sqlite, std::string graphId, std::string masterIP, - int threadPriority); - - static string isFileAccessibleToWorker(std::string graphId, std::string partitionId, std::string aggregatorHostName, - std::string aggregatorPort, std::string masterIP, std::string fileType, - std::string fileName); + static long getTriangleCount( + int graphId, std::string host, int port, int dataPort, int partitionId, std::string masterIP, int uniqueId, + bool isCompositeAggregation, int threadPriority, std::vector> fileCombinations, + std::map *combinationWorkerMap_p, + std::unordered_map>> *triangleTree_p, + std::mutex *triangleTreeMutex_p); static std::string copyCompositeCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, std::string fileName, std::string masterIP); - static string countCompositeCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, - std::string compositeCentralStoreFileList, std::string masterIP, - std::string availableFileList, int threadPriority); - - static std::vector> getWorkerCombination(SQLiteDBInterface *sqlite, std::string graphId); + static std::vector countCompositeCentralStoreTriangles(std::string aggregatorHostName, + std::string aggregatorPort, + std::string compositeCentralStoreFileList, + std::string masterIP, std::string availableFileList, + int threadPriority); static std::string copyCentralStoreToAggregator(std::string aggregatorHostName, std::string aggregatorPort, std::string aggregatorDataPort, int graphId, int partitionId, std::string masterIP); - static string countCentralStoreTriangles(std::string aggregatorHostName, std::string aggregatorPort, - std::string host, std::string partitionId, std::string partitionIdList, - std::string graphId, std::string masterIP, int threadPriority); + static string countCentralStoreTriangles(std::string aggregatorPort, std::string host, std::string partitionId, + std::string partitionIdList, std::string graphId, std::string masterIP, + int threadPriority); static bool proceedOrNot(std::set partitionSet, int partitionId); static void updateMap(int partitionId); - static int updateTriangleTreeAndGetTriangleCount(std::vector triangles); - - static std::vector> fileCombinations; - static std::map combinationWorkerMap; - static std::map>> triangleTree; - private: SQLiteDBInterface *sqlite; PerformanceSQLiteDBInterface *perfDB; diff --git a/src/frontend/core/scheduler/JobScheduler.cpp b/src/frontend/core/scheduler/JobScheduler.cpp index 179e09c9..cfbe21fa 100644 --- a/src/frontend/core/scheduler/JobScheduler.cpp +++ b/src/frontend/core/scheduler/JobScheduler.cpp @@ -38,7 +38,7 @@ void *startScheduler(void *dummyPt) { PerformanceUtil::init(); while (true) { if (jobQueue.size() > 0) { - jobScheduler_Logger.log("##JOB SCHEDULER## Jobs Available for Scheduling", "info"); + jobScheduler_Logger.info("##JOB SCHEDULER## Jobs Available for Scheduling"); std::vector pendingHPJobList; std::vector highPriorityGraphList; @@ -57,9 +57,8 @@ void *startScheduler(void *dummyPt) { } if (pendingHPJobList.size() > 0) { - jobScheduler_Logger.log( - "##JOB SCHEDULER## High Priority Jobs in Queue: " + std::to_string(pendingHPJobList.size()), - "info"); + jobScheduler_Logger.info("##JOB SCHEDULER## High Priority Jobs in Queue: " + + std::to_string(pendingHPJobList.size())); std::string masterIP = pendingHPJobList[0].getMasterIP(); std::string jobType = pendingHPJobList[0].getJobType(); std::string category = pendingHPJobList[0].getParameter(Conts::PARAM_KEYS::CATEGORY); @@ -97,6 +96,7 @@ void *startScheduler(void *dummyPt) { void JobScheduler::init() { pthread_t schedulerThread; pthread_create(&schedulerThread, NULL, startScheduler, this); + pthread_detach(schedulerThread); } void JobScheduler::processJob(JobRequest request, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfDB) { @@ -106,11 +106,13 @@ void JobScheduler::processJob(JobRequest request, SQLiteDBInterface *sqlite, Per void JobScheduler::executeJob(JobRequest request, SQLiteDBInterface *sqlite, PerformanceSQLiteDBInterface *perfDB) { ExecutorFactory *executorFactory = new ExecutorFactory(sqlite, perfDB); AbstractExecutor *abstractExecutor = executorFactory->getExecutor(request); + delete executorFactory; if (abstractExecutor == nullptr) { jobScheduler_Logger.error("abstractExecutor is null"); return; } abstractExecutor->execute(); + delete abstractExecutor; } void JobScheduler::pushJob(JobRequest jobDetails) { jobQueue.push(jobDetails); } @@ -126,7 +128,9 @@ JobResponse JobScheduler::getResult(JobRequest jobRequest) { responseFound = true; } responseVectorMutex.unlock(); - usleep(50000); + if (!responseFound) { + usleep(50000); + } } return jobResponse; diff --git a/src/k8s/K8sInterface.cpp b/src/k8s/K8sInterface.cpp index 9ad7adf4..ac3b9532 100644 --- a/src/k8s/K8sInterface.cpp +++ b/src/k8s/K8sInterface.cpp @@ -87,13 +87,13 @@ v1_service_list_t *K8sInterface::getServiceList(char *labelSelectors) { } v1_deployment_t *K8sInterface::createJasmineGraphWorkerDeployment(int workerId, - const std::string &masterIp, - const std::string &nodeName) const { + const std::string &ip, + const std::string &masterIp) const { std::string definiton = Utils::getJsonStringFromYamlFile(ROOT_DIR "/k8s/worker-deployment.yaml"); definiton = Utils::replaceAll(definiton, "", std::to_string(workerId)); definiton = Utils::replaceAll(definiton, "", masterIp); definiton = Utils::replaceAll(definiton, "", Utils::getJasmineGraphProperty("org.jasminegraph.k8s.image")); - definiton = Utils::replaceAll(definiton, "", nodeName); + definiton = Utils::replaceAll(definiton, "", ip); cJSON *deploymentTemplate = cJSON_Parse(definiton.c_str()); v1_deployment_t *deployment = v1_deployment_parseFromJSON(deploymentTemplate); @@ -173,3 +173,79 @@ v1_node_list_t *K8sInterface::getNodes() { NULL); /* watch */ return node_list; } + +std::string K8sInterface::loadFromConfig(std::string key) { + v1_config_map_t *config_map = + CoreV1API_readNamespacedConfigMap(apiClient, strdup("jasminegraph-config"), namespace_, NULL); + if (config_map->metadata->name == NULL) { + k8s_logger.error("No jasminegraph-config config map."); + return ""; + } + + listEntry_t *data = NULL; + list_ForEach(data, config_map->data) { + auto *pair = static_cast(data->data); + if (strcmp(pair->key, key.c_str()) == 0) { + return std::string(static_cast(pair->value)); + } + } + return ""; +} + +static std::map configMap; +static std::mutex configMapMutex; +std::string K8sInterface::getJasmineGraphConfig(std::string key) { + auto item = configMap.find(key); + if (item == configMap.end()) { + configMapMutex.lock(); + if (configMap.find(key) == configMap.end()) { + k8s_logger.info("Key " + key + " not found in cache. Loading from config."); + configMap[key] = loadFromConfig(key); + } + configMapMutex.unlock(); + item = configMap.find(key); + } + return item->second; +} + +v1_persistent_volume_t *K8sInterface::createJasmineGraphPersistentVolume(int workerId) const { + std::string definition = Utils::getJsonStringFromYamlFile(ROOT_DIR "/k8s/worker-volume.yaml"); + definition = Utils::replaceAll(definition, "", std::to_string(workerId)); + cJSON *volumeTemplate = cJSON_Parse(definition.c_str()); + v1_persistent_volume_t *volume = v1_persistent_volume_parseFromJSON(volumeTemplate); + v1_persistent_volume_t *result = CoreV1API_createPersistentVolume(apiClient, volume, NULL, NULL, NULL, NULL); + return result; +} + +v1_persistent_volume_claim_t *K8sInterface::createJasmineGraphPersistentVolumeClaim(int workerId) const { + std::string definition = Utils::getJsonStringFromYamlFile(ROOT_DIR "/k8s/worker-volume-claim.yaml"); + definition = Utils::replaceAll(definition, "", std::to_string(workerId)); + cJSON *volumeClaimTemplate = cJSON_Parse(definition.c_str()); + v1_persistent_volume_claim_t *volumeClaim = v1_persistent_volume_claim_parseFromJSON(volumeClaimTemplate); + v1_persistent_volume_claim_t *result = CoreV1API_createNamespacedPersistentVolumeClaim(apiClient, + namespace_, + volumeClaim, + NULL, + NULL, + NULL, + NULL); + return result; +} + +v1_persistent_volume_t *K8sInterface::deleteJasmineGraphPersistentVolume(int workerId) const { + std::string volumeName = "jasminegraph-worker" + std::to_string(workerId) + "-data"; + v1_persistent_volume_t *result = CoreV1API_deletePersistentVolume(apiClient, strdup(volumeName.c_str()), NULL, NULL, + NULL, NULL, NULL, NULL); + return result; +} + +v1_persistent_volume_claim_t *K8sInterface::deleteJasmineGraphPersistentVolumeClaim(int workerId) const { + std::string volumeClaimName = "jasminegraph-worker" + std::to_string(workerId) + "-data-claim"; + v1_persistent_volume_claim_t *result = + CoreV1API_deleteNamespacedPersistentVolumeClaim(apiClient, + strdup(volumeClaimName.c_str()), + namespace_, + NULL, NULL, NULL, NULL, NULL, + NULL); + return result; +} diff --git a/src/k8s/K8sInterface.h b/src/k8s/K8sInterface.h index f2b9fab3..b6f649d1 100644 --- a/src/k8s/K8sInterface.h +++ b/src/k8s/K8sInterface.h @@ -36,9 +36,8 @@ class K8sInterface { v1_service_list_t *getServiceList(char *labelSelectors); - v1_deployment_t *createJasmineGraphWorkerDeployment(int workerId, - const std::string &masterIp, - const std::string &nodeName) const; + v1_deployment_t *createJasmineGraphWorkerDeployment(int workerId, const std::string &ip, + const std::string &masterIp) const; v1_status_t *deleteJasmineGraphWorkerDeployment(int workerId) const; @@ -55,6 +54,19 @@ class K8sInterface { std::string getMasterIp(); v1_node_list_t *getNodes(); + + std::string getJasmineGraphConfig(std::string key); + + v1_persistent_volume_t *createJasmineGraphPersistentVolume(int workerId) const; + + v1_persistent_volume_claim_t *createJasmineGraphPersistentVolumeClaim(int workerId) const; + + v1_persistent_volume_t *deleteJasmineGraphPersistentVolume(int workerId) const; + + v1_persistent_volume_claim_t *deleteJasmineGraphPersistentVolumeClaim(int workerId) const; + + private: + std::string loadFromConfig(std::string key); }; #endif // JASMINEGRAPH_K8SINTERFACE_H diff --git a/src/k8s/K8sWorkerController.cpp b/src/k8s/K8sWorkerController.cpp index 3e4df7e6..6f17b95a 100644 --- a/src/k8s/K8sWorkerController.cpp +++ b/src/k8s/K8sWorkerController.cpp @@ -13,86 +13,168 @@ limitations under the License. #include "K8sWorkerController.h" -#include -#include #include -#include "../util/Conts.h" -#include "../util/logger/Logger.h" -#include "../server/JasmineGraphServer.h" +#include +#include Logger controller_logger; +std::vector K8sWorkerController::workerList = {}; +static int TIME_OUT = 900; +static std::vector activeWorkerIds = {}; +std::mutex workerIdMutex; +std::mutex k8sSpawnMutex; +static volatile int nextWorkerId = 0; + +static inline int getNextWorkerId(int count) { + const std::lock_guard lock(workerIdMutex); + int returnId = nextWorkerId; + nextWorkerId += count; + return returnId; +} + K8sWorkerController::K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb) { this->masterIp = std::move(masterIp); - this->numberOfWorkers = numberOfWorkers; + this->numberOfWorkers = 0; + apiClient_setupGlobalEnv(); this->interface = new K8sInterface(); this->metadb = *metadb; +} - // Delete all hosts - metadb->runUpdate("DELETE FROM host"); +K8sWorkerController::~K8sWorkerController() { + delete this->interface; + apiClient_unsetupGlobalEnv(); +} - v1_node_list_t *nodeList = interface->getNodes(); - listEntry_t *listEntry = NULL; - v1_node_t *node = NULL; +static K8sWorkerController *instance = nullptr; - std::string query; - list_ForEach(listEntry, nodeList->items) { - node = static_cast(listEntry->data); - query = "INSERT INTO host (name, ip) VALUES ('" + std::string(node->metadata->name) + "', '')"; - metadb->runInsert(query); +K8sWorkerController *K8sWorkerController::getInstance() { + if (instance == nullptr) { + controller_logger.error("K8sWorkerController is not instantiated"); + throw std::runtime_error("K8sWorkerController is not instantiated"); } + return instance; +} - query = "SELECT idhost, name FROM host"; - for (auto entry : metadb->runSelect(query)) { - nodes[entry.at(1).second] = atoi(entry.at(0).second.c_str()); - } +static std::mutex instanceMutex; +K8sWorkerController *K8sWorkerController::getInstance(std::string masterIp, int numberOfWorkers, + SQLiteDBInterface *metadb) { + if (instance == nullptr) { + instanceMutex.lock(); + if (instance == nullptr) { // double-checking lock + instance = new K8sWorkerController(masterIp, numberOfWorkers, metadb); + try { + instance->maxWorkers = stoi(instance->interface->getJasmineGraphConfig("max_worker_count")); + } catch (std::invalid_argument &e) { + controller_logger.error("Invalid max_worker_count value. Defaulted to 4"); + instance->maxWorkers = 4; + } - // Delete all the workers from the database - metadb->runUpdate("DELETE FROM worker"); - int workersAttached = this->attachExistingWorkers(); - for (int i = workersAttached; i < numberOfWorkers; i++) { - this->spawnWorker(i); + // Delete all the workers from the database + metadb->runUpdate("DELETE FROM worker"); + int workersAttached = instance->attachExistingWorkers(); + if (numberOfWorkers - workersAttached > 0) { + instance->scaleUp(numberOfWorkers - workersAttached); + } + } + instanceMutex.unlock(); } + return instance; } -K8sWorkerController::~K8sWorkerController() { delete this->interface; } - -void K8sWorkerController::spawnWorker(int workerId) { - // TODO (M-I-M-Ishad): Develop criteria to assign the worker to node based on performance metrics - // getting a node randomly - auto it = nodes.begin(); - std::advance(it, rand() % nodes.size()); - std::string hostName = it->first; - int hostId = it->second; - - v1_deployment_t *deployment = - this->interface->createJasmineGraphWorkerDeployment(workerId, this->masterIp, hostName); - if (deployment != nullptr && deployment->metadata != nullptr && deployment->metadata->name != nullptr) { - controller_logger.info("Worker " + std::to_string(workerId) + " deployment created successfully"); +std::string K8sWorkerController::spawnWorker(int workerId) { + k8sSpawnMutex.lock(); + controller_logger.info("Spawning worker " + to_string(workerId)); + auto volume = this->interface->createJasmineGraphPersistentVolume(workerId); + if (volume != nullptr && volume->metadata != nullptr && volume->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume created successfully"); + } else { + controller_logger.error("Worker " + std::to_string(workerId) + " persistent volume creation failed"); + throw std::runtime_error("Worker " + std::to_string(workerId) + " persistent volume creation failed"); + } + auto claim = this->interface->createJasmineGraphPersistentVolumeClaim(workerId); + if (claim != nullptr && claim->metadata != nullptr && claim->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume claim created successfully"); } else { - throw std::runtime_error("Worker " + std::to_string(workerId) + " deployment creation failed"); + controller_logger.error("Worker " + std::to_string(workerId) + " persistent volume claim creation failed"); + throw std::runtime_error("Worker " + std::to_string(workerId) + " persistent volume claim creation failed"); } v1_service_t *service = this->interface->createJasmineGraphWorkerService(workerId); if (service != nullptr && service->metadata != nullptr && service->metadata->name != nullptr) { controller_logger.info("Worker " + std::to_string(workerId) + " service created successfully"); } else { + controller_logger.error("Worker " + std::to_string(workerId) + " service creation failed"); throw std::runtime_error("Worker " + std::to_string(workerId) + " service creation failed"); } + std::string ip(service->spec->cluster_ip); + + v1_deployment_t *deployment = this->interface->createJasmineGraphWorkerDeployment(workerId, ip, this->masterIp); + if (deployment != nullptr && deployment->metadata != nullptr && deployment->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " deployment created successfully"); + } else { + controller_logger.error("Worker " + std::to_string(workerId) + " deployment creation failed"); + throw std::runtime_error("Worker " + std::to_string(workerId) + " deployment creation failed"); + } + k8sSpawnMutex.unlock(); + controller_logger.info("Waiting for worker " + to_string(workerId) + " to respond"); + int waiting = 0; + while (true) { + int sockfd; + struct sockaddr_in serv_addr; + struct hostent *server; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + controller_logger.error("Cannot create socket"); + } + + server = gethostbyname(ip.c_str()); + if (server == NULL) { + controller_logger.error("ERROR, no host named " + ip); + } + + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(Conts::JASMINEGRAPH_INSTANCE_PORT); + + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + waiting += 30; // Added overhead in connection retry attempts + sleep(10); + } else { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + break; + } + + if (waiting >= TIME_OUT) { + controller_logger.error("Error in spawning new worker"); + deleteWorker(workerId); + close(sockfd); + return ""; + } + } + controller_logger.info("Worker " + to_string(workerId) + " responded"); + + JasmineGraphServer::worker worker = { + .hostname = ip, .port = Conts::JASMINEGRAPH_INSTANCE_PORT, .dataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT}; + K8sWorkerController::workerList.push_back(worker); std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " - "VALUES (" + std::to_string(hostId) + ", " + - std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + - std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + - "'" + std::string(service->metadata->name) + "', " + "'" + std::string(service->spec->cluster_ip) + - "', " + std::to_string(workerId) + ")"; + "VALUES ( -1, " + + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + std::string(service->metadata->name) + + "', " + "'" + ip + "', " + std::to_string(workerId) + ")"; int status = metadb.runInsert(insertQuery); if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); } + activeWorkerIds.push_back(workerId); + return ip + ":" + to_string(Conts::JASMINEGRAPH_INSTANCE_PORT); } void K8sWorkerController::deleteWorker(int workerId) { @@ -124,8 +206,23 @@ void K8sWorkerController::deleteWorker(int workerId) { controller_logger.error("Worker " + std::to_string(workerId) + " service deletion failed"); } + auto volume = this->interface->deleteJasmineGraphPersistentVolume(workerId); + if (volume != nullptr && volume->metadata != nullptr && volume->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume deleted successfully"); + } else { + controller_logger.error("Worker " + std::to_string(workerId) + " persistent volume deletion failed"); + } + + auto claim = this->interface->deleteJasmineGraphPersistentVolumeClaim(workerId); + if (claim != nullptr && claim->metadata != nullptr && claim->metadata->name != nullptr) { + controller_logger.info("Worker " + std::to_string(workerId) + " persistent volume claim deleted successfully"); + } else { + controller_logger.error("Worker " + std::to_string(workerId) + " persistent volume claim deletion failed"); + } + std::string deleteQuery = "DELETE FROM worker WHERE idworker = " + std::to_string(workerId); metadb.runUpdate(deleteQuery); + std::remove(activeWorkerIds.begin(), activeWorkerIds.end(), workerId); } int K8sWorkerController::attachExistingWorkers() { @@ -138,8 +235,6 @@ int K8sWorkerController::attachExistingWorkers() { list_ForEach(listEntry, deployment_list->items) { deployment = static_cast(listEntry->data); list_t *labels = deployment->metadata->labels; - std::string nodeName = deployment->spec->_template->spec->node_name; - int hostId = nodes[nodeName]; listEntry_t *label = NULL; list_ForEach(label, labels) { @@ -147,6 +242,9 @@ int K8sWorkerController::attachExistingWorkers() { v1_service_t *service; if (strcmp(pair->key, "workerId") == 0) { int workerId = std::stoi(static_cast(pair->value)); + workerIdMutex.lock(); + nextWorkerId = workerId + 1; + workerIdMutex.unlock(); v1_service_list_t *service_list = this->interface->getServiceList( strdup(("service=jasminegraph-worker,workerId=" + std::to_string(workerId)).c_str())); @@ -156,18 +254,23 @@ int K8sWorkerController::attachExistingWorkers() { service = static_cast(service_list->items->firstEntry->data); } + std::string ip(service->spec->cluster_ip); + JasmineGraphServer::worker worker = {.hostname = ip, + .port = Conts::JASMINEGRAPH_INSTANCE_PORT, + .dataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT}; + K8sWorkerController::workerList.push_back(worker); std::string insertQuery = "INSERT INTO worker (host_idhost, server_port, server_data_port, name, ip, idworker) " - "VALUES ( " + std::to_string(hostId) + ", " + - std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + - std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + - std::string(service->metadata->name) + "', " + "'" + std::string(service->spec->cluster_ip) - + - "', " + std::to_string(workerId) + ")"; + "VALUES ( -1, " + + std::to_string(Conts::JASMINEGRAPH_INSTANCE_PORT) + ", " + + std::to_string(Conts::JASMINEGRAPH_INSTANCE_DATA_PORT) + ", " + "'" + + std::string(service->metadata->name) + "', " + "'" + ip + "', " + std::to_string(workerId) + + ")"; int status = metadb.runInsert(insertQuery); if (status == -1) { controller_logger.error("Worker " + std::to_string(workerId) + " database insertion failed"); } + activeWorkerIds.push_back(workerId); break; } } @@ -182,6 +285,9 @@ std::string K8sWorkerController::getMasterIp() const { return this->masterIp; } int K8sWorkerController::getNumberOfWorkers() const { return numberOfWorkers; } +/* + * @deprecated + */ void K8sWorkerController::setNumberOfWorkers(int newNumberOfWorkers) { if (newNumberOfWorkers > this->numberOfWorkers) { for (int i = this->numberOfWorkers; i < newNumberOfWorkers; i++) { @@ -194,3 +300,28 @@ void K8sWorkerController::setNumberOfWorkers(int newNumberOfWorkers) { } this->numberOfWorkers = newNumberOfWorkers; } + +std::map K8sWorkerController::scaleUp(int count) { + if (this->numberOfWorkers + count > this->maxWorkers) { + count = this->maxWorkers - this->numberOfWorkers; + } + std::map workers; + if (count <= 0) return workers; + controller_logger.info("Scale up with " + to_string(count) + " new workers"); + std::future asyncCalls[count]; + int nextWorkerId = getNextWorkerId(count); + for (int i = 0; i < count; i++) { + asyncCalls[i] = std::async(&K8sWorkerController::spawnWorker, this, nextWorkerId + i); + } + + int success = 0; + for (int i = 0; i < count; i++) { + std::string result = asyncCalls[i].get(); + if (!result.empty()) { + success++; + workers.insert(std::make_pair(to_string(nextWorkerId + i), result)); + } + } + this->numberOfWorkers += success; + return workers; +} diff --git a/src/k8s/K8sWorkerController.h b/src/k8s/K8sWorkerController.h index 4b04e31a..de9d6216 100644 --- a/src/k8s/K8sWorkerController.h +++ b/src/k8s/K8sWorkerController.h @@ -11,14 +11,19 @@ See the License for the specific language governing permissions and limitations under the License. */ +class K8sWorkerController; + #ifndef JASMINEGRAPH_K8SWORKERCONTROLLER_H #define JASMINEGRAPH_K8SWORKERCONTROLLER_H +#include + extern "C" { #include } #include "../metadb/SQLiteDBInterface.h" +#include "../server/JasmineGraphServer.h" #include "./K8sInterface.h" class K8sWorkerController { @@ -27,26 +32,32 @@ class K8sWorkerController { SQLiteDBInterface metadb; std::string masterIp; - int numberOfWorkers; + std::atomic numberOfWorkers; + int maxWorkers; - std::map nodes; + K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); - void spawnWorker(int workerId); + std::string spawnWorker(int workerId); void deleteWorker(int workerId); int attachExistingWorkers(); public: - K8sWorkerController(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); + static std::vector workerList; ~K8sWorkerController(); + static K8sWorkerController *getInstance(std::string masterIp, int numberOfWorkers, SQLiteDBInterface *metadb); + static K8sWorkerController *getInstance(); + std::string getMasterIp() const; int getNumberOfWorkers() const; void setNumberOfWorkers(int newNumberOfWorkers); + + std::map scaleUp(int numberOfWorkers); }; #endif // JASMINEGRAPH_K8SWORKERCONTROLLER_H diff --git a/src/localstore/JasmineGraphHashMapLocalStore.cpp b/src/localstore/JasmineGraphHashMapLocalStore.cpp index 41d6bf43..0d5da271 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.cpp +++ b/src/localstore/JasmineGraphHashMapLocalStore.cpp @@ -86,7 +86,7 @@ void JasmineGraphHashMapLocalStore::toLocalSubGraphMap(const PartEdgeMapStore *e const flatbuffers::Vector &vector = *value; unordered_set valueSet(vector.begin(), vector.end()); - localSubGraphMap.insert(std::make_pair(key, valueSet)); + localSubGraphMap[key] = valueSet; } } @@ -113,23 +113,12 @@ unordered_set JasmineGraphHashMapLocalStore::getVertexSet() { return vertexSet; } -int *JasmineGraphHashMapLocalStore::getOutDegreeDistribution() { - distributionArray = new int[vertexCount]; - int counter = 0; - - for (map>::iterator it = localSubGraphMap.begin(); it != localSubGraphMap.end(); ++it) { - distributionArray[counter] = (it->second).size(); - counter++; - } - return distributionArray; -} - map JasmineGraphHashMapLocalStore::getOutDegreeDistributionHashMap() { map distributionHashMap; for (map>::iterator it = localSubGraphMap.begin(); it != localSubGraphMap.end(); ++it) { long distribution = (it->second).size(); - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } @@ -146,7 +135,7 @@ map JasmineGraphHashMapLocalStore::getInDegreeDistributionHashMap() long previousValue = distMapItr->second; distMapItr->second = previousValue + 1; } else { - distributionHashMap.insert(std::make_pair(*itr, 1)); + distributionHashMap[*itr] = 1; } } } @@ -176,34 +165,25 @@ void JasmineGraphHashMapLocalStore::initialize() {} void JasmineGraphHashMapLocalStore::addVertex(string *attributes) {} -bool JasmineGraphHashMapLocalStore::storeAttributes(std::map> attributeMap, - const string storePath) { - this->localAttributeMap = attributeMap; - bool result = false; +bool JasmineGraphHashMapLocalStore::storeAttributes(const std::map> &attributeMap, + const string &storePath) { flatbuffers::FlatBufferBuilder builder; std::vector> attributeStoreEntriesVector; - std::map>::iterator mapIterator; - for (mapIterator = localAttributeMap.begin(); mapIterator != localAttributeMap.end(); mapIterator++) { + for (auto mapIterator = attributeMap.begin(); mapIterator != attributeMap.end(); mapIterator++) { long key = mapIterator->first; - std::vector attributeVector = mapIterator->second; - - auto flatbufferVector = builder.CreateVectorOfStrings(attributeVector); + auto flatbufferVector = builder.CreateVectorOfStrings(mapIterator->second); auto attributeStoreEntry = CreateAttributeStoreEntry(builder, key, flatbufferVector); attributeStoreEntriesVector.push_back(attributeStoreEntry); } auto flatBuffersAttributeStoreEntriesVector = builder.CreateVectorOfSortedTables(&attributeStoreEntriesVector); - auto attributeStore = CreateAttributeStore(builder, flatBuffersAttributeStoreEntriesVector); - builder.Finish(attributeStore); flatbuffers::SaveFile(storePath.c_str(), (const char *)builder.GetBufferPointer(), (size_t)builder.GetSize(), true); - result = true; - - return result; + return true; } bool JasmineGraphHashMapLocalStore::loadAttributes() { @@ -226,6 +206,7 @@ bool JasmineGraphHashMapLocalStore::loadAttributes() { auto attributeStoreData = GetAttributeStore(data); toLocalAttributeMap(attributeStoreData); + delete data; result = true; @@ -245,7 +226,7 @@ void JasmineGraphHashMapLocalStore::toLocalAttributeMap(const AttributeStore *at for (int j = 0; j < attributesSize; j = j + 1) { attributeVector.push_back(attributes->Get(j)->c_str()); } - localAttributeMap.insert(std::make_pair(key, attributeVector)); + localAttributeMap[key] = attributeVector; } } @@ -267,7 +248,7 @@ void JasmineGraphHashMapLocalStore::toLocalEdgeMap(const PartEdgeMapStore *edgeM auto value = entry->value(); const flatbuffers::Vector &vector = *value; std::vector valueSet(vector.begin(), vector.end()); - edgeMap.insert(std::make_pair(key, valueSet)); + edgeMap[key] = valueSet; } } @@ -292,32 +273,27 @@ bool JasmineGraphHashMapLocalStore::loadPartEdgeMap(const std::string filePath) auto edgeMapStoreData = GetPartEdgeMapStore(data); toLocalEdgeMap(edgeMapStoreData); + delete data; result = true; return result; } -bool JasmineGraphHashMapLocalStore::storePartEdgeMap(std::map> edgeMap, - const std::string savePath) { +bool JasmineGraphHashMapLocalStore::storePartEdgeMap(const std::map> &edgeMap, + const std::string &savePath) { bool result = false; flatbuffers::FlatBufferBuilder builder; std::vector> edgeStoreEntriesVector; - - std::map>::iterator mapIterator; - for (mapIterator = edgeMap.begin(); mapIterator != edgeMap.end(); mapIterator++) { + for (auto mapIterator = edgeMap.begin(); mapIterator != edgeMap.end(); mapIterator++) { int key = mapIterator->first; - std::vector value = mapIterator->second; - std::vector valueVector(value.begin(), value.end()); - auto flatbufferVector = builder.CreateVector(valueVector); + auto flatbufferVector = builder.CreateVector(mapIterator->second); auto edgeStoreEntry = CreatePartEdgeMapStoreEntry(builder, key, flatbufferVector); edgeStoreEntriesVector.push_back(edgeStoreEntry); } auto flatBuffersEdgeStoreEntriesVector = builder.CreateVectorOfSortedTables(&edgeStoreEntriesVector); - auto edgeStore = CreatePartEdgeMapStore(builder, flatBuffersEdgeStoreEntriesVector); - builder.Finish(edgeStore); flatbuffers::SaveFile(savePath.c_str(), (const char *)builder.GetBufferPointer(), (size_t)builder.GetSize(), true); diff --git a/src/localstore/JasmineGraphHashMapLocalStore.h b/src/localstore/JasmineGraphHashMapLocalStore.h index 8b913997..54d1f89b 100644 --- a/src/localstore/JasmineGraphHashMapLocalStore.h +++ b/src/localstore/JasmineGraphHashMapLocalStore.h @@ -42,7 +42,6 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { long vertexCount; long edgeCount; - int *distributionArray; std::string getFileSeparator(); @@ -84,6 +83,7 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { vertexCount = localSubGraphMap.size(); edgeCount = getEdgeCount(); + delete data; return result; } @@ -92,7 +92,7 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { bool storeGraph(); - bool storeAttributes(std::map> attributeMap, const std::string storePath); + static bool storeAttributes(const std::map> &attributeMap, const std::string &storePath); long getEdgeCount(); @@ -102,8 +102,6 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { unordered_set getVertexSet(); - int *getOutDegreeDistribution(); - map getOutDegreeDistributionHashMap(); map getInDegreeDistributionHashMap(); @@ -122,7 +120,7 @@ class JasmineGraphHashMapLocalStore : public JasmineGraphLocalStore { bool loadPartEdgeMap(const std::string filePath); - bool storePartEdgeMap(std::map> edgeMap, const std::string savePath); + static bool storePartEdgeMap(const std::map> &edgeMap, const std::string &savePath); map> getEdgeHashMap(const std::string filePath); }; diff --git a/src/localstore/JasmineGraphLocalStoreFactory.cpp b/src/localstore/JasmineGraphLocalStoreFactory.cpp index 34c6afaa..5b21f7ce 100644 --- a/src/localstore/JasmineGraphLocalStoreFactory.cpp +++ b/src/localstore/JasmineGraphLocalStoreFactory.cpp @@ -17,8 +17,7 @@ JasmineGraphHashMapLocalStore JasmineGraphLocalStoreFactory::load(std::string gr std::string baseDir) { int graphIdentifier = atoi(graphId.c_str()); int partitionIdentifier = atoi(partitionId.c_str()); - JasmineGraphHashMapLocalStore hashMapLocalStore = - *new JasmineGraphHashMapLocalStore(graphIdentifier, partitionIdentifier, baseDir); + JasmineGraphHashMapLocalStore hashMapLocalStore(graphIdentifier, partitionIdentifier, baseDir); return hashMapLocalStore; } diff --git a/src/metadb/SQLiteDBInterface.cpp b/src/metadb/SQLiteDBInterface.cpp index af2a4272..4a815618 100644 --- a/src/metadb/SQLiteDBInterface.cpp +++ b/src/metadb/SQLiteDBInterface.cpp @@ -31,7 +31,7 @@ SQLiteDBInterface::SQLiteDBInterface(string databaseLocation) { int SQLiteDBInterface::init() { if (!Utils::fileExists(this->databaseLocation.c_str())) { - if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "src/metadb/ddl.sql") != 0) { + if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "ddl/metadb.sql") != 0) { return -1; } } diff --git a/src/ml/federated/.keep b/src/ml/federated/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp index 22f3fed0..31789d0a 100644 --- a/src/ml/trainer/JasmineGraphTrainingSchedular.cpp +++ b/src/ml/trainer/JasmineGraphTrainingSchedular.cpp @@ -69,7 +69,7 @@ map> JasmineGraphTrainingSchedular::schedulePartition int vertexcount; int centralVertexCount; trainScheduler_logger.log("Scheduling training order for each worker", "info"); - for (std::vector>::iterator j = (hostData.begin()); j != hostData.end(); ++j) { + for (std::vector>::iterator j = hostData.begin(); j != hostData.end(); ++j) { sqlStatement = "SELECT idpartition, vertexcount, central_vertexcount, graph_idgraph FROM partition INNER JOIN " "(SELECT host_idhost, partition_idpartition, partition_graph_idgraph, worker_idworker FROM " @@ -104,7 +104,7 @@ map> JasmineGraphTrainingSchedular::schedulePartition } std::map scheduledPartitionSets = packPartitionsToMemory(memoryEstimationForEachPartition, availableMemory); - scheduleForEachHost.insert(make_pair(j->second, scheduledPartitionSets)); + scheduleForEachHost[j->second] = scheduledPartitionSets; } refToSqlite->finalize(); delete refToSqlite; @@ -270,9 +270,8 @@ static map>> scheduleGradientPassingTraining graphID + " AND a.host_idhost = " + j->first; std::vector>> results = refToSqlite->runSelect(sqlStatement); - vector> partitionMetadata; // Vector for node count, edge count, feature count for memory - // estimation of each partition map partitionWorkerMap; + map> partitionMetadataMap; // Iterate through partition list and get partition details trainScheduler_logger.log("Commence schedule creation for host" + j->second, "info"); for (std::vector>>::iterator i = results.begin(); i != results.end(); ++i) { @@ -295,12 +294,26 @@ static map>> scheduleGradientPassingTraining partitionValues.push_back(featurecount); partitionValues.push_back(featureSize); - partitionMetadata.push_back(partitionValues); + if (partitionMetadataMap.find(partitionID) == partitionMetadataMap.end()) { + partitionMetadataMap[partitionID] = partitionValues; + } // Store assigned worker per partition workerid = stoi(rowData.at(5).second); - partitionWorkerMap[partitionID] = workerid; + if (partitionWorkerMap.find(partitionID) == partitionWorkerMap.end()) { + partitionWorkerMap[partitionID] = workerid; + } else { + if (partitionWorkerMap[partitionID] > workerid) { + partitionWorkerMap[partitionID] = workerid; + } + } + } + vector> partitionMetadata; // Vector for node count, edge count, feature count for memory + // estimation of each partition + for (auto it = partitionMetadataMap.begin(); it != partitionMetadataMap.end(); it++) { + partitionMetadata.push_back(it->second); } + partitionMetadataMap.clear(); long availableMemory = getAvailableMemory(j->second); // Host memory (in KB) @@ -310,7 +323,7 @@ static map>> scheduleGradientPassingTraining // Get schedule for host map> scheduledPartitionSets = schedulePartitionsBestFit(partitionMemoryList, partitionWorkerMap, availableMemory); - scheduleForEachHost.insert(make_pair(j->second, scheduledPartitionSets)); + scheduleForEachHost[j->second] = scheduledPartitionSets; } refToSqlite->finalize(); delete refToSqlite; diff --git a/src/nativestore/DataPublisher.cpp b/src/nativestore/DataPublisher.cpp index f0e1da00..2fafa2ad 100644 --- a/src/nativestore/DataPublisher.cpp +++ b/src/nativestore/DataPublisher.cpp @@ -28,7 +28,7 @@ DataPublisher::DataPublisher(int worker_port, std::string worker_address) { server = gethostbyname(worker_address.c_str()); if (server == NULL) { - data_publisher_logger.log("ERROR, no host named " + worker_address, "error"); + data_publisher_logger.error("ERROR, no host named " + worker_address); pthread_exit(NULL); } @@ -44,7 +44,10 @@ DataPublisher::DataPublisher(int worker_port, std::string worker_address) { } } -DataPublisher::~DataPublisher() { close(sock); } +DataPublisher::~DataPublisher() { + Utils::send_str_wrapper(sock, JasmineGraphInstanceProtocol::CLOSE); + close(sock); +} void DataPublisher::publish(std::string message) { char receiver_buffer[MAX_STREAMING_DATA_LENGTH] = {0}; diff --git a/src/nativestore/NodeManager.cpp b/src/nativestore/NodeManager.cpp index f2565007..ad404e52 100644 --- a/src/nativestore/NodeManager.cpp +++ b/src/nativestore/NodeManager.cpp @@ -83,7 +83,7 @@ NodeManager::NodeManager(GraphConfig gConfig) { // std::ios::binary); // TODO (tmkasun): set PropertyLink nextPropertyIndex after validating by modulus check from file number of bytes - node_manager_logger.log("NodesDB, PropertiesDB, and RelationsDB files opened (or created) successfully.", "info"); + node_manager_logger.info("NodesDB, PropertiesDB, and RelationsDB files opened (or created) successfully."); // unsigned int nextAddress; // unsigned int propertyBlockAddress = 0; // PropertyLink::propertiesDB->seekg(propertyBlockAddress * PropertyLink::PROPERTY_BLOCK_SIZE); diff --git a/src/nativestore/PropertyLink.cpp b/src/nativestore/PropertyLink.cpp index 4c450b5f..287bde6c 100644 --- a/src/nativestore/PropertyLink.cpp +++ b/src/nativestore/PropertyLink.cpp @@ -126,7 +126,6 @@ unsigned int PropertyLink::insert(std::string name, const char* value) { property_link_logger.warn("Property key/name already exist key = " + std::string(name)); return this->blockAddress; } else if (this->nextPropAddress) { // Traverse to the edge/end of the link list - property_link_logger.log("Next Called", "info"); return this->next()->insert(name, value); } else { // No next link means end of the link, Now add the new link // property_link_logger.debug("Next prop index = " + std::to_string(PropertyLink::nextPropertyIndex)); diff --git a/src/partitioner/local/JSONParser.cpp b/src/partitioner/local/JSONParser.cpp index a5bb23f0..e878d6ba 100644 --- a/src/partitioner/local/JSONParser.cpp +++ b/src/partitioner/local/JSONParser.cpp @@ -103,9 +103,6 @@ static void attributeFileCreate(std::map &vertexToIDMap, std::map &vertexToIDMap, std::map &fieldsMap, std::string inputFilePath, string outputFilePath) { - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/"); - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp"); - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/JSONParser"); Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/JSONParser/output"); auto edgeListFuture = std::async(createEdgeList, inputFilePath, outputFilePath); @@ -188,7 +185,7 @@ static std::map countFileds(std::string inputFilePath) { continue; } if (fieldCounts.find(field) == fieldCounts.end()) { - fieldCounts.insert(make_pair(field, 1)); + fieldCounts[field] = 1; } else { fieldCounts[field]++; } @@ -202,7 +199,7 @@ static std::map countFileds(std::string inputFilePath) { std::string field = it->first; if (it->second > 821) { if (fieldsMap.find(field) == fieldsMap.end()) { - fieldsMap.insert(make_pair(field, field_counter)); + fieldsMap[field] = field_counter; field_counter++; } } diff --git a/src/partitioner/local/MetisPartitioner.cpp b/src/partitioner/local/MetisPartitioner.cpp index f3548cae..83094791 100644 --- a/src/partitioner/local/MetisPartitioner.cpp +++ b/src/partitioner/local/MetisPartitioner.cpp @@ -39,7 +39,6 @@ void MetisPartitioner::loadDataSet(string inputFilePath, int graphID) { // Have to call createDirectory twice since it does not support recursive directory creation. Could use // boost::filesystem for path creation - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/"); Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp"); Utils::createDirectory(this->outputFilePath); @@ -212,9 +211,7 @@ std::vector> MetisPartitioner::partitioneWithGPMetis( char buffer[128]; std::string result = ""; - std::string metisBinDir = Utils::getJasmineGraphProperty("org.jasminegraph.partitioner.metis.bin"); - string metisCommand = - metisBinDir + "/gpmetis " + this->outputFilePath + "/grf " + to_string(this->nParts) + " 2>&1"; + string metisCommand = "gpmetis " + this->outputFilePath + "/grf " + to_string(this->nParts) + " 2>&1"; partitioner_logger.log("metisCommand " + metisCommand, "info"); FILE *input = popen(metisCommand.c_str(), "r"); @@ -292,7 +289,7 @@ std::vector> MetisPartitioner::partitioneWithGPMetis( std::to_string(this->edgeCount) + "' WHERE idgraph = '" + std::to_string(this->graphID) + "'"; this->sqlite->runUpdate(sqlStatement); - this->fullFileList.push_back(this->partitionFileList); + this->fullFileList.push_back(this->partitionFileMap); this->fullFileList.push_back(this->centralStoreFileList); this->fullFileList.push_back(this->centralStoreDuplicateFileList); this->fullFileList.push_back(this->partitionAttributeFileList); @@ -316,7 +313,7 @@ void MetisPartitioner::createPartitionFiles(std::map partMap) { edgeMap = GetConfig::getEdgeMap(); articlesMap = GetConfig::getAttributesMap(); - std::thread *threadList = new std::thread[nParts]; + std::thread threadList[nParts]; int count = 0; for (int part = 0; part < nParts; part++) { populatePartMaps(partMap, part); @@ -358,32 +355,23 @@ void MetisPartitioner::createPartitionFiles(std::map partMap) { if (graphAttributeType == Conts::GRAPH_WITH_TEXT_ATTRIBUTES || graphType == Conts::GRAPH_TYPE_RDF) { threadCount = nParts * 5; } - std::thread *threads = new std::thread[threadCount]; + std::thread threads[threadCount]; count = 0; for (int part = 0; part < nParts; part++) { - // threads[count] = std::thread(&MetisPartitioner::writePartitionFiles, this, part); - threads[count] = std::thread(&MetisPartitioner::writeSerializedPartitionFiles, this, part); - count++; - // threads[count] = std::thread(&MetisPartitioner::writeMasterFiles, this, part); - threads[count] = std::thread(&MetisPartitioner::writeSerializedMasterFiles, this, part); - count++; - threads[count] = std::thread(&MetisPartitioner::writeSerializedDuplicateMasterFiles, this, part); - count++; + threads[count++] = std::thread(&MetisPartitioner::writeSerializedPartitionFiles, this, part); + threads[count++] = std::thread(&MetisPartitioner::writeSerializedMasterFiles, this, part); + threads[count++] = std::thread(&MetisPartitioner::writeSerializedDuplicateMasterFiles, this, part); if (graphAttributeType == Conts::GRAPH_WITH_TEXT_ATTRIBUTES) { - threads[count] = std::thread(&MetisPartitioner::writeTextAttributeFilesForPartitions, this, part); - count++; - threads[count] = std::thread(&MetisPartitioner::writeTextAttributeFilesForMasterParts, this, part); - count++; + threads[count++] = std::thread(&MetisPartitioner::writeTextAttributeFilesForPartitions, this, part); + threads[count++] = std::thread(&MetisPartitioner::writeTextAttributeFilesForMasterParts, this, part); } if (graphType == Conts::GRAPH_TYPE_RDF) { - threads[count] = std::thread(&MetisPartitioner::writeRDFAttributeFilesForPartitions, this, part); - count++; - threads[count] = std::thread(&MetisPartitioner::writeRDFAttributeFilesForMasterParts, this, part); - count++; + threads[count++] = std::thread(&MetisPartitioner::writeRDFAttributeFilesForPartitions, this, part); + threads[count++] = std::thread(&MetisPartitioner::writeRDFAttributeFilesForMasterParts, this, part); } } - for (int tc = 0; tc < threadCount; tc++) { + for (int tc = 0; tc < count; tc++) { threads[tc].join(); } partitioner_logger.log("Writing to files completed", "info"); @@ -527,7 +515,7 @@ void MetisPartitioner::createPartitionFiles(std::map partMap) { } std::vector::iterator compositeGraphIdListIterator; - std::thread *compositeCopyThreads = new std::thread[threadCount]; + std::thread compositeCopyThreads[threadCount]; int compositeCopyCount = 0; for (compositeGraphIdListIterator = compositeGraphIdList.begin(); @@ -665,13 +653,12 @@ void MetisPartitioner::writeSerializedPartitionFiles(int part) { std::map> partEdgeMap = partitionedLocalGraphStorageMap[part]; - JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); - hashMapLocalStore->storePartEdgeMap(partEdgeMap, outputFilePart); + JasmineGraphHashMapLocalStore::storePartEdgeMap(partEdgeMap, outputFilePart); // Compress part files Utils::compressFile(outputFilePart); partFileMutex.lock(); - partitionFileList.insert(make_pair(part, outputFilePart + ".gz")); + partitionFileMap[part] = outputFilePart + ".gz"; partFileMutex.unlock(); partitioner_logger.log("Serializing done for local part " + to_string(part), "info"); } @@ -682,12 +669,11 @@ void MetisPartitioner::writeSerializedMasterFiles(int part) { std::map> partMasterEdgeMap = masterGraphStorageMap[part]; - JasmineGraphHashMapCentralStore *hashMapCentralStore = new JasmineGraphHashMapCentralStore(); - hashMapCentralStore->storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); + JasmineGraphHashMapCentralStore::storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); Utils::compressFile(outputFilePartMaster); masterFileMutex.lock(); - centralStoreFileList.insert(make_pair(part, outputFilePartMaster + ".gz")); + centralStoreFileList[part] = outputFilePartMaster + ".gz"; masterFileMutex.unlock(); partitioner_logger.log("Serializing done for central part " + to_string(part), "info"); } @@ -698,103 +684,15 @@ void MetisPartitioner::writeSerializedDuplicateMasterFiles(int part) { std::map> partMasterEdgeMap = duplicateMasterGraphStorageMap[part]; - JasmineGraphHashMapCentralStore *hashMapCentralStore = new JasmineGraphHashMapCentralStore(); - hashMapCentralStore->storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); + JasmineGraphHashMapCentralStore::storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); Utils::compressFile(outputFilePartMaster); masterFileMutex.lock(); - centralStoreDuplicateFileList.insert(make_pair(part, outputFilePartMaster + ".gz")); + centralStoreDuplicateFileList[part] = outputFilePartMaster + ".gz"; masterFileMutex.unlock(); partitioner_logger.log("Serializing done for duplicate central part " + to_string(part), "info"); } -void MetisPartitioner::writePartitionFiles(int part) { - string outputFilePart = outputFilePath + "/" + std::to_string(this->graphID) + "_" + std::to_string(part); - - std::map> partEdgeMap = partitionedLocalGraphStorageMap[part]; - - if (!partEdgeMap.empty()) { - std::ofstream localFile(outputFilePart); - - if (localFile.is_open()) { - for (auto it = partEdgeMap.begin(); it != partEdgeMap.end(); ++it) { - int vertex = it->first; - std::vector destinationSet = it->second; - - if (!destinationSet.empty()) { - for (std::vector::iterator itr = destinationSet.begin(); itr != destinationSet.end(); ++itr) { - string edge; - - if (graphType == Conts::GRAPH_TYPE_RDF) { - auto entry = edgeMap.find(make_pair(vertex, (*itr))); - long article_id = entry->second; - - edge = std::to_string(vertex) + " " + std::to_string((*itr)) + " " + - std::to_string(article_id); - } else { - edge = std::to_string(vertex) + " " + std::to_string((*itr)); - } - localFile << edge; - localFile << "\n"; - } - } - } - } - localFile.flush(); - localFile.close(); - } - - // Compress part files - Utils::compressFile(outputFilePart); - partFileMutex.lock(); - partitionFileList.insert(make_pair(part, outputFilePart + ".gz")); - partFileMutex.unlock(); -} - -void MetisPartitioner::writeMasterFiles(int part) { - string outputFilePartMaster = - outputFilePath + "/" + std::to_string(this->graphID) + "_centralstore_" + std::to_string(part); - - std::map> partMasterEdgeMap = masterGraphStorageMap[part]; - - if (!partMasterEdgeMap.empty()) { - std::ofstream masterFile(outputFilePartMaster); - - if (masterFile.is_open()) { - for (auto it = partMasterEdgeMap.begin(); it != partMasterEdgeMap.end(); ++it) { - int vertex = it->first; - std::vector destinationSet = it->second; - - if (!destinationSet.empty()) { - for (std::vector::iterator itr = destinationSet.begin(); itr != destinationSet.end(); ++itr) { - string edge; - - if (graphType == Conts::GRAPH_TYPE_RDF) { - auto entry = edgeMap.find(make_pair(vertex, (*itr))); - long article_id = entry->second; - - edge = std::to_string(vertex) + " " + std::to_string((*itr)) + " " + - std::to_string(article_id); - - } else { - edge = std::to_string(vertex) + " " + std::to_string((*itr)); - } - masterFile << edge; - masterFile << "\n"; - } - } - } - } - masterFile.flush(); - masterFile.close(); - } - - Utils::compressFile(outputFilePartMaster); - masterFileMutex.lock(); - centralStoreFileList.insert(make_pair(part, outputFilePartMaster + ".gz")); - masterFileMutex.unlock(); -} - void MetisPartitioner::writeTextAttributeFilesForPartitions(int part) { string attributeFilePart = outputFilePath + "/" + std::to_string(this->graphID) + "_attributes_" + std::to_string(part); @@ -826,7 +724,7 @@ void MetisPartitioner::writeTextAttributeFilesForPartitions(int part) { Utils::compressFile(attributeFilePart); partAttrFileMutex.lock(); - partitionAttributeFileList.insert(make_pair(part, attributeFilePart + ".gz")); + partitionAttributeFileList[part] = attributeFilePart + ".gz"; partAttrFileMutex.unlock(); partitioner_logger.log("Attribute writing done for local part " + to_string(part), "info"); } @@ -861,7 +759,7 @@ void MetisPartitioner::writeTextAttributeFilesForMasterParts(int part) { Utils::compressFile(attributeFilePartMaster); masterAttrFileMutex.lock(); - centralStoreAttributeFileList.insert(make_pair(part, attributeFilePartMaster + ".gz")); + centralStoreAttributeFileList[part] = attributeFilePartMaster + ".gz"; masterAttrFileMutex.unlock(); partitioner_logger.log("Attribute writing done for central part " + to_string(part), "info"); } @@ -890,12 +788,11 @@ void MetisPartitioner::writeRDFAttributeFilesForPartitions(int part) { } } - JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); - hashMapLocalStore->storeAttributes(partitionedEdgeAttributes, attributeFilePart); + JasmineGraphHashMapLocalStore::storeAttributes(partitionedEdgeAttributes, attributeFilePart); Utils::compressFile(attributeFilePart); partAttrFileMutex.lock(); - partitionAttributeFileList.insert(make_pair(part, attributeFilePart + ".gz")); + partitionAttributeFileList[part] = attributeFilePart + ".gz"; partAttrFileMutex.unlock(); } @@ -923,12 +820,11 @@ void MetisPartitioner::writeRDFAttributeFilesForMasterParts(int part) { } } - JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); - hashMapLocalStore->storeAttributes(centralStoreEdgeAttributes, attributeFilePartMaster); + JasmineGraphHashMapLocalStore::storeAttributes(centralStoreEdgeAttributes, attributeFilePartMaster); Utils::compressFile(attributeFilePartMaster); masterAttrFileMutex.lock(); - centralStoreAttributeFileList.insert(make_pair(part, attributeFilePartMaster + ".gz")); + centralStoreAttributeFileList[part] = attributeFilePartMaster + ".gz"; masterAttrFileMutex.unlock(); } @@ -974,13 +870,13 @@ string MetisPartitioner::reformatDataSet(string inputFilePath, int graphID) { secondVertex = std::stoi(vertexTwo); if (vertexToIDMap.find(firstVertex) == vertexToIDMap.end()) { - vertexToIDMap.insert(make_pair(firstVertex, idCounter)); - idToVertexMap.insert(make_pair(idCounter, firstVertex)); + vertexToIDMap[firstVertex] = idCounter; + idToVertexMap[idCounter] = firstVertex; idCounter++; } if (vertexToIDMap.find(secondVertex) == vertexToIDMap.end()) { - vertexToIDMap.insert(make_pair(secondVertex, idCounter)); - idToVertexMap.insert(make_pair(idCounter, secondVertex)); + vertexToIDMap[secondVertex] = idCounter; + idToVertexMap[idCounter] = secondVertex; idCounter++; } @@ -1099,6 +995,7 @@ void MetisPartitioner::writeSerializedCompositeMasterFiles(std::string part) { JasmineGraphHashMapCentralStore *hashMapCentralStore = new JasmineGraphHashMapCentralStore(); hashMapCentralStore->storePartEdgeMap(partMasterEdgeMap, outputFilePartMaster); + delete hashMapCentralStore; std::vector graphIds = Utils::split(part, '_'); std::vector::iterator graphIdIterator; @@ -1107,7 +1004,7 @@ void MetisPartitioner::writeSerializedCompositeMasterFiles(std::string part) { masterFileMutex.lock(); for (graphIdIterator = graphIds.begin(); graphIdIterator != graphIds.end(); ++graphIdIterator) { std::string graphId = *graphIdIterator; - compositeCentralStoreFileList.insert(make_pair(std::atoi(graphId.c_str()), outputFilePartMaster + ".gz")); + compositeCentralStoreFileList[std::atoi(graphId.c_str())] = outputFilePartMaster + ".gz"; } masterFileMutex.unlock(); partitioner_logger.log("Serializing done for central part " + part, "info"); diff --git a/src/partitioner/local/MetisPartitioner.h b/src/partitioner/local/MetisPartitioner.h index 5b581fa9..6b28c634 100644 --- a/src/partitioner/local/MetisPartitioner.h +++ b/src/partitioner/local/MetisPartitioner.h @@ -70,7 +70,7 @@ class MetisPartitioner { int smallestVertex = std::numeric_limits::max(); string graphAttributeType; - std::map partitionFileList; + std::map partitionFileMap; std::map centralStoreFileList; std::map compositeCentralStoreFileList; std::map centralStoreDuplicateFileList; @@ -100,10 +100,6 @@ class MetisPartitioner { void populatePartMaps(std::map partMap, int part); - void writePartitionFiles(int part); - - void writeMasterFiles(int part); - void writeSerializedMasterFiles(int part); void writeSerializedCompositeMasterFiles(std::string part); diff --git a/src/partitioner/local/RDFParser.cpp b/src/partitioner/local/RDFParser.cpp index dfb11dfc..125857ab 100644 --- a/src/partitioner/local/RDFParser.cpp +++ b/src/partitioner/local/RDFParser.cpp @@ -299,9 +299,6 @@ void GetConfig::readConfigFile(string &configFile, int graphId) { void GetConfig::writeEdgesToFile() { ofstream file; - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/"); - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/"); - Utils::createDirectory(Utils::getHomeDir() + "/.jasminegraph/tmp/" + to_string(this->graphID)); file.open(Utils::getHomeDir() + "/.jasminegraph/tmp/" + std::to_string(this->graphID) + "/" + std::to_string(this->graphID)); for (int i = 0; i < edgelist.size(); i++) { diff --git a/src/performance/metrics/PerformanceUtil.cpp b/src/performance/metrics/PerformanceUtil.cpp index be1487fd..fe17a8ec 100644 --- a/src/performance/metrics/PerformanceUtil.cpp +++ b/src/performance/metrics/PerformanceUtil.cpp @@ -16,6 +16,8 @@ limitations under the License. using namespace std::chrono; std::map> resourceUsageMap; +static size_t write_callback(void *contents, size_t size, size_t nmemb, std::string *output); + Logger scheduler_logger; SQLiteDBInterface *sqlLiteDB; PerformanceSQLiteDBInterface *perfDb; @@ -33,58 +35,39 @@ void PerformanceUtil::init() { } int PerformanceUtil::collectPerformanceStatistics() { - vector hostList = Utils::getHostListFromProperties(); - int hostListSize = hostList.size(); - int counter = 0; - std::vector> intermRes; - PlacesToNodeMapper placesToNodeMapper; + // Host level + double cpuUsage = StatisticCollector::getCpuUsage(); + Utils::send_job("", "cpu_usage", std::to_string(cpuUsage)); - std::string placeLoadQuery = - "select ip, user, server_port, is_master, is_host_reporter,host_idhost,idplace from place"; - std::vector>> placeList = perfDb->runSelect(placeLoadQuery); - std::vector>>::iterator placeListIterator; + long rxBytes = StatisticCollector::getRXBytes(); + Utils::send_job("", "rx_bytes", std::to_string(rxBytes)); - for (placeListIterator = placeList.begin(); placeListIterator != placeList.end(); ++placeListIterator) { - vector> place = *placeListIterator; - std::string host; - std::string requestResourceAllocation = "false"; + long txBytes = StatisticCollector::getTXBytes(); + Utils::send_job("", "tx_bytes", std::to_string(txBytes)); - std::string ip = place.at(0).second; - std::string user = place.at(1).second; - std::string serverPort = place.at(2).second; - std::string isMaster = place.at(3).second; - std::string isHostReporter = place.at(4).second; - std::string hostId = place.at(5).second; - std::string placeId = place.at(6).second; + long totalMemoryUsage = StatisticCollector::getTotalMemoryUsage(); + Utils::send_job("", "total_memory", std::to_string(totalMemoryUsage)); - if (ip.find("localhost") != std::string::npos) { - host = "localhost"; - } else { - host = user + "@" + ip; - } + long usedSwapSpace = StatisticCollector::getUsedSwapSpace(); + Utils::send_job("", "used_swap_space", std::to_string(usedSwapSpace)); - if (isHostReporter.find("true") != std::string::npos) { - std::string hostSearch = "select total_cpu_cores,total_memory from host where idhost='" + hostId + "'"; - std::vector>> hostAllocationList = perfDb->runSelect(hostSearch); + double currentLoadAverage = StatisticCollector::getLoadAverage(); + Utils::send_job("", "load_average", std::to_string(currentLoadAverage)); - vector> firstHostAllocation = hostAllocationList.at(0); + long totalSwapSpace = StatisticCollector::getTotalSwapSpace(); + Utils::send_job("", "total_swap_space", std::to_string(totalSwapSpace)); - std::string totalCPUCores = firstHostAllocation.at(0).second; - std::string totalMemory = firstHostAllocation.at(1).second; + // Per process + long memoryUsage = StatisticCollector::getMemoryUsageByProcess(); + Utils::send_job("", "memory_usage", std::to_string(memoryUsage)); - if (totalCPUCores.empty() || totalMemory.empty()) { - requestResourceAllocation = "true"; - } - } + int threadCount = StatisticCollector::getThreadCount(); + Utils::send_job("", "thread_count", std::to_string(threadCount)); - if (isMaster.find("true") != std::string::npos) { - collectLocalPerformanceData(isHostReporter, requestResourceAllocation, hostId, placeId); - } else { - collectRemotePerformanceData(host, atoi(serverPort.c_str()), isHostReporter, requestResourceAllocation, - hostId, placeId); - } - } + int socketCount = StatisticCollector::getSocketCount(); + Utils::send_job("", "socket_count", std::to_string(socketCount)); + scheduler_logger.info("Pushed performance metrics"); return 0; } @@ -149,342 +132,11 @@ int PerformanceUtil::collectSLAResourceConsumption(std::vector placeList, return 0; } -std::vector PerformanceUtil::retrieveCurrentResourceUtilization(std::string masterIP) { - std::string placeLoadQuery = - "select ip, user, server_port, is_master, is_host_reporter,host_idhost,idplace from place"; - std::vector>> placeList = perfDb->runSelect(placeLoadQuery); - std::vector>>::iterator placeListIterator; - std::vector placeResourceConsumptionList; - ResourceConsumption resourceConsumption; - - for (placeListIterator = placeList.begin(); placeListIterator != placeList.end(); ++placeListIterator) { - vector> place = *placeListIterator; - std::string host; - std::string requestResourceAllocation = "false"; - - std::string ip = place.at(0).second; - std::string user = place.at(1).second; - std::string serverPort = place.at(2).second; - std::string isMaster = place.at(3).second; - std::string isHostReporter = place.at(4).second; - std::string hostId = place.at(5).second; - std::string placeId = place.at(6).second; - - if (ip.find("localhost") != std::string::npos || ip.compare(masterIP) == 0) { - host = "localhost"; - } else { - host = user + "@" + ip; - } - - if (isHostReporter.find("true") != std::string::npos) { - std::string hostSearch = "select total_cpu_cores,total_memory from host where idhost='" + hostId + "'"; - std::vector>> hostAllocationList = perfDb->runSelect(hostSearch); - - vector> firstHostAllocation = hostAllocationList.at(0); - - std::string totalCPUCores = firstHostAllocation.at(0).second; - std::string totalMemory = firstHostAllocation.at(1).second; - - if (totalCPUCores.empty() || totalMemory.empty()) { - requestResourceAllocation = "true"; - } - } - - if ((isMaster.find("true") != std::string::npos || host == "localhost" || host.compare(masterIP) == 0) && - isHostReporter.find("true") != std::string::npos) { - resourceConsumption = retrieveLocalResourceConsumption(host, placeId); - placeResourceConsumptionList.push_back(resourceConsumption); - } else if (isHostReporter.find("true") != std::string::npos) { - resourceConsumption = retrieveRemoteResourceConsumption(host, atoi(serverPort.c_str()), hostId, placeId); - placeResourceConsumptionList.push_back(resourceConsumption); - } - } - - return placeResourceConsumptionList; -} - -void PerformanceUtil::collectRemotePerformanceData(std::string host, int port, std::string isVMStatManager, - std::string isResourceAllocationRequired, std::string hostId, - std::string placeId) { - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - scheduler_logger.error("ERROR, no host named " + host); - return; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return; - } - - bzero(data, 301); - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - string server_host = Utils::getJasmineGraphProperty("org.jasminegraph.server.host"); - write(sockfd, server_host.c_str(), server_host.size()); - scheduler_logger.log("Sent : " + server_host, "info"); - - write(sockfd, JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.c_str(), - JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - // scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - write(sockfd, isVMStatManager.c_str(), isVMStatManager.size()); - scheduler_logger.log("Sent : VM Manager Status " + isVMStatManager, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isResourceAllocationRequired.c_str(), isResourceAllocationRequired.size()); - scheduler_logger.log("Sent : Resource Allocation Requested " + isResourceAllocationRequired, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - scheduler_logger.log("Performance Response " + response, "info"); - - std::vector strArr = Utils::split(response, ','); - - std::string processTime = strArr[0]; - std::string memoryUsage = strArr[1]; - std::string cpuUsage = strArr[2]; - std::string loadAverage = strArr[3]; - - if (isVMStatManager == "true" && strArr.size() > 4) { - std::string memoryConsumption = strArr[4]; - std::string cpuUsage = strArr[5]; - string vmPerformanceSql = - "insert into host_performance_data (date_time, memory_usage, cpu_usage, idhost) values ('" + - processTime + "','" + memoryConsumption + "','" + cpuUsage + "','" + hostId + "')"; - - perfDb->runInsert(vmPerformanceSql); - - if (isResourceAllocationRequired == "true") { - std::string totalMemory = strArr[6]; - std::string totalCores = strArr[7]; - string allocationUpdateSql = "update host set total_cpu_cores='" + totalCores + - "',total_memory='" + totalMemory + "' where idhost='" + hostId + - "'"; - - perfDb->runUpdate(allocationUpdateSql); - } - } - - string placePerfSql = - "insert into place_performance_data (idplace, memory_usage, cpu_usage, date_time) values ('" + - placeId + "','" + memoryUsage + "','" + cpuUsage + "','" + processTime + "')"; - - perfDb->runInsert(placePerfSql); - } - } - } -} - -void PerformanceUtil::collectLocalPerformanceData(std::string isVMStatManager, std::string isResourceAllocationRequired, - std::string hostId, std::string placeId) { - StatisticCollector statisticCollector; - statisticCollector.init(); - - int memoryUsage = statisticCollector.getMemoryUsageByProcess(); - double cpuUsage = statisticCollector.getCpuUsage(); - - auto executedTime = std::chrono::system_clock::now(); - std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); - std::string reportTimeString(std::ctime(&reportTime)); - reportTimeString = Utils::trim_copy(reportTimeString); - - if (isVMStatManager.find("true") != std::string::npos) { - std::string vmLevelStatistics = - statisticCollector.collectVMStatistics(isVMStatManager, isResourceAllocationRequired); - std::vector strArr = Utils::split(vmLevelStatistics, ','); - - string totalMemoryUsed = strArr[0]; - string totalCPUUsage = strArr[1]; - - string vmPerformanceSql = - "insert into host_performance_data (date_time, memory_usage, cpu_usage, idhost) values ('" + - reportTimeString + "','" + totalMemoryUsed + "','" + totalCPUUsage + "','" + hostId + "')"; - - perfDb->runInsert(vmPerformanceSql); - - if (isResourceAllocationRequired == "true") { - std::string totalMemory = strArr[2]; - std::string totalCores = strArr[3]; - string allocationUpdateSql = "update host set total_cpu_cores='" + totalCores + "',total_memory='" + - totalMemory + "' where idhost='" + hostId + "'"; - - perfDb->runUpdate(allocationUpdateSql); - } - } - - string placePerfSql = "insert into place_performance_data (idplace, memory_usage, cpu_usage, date_time) values ('" + - placeId + "','" + to_string(memoryUsage) + "','" + to_string(cpuUsage) + "','" + - reportTimeString + "')"; - - perfDb->runInsert(placePerfSql); -} - -int PerformanceUtil::collectRemoteSLAResourceUtilization(std::string host, int port, std::string isVMStatManager, - std::string isResourceAllocationRequired, std::string placeId, - int elapsedTime, std::string masterIP) { - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - std::string graphSlaId; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return 0; - } - - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - scheduler_logger.error("ERROR, no host named " + host); - return 0; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return 0; - } - - bzero(data, 301); - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - write(sockfd, masterIP.c_str(), masterIP.size()); - scheduler_logger.log("Sent : " + masterIP, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { - write(sockfd, JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.c_str(), - JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + response, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isVMStatManager.c_str(), isVMStatManager.size()); - scheduler_logger.log("Sent : VM Manager Status " + isVMStatManager, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isResourceAllocationRequired.c_str(), isResourceAllocationRequired.size()); - scheduler_logger.log("Sent : Resource Allocation Requested " + isResourceAllocationRequired, - "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - scheduler_logger.log("Performance Response " + response, "info"); - - std::vector strArr = Utils::split(response, ','); - - std::string processTime = strArr[0]; - std::string memoryUsage = strArr[1]; - std::string cpuUsage = strArr[2]; - std::string loadAverage = strArr[3]; - - ResourceUsageInfo resourceUsageInfo; - resourceUsageInfo.elapsedTime = std::to_string(elapsedTime); - resourceUsageInfo.loadAverage = loadAverage; - resourceUsageInfo.memoryUsage = memoryUsage; - - if (!resourceUsageMap[placeId].empty()) { - resourceUsageMap[placeId].push_back(resourceUsageInfo); - } else { - std::vector resourceUsageVector; - - resourceUsageVector.push_back(resourceUsageInfo); - - resourceUsageMap[placeId] = resourceUsageVector; - } - return 1; - } - } - } - } - return 0; -} - void PerformanceUtil::collectLocalSLAResourceUtilization(std::string graphId, std::string placeId, std::string command, std::string category, int elapsedTime, bool autoCalibrate) { - StatisticCollector statisticCollector; - statisticCollector.init(); string graphSlaId; - double loadAverage = statisticCollector.getLoadAverage(); + double loadAverage = StatisticCollector::getLoadAverage(); auto executedTime = std::chrono::system_clock::now(); std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); @@ -510,115 +162,6 @@ void PerformanceUtil::collectLocalSLAResourceUtilization(std::string graphId, st } } -ResourceConsumption PerformanceUtil::retrieveLocalResourceConsumption(std::string host, std::string placeId) { - ResourceConsumption placeResourceConsumption; - - StatisticCollector statisticCollector; - statisticCollector.init(); - - int memoryUsage = statisticCollector.getMemoryUsageByProcess(); - double cpuUsage = statisticCollector.getCpuUsage(); - - placeResourceConsumption.memoryUsage = memoryUsage; - placeResourceConsumption.host = host; - - return placeResourceConsumption; -} - -ResourceConsumption PerformanceUtil::retrieveRemoteResourceConsumption(std::string host, int port, std::string hostId, - std::string placeId) { - ResourceConsumption placeResourceConsumption; - - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - std::string graphSlaId; - std::string isVMStatManager = "false"; - std::string isResourceAllocationRequired = "false"; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return placeResourceConsumption; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - scheduler_logger.error("ERROR, no host named " + host); - return placeResourceConsumption; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return placeResourceConsumption; - } - - bzero(data, 301); - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - string server_host = Utils::getJasmineGraphProperty("org.jasminegraph.server.host"); - write(sockfd, server_host.c_str(), server_host.size()); - scheduler_logger.log("Sent : " + server_host, "info"); - - write(sockfd, JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.c_str(), - JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isVMStatManager.c_str(), isVMStatManager.size()); - scheduler_logger.log("Sent : VM Manager Status " + isVMStatManager, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - write(sockfd, isResourceAllocationRequired.c_str(), isResourceAllocationRequired.size()); - scheduler_logger.log("Sent : Resource Allocation Requested " + isResourceAllocationRequired, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - scheduler_logger.log("Performance Response " + response, "info"); - - std::vector strArr = Utils::split(response, ','); - - std::string processTime = strArr[0]; - std::string memoryUsage = strArr[1]; - std::string cpuUsage = strArr[2]; - - placeResourceConsumption.memoryUsage = atoi(memoryUsage.c_str()); - placeResourceConsumption.host = host; - } - } - } - return placeResourceConsumption; -} - std::vector PerformanceUtil::getResourceAvailableTime(std::vector graphIdList, std::string command, std::string category, std::string masterIP, std::vector &pendingHPJobList) { @@ -894,11 +437,7 @@ void PerformanceUtil::adjustAggregateLoadMap(std::map loadVector = Utils::split(loadString, ','); - std::vector::iterator loadVectorIterator; string valuesString; std::string query = "SELECT id from graph_sla where graph_id='" + graphId + "' and partition_count='" + std::to_string(partitionCount) + "' and id_sla_category='" + slaCategoryId + "';"; @@ -1009,13 +548,15 @@ void PerformanceUtil::updateRemoteResourceConsumption(PerformanceSQLiteDBInterfa "elapsed_time) " "values "; - for (loadVectorIterator = loadVector.begin(); loadVectorIterator != loadVector.end(); + for (auto loadVectorIterator = loadVector.begin(); loadVectorIterator != loadVector.end(); ++loadVectorIterator) { std::string loadAverage = *loadVectorIterator; valuesString += "('" + graphSlaId + "','" + placeId + "', '','" + loadAverage + "','" + std::to_string(elapsedTime * 1000) + "'),"; elapsedTime += 5; + + Utils::send_job("loadAverageSLARem", "load_average_rem", loadAverage); } valuesString = valuesString.substr(0, valuesString.length() - 1); @@ -1115,92 +656,16 @@ void PerformanceUtil::initiateCollectingRemoteSLAResourceUtilization(std::string } } } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); } std::string PerformanceUtil::requestRemoteLoadAverages(std::string host, int port, std::string isVMStatManager, std::string isResourceAllocationRequired, std::string placeId, int elapsedTime, std::string masterIP) { - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - std::string graphSlaId; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return 0; - } - - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - scheduler_logger.error("ERROR, no host named " + host); - return 0; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return 0; - } - - bzero(data, 301); - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - write(sockfd, masterIP.c_str(), masterIP.size()); - scheduler_logger.log("Sent : " + masterIP, "info"); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - scheduler_logger.log("Received : " + JasmineGraphInstanceProtocol::OK, "info"); - - if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { - write(sockfd, JasmineGraphInstanceProtocol::REQUEST_COLLECTED_STATS.c_str(), - JasmineGraphInstanceProtocol::REQUEST_COLLECTED_STATS.size()); - scheduler_logger.log("Sent : " + JasmineGraphInstanceProtocol::REQUEST_COLLECTED_STATS, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - string status = response.substr(response.size() - 5); - std::string result = response.substr(0, response.size() - 5); - - while (status == "/SEND") { - write(sockfd, status.c_str(), status.size()); - - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - status = response.substr(response.size() - 5); - std::string loadAverageString = response.substr(0, response.size() - 5); - result = result + loadAverageString; - } - response = result; - } - } - - return response; + // master should not request load averages directly from workers. + // Instead it should get from Prometheus. + return ""; } double PerformanceUtil::getAggregatedLoadAverage(std::string graphId, std::string placeId, std::string command, @@ -1265,7 +730,8 @@ double PerformanceUtil::getAggregatedLoadAverage(std::string graphId, std::strin "' and graph_place_sla_performance.place_id='" + placeId + "' and sla_category.category='" + category + "' and graph_place_sla_performance.elapsed_time > '" + std::to_string(adjustedElapsedTime) + - "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time LIMIT 2;"; + "' order by graph_place_sla_performance.place_id,graph_place_sla_performance.elapsed_time" + + " LIMIT 2;"; std::vector>> loadAvgResults = perfDb->runSelect(slaLoadQuery); diff --git a/src/performance/metrics/PerformanceUtil.h b/src/performance/metrics/PerformanceUtil.h index 7e18bb5e..83cc038d 100644 --- a/src/performance/metrics/PerformanceUtil.h +++ b/src/performance/metrics/PerformanceUtil.h @@ -29,7 +29,6 @@ limitations under the License. #include "../../performancedb/PerformanceSQLiteDBInterface.h" #include "../../server/JasmineGraphInstanceProtocol.h" #include "../../util/Conts.h" -#include "../../util/PlacesToNodeMapper.h" #include "../../util/Utils.h" #include "../../util/logger/Logger.h" #include "StatisticCollector.h" @@ -60,16 +59,15 @@ class PerformanceUtil { static int collectSLAResourceConsumption(std::vector placeList, std::string graphId, std::string command, std::string category, std::string masterIP, int elapsedTime, bool autoCalibrate); - static std::vector retrieveCurrentResourceUtilization(std::string masterIP); static std::vector getResourceAvailableTime(std::vector graphIdList, std::string command, std::string category, std::string masterIP, std::vector& pendingHPJobList); static void logLoadAverage(); static std::vector getHostReporterList(); - static void updateResourceConsumption(PerformanceSQLiteDBInterface *performanceDb, std::string graphId, + static void updateResourceConsumption(PerformanceSQLiteDBInterface* performanceDb, std::string graphId, int partitionCount, std::vector placeList, std::string slaCategoryId); - static void updateRemoteResourceConsumption(PerformanceSQLiteDBInterface *performanceDb, std::string graphId, + static void updateRemoteResourceConsumption(PerformanceSQLiteDBInterface* performanceDb, std::string graphId, int partitionCount, std::vector placeList, std::string slaCategoryId, std::string masterIP); static std::string getSLACategoryId(std::string command, std::string category); @@ -84,19 +82,8 @@ class PerformanceUtil { std::string category, int elapsedTime); private: - static void collectRemotePerformanceData(std::string host, int port, std::string isVMStatManager, - std::string isResourceAllocationRequired, std::string hostId, - std::string placeId); - static void collectLocalPerformanceData(std::string isVMStatManager, std::string isResourceAllocationRequired, - std::string hostId, std::string placeId); - static int collectRemoteSLAResourceUtilization(std::string host, int port, std::string isVMStatManager, - std::string isResourceAllocationRequired, std::string placeId, - int elapsedTime, std::string masterIP); static void collectLocalSLAResourceUtilization(std::string graphId, std::string placeId, std::string command, std::string category, int elapsedTime, bool autoCalibrate); - static ResourceConsumption retrieveRemoteResourceConsumption(std::string host, int port, std::string hostId, - std::string placeId); - static ResourceConsumption retrieveLocalResourceConsumption(std::string hostId, std::string placeId); static void adjustAggregateLoadMap(std::map>& aggregateLoadAvgMap, std::map>& newJobLoadAvgMap, long newJobAcceptanceTime); diff --git a/src/performance/metrics/StatisticCollector.cpp b/src/performance/metrics/StatisticCollector.cpp index 10639555..db31ac4b 100644 --- a/src/performance/metrics/StatisticCollector.cpp +++ b/src/performance/metrics/StatisticCollector.cpp @@ -13,97 +13,203 @@ limitations under the License. #include "StatisticCollector.h" -static clock_t lastCPU, lastSysCPU, lastUserCPU; +Logger stat_logger; static int numProcessors; +static long parseLine(char *line); +static long getSwapSpace(const char *type); + +#define LINE_BUF_SIZE 128 + int StatisticCollector::init() { - FILE* file; + FILE *file; struct tms timeSample; - char line[128]; - - lastCPU = times(&timeSample); - lastSysCPU = timeSample.tms_stime; - lastUserCPU = timeSample.tms_utime; + char line[LINE_BUF_SIZE]; file = fopen("/proc/cpuinfo", "r"); + if (!file) { + stat_logger.error("Cannot open /proc/cpuinfo"); + exit(-1); + } numProcessors = 0; - while (fgets(line, 128, file) != NULL) { + while (fgets(line, LINE_BUF_SIZE, file) != NULL) { if (strncmp(line, "processor", 9) == 0) numProcessors++; } fclose(file); return 0; } -int StatisticCollector::getMemoryUsageByProcess() { - FILE* file = fopen("/proc/self/status", "r"); - int result = -1; - char line[128]; +long StatisticCollector::getMemoryUsageByProcess() { + FILE *file = fopen("/proc/self/status", "r"); + long result = -1; + char line[LINE_BUF_SIZE]; - while (fgets(line, 128, file) != NULL) { + while (fgets(line, LINE_BUF_SIZE, file) != NULL) { if (strncmp(line, "VmSize:", 7) == 0) { result = parseLine(line); break; } } fclose(file); - std::cout << "Memory Usage: " + std::to_string(result) << std::endl; return result; } -int StatisticCollector::parseLine(char* line) { - int i = strlen(line); - const char* p = line; - while (*p < '0' || *p > '9') p++; - line[i - 3] = '\0'; - i = atoi(p); - return i; +int StatisticCollector::getThreadCount() { + FILE *file = fopen("/proc/self/stat", "r"); + long result; + char line[LINE_BUF_SIZE]; + + for (int i = 0; i < 20; i++) { + if (fscanf(file, "%127s%*c", line) < 0) { + fclose(file); + return -1; + } + } + fclose(file); + result = strtol(line, NULL, 10); + if (result <= 0 || result > 0xfffffffffffffffL) return -1; + return result; } -double StatisticCollector::getCpuUsage() { - struct tms timeSample; - clock_t now; - double percent; - - now = times(&timeSample); - if (now <= lastCPU || timeSample.tms_stime < lastSysCPU || timeSample.tms_utime < lastUserCPU) { - // Overflow detection. Just skip this value. - percent = -1.0; - } else { - percent = (timeSample.tms_stime - lastSysCPU) + (timeSample.tms_utime - lastUserCPU); - percent /= (now - lastCPU); - percent /= numProcessors; - percent *= 100; +static long getSwapSpace(int field) { + FILE *file = fopen("/proc/swaps", "r"); + long result = -1; + char line[LINE_BUF_SIZE]; + + fgets(line, LINE_BUF_SIZE, file); + + while (fgets(line, LINE_BUF_SIZE, file) != NULL) { + char *value; + char *save = NULL; + for (int i = 0; i < field; i++) { + if (i == 0) { + value = strtok_r(line, " ", &save); + } else { + value = strtok_r(NULL, "\t", &save); + } + } + long used = strtol(value, NULL, 10); + if (used < 0 || used > 0xfffffffffffffffL) { + continue; + } + if (result >= 0) { + result += used; + } else { + result = used; + } } - lastCPU = now; - lastSysCPU = timeSample.tms_stime; - lastUserCPU = timeSample.tms_utime; + fclose(file); - return percent; + return result; } -std::string StatisticCollector::collectVMStatistics(std::string isVMStatManager, - std::string isTotalAllocationRequired) { - std::string vmLevelStatistics; +long StatisticCollector::getUsedSwapSpace() { + long result = getSwapSpace(4); + return result; +} - if (isVMStatManager == "true") { - long totalMemoryUsed = getTotalMemoryUsage(); - double totalCPUUsage = getTotalCpuUsage(); +long StatisticCollector::getTotalSwapSpace() { + long result = getSwapSpace(3); + return result; +} - std::stringstream stream; - stream << std::fixed << std::setprecision(2) << totalCPUUsage; - std::string cpuUsageString = stream.str(); +long StatisticCollector::getRXBytes() { + FILE *file = fopen("/sys/class/net/eth0/statistics/rx_bytes", "r"); + long result = -1; + fscanf(file, "%li", &result); + fclose(file); + return result; +} - vmLevelStatistics = std::to_string(totalMemoryUsed) + "," + cpuUsageString + ","; +long StatisticCollector::getTXBytes() { + FILE *file = fopen("/sys/class/net/eth0/statistics/tx_bytes", "r"); + long result = -1; + fscanf(file, "%li", &result); + fclose(file); + return result; +} + +int StatisticCollector::getSocketCount() { + DIR *d = opendir("/proc/self/fd"); + if (!d) { + puts("Error opening directory /proc/self/fd"); + return -1; } + const struct dirent *dir; + char path[64]; + char link_buf[1024]; + int count = 0; + while ((dir = readdir(d)) != NULL) { + const char *filename = dir->d_name; + if (filename[0] < '0' || '9' < filename[0]) continue; + sprintf(path, "/proc/self/fd/%s", filename); + size_t len = readlink(path, link_buf, sizeof(link_buf) - 1); + link_buf[len] = 0; + if (len > 0 && strncmp("socket:", link_buf, 7) == 0) { + count++; + } + } + (void)closedir(d); + return count; +} + +static long parseLine(char *line) { + int i = strlen(line); + const char *p = line; + while (*p < '0' || *p > '9') p++; + line[i - 3] = '\0'; + long val = strtol(p, NULL, 10); + if (val < 0 || val > 0xfffffffffffffffL) return -1; + return val; +} - if (isTotalAllocationRequired == "true") { - long totalMemory = getTotalMemoryAllocated(); - int totalCoresAvailable = getTotalNumberofCores(); +static void getCpuCycles(long long *totalp, long long *idlep) { + *totalp = 0; + *idlep = 0; + FILE *fp = fopen("/proc/stat", "r"); + if (!fp) return; + char line[1024]; + fscanf(fp, "%[^\r\n]%*c", line); + fclose(fp); - vmLevelStatistics = vmLevelStatistics + std::to_string(totalMemory) + "," + std::to_string(totalCoresAvailable); + char *p = line; + while (*p < '0' || *p > '9') p++; + long long total = 0; + long long idle = 0; + char *end_ptr = p; + for (int field = 1; field <= 10; field++) { + while (*p < '0' || *p > '9') { + if (!(*p)) break; + p++; + } + if (!(*p)) break; + long long value = strtoll(p, &end_ptr, 10); + p = end_ptr; + if (value < 0) { + stat_logger.error("Value is " + to_string(value) + " for line " + string(line)); + } + if (field == 4) { + idle += value; + } + total += value; } + *totalp = total; + *idlep = idle; +} - return vmLevelStatistics; +double StatisticCollector::getCpuUsage() { + long long total1; + long long idle1; + getCpuCycles(&total1, &idle1); + sleep(5); + long long total2; + long long idle2; + getCpuCycles(&total2, &idle2); + + long long diffTotal = total2 - total1; + long long diffIdle = idle2 - idle1; + + return (diffTotal - diffIdle) / (double)diffTotal; } long StatisticCollector::getTotalMemoryAllocated() { @@ -136,6 +242,7 @@ long StatisticCollector::getTotalMemoryUsage() { unsigned long cached; unsigned long sReclaimable; unsigned long memUsage; + while (file >> token) { if (token == "MemTotal:") { file >> memTotal; @@ -151,6 +258,7 @@ long StatisticCollector::getTotalMemoryUsage() { file.ignore(std::numeric_limits::max(), '\n'); } memUsage = memTotal - (memFree + buffers + cached + sReclaimable); + return memUsage; } @@ -162,7 +270,7 @@ double StatisticCollector::getTotalCpuUsage() { int count = 0; double totalCPUUsage = 0; - FILE* input = popen(mpstatCommand.c_str(), "r"); + FILE *input = popen(mpstatCommand.c_str(), "r"); if (input) { // read the input @@ -204,11 +312,9 @@ double StatisticCollector::getTotalCpuUsage() { } double StatisticCollector::getLoadAverage() { - double averages[3]; - - getloadavg(averages, 3); - - return averages[0]; + double loadAvg; + getloadavg(&loadAvg, 1); + return loadAvg; } void StatisticCollector::logLoadAverage(std::string name) { @@ -221,7 +327,6 @@ void StatisticCollector::logLoadAverage(std::string name) { performanceUtil.init(); start = time(0); - while (true) { if (isStatCollect) { std::this_thread::sleep_for(std::chrono::seconds(60)); diff --git a/src/performance/metrics/StatisticCollector.h b/src/performance/metrics/StatisticCollector.h index d6cae31f..d2550d70 100644 --- a/src/performance/metrics/StatisticCollector.h +++ b/src/performance/metrics/StatisticCollector.h @@ -40,11 +40,15 @@ class StatisticCollector { static const int BUFFER_SIZE = 128; public: - int init(); - static int getMemoryUsageByProcess(); - static int parseLine(char* line); + static int init(); + static long getMemoryUsageByProcess(); + static int getThreadCount(); + static long getUsedSwapSpace(); + static long getTotalSwapSpace(); + static long getRXBytes(); + static long getTXBytes(); + static int getSocketCount(); static double getCpuUsage(); - static std::string collectVMStatistics(std::string isVMStatManager, std::string isTotalAllocationRequired); static long getTotalMemoryAllocated(); static int getTotalNumberofCores(); static long getTotalMemoryUsage(); diff --git a/src/performancedb/PerformanceSQLiteDBInterface.cpp b/src/performancedb/PerformanceSQLiteDBInterface.cpp index f89b72c2..a61aa6f1 100644 --- a/src/performancedb/PerformanceSQLiteDBInterface.cpp +++ b/src/performancedb/PerformanceSQLiteDBInterface.cpp @@ -23,18 +23,17 @@ Logger perfdb_logger; int PerformanceSQLiteDBInterface::init() { if (!Utils::fileExists(this->databaseLocation.c_str())) { - if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "src/performancedb/ddl.sql") != 0) { + if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "ddl/perfdb.sql") != 0) { return -1; } } - int rc = - sqlite3_open(this->databaseLocation.c_str(), &database); + int rc = sqlite3_open(this->databaseLocation.c_str(), &database); if (rc) { - perfdb_logger.log("Cannot open database: " + string(sqlite3_errmsg(database)), "error"); + perfdb_logger.error("Cannot open database: " + string(sqlite3_errmsg(database))); return -1; } - perfdb_logger.log("Database opened successfully", "info"); + perfdb_logger.info("Database opened successfully"); return 0; } diff --git a/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp b/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp index 3a8045e8..58e3245a 100644 --- a/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp +++ b/src/query/algorithms/linkprediction/JasminGraphLinkPredictor.cpp @@ -44,7 +44,7 @@ void JasminGraphLinkPredictor::initiateLinkPrediction(std::string graphID, std:: selectedHostPartitions = (graphPartitionedHosts[it->first]).partitionID; selectedHostPartitionsNo = selectedHostPartitions.size(); } else { - remainHostMap.insert(std::pair(it->first, it->second)); + remainHostMap[it->first] = it->second; } count++; } @@ -194,7 +194,7 @@ int JasminGraphLinkPredictor::sendQueryToWorker(std::string host, int port, int if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { predictor_logger.log("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT, "info"); predictor_logger.log("Going to send file through service", "info"); - JasmineGraphServer::sendFileThroughService(host, dataPort, fileName, filePath, masterIP); + Utils::sendFileThroughService(host, dataPort, fileName, filePath); } } } diff --git a/src/query/algorithms/triangles/Triangles.cpp b/src/query/algorithms/triangles/Triangles.cpp index 30cf1a06..00b08dbd 100644 --- a/src/query/algorithms/triangles/Triangles.cpp +++ b/src/query/algorithms/triangles/Triangles.cpp @@ -24,37 +24,32 @@ limitations under the License. Logger triangle_logger; -long Triangles::run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCentralStore centralStore, - JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore, std::string hostName) { +long Triangles::run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string hostName) { return run(graphDB, centralStore, duplicateCentralStore, NULL, NULL, 0); } -long Triangles::run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCentralStore centralStore, - JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore, std::string graphId, +long Triangles::run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string graphId, std::string partitionId, int threadPriority) { - triangle_logger.log("###TRIANGLE### Triangle Counting: Started", "info"); + triangle_logger.info("###TRIANGLE### Triangle Counting: Started"); map> localSubGraphMap = graphDB.getUnderlyingHashMap(); map> centralDBSubGraphMap = centralStore.getUnderlyingHashMap(); map> duplicateCentralDBSubGraphMap = duplicateCentralStore.getUnderlyingHashMap(); map degreeDistribution = graphDB.getOutDegreeDistributionHashMap(); map centralDBDegreeDistribution = centralStore.getOutDegreeDistributionHashMap(); map centralDuplicateDBDegreeDistribution = duplicateCentralStore.getOutDegreeDistributionHashMap(); - std::map> degreeMap; - - std::map::iterator it; - std::map::iterator centralDBDegreeDistributionIterator; - std::map::iterator centralDuplicateDBDegreeDistributionIterator; auto mergeBbegin = std::chrono::high_resolution_clock::now(); - for (centralDuplicateDBDegreeDistributionIterator = centralDuplicateDBDegreeDistribution.begin(); + for (auto centralDuplicateDBDegreeDistributionIterator = centralDuplicateDBDegreeDistribution.begin(); centralDuplicateDBDegreeDistributionIterator != centralDuplicateDBDegreeDistribution.end(); ++centralDuplicateDBDegreeDistributionIterator) { long centralDuplicateDBStartVid = centralDuplicateDBDegreeDistributionIterator->first; - unordered_set centralDBSecondVertexSet = centralDBSubGraphMap[centralDuplicateDBStartVid]; - unordered_set duplicateSecondVertexSet = duplicateCentralDBSubGraphMap[centralDuplicateDBStartVid]; - std::set result; + unordered_set ¢ralDBSecondVertexSet = centralDBSubGraphMap[centralDuplicateDBStartVid]; + const unordered_set &duplicateSecondVertexSet = duplicateCentralDBSubGraphMap[centralDuplicateDBStartVid]; + unordered_set result; std::set_difference(duplicateSecondVertexSet.begin(), duplicateSecondVertexSet.end(), centralDBSecondVertexSet.begin(), centralDBSecondVertexSet.end(), @@ -62,12 +57,12 @@ long Triangles::run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCe if (result.size() > 0) { centralDBDegreeDistribution[centralDuplicateDBStartVid] += result.size(); - centralDBSubGraphMap[centralDuplicateDBStartVid].insert(result.begin(), result.end()); + centralDBSecondVertexSet.insert(result.begin(), result.end()); } } // Merging Local Store and Workers central stores before starting triangle count - for (centralDBDegreeDistributionIterator = centralDBDegreeDistribution.begin(); + for (auto centralDBDegreeDistributionIterator = centralDBDegreeDistribution.begin(); centralDBDegreeDistributionIterator != centralDBDegreeDistribution.end(); ++centralDBDegreeDistributionIterator) { long centralDBStartVid = centralDBDegreeDistributionIterator->first; @@ -82,7 +77,7 @@ long Triangles::run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCe auto mergeDur = mergeEnd - mergeBbegin; auto mergeMsDuration = std::chrono::duration_cast(mergeDur).count(); - triangle_logger.log(" Merge time Taken: " + std::to_string(mergeMsDuration) + " milliseconds", "info"); + triangle_logger.info(" Merge time Taken: " + std::to_string(mergeMsDuration) + " milliseconds"); const TriangleResult &triangleResult = countTriangles(localSubGraphMap, degreeDistribution, false); return triangleResult.count; @@ -105,23 +100,24 @@ TriangleResult Triangles::countTriangles(map> ¢ral long triangleCount = 0; std::unordered_map>> triangleTree; + // TODO(thevindu-w): Make centralstore undirected and prevent saving triangles in-memory for (auto iterator = degreeMap.begin(); iterator != degreeMap.end(); ++iterator) { - std::set &vertices = iterator->second; + auto &vertices = iterator->second; for (auto verticesIterator = vertices.begin(); verticesIterator != vertices.end(); ++verticesIterator) { long temp = *verticesIterator; - std::unordered_set &unorderedUSet = centralStore[temp]; + auto &unorderedUSet = centralStore[temp]; for (auto uSetIterator = unorderedUSet.begin(); uSetIterator != unorderedUSet.end(); ++uSetIterator) { long u = *uSetIterator; if (temp == u) continue; - std::unordered_set &unorderedNuSet = centralStore[u]; + auto &unorderedNuSet = centralStore[u]; for (auto nuSetIterator = unorderedNuSet.begin(); nuSetIterator != unorderedNuSet.end(); ++nuSetIterator) { long nu = *nuSetIterator; if (temp == nu) continue; if (u == nu) continue; - std::unordered_set ¢ralStoreNu = centralStore[nu]; + auto ¢ralStoreNu = centralStore[nu]; if ((unorderedUSet.find(nu) != unorderedUSet.end()) || (centralStoreNu.find(temp) != centralStoreNu.end())) { long varOne = temp; @@ -152,6 +148,7 @@ TriangleResult Triangles::countTriangles(map> ¢ral set2.insert(varThree); triangleCount++; if (returnTriangles) { + // TODO(thevindu-w): Flush to file on count exceeds value triangleStream << varOne << "," << varTwo << "," << varThree << ":"; } } @@ -159,6 +156,7 @@ TriangleResult Triangles::countTriangles(map> ¢ral triangleTree[varOne][varTwo].insert(varThree); triangleCount++; if (returnTriangles) { + // TODO(thevindu-w): Flush to file on count exceeds value triangleStream << varOne << "," << varTwo << "," << varThree << ":"; } } @@ -167,6 +165,7 @@ TriangleResult Triangles::countTriangles(map> ¢ral } } } + triangleTree.clear(); TriangleResult result; diff --git a/src/query/algorithms/triangles/Triangles.h b/src/query/algorithms/triangles/Triangles.h index 66586867..03728c89 100644 --- a/src/query/algorithms/triangles/Triangles.h +++ b/src/query/algorithms/triangles/Triangles.h @@ -37,15 +37,15 @@ class JasmineGraphHashMapDuplicateCentralStore; class Triangles { public: - static long run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCentralStore centralStore, - JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore, std::string hostName); + static long run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string hostName); - static long run(JasmineGraphHashMapLocalStore graphDB, JasmineGraphHashMapCentralStore centralStore, - JasmineGraphHashMapDuplicateCentralStore duplicateCentralStore, std::string graphId, + static long run(JasmineGraphHashMapLocalStore &graphDB, JasmineGraphHashMapCentralStore ¢ralStore, + JasmineGraphHashMapDuplicateCentralStore &duplicateCentralStore, std::string graphId, std::string partitionId, int threadPriority); - static TriangleResult countTriangles(map> ¢ralStore, - map &distributionMap, bool returnTriangles); + static TriangleResult countTriangles(map> ¢ralStore, map &distributionMap, + bool returnTriangles); }; #endif // JASMINEGRAPH_TRIANGLES_H diff --git a/src/server/JasmineGraphInstance.cpp b/src/server/JasmineGraphInstance.cpp index cf07e87d..0c585eee 100644 --- a/src/server/JasmineGraphInstance.cpp +++ b/src/server/JasmineGraphInstance.cpp @@ -21,7 +21,7 @@ Logger graphInstance_logger; void *runInstanceService(void *dummyPt) { JasmineGraphInstance *refToInstance = (JasmineGraphInstance *)dummyPt; refToInstance->instanceService = new JasmineGraphInstanceService(); - refToInstance->instanceService->run(refToInstance->profile, refToInstance->masterHostName, refToInstance->hostName, + refToInstance->instanceService->run(refToInstance->masterHostName, refToInstance->hostName, refToInstance->serverPort, refToInstance->serverDataPort); return NULL; } @@ -33,17 +33,25 @@ void *runFileTransferService(void *dummyPt) { return NULL; } -int JasmineGraphInstance::start_running(string profile, string hostName, string masterHost, int serverPort, - int serverDataPort, string enableNmon) { +int JasmineGraphInstance::start_running(string hostName, string masterHost, int serverPort, int serverDataPort, + string enableNmon) { graphInstance_logger.info("Worker started"); this->hostName = hostName; - this->profile = profile; this->masterHostName = masterHost; this->serverPort = serverPort; this->serverDataPort = serverDataPort; this->enableNmon = enableNmon; + if (Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"))) { + graphInstance_logger.error("Could not create directory: " + + Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); + } + if (Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"))) { + graphInstance_logger.error("Could not create directory: " + + Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder")); + } + startNmonAnalyzer(enableNmon, serverPort); pthread_t instanceCommunicatorThread; @@ -59,116 +67,6 @@ int JasmineGraphInstance::start_running(string profile, string hostName, string return 0; } -bool JasmineGraphInstance::acknowledgeMaster(string masterHost, string workerIP, string workerPort) { - int sockfd; - char data[301]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return false; - } - - if (masterHost.find('@') != std::string::npos) { - masterHost = Utils::split(masterHost, '@')[1]; - } - - graphInstance_logger.log("###INSTANCE### Get Host By Name : " + masterHost, "info"); - - server = gethostbyname(masterHost.c_str()); - if (server == NULL) { - graphInstance_logger.error("ERROR, no host named " + masterHost); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(Conts::JASMINEGRAPH_BACKEND_PORT); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting" << std::endl; - return false; - } - - bzero(data, 301); - int result_wr = - write(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE.c_str(), JasmineGraphInstanceProtocol::HANDSHAKE.size()); - - if (result_wr < 0) { - graphInstance_logger.log("Error writing to socket", "error"); - } - - graphInstance_logger.log("Sent : " + JasmineGraphInstanceProtocol::HANDSHAKE, "info"); - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) == 0) { - graphInstance_logger.log("Received : " + JasmineGraphInstanceProtocol::HANDSHAKE_OK, "info"); - - result_wr = write(sockfd, workerIP.c_str(), workerIP.size()); - - if (result_wr < 0) { - graphInstance_logger.log("Error writing to socket", "error"); - } - - graphInstance_logger.log("Sent : " + workerIP, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) == 0) { - graphInstance_logger.log("Received : " + JasmineGraphInstanceProtocol::HOST_OK, "info"); - - result_wr = write(sockfd, JasmineGraphInstanceProtocol::ACKNOWLEDGE_MASTER.c_str(), - JasmineGraphInstanceProtocol::ACKNOWLEDGE_MASTER.size()); - - if (result_wr < 0) { - graphInstance_logger.log("Error writing to socket", "error"); - } - - graphInstance_logger.log("Sent : " + JasmineGraphInstanceProtocol::ACKNOWLEDGE_MASTER, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - response = Utils::trim_copy(response); - - if (response.compare(JasmineGraphInstanceProtocol::WORKER_INFO_SEND) == 0) { - std::string workerInfo = workerIP + "|" + workerPort; - result_wr = write(sockfd, workerInfo.c_str(), workerInfo.size()); - - if (result_wr < 0) { - graphInstance_logger.log("Error writing to socket", "error"); - } - - graphInstance_logger.log("Sent : " + workerInfo, "info"); - bzero(data, 301); - read(sockfd, data, 300); - response = (data); - - if (response.compare(JasmineGraphInstanceProtocol::UPDATE_DONE) == 0) { - return true; - } - } - - if (response.compare(JasmineGraphInstanceProtocol::OK) == 0) { - return true; - } - } - - return false; - } - return false; -} - void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) { if (enableNmon == "true") { std::string nmonFileLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.nmon.file.location"); @@ -176,7 +74,7 @@ void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) std::string snapshotGap = Utils::getJasmineGraphProperty("org.jasminegraph.server.nmon.snapshot.gap"); std::string nmonFileName = nmonFileLocation + "nmon.log." + std::to_string(serverPort); std::string nmonStartupCommand = - "nmon_x86_64_ubuntu18 -c " + numberOfSnapshots + " -s " + snapshotGap + " -T -F " + nmonFileName; + "nmon -c " + numberOfSnapshots + " -s " + snapshotGap + " -T -F " + nmonFileName; char buffer[BUFFER_SIZE]; std::string result = ""; @@ -191,84 +89,10 @@ void JasmineGraphInstance::startNmonAnalyzer(string enableNmon, int serverPort) } } if (!result.empty()) { - graphInstance_logger.log("Error in performance database backup process", "error"); + graphInstance_logger.error("Error in performance database backup process"); } pclose(input); } } } - -bool JasmineGraphInstance::isRunning() { return true; } - -bool JasmineGraphInstance::sendFileThroughService(std::string host, int dataPort, std::string fileName, - std::string filePath) { - int sockfd; - char data[301]; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - std::cerr << "Cannot create socket" << std::endl; - return false; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - graphInstance_logger.error("ERROR, no host named " + host); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(dataPort); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - std::cerr << "ERROR connecting to port " << dataPort << std::endl; - return false; - } - - fileName = "jasminegraph-local_trained_model_store/" + fileName; - write(sockfd, fileName.c_str(), fileName.size()); - - bzero(data, 301); - read(sockfd, data, 300); - string response = (data); - response = Utils::trim_copy(response); - if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE) == 0) { - std::cout << "Sending file " << filePath << " through port " << dataPort << std::endl; - - FILE *fp = fopen(filePath.c_str(), "r"); - if (fp == NULL) { - printf("Error opening file\n"); - close(sockfd); - return false; - } - for (;;) { - unsigned char buff[1024]; - int nread = fread(buff, 1, sizeof(buff), fp); - - /* If read was success, send data. */ - if (nread > 0) { - write(sockfd, buff, nread); - } - - if (nread < sizeof(buff)) { - if (feof(fp)) - if (ferror(fp)) { - printf("Error reading\n"); - return false; - } - break; - } - } - - fclose(fp); - close(sockfd); - return true; - } - return false; -} diff --git a/src/server/JasmineGraphInstance.h b/src/server/JasmineGraphInstance.h index 964f1376..4760eccc 100644 --- a/src/server/JasmineGraphInstance.h +++ b/src/server/JasmineGraphInstance.h @@ -31,10 +31,7 @@ class JasmineGraphInstance { static const int BUFFER_SIZE = 128; public: - int start_running(string profile, string hostName, string masterHost, int serverPort, int serverDataPort, - string enableNmon); - - bool acknowledgeMaster(string masterHost, string workerIP, string workerPort); + int start_running(string hostName, string masterHost, int serverPort, int serverDataPort, string enableNmon); void startNmonAnalyzer(string enableNmon, int serverPort); @@ -44,10 +41,7 @@ class JasmineGraphInstance { void shutdown(); - bool isRunning(); - string hostName; - string profile; string masterHostName; int serverPort; int serverDataPort; @@ -55,7 +49,6 @@ class JasmineGraphInstance { JasmineGraphInstanceService *instanceService; JasmineGraphInstanceFileTransferService *ftpService; - static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath); }; #endif // JASMINEGRAPH_JASMINEGRAPHINSTANCE_H diff --git a/src/server/JasmineGraphInstanceFileTransferService.cpp b/src/server/JasmineGraphInstanceFileTransferService.cpp index 5ab602ac..77ad885a 100644 --- a/src/server/JasmineGraphInstanceFileTransferService.cpp +++ b/src/server/JasmineGraphInstanceFileTransferService.cpp @@ -13,6 +13,7 @@ limitations under the License. #include "JasmineGraphInstanceFileTransferService.h" +#include "../server/JasmineGraphInstanceProtocol.h" #include "../util/Utils.h" #include "../util/logger/Logger.h" @@ -24,26 +25,30 @@ void *filetransferservicesession(void *dummyPt) { filetransferservicesessionargs *sessionargs = (filetransferservicesessionargs *)dummyPt; int connFd = sessionargs->connFd; delete sessionargs; - char data[301]; - bzero(data, 301); - read(connFd, data, 300); - string fileName = (data); + char data[INSTANCE_DATA_LENGTH + 1]; + string fileName = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH); string filePathWithName = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; - write(connFd, JasmineGraphInstanceProtocol::SEND_FILE.c_str(), JasmineGraphInstanceProtocol::SEND_FILE.size()); - int bytesReceived = 0; - char buffer[1024]; + Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::SEND_FILE_LEN); + string fsizeStr = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + int fsize = stoi(fsizeStr); + Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::SEND_FILE); + char buffer[4096]; + file_service_logger.info("File transfer started for file: " + fileName); std::ofstream file(filePathWithName, std::ios::out | std::ios::binary); - do { - bytesReceived = recv(connFd, buffer, sizeof(buffer), 0); + while (fsize > 0) { + int bytesReceived = recv(connFd, buffer, sizeof(buffer), 0); if (bytesReceived > 0) { file.write(buffer, bytesReceived); - // printf("Buffer: %.*s\n", connFd, buffer); - // or: printf("Buffer: %*.*s\n", bytes_read, bytes_read, buffer); + fsize -= bytesReceived; + } else if (bytesReceived == 0) { + file_service_logger.error("File transfer failed for file: " + fileName); + break; } - } while (bytesReceived > 0); + } file.close(); + if (fsize == 0) file_service_logger.info("File transfer completed for file: " + fileName); return NULL; } @@ -80,21 +85,29 @@ void JasmineGraphInstanceFileTransferService::run(int dataPort) { } int connFd; listen(listenFd, 10); + file_service_logger.info("Worker FileTransfer Service listening on port " + to_string(dataPort)); len = sizeof(clntAdd); while (true) { - file_service_logger.log("Worker FileTransfer Service listening on port " + to_string(dataPort), "info"); connFd = accept(listenFd, (struct sockaddr *)&clntAdd, &len); if (connFd < 0) { - file_service_logger.log("Cannot accept connection to port " + to_string(dataPort), "error"); + file_service_logger.info("Cannot accept connection to port " + to_string(dataPort)); continue; } - file_service_logger.log("Connection successful to port " + to_string(dataPort), "info"); - filetransferservicesessionargs *sessionargs = new filetransferservicesessionargs; - sessionargs->connFd = connFd; - pthread_t pt; - pthread_create(&pt, NULL, filetransferservicesession, sessionargs); + file_service_logger.info("Connection successful to port " + to_string(dataPort)); + + pid_t pid = fork(); + if (pid == 0) { + close(listenFd); + filetransferservicesessionargs *sessionargs = new filetransferservicesessionargs; + sessionargs->connFd = connFd; + filetransferservicesession(sessionargs); + break; + } else { + close(connFd); + } } + exit(0); } diff --git a/src/server/JasmineGraphInstanceProtocol.cpp b/src/server/JasmineGraphInstanceProtocol.cpp index ba2712f1..c427981f 100644 --- a/src/server/JasmineGraphInstanceProtocol.cpp +++ b/src/server/JasmineGraphInstanceProtocol.cpp @@ -18,7 +18,6 @@ const string JasmineGraphInstanceProtocol::HANDSHAKE_OK = "hske-ok"; const string JasmineGraphInstanceProtocol::HOST_OK = "hst-ok"; const string JasmineGraphInstanceProtocol::BATCH_UPLOAD_COMPOSITE_CENTRAL = "upload-g-c-c"; const string JasmineGraphInstanceProtocol::CLOSE = "close"; -const string JasmineGraphInstanceProtocol::CLOSE_ACK = "close-ok"; const string JasmineGraphInstanceProtocol::SHUTDOWN = "shdn"; const string JasmineGraphInstanceProtocol::SHUTDOWN_ACK = "shdn-ok"; const string JasmineGraphInstanceProtocol::READY = "ready"; @@ -62,7 +61,6 @@ const string JasmineGraphInstanceProtocol::SEND_COMPOSITE_CENTRALSTORE_TO_AGGREG const string JasmineGraphInstanceProtocol::AGGREGATE_CENTRALSTORE_TRIANGLES = "aggregate"; const string JasmineGraphInstanceProtocol::AGGREGATE_STREAMING_CENTRALSTORE_TRIANGLES = "aggregate-streaming-central"; const string JasmineGraphInstanceProtocol::AGGREGATE_COMPOSITE_CENTRALSTORE_TRIANGLES = "aggregate-composite"; -const string JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS = "perf-stat"; const string JasmineGraphInstanceProtocol::START_STAT_COLLECTION = "begin-stat"; const string JasmineGraphInstanceProtocol::REQUEST_COLLECTED_STATS = "request-stat"; const string JasmineGraphInstanceProtocol::INITIATE_TRAIN = "initiate-train"; @@ -79,9 +77,11 @@ const string JasmineGraphInstanceProtocol::CHECK_FILE_ACCESSIBLE = "check-file-a const string JasmineGraphInstanceProtocol::SEND_FILE_TYPE = "send-file-type"; const string JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_AGGREGATE = "file-type-centralstore-aggregate"; const string JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_COMPOSITE = "file-type-centralstore-composite"; +const string JasmineGraphInstanceProtocol::FILE_TYPE_DATA = "file-type-data"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_START = "stream-start"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_START_ACK = "stream-start-ack"; const string JasmineGraphInstanceProtocol::SEND_PRIORITY = "send-priority"; +const string JasmineGraphInstanceProtocol::PUSH_PARTITION = "push-partition"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_C_length_ACK = "stream-c-length-ack"; const string JasmineGraphInstanceProtocol::GRAPH_STREAM_END_OF_EDGE = "\r\n"; // CRLF equivelent in HTTP const string JasmineGraphInstanceProtocol::GRAPH_CSV_STREAM_START = "csv-stream-start"; diff --git a/src/server/JasmineGraphInstanceProtocol.h b/src/server/JasmineGraphInstanceProtocol.h index 9c5fbae9..e52c46ce 100644 --- a/src/server/JasmineGraphInstanceProtocol.h +++ b/src/server/JasmineGraphInstanceProtocol.h @@ -76,12 +76,12 @@ class JasmineGraphInstanceProtocol { static const string AGGREGATE_CENTRALSTORE_TRIANGLES; static const string AGGREGATE_STREAMING_CENTRALSTORE_TRIANGLES; static const string AGGREGATE_COMPOSITE_CENTRALSTORE_TRIANGLES; - static const string PERFORMANCE_STATISTICS; static const string START_STAT_COLLECTION; static const string REQUEST_COLLECTED_STATS; static const string INITIATE_TRAIN; static const string INITIATE_PREDICT; static const string SEND_HOSTS; + static const string PUSH_PARTITION; static const string INITIATE_MODEL_COLLECTION; static const string INITIATE_FRAGMENT_RESOLUTION; static const string FRAGMENT_RESOLUTION_CHK; @@ -99,6 +99,7 @@ class JasmineGraphInstanceProtocol { static const string SEND_FILE_TYPE; static const string FILE_TYPE_CENTRALSTORE_AGGREGATE; static const string FILE_TYPE_CENTRALSTORE_COMPOSITE; + static const string FILE_TYPE_DATA; static const string GRAPH_STREAM_START; static const string GRAPH_STREAM_START_ACK; static const string GRAPH_CSV_STREAM_START; diff --git a/src/server/JasmineGraphInstanceService.cpp b/src/server/JasmineGraphInstanceService.cpp index 6927a19f..0c706f71 100644 --- a/src/server/JasmineGraphInstanceService.cpp +++ b/src/server/JasmineGraphInstanceService.cpp @@ -36,7 +36,6 @@ using namespace std; Logger instance_logger; pthread_mutex_t file_lock; pthread_mutex_t map_lock; -StatisticCollector collector; int JasmineGraphInstanceService::partitionCounter = 0; std::map> JasmineGraphInstanceService::iterationData; const string JasmineGraphInstanceService::END_OF_MESSAGE = "eom"; @@ -62,33 +61,33 @@ static void delete_graph_command(int connFd, bool *loop_exit_p); static void delete_graph_fragment_command(int connFd, bool *loop_exit_p); static void duplicate_centralstore_command(int connFd, int serverPort, bool *loop_exit_p); static void worker_in_degree_distribution_command( - int connFd, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p); + int connFd, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void in_degree_distribution_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p); + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void worker_out_degree_distribution_command( - int connFd, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p); + int connFd, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void out_degree_distribution_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p); + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void page_rank_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void worker_page_rank_distribution_command( - int connFd, int serverPort, std::map graphDBMapCentralStores, + int connFd, int serverPort, std::map &graphDBMapCentralStores, bool *loop_exit_p); static void egonet_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void worker_egonet_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p); static void triangles_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::map graphDBMapDuplicateCentralStores, + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, bool *loop_exit_p); static void streaming_triangles_command( int connFd, int serverPort, std::map &incrementalLocalStoreMap, @@ -100,7 +99,6 @@ static void aggregate_streaming_centralstore_triangles_command( int connFd, std::map &incrementalLocalStoreMap, bool *loop_exit_p); static void aggregate_composite_centralstore_triangles_command(int connFd, bool *loop_exit_p); -static void performance_statistics_command(int connFd, bool *loop_exit_p); static void initiate_files_command(int connFd, bool *loop_exit_p); static void initiate_fed_predict_command(int connFd, bool *loop_exit_p); static void initiate_server_command(int connFd, bool *loop_exit_p); @@ -120,9 +118,17 @@ static void send_priority_command(int connFd, bool *loop_exit_p); static std::string initiate_command_common(int connFd, bool *loop_exit_p); static void batch_upload_common(int connFd, bool *loop_exit_p, bool batch_upload); static void degree_distribution_common(int connFd, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p, bool in); +static void push_partition_command(int connFd, bool *loop_exit_p); +static void push_file_command(int connFd, bool *loop_exit_p); +long countLocalTriangles( + std::string graphId, std::string partitionId, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, + int threadPriority); char *converter(const std::string &s) { char *pc = new char[s.size() + 1]; @@ -135,32 +141,29 @@ void *instanceservicesession(void *dummyPt) { instanceservicesessionargs sessionargs = *sessionargs_p; delete sessionargs_p; int connFd = sessionargs.connFd; - std::map graphDBMapLocalStores = sessionargs.graphDBMapLocalStores; - std::map graphDBMapCentralStores = + std::map *graphDBMapLocalStores = sessionargs.graphDBMapLocalStores; + std::map *graphDBMapCentralStores = sessionargs.graphDBMapCentralStores; - std::map graphDBMapDuplicateCentralStores = + std::map *graphDBMapDuplicateCentralStores = sessionargs.graphDBMapDuplicateCentralStores; - std::map incrementalLocalStoreMap = - sessionargs.incrementalLocalStore; + std::map &incrementalLocalStoreMap = + *(sessionargs.incrementalLocalStore); InstanceStreamHandler streamHandler(incrementalLocalStoreMap); string serverName = sessionargs.host; string masterHost = sessionargs.masterHost; - string profile = sessionargs.profile; int serverPort = sessionargs.port; int serverDataPort = sessionargs.dataPort; instance_logger.info("New service session started on thread:" + to_string(pthread_self()) + " connFd:" + to_string(connFd)); - collector.init(); - - Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); char data[DATA_BUFFER_SIZE]; bool loop_exit = false; while (!loop_exit) { string line = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, true); if (line.empty()) { + sleep(1); continue; } line = Utils::trim_copy(line); @@ -191,26 +194,27 @@ void *instanceservicesession(void *dummyPt) { } else if (line.compare(JasmineGraphInstanceProtocol::DP_CENTRALSTORE) == 0) { duplicate_centralstore_command(connFd, serverPort, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::WORKER_IN_DEGREE_DISTRIBUTION) == 0) { - worker_in_degree_distribution_command(connFd, graphDBMapLocalStores, graphDBMapCentralStores, &loop_exit); + worker_in_degree_distribution_command(connFd, *graphDBMapLocalStores, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::IN_DEGREE_DISTRIBUTION) == 0) { - in_degree_distribution_command(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, + in_degree_distribution_command(connFd, serverPort, *graphDBMapLocalStores, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::WORKER_OUT_DEGREE_DISTRIBUTION) == 0) { - worker_out_degree_distribution_command(connFd, graphDBMapLocalStores, graphDBMapCentralStores, &loop_exit); + worker_out_degree_distribution_command(connFd, *graphDBMapLocalStores, *graphDBMapCentralStores, + &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::OUT_DEGREE_DISTRIBUTION) == 0) { - out_degree_distribution_command(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, + out_degree_distribution_command(connFd, serverPort, *graphDBMapLocalStores, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::PAGE_RANK) == 0) { - page_rank_command(connFd, serverPort, graphDBMapCentralStores, &loop_exit); + page_rank_command(connFd, serverPort, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::WORKER_PAGE_RANK_DISTRIBUTION) == 0) { - worker_page_rank_distribution_command(connFd, serverPort, graphDBMapCentralStores, &loop_exit); + worker_page_rank_distribution_command(connFd, serverPort, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::EGONET) == 0) { - egonet_command(connFd, serverPort, graphDBMapCentralStores, &loop_exit); + egonet_command(connFd, serverPort, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::WORKER_EGO_NET) == 0) { - worker_egonet_command(connFd, serverPort, graphDBMapCentralStores, &loop_exit); + worker_egonet_command(connFd, serverPort, *graphDBMapCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::TRIANGLES) == 0) { - triangles_command(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, - graphDBMapDuplicateCentralStores, &loop_exit); + triangles_command(connFd, serverPort, *graphDBMapLocalStores, *graphDBMapCentralStores, + *graphDBMapDuplicateCentralStores, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::INITIATE_STREAMING_TRIAN) == 0) { streaming_triangles_command(connFd, serverPort, incrementalLocalStoreMap, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::SEND_CENTRALSTORE_TO_AGGREGATOR) == 0) { @@ -223,8 +227,6 @@ void *instanceservicesession(void *dummyPt) { aggregate_streaming_centralstore_triangles_command(connFd, incrementalLocalStoreMap, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::AGGREGATE_COMPOSITE_CENTRALSTORE_TRIANGLES) == 0) { aggregate_composite_centralstore_triangles_command(connFd, &loop_exit); - } else if (line.compare(JasmineGraphInstanceProtocol::PERFORMANCE_STATISTICS) == 0) { - performance_statistics_command(connFd, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::INITIATE_FILES) == 0) { initiate_files_command(connFd, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::INITIATE_FED_PREDICT) == 0) { @@ -257,6 +259,8 @@ void *instanceservicesession(void *dummyPt) { graph_stream_start_command(connFd, streamHandler, &loop_exit); } else if (line.compare(JasmineGraphInstanceProtocol::SEND_PRIORITY) == 0) { send_priority_command(connFd, &loop_exit); + } else if (line.compare(JasmineGraphInstanceProtocol::PUSH_PARTITION) == 0) { + push_partition_command(connFd, &loop_exit); } else { instance_logger.error("Invalid command"); loop_exit = true; @@ -269,8 +273,7 @@ void *instanceservicesession(void *dummyPt) { JasmineGraphInstanceService::JasmineGraphInstanceService() {} -void JasmineGraphInstanceService::run(string profile, string masterHost, string host, int serverPort, - int serverDataPort) { +void JasmineGraphInstanceService::run(string masterHost, string host, int serverPort, int serverDataPort) { int listenFd; socklen_t len; struct sockaddr_in svrAdd; @@ -310,6 +313,9 @@ void JasmineGraphInstanceService::run(string profile, string masterHost, string std::map graphDBMapDuplicateCentralStores; std::map incrementalLocalStore; + std::thread perfThread = std::thread(&PerformanceUtil::collectPerformanceStatistics); + perfThread.detach(); + instance_logger.info("Worker listening on port " + to_string(serverPort)); while (true) { int connFd = accept(listenFd, (struct sockaddr *)&clntAdd, &len); @@ -319,22 +325,29 @@ void JasmineGraphInstanceService::run(string profile, string masterHost, string continue; } instance_logger.info("Connection successful to port " + to_string(serverPort)); - instanceservicesessionargs *serviceArguments_p = new instanceservicesessionargs; - serviceArguments_p->graphDBMapLocalStores = graphDBMapLocalStores; - serviceArguments_p->graphDBMapCentralStores = graphDBMapCentralStores; - serviceArguments_p->graphDBMapDuplicateCentralStores = graphDBMapDuplicateCentralStores; - serviceArguments_p->incrementalLocalStore = incrementalLocalStore; - serviceArguments_p->profile = profile; - serviceArguments_p->masterHost = masterHost; - serviceArguments_p->port = serverPort; - serviceArguments_p->dataPort = serverDataPort; - serviceArguments_p->host = host; - serviceArguments_p->connFd = connFd; - pthread_t pt; - pthread_create(&pt, NULL, instanceservicesession, serviceArguments_p); + + pid_t pid = fork(); + if (pid == 0) { + close(listenFd); + instanceservicesessionargs *serviceArguments_p = new instanceservicesessionargs; + serviceArguments_p->graphDBMapLocalStores = &graphDBMapLocalStores; + serviceArguments_p->graphDBMapCentralStores = &graphDBMapCentralStores; + serviceArguments_p->graphDBMapDuplicateCentralStores = &graphDBMapDuplicateCentralStores; + serviceArguments_p->incrementalLocalStore = &incrementalLocalStore; + serviceArguments_p->masterHost = masterHost; + serviceArguments_p->port = serverPort; + serviceArguments_p->dataPort = serverDataPort; + serviceArguments_p->host = host; + serviceArguments_p->connFd = connFd; + instanceservicesession(serviceArguments_p); + break; + } else { + close(connFd); + } } pthread_mutex_destroy(&file_lock); + exit(0); // FIXME: Cleanup before exit. } int deleteGraphPartition(std::string graphID, std::string partitionID) { @@ -376,7 +389,6 @@ void removeGraphFragments(std::string graphID) { } void writeCatalogRecord(string record) { - Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder")); string catalogFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/catalog.txt"; ofstream outfile; @@ -387,9 +399,9 @@ void writeCatalogRecord(string record) { long countLocalTriangles( std::string graphId, std::string partitionId, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::map graphDBMapDuplicateCentralStores, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, int threadPriority) { long result; @@ -397,35 +409,30 @@ long countLocalTriangles( std::string graphIdentifier = graphId + "_" + partitionId; std::string centralGraphIdentifier = graphId + "_centralstore_" + partitionId; std::string duplicateCentralGraphIdentifier = graphId + "_centralstore_dp_" + partitionId; - JasmineGraphHashMapLocalStore graphDB; - JasmineGraphHashMapCentralStore centralGraphDB; - JasmineGraphHashMapDuplicateCentralStore duplicateCentralGraphDB; - std::map::iterator localMapIterator = - graphDBMapLocalStores.find(graphIdentifier); - std::map::iterator centralStoreIterator = - graphDBMapCentralStores.find(graphIdentifier); - std::map::iterator duplicateCentralStoreIterator = - graphDBMapDuplicateCentralStores.find(graphIdentifier); + auto localMapIterator = graphDBMapLocalStores.find(graphIdentifier); + auto centralStoreIterator = graphDBMapCentralStores.find(graphIdentifier); + auto duplicateCentralStoreIterator = graphDBMapDuplicateCentralStores.find(graphIdentifier); if (localMapIterator == graphDBMapLocalStores.end() && JasmineGraphInstanceService::isGraphDBExists(graphId, partitionId)) { JasmineGraphInstanceService::loadLocalStore(graphId, partitionId, graphDBMapLocalStores); } - graphDB = graphDBMapLocalStores[graphIdentifier]; + JasmineGraphHashMapLocalStore graphDB = graphDBMapLocalStores[graphIdentifier]; if (centralStoreIterator == graphDBMapCentralStores.end() && JasmineGraphInstanceService::isInstanceCentralStoreExists(graphId, partitionId)) { JasmineGraphInstanceService::loadInstanceCentralStore(graphId, partitionId, graphDBMapCentralStores); } - centralGraphDB = graphDBMapCentralStores[centralGraphIdentifier]; + JasmineGraphHashMapCentralStore centralGraphDB = graphDBMapCentralStores[centralGraphIdentifier]; if (duplicateCentralStoreIterator == graphDBMapDuplicateCentralStores.end() && JasmineGraphInstanceService::isInstanceDuplicateCentralStoreExists(graphId, partitionId)) { JasmineGraphInstanceService::loadInstanceDuplicateCentralStore(graphId, partitionId, graphDBMapDuplicateCentralStores); } - duplicateCentralGraphDB = graphDBMapDuplicateCentralStores[duplicateCentralGraphIdentifier]; + JasmineGraphHashMapDuplicateCentralStore duplicateCentralGraphDB = + graphDBMapDuplicateCentralStores[duplicateCentralGraphIdentifier]; result = Triangles::run(graphDB, centralGraphDB, duplicateCentralGraphDB, graphId, partitionId, threadPriority); @@ -472,31 +479,31 @@ JasmineGraphIncrementalLocalStore *JasmineGraphInstanceService::loadStreamingSto std::string folderLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); JasmineGraphIncrementalLocalStore *jasmineGraphStreamingLocalStore = new JasmineGraphIncrementalLocalStore(stoi(graphId), stoi(partitionId), openMode); - graphDBMapStreamingStores.insert(std::make_pair(graphIdentifier, jasmineGraphStreamingLocalStore)); + graphDBMapStreamingStores[graphIdentifier] = jasmineGraphStreamingLocalStore; instance_logger.info("###INSTANCE### Loading Local Store : Completed"); return jasmineGraphStreamingLocalStore; } + void JasmineGraphInstanceService::loadLocalStore( std::string graphId, std::string partitionId, std::map &graphDBMapLocalStores) { instance_logger.info("###INSTANCE### Loading Local Store : Started"); std::string graphIdentifier = graphId + "_" + partitionId; std::string folderLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); - JasmineGraphHashMapLocalStore *jasmineGraphHashMapLocalStore = - new JasmineGraphHashMapLocalStore(stoi(graphId), stoi(partitionId), folderLocation); - jasmineGraphHashMapLocalStore->loadGraph(); - graphDBMapLocalStores.insert(std::make_pair(graphIdentifier, *jasmineGraphHashMapLocalStore)); + JasmineGraphHashMapLocalStore jasmineGraphHashMapLocalStore(stoi(graphId), stoi(partitionId), folderLocation); + jasmineGraphHashMapLocalStore.loadGraph(); + graphDBMapLocalStores[graphIdentifier] = jasmineGraphHashMapLocalStore; instance_logger.info("###INSTANCE### Loading Local Store : Completed"); } + void JasmineGraphInstanceService::loadInstanceCentralStore( std::string graphId, std::string partitionId, std::map &graphDBMapCentralStores) { instance_logger.info("###INSTANCE### Loading central Store : Started"); std::string graphIdentifier = graphId + "_centralstore_" + partitionId; - JasmineGraphHashMapCentralStore *jasmineGraphHashMapCentralStore = - new JasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); - jasmineGraphHashMapCentralStore->loadGraph(); - graphDBMapCentralStores.insert(std::make_pair(graphIdentifier, *jasmineGraphHashMapCentralStore)); + JasmineGraphHashMapCentralStore jasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); + jasmineGraphHashMapCentralStore.loadGraph(); + graphDBMapCentralStores[graphIdentifier] = jasmineGraphHashMapCentralStore; instance_logger.info("###INSTANCE### Loading central Store : Completed"); } @@ -504,46 +511,43 @@ void JasmineGraphInstanceService::loadInstanceDuplicateCentralStore( std::string graphId, std::string partitionId, std::map &graphDBMapDuplicateCentralStores) { std::string graphIdentifier = graphId + "_centralstore_dp_" + partitionId; - JasmineGraphHashMapDuplicateCentralStore *jasmineGraphHashMapCentralStore = - new JasmineGraphHashMapDuplicateCentralStore(stoi(graphId), stoi(partitionId)); - jasmineGraphHashMapCentralStore->loadGraph(); - graphDBMapDuplicateCentralStores.insert(std::make_pair(graphIdentifier, *jasmineGraphHashMapCentralStore)); + JasmineGraphHashMapDuplicateCentralStore jasmineGraphHashMapCentralStore(stoi(graphId), stoi(partitionId)); + jasmineGraphHashMapCentralStore.loadGraph(); + graphDBMapDuplicateCentralStores[graphIdentifier] = jasmineGraphHashMapCentralStore; } -JasmineGraphHashMapCentralStore JasmineGraphInstanceService::loadCentralStore(std::string centralStoreFileName) { +JasmineGraphHashMapCentralStore *JasmineGraphInstanceService::loadCentralStore(std::string centralStoreFileName) { instance_logger.info("###INSTANCE### Loading Central Store File : Started " + centralStoreFileName); JasmineGraphHashMapCentralStore *jasmineGraphHashMapCentralStore = new JasmineGraphHashMapCentralStore(); jasmineGraphHashMapCentralStore->loadGraph(centralStoreFileName); instance_logger.info("###INSTANCE### Loading Central Store File : Completed"); - return *jasmineGraphHashMapCentralStore; + return jasmineGraphHashMapCentralStore; } -string JasmineGraphInstanceService::aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, - std::string partitionIdList, int threadPriority) { +static string aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, std::string partitionIdList, + int threadPriority) { instance_logger.info("###INSTANCE### Started Aggregating Central Store Triangles"); - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); + std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::vector fileNames; map> aggregatedCentralStore; std::string centralGraphIdentifier = graphId + "_centralstore_" + partitionId; std::string dataFolder = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string workerCentralStoreFile = dataFolder + "/" + centralGraphIdentifier; instance_logger.info("###INSTANCE### Loading Central Store : Started " + workerCentralStoreFile); - JasmineGraphHashMapCentralStore workerCentralStore = + JasmineGraphHashMapCentralStore *workerCentralStore = JasmineGraphInstanceService::loadCentralStore(workerCentralStoreFile); instance_logger.info("###INSTANCE### Loading Central Store : Completed"); - map> workerCentralGraphMap = workerCentralStore.getUnderlyingHashMap(); - - map>::iterator workerCentalGraphIterator; + const auto &workerCentralGraphMap = workerCentralStore->getUnderlyingHashMap(); - for (workerCentalGraphIterator = workerCentralGraphMap.begin(); + for (auto workerCentalGraphIterator = workerCentralGraphMap.begin(); workerCentalGraphIterator != workerCentralGraphMap.end(); ++workerCentalGraphIterator) { long startVid = workerCentalGraphIterator->first; - unordered_set endVidSet = workerCentalGraphIterator->second; + const unordered_set &endVidSet = workerCentalGraphIterator->second; - unordered_set aggregatedEndVidSet = aggregatedCentralStore[startVid]; + unordered_set &aggregatedEndVidSet = aggregatedCentralStore[startVid]; aggregatedEndVidSet.insert(endVidSet.begin(), endVidSet.end()); - aggregatedCentralStore[startVid] = aggregatedEndVidSet; } + delete workerCentralStore; std::vector paritionIdList = Utils::split(partitionIdList, ','); std::vector::iterator partitionIdListIterator; @@ -553,22 +557,21 @@ string JasmineGraphInstanceService::aggregateCentralStoreTriangles(std::string g std::string aggregatePartitionId = *partitionIdListIterator; std::string centralGraphIdentifier = graphId + "_centralstore_" + aggregatePartitionId; - std::string centralStoreFile = aggregatorFilePath + "/" + centralGraphIdentifier; + std::string centralStoreFile = aggregatorDirPath + "/" + centralGraphIdentifier; if (access(centralStoreFile.c_str(), R_OK) == 0) { - JasmineGraphHashMapCentralStore centralStore = + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); - map> centralGraphMap = centralStore.getUnderlyingHashMap(); - map>::iterator centralGraphMapIterator; + const auto ¢ralGraphMap = centralStore->getUnderlyingHashMap(); - for (centralGraphMapIterator = centralGraphMap.begin(); centralGraphMapIterator != centralGraphMap.end(); - ++centralGraphMapIterator) { + for (auto centralGraphMapIterator = centralGraphMap.begin(); + centralGraphMapIterator != centralGraphMap.end(); ++centralGraphMapIterator) { long startVid = centralGraphMapIterator->first; - unordered_set endVidSet = centralGraphMapIterator->second; + const unordered_set &endVidSet = centralGraphMapIterator->second; - unordered_set aggregatedEndVidSet = aggregatedCentralStore[startVid]; + unordered_set &aggregatedEndVidSet = aggregatedCentralStore[startVid]; aggregatedEndVidSet.insert(endVidSet.begin(), endVidSet.end()); - aggregatedCentralStore[startVid] = aggregatedEndVidSet; } + delete centralStore; } } @@ -585,7 +588,7 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: std::string availableFileList, int threadPriority) { instance_logger.info("###INSTANCE### Started Aggregating Composite Central Store Triangles"); - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); + std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); std::string dataFolder = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); map> aggregatedCompositeCentralStore; @@ -602,21 +605,20 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: std::string availableCompositeFile = dataFolder + "/" + rawFileName; if (access(availableCompositeFile.c_str(), R_OK) == 0) { - JasmineGraphHashMapCentralStore centralStore = + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(availableCompositeFile); - map> compositeCentralGraphMap = centralStore.getUnderlyingHashMap(); - map>::iterator compositeCentralGraphMapIterator; + const auto &compositeCentralGraphMap = centralStore->getUnderlyingHashMap(); - for (compositeCentralGraphMapIterator = compositeCentralGraphMap.begin(); + for (auto compositeCentralGraphMapIterator = compositeCentralGraphMap.begin(); compositeCentralGraphMapIterator != compositeCentralGraphMap.end(); ++compositeCentralGraphMapIterator) { long startVid = compositeCentralGraphMapIterator->first; - unordered_set endVidSet = compositeCentralGraphMapIterator->second; + const unordered_set &endVidSet = compositeCentralGraphMapIterator->second; - unordered_set aggregatedEndVidSet = aggregatedCompositeCentralStore[startVid]; + unordered_set &aggregatedEndVidSet = aggregatedCompositeCentralStore[startVid]; aggregatedEndVidSet.insert(endVidSet.begin(), endVidSet.end()); - aggregatedCompositeCentralStore[startVid] = aggregatedEndVidSet; } + delete centralStore; } } @@ -627,22 +629,20 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: size_t lastindex = compositeCentralStoreFileName.find_last_of("."); string rawFileName = compositeCentralStoreFileName.substr(0, lastindex); - std::string compositeCentralStoreFile = aggregatorFilePath + "/" + rawFileName; + std::string compositeCentralStoreFile = aggregatorDirPath + "/" + rawFileName; if (access(compositeCentralStoreFile.c_str(), R_OK) == 0) { - JasmineGraphHashMapCentralStore centralStore = + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(compositeCentralStoreFile); - map> centralGraphMap = centralStore.getUnderlyingHashMap(); - map>::iterator centralGraphMapIterator; - - for (centralGraphMapIterator = centralGraphMap.begin(); centralGraphMapIterator != centralGraphMap.end(); - ++centralGraphMapIterator) { + const auto ¢ralGraphMap = centralStore->getUnderlyingHashMap(); + for (auto centralGraphMapIterator = centralGraphMap.begin(); + centralGraphMapIterator != centralGraphMap.end(); ++centralGraphMapIterator) { long startVid = centralGraphMapIterator->first; - unordered_set endVidSet = centralGraphMapIterator->second; + const unordered_set &endVidSet = centralGraphMapIterator->second; - unordered_set aggregatedEndVidSet = aggregatedCompositeCentralStore[startVid]; + unordered_set &aggregatedEndVidSet = aggregatedCompositeCentralStore[startVid]; aggregatedEndVidSet.insert(endVidSet.begin(), endVidSet.end()); - aggregatedCompositeCentralStore[startVid] = aggregatedEndVidSet; } + delete centralStore; } } @@ -653,42 +653,22 @@ string JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(std: TriangleResult triangleResult = Triangles::countTriangles(aggregatedCompositeCentralStore, distributionHashMap, true); - std::string triangles = triangleResult.triangles; - - return triangles; + return triangleResult.triangles; } -map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map> graphMap) { +map JasmineGraphInstanceService::getOutDegreeDistributionHashMap(map> &graphMap) { map distributionHashMap; for (map>::iterator it = graphMap.begin(); it != graphMap.end(); ++it) { long distribution = (it->second).size(); - distributionHashMap.insert(std::make_pair(it->first, distribution)); + distributionHashMap[it->first] = distribution; } return distributionHashMap; } -string JasmineGraphInstanceService::requestPerformanceStatistics(std::string isVMStatManager, - std::string isResourceAllocationRequested) { - int memoryUsage = collector.getMemoryUsageByProcess(); - double cpuUsage = collector.getCpuUsage(); - double loadAverage = collector.getLoadAverage(); - std::string vmLevelStatistics = collector.collectVMStatistics(isVMStatManager, isResourceAllocationRequested); - auto executedTime = std::chrono::system_clock::now(); - std::time_t reportTime = std::chrono::system_clock::to_time_t(executedTime); - std::string reportTimeString(std::ctime(&reportTime)); - reportTimeString = Utils::trim_copy(reportTimeString); - std::string usageString = - reportTimeString + "," + to_string(memoryUsage) + "," + to_string(cpuUsage) + "," + to_string(loadAverage); - if (!vmLevelStatistics.empty()) { - usageString = usageString + "," + vmLevelStatistics; - } - return usageString; -} - void JasmineGraphInstanceService::collectTrainedModels( instanceservicesessionargs *sessionargs, std::string graphID, - std::map graphPartitionedHosts, int totalPartitions) { + std::map &graphPartitionedHosts, int totalPartitions) { int total_threads = totalPartitions; std::thread *workerThreads = new std::thread[total_threads]; int count = 0; @@ -840,15 +820,13 @@ int JasmineGraphInstanceService::collectTrainedModelThreadFunction(instanceservi } else { instance_logger.error("There was an error in the model collection process and the response is :: " + response); } - + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return 0; } void JasmineGraphInstanceService::createPartitionFiles(std::string graphID, std::string partitionID, std::string fileType) { - Utils::createDirectory(Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder")); - JasmineGraphHashMapLocalStore *hashMapLocalStore = new JasmineGraphHashMapLocalStore(); string inputFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + graphID + "_" + partitionID; string outputFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder") + @@ -859,7 +837,8 @@ void JasmineGraphInstanceService::createPartitionFiles(std::string graphID, std: outputFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder") + "/" + graphID + "_centralstore_" + partitionID; } - std::map> partEdgeMap = hashMapLocalStore->getEdgeHashMap(inputFilePath); + JasmineGraphHashMapLocalStore hashMapLocalStore; + std::map> partEdgeMap = hashMapLocalStore.getEdgeHashMap(inputFilePath); if (partEdgeMap.empty()) { return; @@ -965,22 +944,21 @@ void JasmineGraphInstanceService::trainPartition(string trainData) { } bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int graphID, int partitionID, - std::vector workerSockets, std::string masterIP) { - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - std::string dataFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + std::vector &workerSockets, std::string masterIP) { + std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); + std::string dataDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string centralGraphIdentifierUnCompressed = to_string(graphID) + "_centralstore_" + to_string(partitionID); - std::string centralStoreFileUnCompressed = dataFilePath + "/" + centralGraphIdentifierUnCompressed; - std::string centralStoreFileUnCompressedDestination = aggregatorFilePath + "/" + centralGraphIdentifierUnCompressed; + std::string centralStoreFileUnCompressed = dataDirPath + "/" + centralGraphIdentifierUnCompressed; + std::string centralStoreFileUnCompressedDestination = aggregatorDirPath + "/" + centralGraphIdentifierUnCompressed; // temporary copy the central store into the aggregate folder in order to compress and send - Utils::copyFile(centralStoreFileUnCompressed, aggregatorFilePath); + Utils::copyToDirectory(centralStoreFileUnCompressed, aggregatorDirPath); // compress the central store file before sending Utils::compressFile(centralStoreFileUnCompressedDestination); - std::string centralGraphIdentifier = to_string(graphID) + "_centralstore_" + to_string(partitionID) + ".gz"; - std::string centralStoreFile = aggregatorFilePath + "/" + centralGraphIdentifier; + std::string centralStoreFile = centralStoreFileUnCompressedDestination + ".gz"; instance_logger.info("###INSTANCE### centralstore " + centralStoreFile); char data[DATA_BUFFER_SIZE]; @@ -994,13 +972,13 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (workerSocketPair.size() != 4) { instance_logger.error("Received worker socket information is invalid "); - return 0; + return false; } struct stat fileStat; if (stat(centralStoreFile.c_str(), &fileStat) != 0) { instance_logger.error("stat() failed on " + centralStoreFile); - return 0; + return false; } if (!S_ISREG(fileStat.st_mode)) { instance_logger.error(centralStoreFile + " is not a regular file."); @@ -1025,7 +1003,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { instance_logger.error("Cannot create socket"); - return 0; + return false; } if (host.find('@') != std::string::npos) { @@ -1035,7 +1013,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int server = gethostbyname(host.c_str()); if (server == NULL) { instance_logger.error("ERROR, no host named " + host); - return 0; + return false; } bzero((char *)&serv_addr, sizeof(serv_addr)); @@ -1043,7 +1021,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return 0; + return false; } if (Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::HANDSHAKE)) { @@ -1054,6 +1032,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::HANDSHAKE_OK) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::HANDSHAKE_OK + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1067,6 +1046,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::HOST_OK) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::HOST_OK + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1080,6 +1060,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::OK + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1097,6 +1078,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_NAME) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::SEND_FILE_NAME + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1110,6 +1092,7 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_LEN) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::SEND_FILE_LEN + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1122,13 +1105,14 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) != 0) { instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::SEND_FILE_CONT + " ; Received: " + response); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } instance_logger.info("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT); instance_logger.info("Going to send file through service"); - JasmineGraphInstanceService::sendFileThroughService(host, dataPort, fileName, centralStoreFile, masterIP); + Utils::sendFileThroughService(host, dataPort, fileName, centralStoreFile); int count = 0; while (true) { @@ -1150,7 +1134,6 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int break; } else { instance_logger.error("Incorrect response. Received: " + response); - close(sockfd); goto END_OUTER_LOOP; } }; @@ -1171,91 +1154,20 @@ bool JasmineGraphInstanceService::duplicateCentralStore(int thisWorkerPort, int break; } else { instance_logger.error("Incorrect response. Received: " + response); - close(sockfd); goto END_OUTER_LOOP; } } END_OUTER_LOOP: + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); } - return 0; -} - -bool JasmineGraphInstanceService::sendFileThroughService(std::string host, int dataPort, std::string fileName, - std::string filePath, std::string masterIP) { - int sockfd; - char data[DATA_BUFFER_SIZE]; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - instance_logger.error("Cannot create socket"); - return false; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - instance_logger.error("ERROR, no host named " + host); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(dataPort); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return false; - } - - Utils::send_str_wrapper(sockfd, fileName); - - string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::SEND_FILE) != 0) { - instance_logger.error("Incorrect response. Expected: " + JasmineGraphInstanceProtocol::SEND_FILE + - " ; Received: " + response); - close(sockfd); - return false; - } - FILE *fp = fopen(filePath.c_str(), "r"); - if (fp == NULL) { - instance_logger.error("Error opening file: " + filePath); - close(sockfd); - return false; - } - - for (;;) { - unsigned char buff[INSTANCE_FILE_BUFFER_LENGTH] = {0}; - int nread = fread(buff, 1, INSTANCE_FILE_BUFFER_LENGTH, fp); - instance_logger.info("Bytes read " + to_string(nread)); - - /* If read was success, send data. */ - if (nread > 0) { - Utils::send_wrapper(sockfd, (char *)buff, nread); - } - - if (nread < INSTANCE_FILE_BUFFER_LENGTH) { - if (feof(fp)) { - instance_logger.info("Reading completed for file: " + filePath); - } - if (ferror(fp)) { - instance_logger.error("Error reading file: " + filePath); - } - break; - } - } - - fclose(fp); - close(sockfd); return true; } map calculateOutDegreeDist(string graphID, string partitionID, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::vector workerSockets) { + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::vector &workerSockets) { map degreeDistribution = calculateLocalOutDegreeDist(graphID, partitionID, graphDBMapLocalStores, graphDBMapCentralStores); @@ -1276,8 +1188,8 @@ map calculateOutDegreeDist(string graphID, string partitionID, int s } map calculateLocalOutDegreeDist( - string graphID, string partitionID, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores) { + string graphID, string partitionID, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores) { auto t_start = std::chrono::high_resolution_clock::now(); JasmineGraphHashMapLocalStore graphDB; @@ -1323,8 +1235,8 @@ map calculateLocalOutDegreeDist( } map calculateLocalInDegreeDist( - string graphID, string partitionID, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores) { + string graphID, string partitionID, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores) { JasmineGraphHashMapLocalStore graphDB; std::map::iterator it; @@ -1342,9 +1254,9 @@ map calculateLocalInDegreeDist( } map calculateInDegreeDist(string graphID, string partitionID, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::vector workerSockets, string workerList) { + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::vector &workerSockets, string workerList) { auto t_start = std::chrono::high_resolution_clock::now(); map degreeDistribution = @@ -1381,7 +1293,7 @@ map calculateInDegreeDist(string graphID, string partitionID, int se long degreeDistributionValue = degreeDistributionLocalItr->second; degreeDistribution[degreeDistributionLocalItr->first] = degreeDistributionValue + its->second; } else { - degreeDistribution.insert(std::make_pair(its->first, its->second)); + degreeDistribution[its->first] = its->second; } } @@ -1414,7 +1326,7 @@ map calculateInDegreeDist(string graphID, string partitionID, int se map>> calculateLocalEgoNet(string graphID, string partitionID, int serverPort, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - std::vector workerSockets) { + std::vector &workerSockets) { std::map>> egonetMap; map> centralGraphMap = centralDB.getUnderlyingHashMap(); @@ -1424,7 +1336,7 @@ map>> calculateLocalEgoNet(string graphID, s unordered_set neighbours = it->second; map> individualEgoNet; - individualEgoNet.insert(std::make_pair(it->first, neighbours)); + individualEgoNet[it->first] = neighbours; for (unordered_set::iterator neighbour = neighbours.begin(); neighbour != neighbours.end(); ++neighbour) { unordered_set neighboursOfNeighboursInSameEgoNet; @@ -1441,10 +1353,10 @@ map>> calculateLocalEgoNet(string graphID, s } } } - individualEgoNet.insert(std::make_pair(*neighbour, neighboursOfNeighboursInSameEgoNet)); + individualEgoNet[*neighbour] = neighboursOfNeighboursInSameEgoNet; } - egonetMap.insert(std::make_pair(it->first, individualEgoNet)); + egonetMap[it->first] = individualEgoNet; } for (map>::iterator it = centralGraphMap.begin(); it != centralGraphMap.end(); ++it) { @@ -1457,7 +1369,7 @@ map>> calculateLocalEgoNet(string graphID, s vertexMapFromCentralStore.insert( std::make_pair(it->first, distribution)); // Here we do not have the relation information among neighbours - egonetMap.insert(std::make_pair(it->first, vertexMapFromCentralStore)); + egonetMap[it->first] = vertexMapFromCentralStore; } else { map> egonetSubGraph = egonetMapItr->second; @@ -1479,10 +1391,10 @@ map>> calculateLocalEgoNet(string graphID, s workerSocketPair.push_back(intermediate); } - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + std::string dataDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string centralGraphIdentifier = graphID + "_centralstore_" + workerSocketPair[2]; - std::string centralStoreFile = aggregatorFilePath + "/" + centralGraphIdentifier; + std::string centralStoreFile = dataDirPath + "/" + centralGraphIdentifier; instance_logger.info("###INSTANCE### centralstore " + centralStoreFile); struct stat centralStoreFileStat; @@ -1494,28 +1406,27 @@ map>> calculateLocalEgoNet(string graphID, s instance_logger.error(centralStoreFile + " is not a regular file"); continue; } - JasmineGraphHashMapCentralStore centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); - map> centralGraphMap = centralStore.getUnderlyingHashMap(); + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); + const auto ¢ralGraphMap = centralStore->getUnderlyingHashMap(); - for (map>::iterator centralGraphMapIterator = centralGraphMap.begin(); - centralGraphMapIterator != centralGraphMap.end(); ++centralGraphMapIterator) { + for (auto centralGraphMapIterator = centralGraphMap.begin(); centralGraphMapIterator != centralGraphMap.end(); + ++centralGraphMapIterator) { long startVid = centralGraphMapIterator->first; - unordered_set endVidSet = centralGraphMapIterator->second; + const unordered_set &endVidSet = centralGraphMapIterator->second; for (auto itr = endVidSet.begin(); itr != endVidSet.end(); ++itr) { - map>>::iterator egonetMapItr = egonetMap.find(*itr); - + auto egonetMapItr = egonetMap.find(*itr); if (egonetMapItr != egonetMap.end()) { - map> egonetSubGraph = egonetMapItr->second; - map>::iterator egonetSubGraphItr = egonetSubGraph.find(*itr); + map> &egonetSubGraph = egonetMapItr->second; + auto egonetSubGraphItr = egonetSubGraph.find(*itr); if (egonetSubGraphItr != egonetSubGraph.end()) { - unordered_set egonetSubGraphNeighbours = egonetSubGraphItr->second; + unordered_set &egonetSubGraphNeighbours = egonetSubGraphItr->second; egonetSubGraphNeighbours.insert(startVid); - egonetSubGraphItr->second = egonetSubGraphNeighbours; } } } } + delete centralStore; } return egonetMap; @@ -1595,6 +1506,7 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine } if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::WORKER_EGO_NET)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1603,12 +1515,14 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine string response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Error reading from socket"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } instance_logger.info("Received : " + response); if (Utils::send_str_wrapper(sockfd, graphID)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1616,6 +1530,7 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Error reading from socket"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1625,6 +1540,7 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine string egonetString; if (!Utils::send_str_wrapper(sockfd, workerSocketPair[2])) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1633,11 +1549,13 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Error reading from socket"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } if (!Utils::send_str_wrapper(sockfd, workerList)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -1646,9 +1564,11 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine response = Utils::read_str_trim_wrapper(sockfd, data, INSTANCE_DATA_LENGTH); if (response.compare(JasmineGraphInstanceProtocol::OK) != 0) { instance_logger.error("Error reading from socket"); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); } } @@ -1656,7 +1576,7 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine map calculateLocalPageRank(string graphID, double alpha, string partitionID, int serverPort, int top_k_page_rank_value, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, - JasmineGraphHashMapCentralStore centralDB, std::vector workerSockets, + JasmineGraphHashMapCentralStore centralDB, std::vector &workerSockets, int iterations) { auto t_start = std::chrono::high_resolution_clock::now(); @@ -1674,7 +1594,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa for (auto itr = endVidSet.begin(); itr != endVidSet.end(); ++itr) { if (localGraphMap.find(*itr) == localGraphMap.end()) { unordered_set valueSet; - localGraphMap.insert(std::make_pair(*itr, valueSet)); + localGraphMap[*itr] = valueSet; } } } @@ -1706,13 +1626,13 @@ map calculateLocalPageRank(string graphID, double alpha, string pa std::map inDegreeDistribution; - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + std::string dataDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string partitionCount = Utils::getJasmineGraphProperty("org.jasminegraph.server.npartitions"); int parCount = std::stoi(partitionCount); for (int partitionID = 0; partitionID < parCount; ++partitionID) { - std::string iddFilePath = aggregatorFilePath + "/" + graphID + "_idd_" + std::to_string(partitionID); + std::string iddFilePath = dataDirPath + "/" + graphID + "_idd_" + std::to_string(partitionID); std::ifstream dataFile; dataFile.open(iddFilePath); @@ -1754,7 +1674,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa if (inDegreeDistributionItr != inDegreeDistribution.end()) { long inDegree = inDegreeDistributionItr->second; double authorityScore = (alpha * 1 + mu) * inDegree; - rankMap.insert(std::make_pair(inDegreeDistributionItr->first, authorityScore)); + rankMap[inDegreeDistributionItr->first] = authorityScore; } } @@ -1770,7 +1690,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa if (rankMapItr != rankMap.end()) { existingParentRank = rankMapItr->second; } else { - rankMap.insert(std::make_pair(startVid, existingParentRank)); + rankMap[startVid] = existingParentRank; } long degree = endVidSet.size(); @@ -1788,7 +1708,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa rankMapItr->second = finalRank; } else { finalRank = existingChildRank + distributedRank; - rankMap.insert(std::make_pair(itr, finalRank)); + rankMap[itr] = finalRank; } } } @@ -1803,7 +1723,7 @@ map calculateLocalPageRank(string graphID, double alpha, string pa } else { int count = 0; for (map::iterator rankMapItr = rankMap.begin(); rankMapItr != rankMap.end(); ++rankMapItr) { - finalPageRankResults.insert(std::make_pair(rankMapItr->first, rankMapItr->second)); + finalPageRankResults[rankMapItr->first] = rankMapItr->second; count++; } } @@ -1824,8 +1744,8 @@ map calculateLocalPageRank(string graphID, double alpha, string pa map> getEdgesWorldToLocal(string graphID, string partitionID, int serverPort, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - map> graphVertexMap, - std::vector workerSockets) { + map> &graphVertexMap, + std::vector &workerSockets) { map> worldToLocalVertexMap; for (vector::iterator workerIt = workerSockets.begin(); workerIt != workerSockets.end(); ++workerIt) { std::vector workerSocketPair; @@ -1835,40 +1755,39 @@ map> getEdgesWorldToLocal(string graphID, string parti workerSocketPair.push_back(intermediate); } - std::string aggregatorFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + std::string dataDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); std::string centralGraphIdentifier = graphID + "_centralstore_" + workerSocketPair[2]; - std::string centralStoreFile = aggregatorFilePath + "/" + centralGraphIdentifier; + std::string centralStoreFile = dataDirPath + "/" + centralGraphIdentifier; instance_logger.info("###INSTANCE### centralstore " + centralStoreFile); if (access(centralStoreFile.c_str(), R_OK) != 0) { instance_logger.error("Read permission denied for " + centralStoreFile); continue; } - JasmineGraphHashMapCentralStore centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); - map> centralGraphMap = centralStore.getUnderlyingHashMap(); + JasmineGraphHashMapCentralStore *centralStore = JasmineGraphInstanceService::loadCentralStore(centralStoreFile); + const auto ¢ralGraphMap = centralStore->getUnderlyingHashMap(); - for (map>::iterator centralGraphMapIterator = centralGraphMap.begin(); - centralGraphMapIterator != centralGraphMap.end(); ++centralGraphMapIterator) { + for (auto centralGraphMapIterator = centralGraphMap.begin(); centralGraphMapIterator != centralGraphMap.end(); + ++centralGraphMapIterator) { long startVid = centralGraphMapIterator->first; - unordered_set endVidSet = centralGraphMapIterator->second; + const unordered_set &endVidSet = centralGraphMapIterator->second; for (auto itr = endVidSet.begin(); itr != endVidSet.end(); ++itr) { if (graphVertexMap.find(*itr) != graphVertexMap.end()) { - map>::iterator toIDIterator = worldToLocalVertexMap.find(*itr); - + auto toIDIterator = worldToLocalVertexMap.find(*itr); if (toIDIterator != worldToLocalVertexMap.end()) { - unordered_set fromIDs = toIDIterator->second; + unordered_set &fromIDs = toIDIterator->second; fromIDs.insert(startVid); - toIDIterator->second = fromIDs; } else { unordered_set fromIDs; fromIDs.insert(startVid); - worldToLocalVertexMap.insert(std::make_pair(*itr, fromIDs)); + worldToLocalVertexMap[*itr] = fromIDs; } } } } + delete centralStore; } return worldToLocalVertexMap; @@ -1877,17 +1796,17 @@ map> getEdgesWorldToLocal(string graphID, string parti void JasmineGraphInstanceService::startCollectingLoadAverage() { int elapsedTime = 0; time_t start; - time_t end; - StatisticCollector statisticCollector; start = time(0); - while (collectValid) { - if (time(0) - start == Conts::LOAD_AVG_COLLECTING_GAP) { + time_t elapsed = time(0) - start; + if (elapsed >= Conts::LOAD_AVG_COLLECTING_GAP) { elapsedTime += Conts::LOAD_AVG_COLLECTING_GAP * 1000; - double loadAgerage = statisticCollector.getLoadAverage(); + double loadAgerage = StatisticCollector::getLoadAverage(); loadAverageVector.push_back(std::to_string(loadAgerage)); start = start + Conts::LOAD_AVG_COLLECTING_GAP; + } else { + sleep(Conts::LOAD_AVG_COLLECTING_GAP - elapsed); } } } @@ -2056,12 +1975,10 @@ static void handshake_command(int connFd, bool *loop_exit_p) { static inline void close_command(int connFd, bool *loop_exit_p) { *loop_exit_p = true; - Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::CLOSE_ACK); close(connFd); } static inline void shutdown_command(int connFd) { - Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::CLOSE_ACK); close(connFd); exit(0); } @@ -2148,9 +2065,13 @@ static void batch_upload_common(int connFd, bool *loop_exit_p, bool batch_upload instance_logger.info("File received and saved to " + fullFilePath); *loop_exit_p = true; - Utils::unzipFile(fullFilePath); - size_t lastindex = fileName.find_last_of("."); - string rawname = fileName.substr(0, lastindex); + string rawname = fileName; + if (fullFilePath.compare(fullFilePath.size() - 3, 3, ".gz") == 0) { + Utils::unzipFile(fullFilePath); + size_t lastindex = fileName.find_last_of("."); + rawname = fileName.substr(0, lastindex); + } + fullFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + rawname; if (batch_upload) { @@ -2304,8 +2225,8 @@ static void duplicate_centralstore_command(int connFd, int serverPort, bool *loo } static void worker_in_degree_distribution_command( - int connFd, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p) { + int connFd, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; @@ -2402,8 +2323,8 @@ static void worker_in_degree_distribution_command( } static void degree_distribution_common(int connFd, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p, bool in) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2454,14 +2375,14 @@ static void degree_distribution_common(int connFd, int serverPort, } static void in_degree_distribution_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p) { + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { degree_distribution_common(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, loop_exit_p, true); } static void worker_out_degree_distribution_command( - int connFd, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p) { + int connFd, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; @@ -2496,13 +2417,13 @@ static void worker_out_degree_distribution_command( } static void out_degree_distribution_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, bool *loop_exit_p) { + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { degree_distribution_common(connFd, serverPort, graphDBMapLocalStores, graphDBMapCentralStores, loop_exit_p, false); } static void page_rank_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2609,10 +2530,10 @@ static void page_rank_command(int connFd, int serverPort, map::iterator pageRankValue = pageRankResults.find(startVid); if (pageRankValue == pageRankResults.end()) { - pageRankLocalstore.insert(std::make_pair(startVid, 0.0)); + pageRankLocalstore[startVid] = 0.0; } else { double value = pageRankValue->second; - pageRankLocalstore.insert(std::make_pair(startVid, value)); + pageRankLocalstore[startVid] = value; } for (auto a = endVidSet.begin(); a != endVidSet.end(); ++a) { @@ -2620,10 +2541,10 @@ static void page_rank_command(int connFd, int serverPort, map::iterator pageRankValue = pageRankResults.find(endVid); if (pageRankLocalstore.find(endVid) == pageRankLocalstore.end()) { if (pageRankValue == pageRankResults.end()) { - pageRankLocalstore.insert(std::make_pair(endVid, 0.0)); + pageRankLocalstore[endVid] = 0.0; } else { double value = pageRankValue->second; - pageRankLocalstore.insert(std::make_pair(endVid, value)); + pageRankLocalstore[endVid] = value; } } } @@ -2649,13 +2570,13 @@ static void page_rank_command(int connFd, int serverPort, *loop_exit_p = true; return; } - instance_logger.log("Sent : " + JasmineGraphInstanceProtocol::OK, "info"); + instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); instance_logger.info("Finish : Calculate Local PageRank."); } static void worker_page_rank_distribution_command( - int connFd, int serverPort, std::map graphDBMapCentralStores, + int connFd, int serverPort, std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2761,10 +2682,10 @@ static void worker_page_rank_distribution_command( map::iterator pageRankValue = pageRankResults.find(startVid); if (pageRankValue == pageRankResults.end()) { - pageRankLocalstore.insert(std::make_pair(startVid, 1.0)); + pageRankLocalstore[startVid] = 1.0; } else { double value = pageRankValue->second; - pageRankLocalstore.insert(std::make_pair(startVid, value)); + pageRankLocalstore[startVid] = value; } } @@ -2783,7 +2704,7 @@ static void worker_page_rank_distribution_command( } static void egonet_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2832,7 +2753,7 @@ static void egonet_command(int connFd, int serverPort, } static void worker_egonet_command(int connFd, int serverPort, - std::map graphDBMapCentralStores, + std::map &graphDBMapCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2914,9 +2835,9 @@ static void worker_egonet_command(int connFd, int serverPort, } static void triangles_command( - int connFd, int serverPort, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::map graphDBMapDuplicateCentralStores, + int connFd, int serverPort, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::map &graphDBMapDuplicateCentralStores, bool *loop_exit_p) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -2953,6 +2874,8 @@ static void triangles_command( threadPriorityMutex.unlock(); } + std::thread perfThread = std::thread(&PerformanceUtil::collectPerformanceStatistics); + perfThread.detach(); long localCount = countLocalTriangles(graphID, partitionId, graphDBMapLocalStores, graphDBMapCentralStores, graphDBMapDuplicateCentralStores, threadPriority); @@ -3140,32 +3063,10 @@ static void send_centralstore_to_aggregator_command(int connFd, bool *loop_exit_ fullFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + rawname; std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - if (access(aggregatorDirPath.c_str(), F_OK)) { - std::string createDirCommand = "mkdir -p " + aggregatorDirPath; - if (system(createDirCommand.c_str())) { - instance_logger.error("Creating directory " + aggregatorDirPath + " failed"); - } - } - - std::string copyCommand = "cp " + fullFilePath + " " + aggregatorDirPath; - if (system(copyCommand.c_str())) { + if (Utils::copyToDirectory(fullFilePath, aggregatorDirPath)) { instance_logger.error("Copying " + fullFilePath + " into " + aggregatorDirPath + " failed"); } - std::string movedFullFilePath = aggregatorDirPath + "/" + rawname; - - while (!Utils::fileExists(movedFullFilePath)) { - string response = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK) == 0) { - instance_logger.info("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); - } - } - line = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); if (line.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK) == 0) { instance_logger.info("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); @@ -3242,32 +3143,10 @@ static void send_composite_centralstore_to_aggregator_command(int connFd, bool * fullFilePath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + rawname; std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - if (access(aggregatorDirPath.c_str(), F_OK)) { - std::string createDirCommand = "mkdir -p " + aggregatorDirPath; - if (system(createDirCommand.c_str())) { - instance_logger.error("Creating directory " + aggregatorDirPath + " failed"); - } - } - - std::string copyCommand = "cp " + fullFilePath + " " + aggregatorDirPath; - if (system(copyCommand.c_str())) { + if (Utils::copyToDirectory(fullFilePath, aggregatorDirPath)) { instance_logger.error("Copying " + fullFilePath + " into " + aggregatorDirPath + " failed"); } - std::string movedFullFilePath = aggregatorDirPath + "/" + rawname; - - while (!Utils::fileExists(movedFullFilePath)) { - string response = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK) == 0) { - instance_logger.info("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); - } - } - line = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); if (line.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK) == 0) { instance_logger.info("Received : " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); @@ -3323,8 +3202,8 @@ static void aggregate_centralstore_triangles_command(int connFd, bool *loop_exit threadPriorityMutex.unlock(); } - std::string aggregatedTriangles = JasmineGraphInstanceService::aggregateCentralStoreTriangles( - graphId, partitionId, partitionIdList, threadPriority); + const std::string &aggregatedTriangles = + aggregateCentralStoreTriangles(graphId, partitionId, partitionIdList, threadPriority); if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { threadPriorityMutex.lock(); @@ -3353,7 +3232,7 @@ static void aggregate_centralstore_triangles_command(int connFd, bool *loop_exit std::string chunk = chunksVector.at(loopCount); if (!Utils::send_str_wrapper(connFd, chunk)) { *loop_exit_p = true; - return; + break; } } else { string chunkStatus = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); @@ -3363,6 +3242,8 @@ static void aggregate_centralstore_triangles_command(int connFd, bool *loop_exit } } } + chunksVector.clear(); + chunksVector.shrink_to_fit(); } static void aggregate_streaming_centralstore_triangles_command( @@ -3522,7 +3403,7 @@ static void aggregate_composite_centralstore_triangles_command(int connFd, bool threadPriorityMutex.unlock(); } - std::string aggregatedTriangles = + const std::string &aggregatedTriangles = JasmineGraphInstanceService::aggregateCompositeCentralStoreTriangles(response, availableFiles, threadPriority); if (threadPriority > Conts::DEFAULT_THREAD_PRIORITY) { @@ -3552,7 +3433,7 @@ static void aggregate_composite_centralstore_triangles_command(int connFd, bool std::string chunk = chunksVector.at(loopCount); if (!Utils::send_str_wrapper(connFd, chunk)) { *loop_exit_p = true; - return; + break; } } else { string chunkStatus = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); @@ -3562,32 +3443,8 @@ static void aggregate_composite_centralstore_triangles_command(int connFd, bool } } } -} - -static void performance_statistics_command(int connFd, bool *loop_exit_p) { - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); - - char data[DATA_BUFFER_SIZE]; - string isVMStatManager = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - - if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); - - string isResourceAllocationRequired = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); - - std::string memoryUsage = - JasmineGraphInstanceService::requestPerformanceStatistics(isVMStatManager, isResourceAllocationRequired); - if (!Utils::send_str_wrapper(connFd, memoryUsage)) { - *loop_exit_p = true; - return; - } + chunksVector.clear(); + chunksVector.shrink_to_fit(); } static void initiate_files_command(int connFd, bool *loop_exit_p) { @@ -3611,7 +3468,7 @@ static void initiate_files_command(int connFd, bool *loop_exit_p) { } } - std::thread *workerThreads = new std::thread[2]; + std::thread workerThreads[2]; workerThreads[0] = std::thread(&JasmineGraphInstanceService::createPartitionFiles, graphID, partitionID, "local"); workerThreads[1] = std::thread(&JasmineGraphInstanceService::createPartitionFiles, graphID, partitionID, "centralstore"); @@ -3679,9 +3536,7 @@ static void request_collected_stats_command(int connFd, bool *collectValid_p, bo collectValid = false; std::string loadAverageString; - std::vector::iterator loadVectorIterator; - - for (loadVectorIterator = loadAverageVector.begin(); loadVectorIterator != loadAverageVector.end(); + for (auto loadVectorIterator = loadAverageVector.begin(); loadVectorIterator != loadAverageVector.end(); ++loadVectorIterator) { std::string tempLoadAverage = *loadVectorIterator; loadAverageString = loadAverageString + "," + tempLoadAverage; @@ -3691,7 +3546,6 @@ static void request_collected_stats_command(int connFd, bool *collectValid_p, bo loadAverageString = loadAverageString.substr(1, loadAverageString.length() - 1); std::vector chunksVector; - for (unsigned i = 0; i < loadAverageString.length(); i += CHUNK_OFFSET) { std::string chunk = loadAverageString.substr(i, CHUNK_OFFSET); if (i + CHUNK_OFFSET < loadAverageString.length()) { @@ -3704,13 +3558,11 @@ static void request_collected_stats_command(int connFd, bool *collectValid_p, bo for (int loopCount = 0; loopCount < chunksVector.size(); loopCount++) { std::string chunk; - if (loopCount == 0) { - chunk = chunksVector.at(loopCount); - } else { + if (loopCount > 0) { char data[DATA_BUFFER_SIZE]; string chunkStatus = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); - chunk = chunksVector.at(loopCount); } + chunk = chunksVector.at(loopCount); if (!Utils::send_str_wrapper(connFd, chunk)) { *loop_exit_p = true; return; @@ -3739,7 +3591,7 @@ static void initiate_train_command(int connFd, bool *loop_exit_p) { } } - std::thread *workerThreads = new std::thread[2]; + std::thread workerThreads[2]; workerThreads[0] = std::thread(&JasmineGraphInstanceService::createPartitionFiles, graphID, partitionID, "local"); workerThreads[1] = std::thread(&JasmineGraphInstanceService::createPartitionFiles, graphID, partitionID, "centralstore"); @@ -3863,7 +3715,7 @@ static void initiate_predict_command(int connFd, instanceservicesessionargs *ses Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; int fileSize = stoi(size); string line; - while (Utils::fileExists(fullFilePath) && Utils::getFileSize(fullFilePath) < fileSize) { + while (!Utils::fileExists(fullFilePath) || Utils::getFileSize(fullFilePath) < fileSize) { line = Utils::read_str_wrapper(connFd, data, INSTANCE_DATA_LENGTH, false); if (line.compare(JasmineGraphInstanceProtocol::FILE_RECV_CHK) == 0) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::FILE_RECV_WAIT)) { @@ -3969,8 +3821,8 @@ static void initiate_model_collection_command(int connFd, bool *loop_exit_p) { if (line.compare(JasmineGraphInstanceProtocol::SEND_FILE_CONT) == 0) { instance_logger.info("Received : " + JasmineGraphInstanceProtocol::SEND_FILE_CONT); instance_logger.info("Going to send file through service"); - JasmineGraphInstance::sendFileThroughService(serverHostName, stoi(serverHostDataPort), fileName, - filePath); + fileName = "jasminegraph-local_trained_model_store/" + fileName; + Utils::sendFileThroughService(serverHostName, stoi(serverHostDataPort), fileName, filePath); } } } @@ -4116,6 +3968,8 @@ static void check_file_accessible_command(int connFd, bool *loop_exit_p) { return; } instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::SEND_FILE_TYPE); + string fullFilePath; + string result = "false"; char data[DATA_BUFFER_SIZE]; string fileType = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); @@ -4139,19 +3993,7 @@ static void check_file_accessible_command(int connFd, bool *loop_exit_p) { string aggregateLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); string fileName = graphId + "_centralstore_" + partitionId; - string fullFilePath = aggregateLocation + "/" + fileName; - string result = "false"; - - bool fileExists = Utils::fileExists(fullFilePath); - if (fileExists) { - result = "true"; - } - - if (!Utils::send_str_wrapper(connFd, result)) { - *loop_exit_p = true; - return; - } - instance_logger.info("Sent : " + result); + fullFilePath = aggregateLocation + "/" + fileName; } else if (fileType.compare(JasmineGraphInstanceProtocol::FILE_TYPE_CENTRALSTORE_COMPOSITE) == 0) { if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; @@ -4163,20 +4005,26 @@ static void check_file_accessible_command(int connFd, bool *loop_exit_p) { instance_logger.info("Received File name: " + fileName); string aggregateLocation = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); - string fullFilePath = aggregateLocation + "/" + fileName; - string result = "false"; - - bool fileExists = Utils::fileExists(fullFilePath); - if (fileExists) { - result = "true"; - } - - if (!Utils::send_str_wrapper(connFd, result)) { + fullFilePath = aggregateLocation + "/" + fileName; + } else if (fileType.compare(JasmineGraphInstanceProtocol::FILE_TYPE_DATA) == 0) { + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { *loop_exit_p = true; return; } - instance_logger.info("Sent : " + result); + instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); + string fileName = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + string dataDir = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); + fullFilePath = dataDir + "/" + fileName; } + instance_logger.info("Checking existance of: " + fullFilePath); + if (Utils::fileExists(fullFilePath)) { + result = "true"; + } + if (!Utils::send_str_wrapper(connFd, result)) { + *loop_exit_p = true; + return; + } + instance_logger.info("Sent : " + result); } static void graph_stream_start_command(int connFd, InstanceStreamHandler &instanceStreamHandler, bool *loop_exit_p) { @@ -4236,9 +4084,58 @@ static void send_priority_command(int connFd, bool *loop_exit_p) { highestPriority = retrievedPriority; } +static void push_partition_command(int connFd, bool *loop_exit_p) { + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { + *loop_exit_p = true; + return; + } + instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); + + char data[DATA_BUFFER_SIZE]; + string hostDataPort = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + instance_logger.info("Received host:dataPort: " + hostDataPort); + std::vector hostPortList = Utils::split(hostDataPort, ':'); + std::string &host = hostPortList[0]; + int port = std::stoi(hostPortList[1]); + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { + *loop_exit_p = true; + return; + } + instance_logger.info("Sent : " + JasmineGraphInstanceProtocol::OK); + + string graphIDPartitionID = Utils::read_str_trim_wrapper(connFd, data, INSTANCE_DATA_LENGTH); + instance_logger.info("Received graphID,partionID: " + graphIDPartitionID); + std::vector graphPartitionList = Utils::split(graphIDPartitionID, ','); + int graphID = std::stoi(graphPartitionList[0]); + int partitionID = std::stoi(graphPartitionList[1]); + + std::vector fileList = {to_string(graphID) + "_" + to_string(partitionID), + to_string(graphID) + "_centralstore_" + to_string(partitionID), + to_string(graphID) + "_centralstore_dp_" + to_string(partitionID)}; + + for (auto it = fileList.begin(); it != fileList.end(); it++) { + std::string fileName = *it; + std::string path = + Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder") + "/" + fileName; + + if (!Utils::sendFileThroughService(host, port, fileName, path)) { + instance_logger.error("Sending failed"); + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::ERROR)) { + *loop_exit_p = true; + } + return; + } + } + if (!Utils::send_str_wrapper(connFd, JasmineGraphInstanceProtocol::OK)) { + *loop_exit_p = true; + return; + } + instance_logger.info("Sent: " + JasmineGraphInstanceProtocol::OK); +} + string JasmineGraphInstanceService::aggregateStreamingCentralStoreTriangles( std::string graphId, std::string partitionId, std::string partitionIdString, std::string centralCountString, - int threadPriority, std::map incrementalLocalStores, + int threadPriority, std::map &incrementalLocalStores, std::string mode) { instance_logger.info("###INSTANCE### Started Aggregating Central Store Triangles"); std::vector incrementalLocalStoreInstances; diff --git a/src/server/JasmineGraphInstanceService.h b/src/server/JasmineGraphInstanceService.h index ecc2117d..86614ba2 100644 --- a/src/server/JasmineGraphInstanceService.h +++ b/src/server/JasmineGraphInstanceService.h @@ -46,39 +46,33 @@ limitations under the License. #include "../util/Utils.h" #include "JasmineGraphInstanceProtocol.h" -void* instanceservicesession(void* dummyPt); +void *instanceservicesession(void *dummyPt); void writeCatalogRecord(string record); int deleteGraphPartition(std::string graphID, std::string partitionID); void removeGraphFragments(std::string graphID); -long countLocalTriangles( - std::string graphId, std::string partitionId, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::map graphDBMapDuplicateCentralStores, - int threadPriority); map calculateOutDegreeDist(string graphID, string partitionID, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::vector workerSockets); + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::vector &workerSockets); map calculateLocalOutDegreeDist( - string graphID, string partitionID, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores); + string graphID, string partitionID, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores); map calculateInDegreeDist(string graphID, string partitionID, int serverPort, - std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores, - std::vector workerSockets, string workerList); + std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores, + std::vector &workerSockets, string workerList); map calculateLocalInDegreeDist( - string graphID, string partitionID, std::map graphDBMapLocalStores, - std::map graphDBMapCentralStores); + string graphID, string partitionID, std::map &graphDBMapLocalStores, + std::map &graphDBMapCentralStores); map>> calculateLocalEgoNet(string graphID, string partitionID, int serverPort, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - std::vector workerSockets); + std::vector &workerSockets); void calculateEgoNet(string graphID, string partitionID, int serverPort, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, string workerList); @@ -86,32 +80,31 @@ void calculateEgoNet(string graphID, string partitionID, int serverPort, Jasmine map calculateLocalPageRank(string graphID, double alpha, string partitionID, int serverPort, int top_k_page_rank_value, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, - JasmineGraphHashMapCentralStore centralDB, std::vector workerSockets, + JasmineGraphHashMapCentralStore centralDB, std::vector &workerSockets, int iterations); map getAuthorityScoresWorldToLocal(string graphID, string partitionID, int serverPort, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - map> graphVertexMap, - std::vector workerSockets, long worldOnlyVertexCount); + map> &graphVertexMap, + std::vector &workerSockets, long worldOnlyVertexCount); map> getEdgesWorldToLocal(string graphID, string partitionID, int serverPort, string graphVertexCount, JasmineGraphHashMapLocalStore localDB, JasmineGraphHashMapCentralStore centralDB, - map> graphVertexMap, - std::vector workerSockets); + map> &graphVertexMap, + std::vector &workerSockets); struct instanceservicesessionargs { - string profile; string masterHost; string host; int connFd; int port; int dataPort; - std::map graphDBMapLocalStores; - std::map graphDBMapCentralStores; - std::map graphDBMapDuplicateCentralStores; - std::map incrementalLocalStore; + std::map *graphDBMapLocalStores; + std::map *graphDBMapCentralStores; + std::map *graphDBMapDuplicateCentralStores; + std::map *incrementalLocalStore; }; class JasmineGraphInstanceService { @@ -120,29 +113,28 @@ class JasmineGraphInstanceService { static const string END_OF_MESSAGE; JasmineGraphInstanceService(); - void run(string profile, string masterHost, string hostName, int serverPort, int serverDataPort); + void run(string masterHost, string hostName, int serverPort, int serverDataPort); static bool isGraphDBExists(std::string graphId, std::string partitionId); static bool isInstanceCentralStoreExists(std::string graphId, std::string partitionId); static bool isInstanceDuplicateCentralStoreExists(std::string graphId, std::string partitionId); static void loadLocalStore(std::string graphId, std::string partitionId, - std::map& graphDBMapLocalStores); - static JasmineGraphIncrementalLocalStore* loadStreamingStore( + std::map &graphDBMapLocalStores); + static JasmineGraphIncrementalLocalStore *loadStreamingStore( std::string graphId, std::string partitionId, - std::map& graphDBMapStreamingStores, std::string openMode); + std::map &graphDBMapStreamingStores, std::string openMode); static void loadInstanceCentralStore( std::string graphId, std::string partitionId, - std::map& graphDBMapCentralStores); + std::map &graphDBMapCentralStores); static void loadInstanceDuplicateCentralStore( std::string graphId, std::string partitionId, - std::map& graphDBMapDuplicateCentralStores); - static JasmineGraphHashMapCentralStore loadCentralStore(std::string centralStoreFileName); + std::map &graphDBMapDuplicateCentralStores); + static JasmineGraphHashMapCentralStore *loadCentralStore(std::string centralStoreFileName); static string aggregateCentralStoreTriangles(std::string graphId, std::string partitionId, std::string partitionIdList, int threadPriority); static string aggregateCompositeCentralStoreTriangles(std::string compositeFileList, std::string availableFileList, int threadPriority); - static map getOutDegreeDistributionHashMap(map> graphMap); - static string requestPerformanceStatistics(std::string isVMStatManager, std::string isResourceAllocationRequested); + static map getOutDegreeDistributionHashMap(map> &graphMap); struct workerPartitions { int port; @@ -151,11 +143,11 @@ class JasmineGraphInstanceService { }; static void collectTrainedModels( - instanceservicesessionargs* sessionargs, std::string graphID, - std::map graphPartitionedHosts, + instanceservicesessionargs *sessionargs, std::string graphID, + std::map &graphPartitionedHosts, int totalPartitions); - static int collectTrainedModelThreadFunction(instanceservicesessionargs* sessionargs, std::string host, int port, + static int collectTrainedModelThreadFunction(instanceservicesessionargs *sessionargs, std::string host, int port, int dataPort, std::string graphID, std::string partition); static void createPartitionFiles(std::string graphID, std::string partitionID, std::string fileType); @@ -181,16 +173,12 @@ class JasmineGraphInstanceService { static std::map> iterationData; static bool duplicateCentralStore(int thisWorkerPort, int graphID, int partitionID, - std::vector workerSockets, std::string masterIP); - - static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath, - std::string masterIP); + std::vector &workerSockets, std::string masterIP); static string aggregateStreamingCentralStoreTriangles( - std::string graphId, std::string partitionId, std::string partitionIdString, - std::string centralCountString, int threadPriority, - std::map incrementalLocalStores, - std::string mode); + std::string graphId, std::string partitionId, std::string partitionIdString, std::string centralCountString, + int threadPriority, std::map &incrementalLocalStores, + std::string mode); static int partitionCounter; diff --git a/src/server/JasmineGraphServer.cpp b/src/server/JasmineGraphServer.cpp index 5474070f..7f6f8751 100644 --- a/src/server/JasmineGraphServer.cpp +++ b/src/server/JasmineGraphServer.cpp @@ -21,10 +21,9 @@ limitations under the License. #include #include +#include "../../globals.h" +#include "../k8s/K8sWorkerController.h" #include "../ml/trainer/JasmineGraphTrainingSchedular.h" -#include "../partitioner/local/MetisPartitioner.h" -#include "../util/Utils.h" -#include "../util/logger/Logger.h" #include "JasmineGraphInstance.h" #include "JasmineGraphInstanceProtocol.h" @@ -34,11 +33,8 @@ static void copyArtifactsToWorkers(const std::string &workerPath, const std::str const std::string &remoteWorker); static void createLogFilePath(const std::string &workerHost, const std::string &workerPath); static void deleteWorkerPath(const std::string &workerHost, const std::string &workerPath); -static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort, - int workerDataPort); +static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort); static void updateMetaDB(int graphID, std::string uploadEndTime); -static bool batchUploadCommon(std::string host, int port, int dataPort, int graphID, std::string filePath, - std::string masterIP, std::string uploadType); static bool batchUploadFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP); static bool batchUploadCentralStore(std::string host, int port, int dataPort, int graphID, std::string filePath, @@ -51,27 +47,27 @@ static bool batchUploadCompositeCentralstoreFile(std::string host, int port, int std::string filePath, std::string masterIP); static bool removeFragmentThroughService(string host, int port, string graphID, string masterIP); static bool removePartitionThroughService(string host, int port, string graphID, string partitionID, string masterIP); -static bool initiateCommon(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP, std::string initType); -static bool initiateTrain(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); -static bool initiatePredict(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); -static bool initiateServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); -static bool initiateClient(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); -static bool initiateAggregator(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); -static bool initiateOrgServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP); +static bool initiateCommon(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP, + std::string initType); +static bool initiateTrain(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); +static bool initiatePredict(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); +static bool initiateServer(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); +static bool initiateClient(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP); +static bool initiateAggregator(std::string host, int port, std::string trainingArgs, int iteration, + std::string masterIP); +static bool initiateOrgServer(std::string host, int port, std::string trainingArgs, int iteration, + std::string masterIP); static void degreeDistributionCommon(std::string graphID, std::string command); +static int getPortByHost(const std::string &host); +static int getDataPortByHost(const std::string &host); +static size_t getWorkerCount(); -static map hostIDMap; -static std::vector hostWorkerMap; -static map> hostPortMap; +static std::vector hostWorkerList; +static unordered_map> hostPortMap; std::map aggregateWeightMap; +static int graphUploadWorkerTracker = 0; + void *runfrontend(void *dummyPt) { JasmineGraphServer *refToServer = (JasmineGraphServer *)dummyPt; refToServer->frontend = new JasmineGraphFrontEnd(refToServer->sqlite, refToServer->performanceSqlite, @@ -98,17 +94,20 @@ JasmineGraphServer::JasmineGraphServer() { this->jobScheduler->init(); } +static std::mutex instanceMutex; JasmineGraphServer *JasmineGraphServer::getInstance() { static JasmineGraphServer *instance = nullptr; - // TODO(thevindu-w): synchronize if (instance == nullptr) { - instance = new JasmineGraphServer(); + instanceMutex.lock(); + if (instance == nullptr) { // double-checking lock + instance = new JasmineGraphServer(); + } + instanceMutex.unlock(); } return instance; } JasmineGraphServer::~JasmineGraphServer() { - server_logger.info("Freeing up server resources."); this->sqlite->finalize(); this->performanceSqlite->finalize(); delete this->sqlite; @@ -116,8 +115,7 @@ JasmineGraphServer::~JasmineGraphServer() { delete this->jobScheduler; } -int JasmineGraphServer::run(std::string profile, std::string masterIp, int numberofWorkers, std::string workerIps, - std::string enableNmon) { +int JasmineGraphServer::run(std::string masterIp, int numberofWorkers, std::string workerIps, std::string enableNmon) { server_logger.info("Running the server..."); std::vector masterPortVector; if (masterIp.empty()) { @@ -126,63 +124,53 @@ int JasmineGraphServer::run(std::string profile, std::string masterIp, int numbe this->masterHost = masterIp; } server_logger.info("masterHost = " + this->masterHost); - this->profile = profile; this->numberOfWorkers = numberofWorkers; this->workerHosts = workerIps; this->enableNmon = enableNmon; - init(); masterPortVector.push_back(Conts::JASMINEGRAPH_FRONTEND_PORT); updateOperationalGraphList(); - if (profile == Conts::PROFILE_K8S) { - k8sWorkerController = new K8sWorkerController(masterIp, numberofWorkers, sqlite); - std::string selectQuery = "select ip,server_port,server_data_port from worker"; - std::vector>> output = this->sqlite->runSelect(selectQuery); - for (std::vector>>::iterator i = output.begin(); i != output.end(); ++i) { - std::vector>::iterator j = (i->begin()); - std::string ip = j->second; - ++j; - std::string port = j->second; - ++j; - std::string dataPort = j->second; - hostWorkerMap.push_back({ip, atoi(port.c_str()), atoi(dataPort.c_str())}); - } - + if (jasminegraph_profile == PROFILE_K8S) { + // Create K8s worker controller + (void)K8sWorkerController::getInstance(masterIp, numberofWorkers, sqlite); + hostWorkerList = K8sWorkerController::workerList; } else { start_workers(); addInstanceDetailsToPerformanceDB(masterHost, masterPortVector, "true"); } + init(); std::thread *myThreads = new std::thread[1]; myThreads[0] = std::thread(StatisticCollector::logLoadAverage, "Load Average"); - sleep(2); + sleep(1); waitForAcknowledgement(numberofWorkers); resolveOperationalGraphs(); initiateAggregateMap(); return 0; } -bool JasmineGraphServer::isRunning() { return true; } - void JasmineGraphServer::init() { pthread_t frontendthread; pthread_t backendthread; pthread_create(&frontendthread, NULL, runfrontend, this); + pthread_detach(frontendthread); pthread_create(&backendthread, NULL, runbackend, this); + pthread_detach(backendthread); } void JasmineGraphServer::start_workers() { + // Not used in K8s mode int hostListModeNWorkers = 0; int numberOfWorkersPerHost; std::vector hostsList; std::string nWorkers; - if (profile == Conts::PROFILE_NATIVE) { + if (jasminegraph_profile == PROFILE_NATIVE) { hostsList = Utils::getHostListFromProperties(); if ((this->numberOfWorkers) == -1) { nWorkers = Utils::getJasmineGraphProperty("org.jasminegraph.server.nworkers"); } enableNmon = Utils::getJasmineGraphProperty("org.jasminegraph.server.enable.nmon"); - } else if (profile == Conts::PROFILE_DOCKER) { + } else if (jasminegraph_profile == PROFILE_DOCKER) { hostsList = getWorkerVector(workerHosts); } @@ -194,7 +182,6 @@ void JasmineGraphServer::start_workers() { this->sqlite->runUpdate("DELETE FROM host"); std::vector::iterator it; - it = hostsList.begin(); std::string hostString = ""; std::string sqlString = "INSERT INTO host (idhost,name,ip,is_public) VALUES "; int counter = 0; @@ -247,8 +234,6 @@ void JasmineGraphServer::start_workers() { this->sqlite->runUpdate("DELETE FROM worker"); int workerIDCounter = 0; - it = hostsList.begin(); - for (it = hostsList.begin(); it < hostsList.end(); it++) { string sqlStatement = "INSERT INTO worker (idworker,host_idhost,name,ip,user,is_public,server_port,server_data_port) VALUES "; @@ -269,10 +254,10 @@ void JasmineGraphServer::start_workers() { while (portCount < numberOfWorkersPerHost) { portVector.push_back(workerPort); dataPortVector.push_back(workerDataPort); - hostWorkerMap.push_back({*it, workerPort, workerDataPort}); + hostWorkerList.push_back({*it, workerPort, workerDataPort}); // FIXME: When there are more than 1 worker in the same host, one workers ports will replace the entries of // other workers port entries in hostPortMap - hostPortMap.insert((pair>(*it, make_pair(workerPort, workerDataPort)))); + hostPortMap[*it] = make_pair(workerPort, workerDataPort); portCount++; // ToDO: Here for the moment we use host name as the IP address as the third parameter. // ToDO: We also keep user as empty string @@ -288,8 +273,8 @@ void JasmineGraphServer::start_workers() { if (hostListModeNWorkers > 0) { portVector.push_back(workerPort); dataPortVector.push_back(workerDataPort); - hostWorkerMap.push_back({*it, workerPort, workerDataPort}); - hostPortMap.insert(((pair>(*it, make_pair(workerPort, workerDataPort))))); + hostWorkerList.push_back({*it, workerPort, workerDataPort}); + hostPortMap[*it] = make_pair(workerPort, workerDataPort); hostListModeNWorkers--; string is_public = "false"; valuesString += "(" + std::to_string(workerIDCounter) + ", " + hostID + ", \"" + hostName + "\", \"" + ip + @@ -322,7 +307,7 @@ void JasmineGraphServer::start_workers() { addHostsToMetaDB(host, workerPortsMap[host], workerDataPortsMap[host]); addInstanceDetailsToPerformanceDB(host, workerPortsMap[host], "false"); myThreads[count] = std::thread(startRemoteWorkers, workerPortsMap[host], workerDataPortsMap[host], host, - profile, masterHost, enableNmon); + masterHost, enableNmon); count++; } @@ -330,7 +315,27 @@ void JasmineGraphServer::start_workers() { myThreads[threadCount].join(); } delete[] myThreads; - hostIDMap = getLiveHostIDList(); +} + +static int getPortByHost(const std::string &host) { + if (jasminegraph_profile == PROFILE_K8S) { + return Conts::JASMINEGRAPH_INSTANCE_PORT; + } + return hostPortMap[host].first; +} + +static int getDataPortByHost(const std::string &host) { + if (jasminegraph_profile == PROFILE_K8S) { + return Conts::JASMINEGRAPH_INSTANCE_DATA_PORT; + } + return hostPortMap[host].second; +} + +static size_t getWorkerCount() { + if (jasminegraph_profile == PROFILE_K8S) { + return K8sWorkerController::workerList.size(); + } + return hostWorkerList.size(); } void JasmineGraphServer::waitForAcknowledgement(int numberOfWorkers) { @@ -352,7 +357,7 @@ void JasmineGraphServer::waitForAcknowledgement(int numberOfWorkers) { } void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, std::vector workerDataPortsVector, - string host, string profile, string masterHost, string enableNmon) { + string host, string masterHost, string enableNmon) { std::string executableFile; std::string workerPath = Utils::getJasmineGraphProperty("org.jasminegraph.worker.path"); std::string artifactPath = Utils::getJasmineGraphProperty("org.jasminegraph.artifact.path"); @@ -366,7 +371,7 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, std::string instanceFolderLocal = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.local"); std::string jasmineGraphExecutableName = Conts::JASMINEGRAPH_EXECUTABLE; - server_logger.info("###MASTER#### Starting remote workers for profile " + profile); + server_logger.info("###MASTER#### Starting remote workers"); if (hasEnding(workerPath, "/")) { executableFile = workerPath + jasmineGraphExecutableName; } else { @@ -380,7 +385,7 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, artifactPath = Utils::getJasmineGraphHome(); } - if (profile == Conts::PROFILE_NATIVE) { + if (jasminegraph_profile == PROFILE_NATIVE) { copyArtifactsToWorkers(workerPath, artifactPath, host); for (int i = 0; i < workerPortsVector.size(); i++) { if (host.find("localhost") != std::string::npos) { @@ -399,7 +404,7 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, _exit(1); } } - } else if (profile == Conts::PROFILE_DOCKER) { + } else if (jasminegraph_profile == PROFILE_DOCKER) { char *env_testing = getenv("TESTING"); bool is_testing = (env_testing != nullptr && strcasecmp(env_testing, "true") == 0); for (int i = 0; i < workerPortsVector.size(); i++) { @@ -427,10 +432,9 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, "docker run -v " + instanceDataFolder + ":" + instanceDataFolder + " -v " + aggregateDataFolder + ":" + aggregateDataFolder + " -v " + nmonFileLocation + ":" + nmonFileLocation + " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + - "/var/tmp/jasminegraph/logs" + " -p " + std::to_string(workerPortsVector.at(i)) + - ":" + std::to_string(workerPortsVector.at(i)) + " -p " + - std::to_string(workerDataPortsVector.at(i)) + ":" + - std::to_string(workerDataPortsVector.at(i)) + " -e WORKER_ID=" + to_string(i) + + "/var/tmp/jasminegraph/logs" + " -p " + std::to_string(workerPortsVector.at(i)) + ":" + + std::to_string(workerPortsVector.at(i)) + " -p " + std::to_string(workerDataPortsVector.at(i)) + + ":" + std::to_string(workerDataPortsVector.at(i)) + " -e WORKER_ID=" + to_string(i) + " jasminegraph:latest --MODE 2 --HOST_NAME " + host + " --MASTERIP " + masterHost + " --SERVER_PORT " + std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; @@ -450,10 +454,9 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, "docker -H ssh://" + host + " run -v " + instanceDataFolder + ":" + instanceDataFolder + " -v " + aggregateDataFolder + ":" + aggregateDataFolder + " -v " + nmonFileLocation + ":" + nmonFileLocation + " -v " + instanceDataFolder + "/" + to_string(i) + "/logs" + ":" + - "/var/tmp/jasminegraph/logs" + " -p " + std::to_string(workerPortsVector.at(i)) + - ":" + std::to_string(workerPortsVector.at(i)) + " -p " + - std::to_string(workerDataPortsVector.at(i)) + ":" + - std::to_string(workerDataPortsVector.at(i)) + " -e WORKER_ID=" + to_string(i) + + "/var/tmp/jasminegraph/logs" + " -p " + std::to_string(workerPortsVector.at(i)) + ":" + + std::to_string(workerPortsVector.at(i)) + " -p " + std::to_string(workerDataPortsVector.at(i)) + + ":" + std::to_string(workerDataPortsVector.at(i)) + " -e WORKER_ID=" + to_string(i) + " jasminegraph:latest --MODE 2 --HOST_NAME " + host + " --MASTERIP " + masterHost + " --SERVER_PORT " + std::to_string(workerPortsVector.at(i)) + " --SERVER_DATA_PORT " + std::to_string(workerDataPortsVector.at(i)) + " --ENABLE_NMON " + enableNmon; @@ -469,7 +472,7 @@ void JasmineGraphServer::startRemoteWorkers(std::vector workerPortsVector, } } -bool JasmineGraphServer::spawnNewWorker(string host, string port, string dataPort, string profile, string masterHost, +bool JasmineGraphServer::spawnNewWorker(string host, string port, string dataPort, string masterHost, string enableNmon) { auto *refToSqlite = new SQLiteDBInterface(); refToSqlite->init(); @@ -522,8 +525,7 @@ bool JasmineGraphServer::spawnNewWorker(string host, string port, string dataPor workerPortsVector.push_back(atoi(port.c_str())); workerDataPortsVector.push_back(atoi(dataPort.c_str())); - JasmineGraphServer::startRemoteWorkers(workerPortsVector, workerDataPortsVector, host, profile, masterHost, - enableNmon); + JasmineGraphServer::startRemoteWorkers(workerPortsVector, workerDataPortsVector, host, masterHost, enableNmon); return true; } @@ -540,19 +542,18 @@ void JasmineGraphServer::resolveOperationalGraphs() { std::vector>> output = this->sqlite->runSelect(sqlStatement); std::map> partitionMap; - for (std::vector>>::iterator i = output.begin(); i != output.end(); ++i) { + for (auto i = output.begin(); i != output.end(); ++i) { int workerID = -1; string graphID; string partitionID; - std::vector>::iterator j = (i->begin()); + auto j = i->begin(); graphID = j->second; ++j; partitionID = j->second; ++j; workerID = std::stoi(j->second); - std::vector partitionList = partitionMap[workerID]; + std::vector &partitionList = partitionMap[workerID]; partitionList.push_back(graphID + "_" + partitionID); - partitionMap[workerID] = partitionList; } int RECORD_AGGREGATION_FREQUENCY = 5; @@ -565,25 +566,23 @@ void JasmineGraphServer::resolveOperationalGraphs() { for (std::vector::iterator x = (it->second).begin(); x != (it->second).end(); ++x) { if (counter >= RECORD_AGGREGATION_FREQUENCY) { - std::vector partitionList = partitionAggregatedMap[workerID]; + std::vector &partitionList = partitionAggregatedMap[workerID]; string data = ss.str(); std::stringstream().swap(ss); counter = 0; data = data.substr(0, data.find_last_of(",")); partitionList.push_back(data); - partitionAggregatedMap[workerID] = partitionList; } ss << x->c_str() << ","; counter++; } - std::vector partitionList = partitionAggregatedMap[workerID]; + std::vector &partitionList = partitionAggregatedMap[workerID]; string data = ss.str(); std::stringstream().swap(ss); counter = 0; data = data.substr(0, data.find_last_of(",")); partitionList.push_back(data); - partitionAggregatedMap[workerID] = partitionList; } sqlStatement = "SELECT idworker,ip,server_port FROM worker"; @@ -595,7 +594,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { string host; int workerPort = -1; string partitionID; - std::vector>::iterator j = (i->begin()); + std::vector>::iterator j = i->begin(); workerID = std::stoi(j->second); ++j; host = j->second; @@ -635,6 +634,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, this->masterHost)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -642,6 +642,7 @@ void JasmineGraphServer::resolveOperationalGraphs() { if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::INITIATE_FRAGMENT_RESOLUTION, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -666,10 +667,12 @@ void JasmineGraphServer::resolveOperationalGraphs() { } } if (!success) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FRAGMENT_RESOLUTION_DONE)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); continue; } @@ -683,14 +686,15 @@ void JasmineGraphServer::resolveOperationalGraphs() { graphIDsFromWorkersSet.insert(atoi(it->c_str())); } } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); } sqlStatement = "SELECT idgraph FROM graph"; std::vector>> output2 = this->sqlite->runSelect(sqlStatement); std::set graphIDsFromMetDBSet; - for (std::vector>>::iterator i = output2.begin(); i != output2.end(); ++i) { - std::vector>::iterator j = (i->begin()); + for (auto i = output2.begin(); i != output2.end(); ++i) { + auto j = i->begin(); graphIDsFromMetDBSet.insert(atoi(j->second.c_str())); } @@ -709,40 +713,43 @@ void JasmineGraphServer::deleteNonOperationalGraphFragment(int graphID) { server_logger.info("Deleting non-operational fragment " + to_string(graphID)); int count = 0; // Define threads for each host - std::thread *deleteThreads = new std::thread[hostPortMap.size()]; + std::thread *deleteThreads = new std::thread[getWorkerCount()]; + std::vector *workerListAll; + if (jasminegraph_profile == PROFILE_K8S) { + workerListAll = &(K8sWorkerController::workerList); + } else { + workerListAll = &hostWorkerList; + } // Iterate through all hosts - for (std::map>::iterator it = hostPortMap.begin(); it != hostPortMap.end(); it++) { + for (auto it = (*workerListAll).begin(); it != (*workerListAll).end(); it++) { // Fetch hostname and port - string hostname = it->first; - int port = (it->second).first; + const string &hostname = it->hostname; + int port = it->port; // Initialize threads for host // Each thread runs the service to remove the given graph ID fragments in their datafolders deleteThreads[count++] = std::thread(removeFragmentThroughService, hostname, port, to_string(graphID), this->masterHost); - sleep(1); - server_logger.info("Deleted graph fragments of graph ID " + to_string(graphID)); } + sleep(1); for (int threadCount = 0; threadCount < count; threadCount++) { if (deleteThreads[threadCount].joinable()) { deleteThreads[threadCount].join(); } } + server_logger.info("Deleted graph fragments of graph ID " + to_string(graphID)); } void JasmineGraphServer::shutdown_workers() { server_logger.info("Shutting down workers"); - std::vector>::iterator mapIterator; - auto *server = JasmineGraphServer::getInstance(); - if (server->profile == Conts::PROFILE_K8S) { - server->k8sWorkerController->setNumberOfWorkers(0); + if (jasminegraph_profile == PROFILE_K8S) { + K8sWorkerController::getInstance()->setNumberOfWorkers(0); return; } - - for (mapIterator = hostWorkerMap.begin(); mapIterator < hostWorkerMap.end(); mapIterator++) { - workers worker = *mapIterator; + for (auto listIterator = hostWorkerList.begin(); listIterator < hostWorkerList.end(); listIterator++) { + worker worker = *listIterator; server_logger.info("Host:" + worker.hostname + " Port:" + to_string(worker.port) + " DPort:" + to_string(worker.dataPort)); @@ -789,70 +796,137 @@ int JasmineGraphServer::shutdown_worker(std::string workerIP, int port) { string response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); server_logger.info("Response: " + response); + close(sockfd); return 0; } +static map scaleK8s(size_t npart) { + std::vector &workerList = K8sWorkerController::workerList; + const map &cpu_map = Utils::getMetricMap("cpu_usage"); + // Convert strings to float + map cpu_loads; + for (auto it = cpu_map.begin(); it != cpu_map.end(); it++) { + cpu_loads[it->first] = (float)atof(it->second.c_str()); + } + + for (auto it = workerList.begin(); it != workerList.end(); it++) { + auto &worker = *it; + // 0.8 depends on num cpu cores and other factors + if (npart > 0 && cpu_loads[worker.hostname + ":" + to_string(worker.port)] < 0.8) npart--; + } + if (npart <= 0) return cpu_loads; + K8sWorkerController *controller = K8sWorkerController::getInstance(); + controller->scaleUp((int)npart); + server_logger.info("Scalled up with " + to_string(npart) + " new workers"); + for (auto it = K8sWorkerController::workerList.begin(); it != K8sWorkerController::workerList.end(); it++) { + auto &worker = *it; + string workerHostPort = worker.hostname + ":" + to_string(worker.port); + if (cpu_loads.find(workerHostPort) == cpu_loads.end()) { + cpu_loads[workerHostPort] = 0.; + } + } + return cpu_loads; +} + +static std::vector getWorkers(size_t npart) { + // TODO: get the workers with lowest load from workerList + std::vector *workerListAll; + map cpu_loads; + if (jasminegraph_profile == PROFILE_K8S) { + std::unique_ptr k8sInterface(new K8sInterface()); + if (k8sInterface->getJasmineGraphConfig("scale_on_adgr") != "true") { + return K8sWorkerController::workerList; + } + workerListAll = &(K8sWorkerController::workerList); + cpu_loads = scaleK8s(npart); + } else { + workerListAll = &hostWorkerList; + for (auto it = hostWorkerList.begin(); it != hostWorkerList.end(); it++) { + auto &worker = *it; + string workerHostPort = worker.hostname + ":" + to_string(worker.port); + cpu_loads[workerHostPort] = 0.; + } + } + size_t len = workerListAll->size(); + std::vector workerList; + for (int i = 0; i < npart; i++) { + JasmineGraphServer::worker worker_min; + float cpu_min = 100.; + for (auto it = (*workerListAll).begin(); it != (*workerListAll).end(); it++) { + auto &worker = *it; + string workerHostPort = worker.hostname + ":" + to_string(worker.port); + float cpu = cpu_loads[workerHostPort]; + if (cpu < cpu_min) { + worker_min = worker; + cpu_min = cpu; + } + } + string workerHostPort = worker_min.hostname + ":" + to_string(worker_min.port); + cpu_loads[workerHostPort] += 0.25; // 0.25 = 1/nproc + workerList.push_back(worker_min); + } + return workerList; +} + void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, vector> fullFileList, std::string masterIP) { server_logger.info("Uploading the graph locally.."); - std::map partitionFileList = fullFileList[0]; - std::map centralStoreFileList = fullFileList[1]; - std::map centralStoreDuplFileList = fullFileList[2]; - std::map compositeCentralStoreFileList = fullFileList[5]; - std::map attributeFileList; - std::map centralStoreAttributeFileList; + std::map partitionFileMap = fullFileList[0]; + std::map centralStoreFileMap = fullFileList[1]; + std::map centralStoreDuplFileMap = fullFileList[2]; + std::map compositeCentralStoreFileMap = fullFileList[5]; + std::map attributeFileMap; + std::map centralStoreAttributeFileMap; if (masterHost.empty()) { masterHost = Utils::getJasmineGraphProperty("org.jasminegraph.server.host"); } - int total_threads = partitionFileList.size() + centralStoreFileList.size() + centralStoreDuplFileList.size() + - compositeCentralStoreFileList.size(); + int total_threads = partitionFileMap.size() + centralStoreFileMap.size() + centralStoreDuplFileMap.size() + + compositeCentralStoreFileMap.size(); if (graphType == Conts::GRAPH_WITH_ATTRIBUTES) { - attributeFileList = fullFileList[3]; - total_threads += attributeFileList.size(); - centralStoreAttributeFileList = fullFileList[4]; - total_threads += centralStoreAttributeFileList.size(); + attributeFileMap = fullFileList[3]; + total_threads += attributeFileMap.size(); + centralStoreAttributeFileMap = fullFileList[4]; + total_threads += centralStoreAttributeFileMap.size(); } int count = 0; int file_count = 0; std::thread *workerThreads = new std::thread[total_threads]; while (count < total_threads) { - std::vector>::iterator mapIterator; - for (mapIterator = hostWorkerMap.begin(); mapIterator < hostWorkerMap.end(); mapIterator++) { - workers worker = *mapIterator; - if (count == total_threads) { + const auto &workerList = getWorkers(partitionFileMap.size()); + while (true) { + if (count >= total_threads) { break; } - std::string partitionFileName = partitionFileList[file_count]; - workerThreads[count] = std::thread(batchUploadFile, worker.hostname, worker.port, worker.dataPort, graphID, - partitionFileName, masterHost); - count++; - copyCentralStoreToAggregateLocation(centralStoreFileList[file_count]); - workerThreads[count] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, - graphID, centralStoreFileList[file_count], masterHost); - count++; - - if (compositeCentralStoreFileList.find(file_count) != compositeCentralStoreFileList.end()) { - copyCentralStoreToAggregateLocation(compositeCentralStoreFileList[file_count]); - workerThreads[count] = + worker worker = workerList[graphUploadWorkerTracker]; + std::string partitionFileName = partitionFileMap[file_count]; + workerThreads[count++] = std::thread(batchUploadFile, worker.hostname, worker.port, worker.dataPort, + graphID, partitionFileName, masterHost); + copyCentralStoreToAggregateLocation(centralStoreFileMap[file_count]); + workerThreads[count++] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, + graphID, centralStoreFileMap[file_count], masterHost); + + if (compositeCentralStoreFileMap.find(file_count) != compositeCentralStoreFileMap.end()) { + copyCentralStoreToAggregateLocation(compositeCentralStoreFileMap[file_count]); + workerThreads[count++] = std::thread(batchUploadCompositeCentralstoreFile, worker.hostname, worker.port, worker.dataPort, - graphID, compositeCentralStoreFileList[file_count], masterHost); - count++; + graphID, compositeCentralStoreFileMap[file_count], masterHost); } - workerThreads[count] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, - graphID, centralStoreDuplFileList[file_count], masterHost); - count++; + workerThreads[count++] = std::thread(batchUploadCentralStore, worker.hostname, worker.port, worker.dataPort, + graphID, centralStoreDuplFileMap[file_count], masterHost); if (graphType == Conts::GRAPH_WITH_ATTRIBUTES) { - workerThreads[count] = std::thread(batchUploadAttributeFile, worker.hostname, worker.port, - worker.dataPort, graphID, attributeFileList[file_count], masterHost); - count++; - workerThreads[count] = + workerThreads[count++] = + std::thread(batchUploadAttributeFile, worker.hostname, worker.port, worker.dataPort, graphID, + attributeFileMap[file_count], masterHost); + workerThreads[count++] = std::thread(batchUploadCentralAttributeFile, worker.hostname, worker.port, worker.dataPort, graphID, - centralStoreAttributeFileList[file_count], masterHost); - count++; + centralStoreAttributeFileMap[file_count], masterHost); } - assignPartitionToWorker(partitionFileName, graphID, worker.hostname, worker.port, worker.dataPort); + assignPartitionToWorker(partitionFileName, graphID, worker.hostname, worker.port); file_count++; + + graphUploadWorkerTracker++; + graphUploadWorkerTracker %= workerList.size(); } } @@ -871,8 +945,7 @@ void JasmineGraphServer::uploadGraphLocally(int graphID, const string graphType, delete[] workerThreads; } -static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort, - int workerDataPort) { +static void assignPartitionToWorker(std::string fileName, int graphId, std::string workerHost, int workerPort) { auto *refToSqlite = new SQLiteDBInterface(); refToSqlite->init(); size_t lastindex = fileName.find_last_of("."); @@ -883,16 +956,15 @@ static void assignPartitionToWorker(std::string fileName, int graphId, std::stri workerHost = Utils::split(workerHost, '@')[1]; } - std::string workerSearchQuery = "select idworker from worker where ip='" + workerHost + "' and server_port='" + - std::to_string(workerPort) + "' and server_data_port='" + - std::to_string(workerDataPort) + "'"; + std::string workerSearchQuery = "SELECT idworker FROM worker WHERE ip='" + workerHost + "' AND server_port='" + + std::to_string(workerPort) + "'"; std::vector>> results = refToSqlite->runSelect(workerSearchQuery); std::string workerID = results[0][0].second; std::string partitionToWorkerQuery = - "insert into worker_has_partition (partition_idpartition, partition_graph_idgraph, worker_idworker) values " + "INSERT INTO worker_has_partition (partition_idpartition, partition_graph_idgraph, worker_idworker) VALUES " "('" + partitionID + "','" + std::to_string(graphId) + "','" + workerID + "')"; @@ -901,131 +973,16 @@ static void assignPartitionToWorker(std::string fileName, int graphId, std::stri delete refToSqlite; } -static bool batchUploadCommon(std::string host, int port, int dataPort, int graphID, std::string filePath, - std::string masterIP, std::string uploadType) { - server_logger.info("Host:" + host + " Port:" + to_string(port) + " DPort:" + to_string(dataPort)); - bool result = true; - int sockfd; - char data[FED_DATA_LENGTH + 1]; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - server_logger.error("Cannot create socket"); - return false; - } - - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - server_logger.error("ERROR, no host named " + host); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return false; - } - - if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { - close(sockfd); - return false; - } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, uploadType, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return false; - } - - std::string fileName = Utils::getFileName(filePath); - int fileSize = Utils::getFileSize(filePath); - std::string fileLength = to_string(fileSize); - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, std::to_string(graphID), - JasmineGraphInstanceProtocol::SEND_FILE_NAME)) { - close(sockfd); - return false; - } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, - JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { - close(sockfd); - return false; - } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileLength, - JasmineGraphInstanceProtocol::SEND_FILE_CONT)) { - close(sockfd); - return false; - } - - server_logger.info("Going to send central store file through file transfer service from master to worker"); - JasmineGraphServer::sendFileThroughService(host, dataPort, fileName, filePath, masterIP); - - string response; - int count = 0; - while (true) { - if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FILE_RECV_CHK)) { - close(sockfd); - return false; - } - server_logger.info("Sent: " + JasmineGraphInstanceProtocol::FILE_RECV_CHK); - - server_logger.info("Checking if file is received"); - response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { - server_logger.info("Received: " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT); - server_logger.info("Checking file status : " + to_string(count)); - count++; - sleep(1); - continue; - } else if (response.compare(JasmineGraphInstanceProtocol::FILE_ACK) == 0) { - server_logger.info("Received: " + JasmineGraphInstanceProtocol::FILE_ACK); - server_logger.info("File transfer completed for file : " + filePath); - break; - } - } - // Next we wait till the batch upload completes - while (true) { - if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK)) { - close(sockfd); - return false; - } - server_logger.info("Sent: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); - - response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); - if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { - server_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); - sleep(1); - continue; - } else if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK) == 0) { - server_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK); - server_logger.info("Batch upload completed"); - break; - } - } - close(sockfd); - return true; -} - static bool batchUploadFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::BATCH_UPLOAD); + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, + JasmineGraphInstanceProtocol::BATCH_UPLOAD); } static bool batchUploadCentralStore(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::BATCH_UPLOAD_CENTRAL); + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CENTRAL); } void JasmineGraphServer::copyCentralStoreToAggregateLocation(std::string filePath) { @@ -1033,10 +990,7 @@ void JasmineGraphServer::copyCentralStoreToAggregateLocation(std::string filePat std::string aggregatorDirPath = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.aggregatefolder"); if (access(aggregatorDirPath.c_str(), F_OK)) { - std::string createDirCommand = "mkdir -p " + aggregatorDirPath; - if (system(createDirCommand.c_str())) { - server_logger.error("Creating directory " + aggregatorDirPath + " failed"); - } + Utils::createDirectory(aggregatorDirPath); } std::string copyCommand = "cp " + filePath + " " + aggregatorDirPath; @@ -1047,87 +1001,25 @@ void JasmineGraphServer::copyCentralStoreToAggregateLocation(std::string filePat static bool batchUploadAttributeFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES); + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, + JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES); } static bool batchUploadCentralAttributeFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES_CENTRAL); + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, + JasmineGraphInstanceProtocol::UPLOAD_RDF_ATTRIBUTES_CENTRAL); } static bool batchUploadCompositeCentralstoreFile(std::string host, int port, int dataPort, int graphID, std::string filePath, std::string masterIP) { - return batchUploadCommon(host, port, dataPort, graphID, filePath, masterIP, - JasmineGraphInstanceProtocol::BATCH_UPLOAD_COMPOSITE_CENTRAL); -} - -bool JasmineGraphServer::sendFileThroughService(std::string host, int dataPort, std::string fileName, - std::string filePath, std::string masterIP) { - int sockfd; - char data[FED_DATA_LENGTH + 1]; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - if (sockfd < 0) { - server_logger.error("Cannot create socket"); - return false; - } - - server = gethostbyname(host.c_str()); - if (server == NULL) { - server_logger.error("ERROR, no host named " + host); - return false; - } - - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(dataPort); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return false; - } - - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, - JasmineGraphInstanceProtocol::SEND_FILE)) { - close(sockfd); - return false; - } - - server_logger.info("Sending file " + filePath + " through port " + std::to_string(dataPort)); - FILE *fp = fopen(filePath.c_str(), "r"); - if (fp == NULL) { - close(sockfd); - return false; - } - - while (true) { - unsigned char buff[1024]; - int nread = fread(buff, 1, sizeof(buff), fp); - - /* If read was success, send data. */ - if (nread > 0) { - write(sockfd, buff, nread); - } - - if (nread < sizeof(buff)) { - if (feof(fp)) server_logger.info("End of file"); - if (ferror(fp)) server_logger.error("Error reading file: " + filePath); - break; - } - } - - fclose(fp); - close(sockfd); - return true; + return Utils::uploadFileToWorker(host, port, dataPort, graphID, filePath, masterIP, + JasmineGraphInstanceProtocol::BATCH_UPLOAD_COMPOSITE_CENTRAL); } static void copyArtifactsToWorkers(const std::string &workerPath, const std::string &artifactLocation, const std::string &remoteWorker) { + // Not used in docker or k8s modes if (artifactLocation.empty() || artifactLocation.find_first_not_of(' ') == artifactLocation.npos) { server_logger.error("Received `" + artifactLocation + "` for `artifactLocation` value!"); return; @@ -1164,6 +1056,7 @@ static void copyArtifactsToWorkers(const std::string &workerPath, const std::str } static void deleteWorkerPath(const std::string &workerHost, const std::string &workerPath) { + // Not used in docker or k8s modes std::string pathDeletionCommand = "rm -rf " + workerPath; if (workerHost.find("localhost") == std::string::npos) { pathDeletionCommand = "ssh " + workerHost + " " + pathDeletionCommand; @@ -1175,6 +1068,7 @@ static void deleteWorkerPath(const std::string &workerHost, const std::string &w } static void createLogFilePath(const std::string &workerHost, const std::string &workerPath) { + // Not used in docker or k8s modes std::string pathCreationCommand = "mkdir -p " + workerPath + "/logs"; if (workerHost.find("localhost") == std::string::npos) { pathCreationCommand = "ssh " + workerHost + " " + pathCreationCommand; @@ -1214,32 +1108,6 @@ void JasmineGraphServer::addHostsToMetaDB(std::string host, std::vector por } } -map JasmineGraphServer::getLiveHostIDList() { - server_logger.info("###MASTER### Loading Live Host ID List"); - map hostIDMap; - std::vector>> v = - this->sqlite->runSelect("SELECT host_idhost,user,ip,server_port FROM worker;"); - string id = v[0][0].second; - for (int i = 0; i < v.size(); i++) { - string id = v[i][0].second; - string user = v[i][1].second; - string ip = v[i][2].second; - string serverPort = v[i][3].second; - - string host = ""; - - if (user == "") { - host = ip + ":" + serverPort; - } else { - host = user + "@" + ip + ":" + serverPort; - } - - hostIDMap.insert(make_pair(host, id)); - } - - return hostIDMap; -} - static void updateMetaDB(int graphID, string uploadEndTime) { std::unique_ptr sqliteDBInterface(new SQLiteDBInterface()); sqliteDBInterface->init(); @@ -1255,8 +1123,8 @@ void JasmineGraphServer::removeGraph(vector> hostHasPartiti int count = 0; std::thread *deleteThreads = new std::thread[hostHasPartition.size()]; for (std::vector>::iterator j = (hostHasPartition.begin()); j != hostHasPartition.end(); ++j) { - deleteThreads[count] = std::thread(removePartitionThroughService, j->first, hostPortMap[j->first].first, - graphID, j->second, masterIP); + deleteThreads[count] = + std::thread(removePartitionThroughService, j->first, getPortByHost(j->first), graphID, j->second, masterIP); count++; } @@ -1309,6 +1177,7 @@ static bool removeFragmentThroughService(string host, int port, string graphID, } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1316,6 +1185,7 @@ static bool removeFragmentThroughService(string host, int port, string graphID, if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::DELETE_GRAPH_FRAGMENT, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1324,7 +1194,7 @@ static bool removeFragmentThroughService(string host, int port, string graphID, close(sockfd); return false; } - + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -1364,12 +1234,14 @@ static bool removePartitionThroughService(string host, int port, string graphID, } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::DELETE_GRAPH, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } @@ -1381,44 +1253,45 @@ static bool removePartitionThroughService(string host, int port, string graphID, } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partitionID, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } -std::vector JasmineGraphServer::getHostWorkerMap() { return hostWorkerMap; } - void JasmineGraphServer::updateOperationalGraphList() { string hosts = ""; string graphIDs = ""; std::vector hostsList; - if (profile == Conts::PROFILE_NATIVE) { + if (jasminegraph_profile == PROFILE_NATIVE) { hostsList = Utils::getHostListFromProperties(); - } else if (profile == "docker") { + } else if (jasminegraph_profile == PROFILE_DOCKER) { hostsList = getWorkerVector(workerHosts); + } else { + return; // TODO(thevindu-w): implement.for k8s } - vector::iterator it; - for (it = hostsList.begin(); it < hostsList.end(); it++) { + for (auto it = hostsList.begin(); it < hostsList.end(); it++) { string host = *it; hosts += ("'" + host + "', "); } hosts = hosts.substr(0, hosts.size() - 2); string sqlStatement = - ("SELECT b.partition_graph_idgraph FROM worker_has_partition AS b " - "JOIN worker WHERE worker.idworker = b.worker_idworker AND worker.name IN " - "(" + - hosts + - ") GROUP BY b.partition_graph_idgraph HAVING COUNT(b.partition_idpartition)= " - "(SELECT COUNT(a.idpartition) FROM partition AS a " - "WHERE a.graph_idgraph = b.partition_graph_idgraph);"); + "SELECT b.partition_graph_idgraph FROM worker_has_partition AS b " + "JOIN worker WHERE worker.idworker = b.worker_idworker AND worker.name IN " + "(" + + hosts + + ") GROUP BY b.partition_graph_idgraph HAVING COUNT(b.partition_idpartition)= " + "(SELECT COUNT(a.idpartition) FROM partition AS a " + "WHERE a.graph_idgraph = b.partition_graph_idgraph);"; std::vector>> v = this->sqlite->runSelect(sqlStatement); for (std::vector>>::iterator i = v.begin(); i != v.end(); ++i) { - for (std::vector>::iterator j = (i->begin()); j != i->end(); ++j) { - graphIDs += (j->second + ", "); + for (std::vector>::iterator j = i->begin(); j != i->end(); ++j) { + graphIDs += j->second + ", "; } } graphIDs = graphIDs.substr(0, graphIDs.size() - 2); @@ -1430,51 +1303,17 @@ void JasmineGraphServer::updateOperationalGraphList() { this->sqlite->runUpdate(sqlStatement2); } -std::map JasmineGraphServer::getWorkerPartitions(string graphID) { - vector> hostHasPartition; - auto *refToSqlite = new SQLiteDBInterface(); - refToSqlite->init(); - map graphPartitionedHosts; - vector>> hostPartitionResults = refToSqlite->runSelect( - "SELECT name, worker_idworker, server_port, server_data_port, partition_idpartition FROM worker_has_partition " - "INNER JOIN worker ON worker_" - "idworker = idworker WHERE partition_graph_idgraph = '" + - graphID + "'"); - - refToSqlite->finalize(); - delete refToSqlite; - - for (std::vector>>::iterator i = hostPartitionResults.begin(); - i != hostPartitionResults.end(); ++i) { - std::vector> rowData = *i; - - string name = rowData.at(0).second; - string workerID = rowData.at(1).second; - int serverPort = std::stoi(rowData.at(2).second); - int serverDataPort = std::stoi(rowData.at(3).second); - string partitionId = rowData.at(4).second; - - server_logger.info("name : " + name + " workerID : " + workerID + " sport : " + std::to_string(serverPort) + - " sdport : " + std::to_string(serverDataPort) + " partitionId : " + partitionId); - graphPartitionedHosts.insert((pair( - workerID, {name, serverPort, serverDataPort, partitionId}))); - } - - return graphPartitionedHosts; -} - std::map JasmineGraphServer::getGraphPartitionedHosts(string graphID) { vector> hostHasPartition; auto *refToSqlite = new SQLiteDBInterface(); refToSqlite->init(); vector>> hostPartitionResults = refToSqlite->runSelect( - "SELECT name, partition_idpartition FROM worker_has_partition INNER JOIN worker ON worker_" - "idworker = idworker WHERE partition_graph_idgraph = '" + + "SELECT name, partition_idpartition FROM worker_has_partition INNER JOIN worker ON " + "worker_idworker = idworker WHERE partition_graph_idgraph = '" + graphID + "'"); refToSqlite->finalize(); delete refToSqlite; - for (vector>>::iterator i = hostPartitionResults.begin(); - i != hostPartitionResults.end(); ++i) { + for (auto i = hostPartitionResults.begin(); i != hostPartitionResults.end(); ++i) { int count = 0; string hostname; string partitionID; @@ -1490,21 +1329,22 @@ std::map JasmineGraphServer::getGr } map> hostPartitions; - for (std::vector>::iterator j = (hostHasPartition.begin()); j != hostHasPartition.end(); ++j) { + for (auto j = hostHasPartition.begin(); j != hostHasPartition.end(); ++j) { string hostname = j->first; - if (hostPartitions.count(hostname) > 0) { - hostPartitions[hostname].push_back(j->second); + auto it = hostPartitions.find(hostname); + if (it != hostPartitions.end()) { + it->second.push_back(j->second); } else { vector vec; vec.push_back(j->second); - hostPartitions.insert((pair>(hostname, vec))); + hostPartitions[hostname] = vec; } } map graphPartitionedHosts; - for (map>::iterator it = (hostPartitions.begin()); it != hostPartitions.end(); ++it) { - graphPartitionedHosts.insert((pair( - it->first, {hostPortMap[it->first].first, hostPortMap[it->first].second, hostPartitions[it->first]}))); + for (map>::iterator it = hostPartitions.begin(); it != hostPartitions.end(); ++it) { + graphPartitionedHosts[it->first] = {getPortByHost(it->first), getDataPortByHost(it->first), + hostPartitions[it->first]}; } return graphPartitionedHosts; @@ -1613,45 +1453,35 @@ void JasmineGraphServer::addInstanceDetailsToPerformanceDB(std::string host, std } static void degreeDistributionCommon(std::string graphID, std::string command) { - std::map graphPartitionedHosts = - JasmineGraphServer::getWorkerPartitions(graphID); - int partition_count = 0; - string partition; - string host; - int port; + std::map graphPartitionedHosts = + JasmineGraphServer::getGraphPartitionedHosts(graphID); std::string workerList; - std::map::iterator workerit; + std::map::iterator workerit; for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; - port = workerPartition.port; + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + string host = workerit->first; + int port = workerPartition.port; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { + std::string partition = *partitionit; + workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } - - workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } workerList.pop_back(); - for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; - port = workerPartition.port; - - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; - } + int sockfd; + char data[FED_DATA_LENGTH + 1]; + bool loop = false; + socklen_t len; + struct sockaddr_in serv_addr; + struct hostent *server; - int sockfd; - char data[FED_DATA_LENGTH + 1]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; + for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + string host = workerit->first; + int port = workerPartition.port; sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { @@ -1665,35 +1495,46 @@ static void degreeDistributionCommon(std::string graphID, std::string command) { continue; } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - continue; - } + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { + std::string partition = *partitionit; + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, command, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, command, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + continue; + } - if (!Utils::send_str_wrapper(sockfd, workerList)) { + if (!Utils::send_str_wrapper(sockfd, workerList)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + continue; + } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); - continue; } - server_logger.info("Sent: " + workerList); } } @@ -1718,47 +1559,40 @@ long JasmineGraphServer::getGraphVertexCount(std::string graphID) { } void JasmineGraphServer::duplicateCentralStore(std::string graphID) { - std::map graphPartitionedHosts = - JasmineGraphServer::getWorkerPartitions(graphID); - int partition_count = 0; - string partition; - string host; - int port; - int dport; + std::map graphPartitionedHosts = + JasmineGraphServer::getGraphPartitionedHosts(graphID); std::string workerList; - std::map::iterator workerit; + std::map::iterator workerit; for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; - port = workerPartition.port; - dport = workerPartition.dataPort; + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + string host = workerit->first; + int port = workerPartition.port; + int dport = workerPartition.dataPort; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { + std::string partition = *partitionit; + workerList.append(host + ":" + std::to_string(port) + ":" + partition + ":" + to_string(dport) + ","); } - - workerList.append(host + ":" + std::to_string(port) + ":" + partition + ":" + to_string(dport) + ","); } workerList.pop_back(); + int sockfd; + char data[FED_DATA_LENGTH + 1]; + bool loop = false; + socklen_t len; + struct sockaddr_in serv_addr; + struct hostent *server; + for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; - port = workerPartition.port; + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + string host = workerit->first; + int port = workerPartition.port; if (host.find('@') != std::string::npos) { host = Utils::split(host, '@')[1]; } - int sockfd; - char data[FED_DATA_LENGTH + 1]; - bool loop = false; - socklen_t len; - struct sockaddr_in serv_addr; - struct hostent *server; - sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd < 0) { @@ -1771,76 +1605,75 @@ void JasmineGraphServer::duplicateCentralStore(std::string graphID) { continue; } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - continue; - } + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { + std::string partition = *partitionit; + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, - JasmineGraphInstanceProtocol::DP_CENTRALSTORE, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, + JasmineGraphInstanceProtocol::DP_CENTRALSTORE, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + continue; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - continue; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + continue; + } - if (!Utils::send_str_wrapper(sockfd, workerList)) { + if (!Utils::send_str_wrapper(sockfd, workerList)) { + close(sockfd); + continue; + } + server_logger.info("Sent: " + workerList); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); - continue; } - server_logger.info("Sent: " + workerList); - close(sockfd); } } void JasmineGraphServer::initiateFiles(std::string graphID, std::string trainingArgs) { int count = 0; map> scheduleForAllHosts = JasmineGraphTrainingSchedular::schedulePartitionTraining(graphID); - std::map graphPartitionedHosts = - this->getWorkerPartitions(graphID); + std::map graphPartitionedHosts = + this->getGraphPartitionedHosts(graphID); int partition_count = 0; - std::map::iterator mapIterator; + std::map::iterator mapIterator; for (mapIterator = graphPartitionedHosts.begin(); mapIterator != graphPartitionedHosts.end(); mapIterator++) { - JasmineGraphServer::workerPartition workerPartition = mapIterator->second; - std::vector partitions; - partitions.push_back(workerPartition.partitionID); - std::vector::iterator it; - for (it = partitions.begin(); it < partitions.end(); it++) { - partition_count++; - } + JasmineGraphServer::workerPartitions workerPartition = mapIterator->second; + partition_count += (int)(workerPartition.partitionID.size()); } std::thread workerThreads[partition_count + 1]; string prefix = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.trainedmodelfolder"); string attr_prefix = Utils::getJasmineGraphProperty("org.jasminegraph.server.instance.datafolder"); - std::map::iterator j; + std::map::iterator j; for (j = graphPartitionedHosts.begin(); j != graphPartitionedHosts.end(); j++) { - JasmineGraphServer::workerPartition workerPartition = j->second; - std::vector partitions; - partitions.push_back(workerPartition.partitionID); - string partitionCount = std::to_string(partitions.size()); + JasmineGraphServer::workerPartitions workerPartition = j->second; std::vector::iterator k; map scheduleOfHost = scheduleForAllHosts[j->first]; - for (k = partitions.begin(); k != partitions.end(); k++) { + for (k = workerPartition.partitionID.begin(); k != workerPartition.partitionID.end(); k++) { int iterationOfPart = scheduleOfHost[stoi(*k)]; - workerThreads[count] = - std::thread(initiateTrain, workerPartition.hostname, workerPartition.port, workerPartition.dataPort, - trainingArgs + " " + *k, iterationOfPart, partitionCount, this->masterHost); - count++; + workerThreads[count++] = std::thread(initiateTrain, j->first, workerPartition.port, trainingArgs + " " + *k, + iterationOfPart, this->masterHost); } } @@ -1865,16 +1698,15 @@ void JasmineGraphServer::initiateCommunication(std::string graphID, std::string workerInstance = workerVector[i]; int serverPort = stoi(workerInstance.port); - int serverDataPort = stoi(workerInstance.dataPort); if (i == 0) { - workerThreads[threadID] = std::thread(initiateServer, workerInstance.hostname, serverPort, serverDataPort, - trainingArgs, fl_clients, to_string(i), masterIP); + workerThreads[threadID] = + std::thread(initiateServer, workerInstance.hostname, serverPort, trainingArgs, fl_clients, masterIP); threadID++; } - workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, serverDataPort, - trainingArgs + " " + to_string(i), fl_clients, to_string(i), masterIP); + workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, + trainingArgs + " " + to_string(i), fl_clients, masterIP); threadID++; } @@ -1898,8 +1730,7 @@ void JasmineGraphServer::initiateOrgCommunication(std::string graphID, std::stri vector workerVector = Utils::getWorkerList(sqlite); if (Utils::getJasmineGraphProperty("org.jasminegraph.fl.aggregator") == "true") { - initiateAggregator("localhost", stoi(workerVector[0].port), stoi(workerVector[0].dataPort), trainingArgs, - fl_clients, "1", masterIP); + initiateAggregator("localhost", stoi(workerVector[0].port), trainingArgs, fl_clients, masterIP); } std::ifstream file(Utils::getJasmineGraphProperty("org.jasminegraph.fl.organization.file")); @@ -1936,16 +1767,15 @@ void JasmineGraphServer::initiateOrgCommunication(std::string graphID, std::stri for (int i = 0; i < workerVector.size(); i++) { workerInstance = workerVector[i]; int serverPort = stoi(workerInstance.port); - int serverDataPort = stoi(workerInstance.dataPort); if (i == 0) { - workerThreads[threadID] = std::thread(initiateOrgServer, workerInstance.hostname, serverPort, - serverDataPort, trainingArgs, fl_clients, to_string(i), masterIP); + workerThreads[threadID] = + std::thread(initiateOrgServer, workerInstance.hostname, serverPort, trainingArgs, fl_clients, masterIP); threadID++; } - workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, serverDataPort, - trainingArgs + " " + to_string(i), fl_clients, to_string(i), masterIP); + workerThreads[threadID] = std::thread(initiateClient, workerInstance.hostname, serverPort, + trainingArgs + " " + to_string(i), fl_clients, masterIP); threadID++; } @@ -1961,38 +1791,29 @@ void JasmineGraphServer::initiateOrgCommunication(std::string graphID, std::stri } void JasmineGraphServer::initiateMerge(std::string graphID, std::string trainingArgs, SQLiteDBInterface *sqlite) { - int count = 0; map> scheduleForAllHosts = JasmineGraphTrainingSchedular::schedulePartitionTraining(graphID); - std::map graphPartitionedHosts = - this->getWorkerPartitions(graphID); + std::map graphPartitionedHosts = + this->getGraphPartitionedHosts(graphID); int partition_count = 0; - std::map::iterator mapIterator; + std::map::iterator mapIterator; for (mapIterator = graphPartitionedHosts.begin(); mapIterator != graphPartitionedHosts.end(); mapIterator++) { - JasmineGraphServer::workerPartition workerPartition = mapIterator->second; - std::vector partitions; - partitions.push_back(workerPartition.partitionID); - std::vector::iterator it; - for (it = partitions.begin(); it < partitions.end(); it++) { - partition_count++; - } + JasmineGraphServer::workerPartitions workerPartition = mapIterator->second; + partition_count += (int)(workerPartition.partitionID.size()); } - std::thread *workerThreads = new std::thread[partition_count + 1]; + std::thread *workerThreads = new std::thread[partition_count]; int fl_clients = stoi(Utils::getJasmineGraphProperty("org.jasminegraph.fl_clients")); - std::map::iterator j; + std::map::iterator j; + int count = 0; for (j = graphPartitionedHosts.begin(); j != graphPartitionedHosts.end(); j++) { - JasmineGraphServer::workerPartition workerPartition = j->second; - std::vector partitions; - partitions.push_back(workerPartition.partitionID); - string partitionCount = std::to_string(partitions.size()); + JasmineGraphServer::workerPartitions workerPartition = j->second; std::vector::iterator k; map scheduleOfHost = scheduleForAllHosts[j->first]; - for (k = partitions.begin(); k != partitions.end(); k++) { + for (k = workerPartition.partitionID.begin(); k != workerPartition.partitionID.end(); k++) { int iterationOfPart = scheduleOfHost[stoi(*k)]; - workerThreads[count] = - std::thread(mergeFiles, workerPartition.hostname, workerPartition.port, workerPartition.dataPort, - trainingArgs + " " + *k, fl_clients, *k, this->masterHost); + workerThreads[count] = std::thread(mergeFiles, j->first, workerPartition.port, trainingArgs + " " + *k, + fl_clients, *k, this->masterHost); count++; } } @@ -2004,8 +1825,8 @@ void JasmineGraphServer::initiateMerge(std::string graphID, std::string training delete[] workerThreads; } -static bool initiateCommon(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP, std::string initType) { +static bool initiateCommon(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP, + std::string initType) { bool result = true; int sockfd; char data[FED_DATA_LENGTH + 1]; @@ -2040,59 +1861,55 @@ static bool initiateCommon(std::string host, int port, int dataPort, std::string } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, initType, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::send_str_wrapper(sockfd, trainingArgs)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } server_logger.info("Sent: " + trainingArgs); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } -static bool initiateTrain(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, +static bool initiateTrain(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_FED_PREDICT); } -static bool initiatePredict(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, - JasmineGraphInstanceProtocol::INITIATE_FILES); +static bool initiatePredict(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_FILES); } -static bool initiateServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, - JasmineGraphInstanceProtocol::INITIATE_SERVER); +static bool initiateServer(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_SERVER); } // todo Remove partCount from parameters as the partition id is being parsed within the training args -static bool initiateClient(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, - JasmineGraphInstanceProtocol::INITIATE_CLIENT); +static bool initiateClient(std::string host, int port, std::string trainingArgs, int iteration, std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_CLIENT); } -static bool initiateAggregator(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, - JasmineGraphInstanceProtocol::INITIATE_AGG); +static bool initiateAggregator(std::string host, int port, std::string trainingArgs, int iteration, + std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_AGG); } -static bool initiateOrgServer(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, - std::string partCount, std::string masterIP) { - return initiateCommon(host, port, dataPort, trainingArgs, iteration, partCount, masterIP, +static bool initiateOrgServer(std::string host, int port, std::string trainingArgs, int iteration, + std::string masterIP) { + return initiateCommon(host, port, trainingArgs, iteration, masterIP, JasmineGraphInstanceProtocol::INITIATE_ORG_SERVER); } @@ -2129,12 +1946,7 @@ bool JasmineGraphServer::receiveGlobalWeights(std::string host, int port, std::s serv_addr.sin_family = AF_INET; bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); serv_addr.sin_port = htons(port); - - while (true) { - if (!(Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0)) { - break; - } - } + Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)); bool isTrue = false; int count = 0; @@ -2193,11 +2005,12 @@ bool JasmineGraphServer::receiveGlobalWeights(std::string host, int port, std::s break; } } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } -bool JasmineGraphServer::mergeFiles(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, +bool JasmineGraphServer::mergeFiles(std::string host, int port, std::string trainingArgs, int iteration, string partCount, std::string masterIP) { bool result = true; int sockfd; @@ -2233,22 +2046,26 @@ bool JasmineGraphServer::mergeFiles(std::string host, int port, int dataPort, st } if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::MERGE_FILES, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::send_str_wrapper(sockfd, trainingArgs)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } server_logger.info("Sent: " + trainingArgs); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } @@ -2256,7 +2073,7 @@ bool JasmineGraphServer::mergeFiles(std::string host, int port, int dataPort, st bool JasmineGraphServer::sendTrainCommand(std::string host, int port, std::string trainingArgs) { bool result = true; int sockfd; - char data[FED_DATA_LENGTH]; + char data[FED_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; @@ -2289,94 +2106,106 @@ bool JasmineGraphServer::sendTrainCommand(std::string host, int port, std::strin if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::INITIATE_TRAIN, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } if (!Utils::send_str_wrapper(sockfd, trainingArgs)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return false; } server_logger.info("Sent: " + trainingArgs); + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); close(sockfd); return true; } void JasmineGraphServer::egoNet(std::string graphID) { - std::map graphPartitionedHosts = - JasmineGraphServer::getWorkerPartitions(graphID); - int partition_count = 0; - string partition; - string host; - int port; - int dataPort; + std::map graphPartitionedHosts = + JasmineGraphServer::getGraphPartitionedHosts(graphID); std::string workerList; - std::map::iterator workerit; - for (workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { - JasmineGraphServer::workerPartition workerPartition = workerit->second; - partition = workerPartition.partitionID; - host = workerPartition.hostname; - port = workerPartition.port; - dataPort = workerPartition.dataPort; + for (auto workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + string host = workerit->first; + int port = workerPartition.port; - if (host.find('@') != std::string::npos) { - host = Utils::split(host, '@')[1]; + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { + std::string partition = *partitionit; + workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } - - workerList.append(host + ":" + std::to_string(port) + ":" + partition + ","); } workerList.pop_back(); int sockfd; - char data[301]; + char data[FED_DATA_LENGTH + 1]; bool loop = false; socklen_t len; struct sockaddr_in serv_addr; struct hostent *server; - sockfd = socket(AF_INET, SOCK_STREAM, 0); + for (auto workerit = graphPartitionedHosts.begin(); workerit != graphPartitionedHosts.end(); workerit++) { + JasmineGraphServer::workerPartitions workerPartition = workerit->second; + string host = workerit->first; + int port = workerPartition.port; + sockfd = socket(AF_INET, SOCK_STREAM, 0); - if (sockfd < 0) { - server_logger.error("Cannot create socket"); - return; - } - server = gethostbyname(host.c_str()); - if (server == NULL) { - server_logger.error("ERROR, no host named " + host); - return; - } + if (sockfd < 0) { + server_logger.error("Cannot create socket"); + return; + } + server = gethostbyname(host.c_str()); + if (server == NULL) { + server_logger.error("ERROR, no host named " + host); + return; + } + + for (std::vector::iterator partitionit = workerPartition.partitionID.begin(); + partitionit != workerPartition.partitionID.end(); partitionit++) { + std::string partition = *partitionit; + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return; + } - bzero((char *)&serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); - serv_addr.sin_port = htons(port); - if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { - return; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::EGONET, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::EGONET, - JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, partition, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; - } + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, workerList, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return; + } - if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, workerList, JasmineGraphInstanceProtocol::OK)) { - close(sockfd); - return; + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + } } - - close(sockfd); } diff --git a/src/server/JasmineGraphServer.h b/src/server/JasmineGraphServer.h index 5931f2b2..b06b3370 100644 --- a/src/server/JasmineGraphServer.h +++ b/src/server/JasmineGraphServer.h @@ -25,19 +25,18 @@ limitations under the License. #include "../backend/JasmineGraphBackend.h" #include "../frontend/JasmineGraphFrontEnd.h" #include "../frontend/core/scheduler/JobScheduler.h" -#include "../k8s/K8sWorkerController.h" #include "../metadb/SQLiteDBInterface.h" #include "../performance/metrics/StatisticCollector.h" #include "../performancedb/PerformanceSQLiteDBInterface.h" #include "../util/Conts.h" #include "../util/Utils.h" -using std::map; +class K8sWorkerController; +using std::map; class JasmineGraphServer { private: map hostPlaceMap; - std::string profile; std::string workerHosts; std::string enableNmon; static const int BUFFER_SIZE = 128; @@ -45,19 +44,16 @@ class JasmineGraphServer { int serverDataPort; std::map> workerPortsMap; std::map> workerDataPortsMap; - K8sWorkerController *k8sWorkerController; JasmineGraphServer(); static void startRemoteWorkers(std::vector workerPortsVector, std::vector workerDataPortsVector, - std::string host, string profile, string masterHost, string enableNmon); + std::string host, string masterHost, string enableNmon); void addHostsToMetaDB(std::string host, std::vector portVector, std::vector dataPortVector); void updateOperationalGraphList(); - std::map getLiveHostIDList(); - static bool hasEnding(std::string const &fullString, std::string const &ending); std::vector getWorkerVector(std::string workerList); void deleteNonOperationalGraphFragment(int graphID); @@ -87,10 +83,7 @@ class JasmineGraphServer { static int shutdown_worker(std::string host, int port); - int run(std::string profile, std::string masterIp, int numberofWorkers, std::string workerIps, - std::string enableNmon); - - bool isRunning(); + int run(std::string masterIp, int numberofWorkers, std::string workerIps, std::string enableNmon); void uploadGraphLocally(int graphID, const std::string graphType, std::vector> fullFileList, std::string masterIP); @@ -102,11 +95,7 @@ class JasmineGraphServer { static void copyCentralStoreToAggregateLocation(std::string filePath); - static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath, - std::string masterIP); - - static bool spawnNewWorker(string host, string port, string dataPort, string profile, string masterHost, - string enableNmon); + static bool spawnNewWorker(string host, string port, string dataPort, string masterHost, string enableNmon); JasmineGraphFrontEnd *frontend; SQLiteDBInterface *sqlite; @@ -116,7 +105,7 @@ class JasmineGraphServer { std::string masterHost; int numberOfWorkers = -1; - struct workers { + struct worker { std::string hostname; int port; int dataPort; @@ -137,12 +126,7 @@ class JasmineGraphServer { // partiton ID. }; - // return hostWorkerMap - static std::vector getHostWorkerMap(); - - static std::map getWorkerPartitions(string graphID); - - std::map getGraphPartitionedHosts(std::string graphID); + static std::map getGraphPartitionedHosts(std::string graphID); static void inDegreeDistribution(std::string graphID); @@ -150,8 +134,6 @@ class JasmineGraphServer { static void duplicateCentralStore(std::string graphID); - static void pageRank(std::string graphID, double alpha, int iterations); - static long getGraphVertexCount(std::string graphID); static void egoNet(std::string graphID); @@ -166,7 +148,7 @@ class JasmineGraphServer { void initiateMerge(std::string graphID, std::string trainingArgs, SQLiteDBInterface *sqlite); - static bool mergeFiles(std::string host, int port, int dataPort, std::string trainingArgs, int iteration, + static bool mergeFiles(std::string host, int port, std::string trainingArgs, int iteration, string partCount, std::string masterIP); static bool receiveGlobalWeights(std::string host, int port, std::string trainingArgs, int iteration, diff --git a/src/streamingdb/StreamingSQLiteDBInterface.cpp b/src/streamingdb/StreamingSQLiteDBInterface.cpp index cf57f537..66a9bfcc 100644 --- a/src/streamingdb/StreamingSQLiteDBInterface.cpp +++ b/src/streamingdb/StreamingSQLiteDBInterface.cpp @@ -23,7 +23,7 @@ Logger streamdb_logger; int StreamingSQLiteDBInterface::init() { if (!Utils::fileExists(this->databaseLocation.c_str())) { - if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "src/streamingdb/ddl.sql") != 0) { + if (Utils::createDatabaseFromDDL(this->databaseLocation.c_str(), ROOT_DIR "ddl/streamingdb.sql") != 0) { streamdb_logger.error("Cannot create database: " + databaseLocation); return -1; } diff --git a/src/util/Conts.cpp b/src/util/Conts.cpp index 4b6b3706..2e373032 100644 --- a/src/util/Conts.cpp +++ b/src/util/Conts.cpp @@ -16,10 +16,6 @@ limitations under the License. std::string Conts::JASMINEGRAPH_EXECUTABLE = "run.sh"; std::string Conts::JASMINEGRAPH_HOME = "JASMINEGRAPH_HOME"; -std::string Conts::PROFILE_NATIVE = "native"; -std::string Conts::PROFILE_DOCKER = "docker"; -std::string Conts::PROFILE_K8S = "k8s"; - std::string Conts::CARRIAGE_RETURN_NEW_LINE = "\r\n"; std::string Conts::GRAPH_TYPE_RDF = "RDF_GRAPH"; diff --git a/src/util/Conts.h b/src/util/Conts.h index a683b990..d19c0253 100644 --- a/src/util/Conts.h +++ b/src/util/Conts.h @@ -31,10 +31,7 @@ extern std::mutex responseVectorMutex; extern bool isStatCollect; extern bool isCalibrating; extern std::vector loadAverageVector; -extern bool collectValid; extern std::map aggregateWeightMap; -extern std::mutex aggregateWeightMutex; -extern std::mutex triangleTreeMutex; struct ProcessInfo { int id; @@ -70,10 +67,6 @@ class Conts { static std::string GRAPH_WITH_JSON_ATTRIBUTES; static std::string GRAPH_WITH_XML_ATTRIBUTES; - static std::string PROFILE_NATIVE; - static std::string PROFILE_DOCKER; - static std::string PROFILE_K8S; - static std::string CARRIAGE_RETURN_NEW_LINE; static std::string diff --git a/src/util/PlacesToNodeMapper.cpp b/src/util/PlacesToNodeMapper.cpp deleted file mode 100644 index 554f9925..00000000 --- a/src/util/PlacesToNodeMapper.cpp +++ /dev/null @@ -1,104 +0,0 @@ -/** -Copyright 2018 JasmineGraph Team -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - */ - -#include "PlacesToNodeMapper.h" - -#include "logger/Logger.h" - -using namespace std; -Logger node_logger; - -std::string PlacesToNodeMapper::getHost(long placeId) { - std::vector hostList = Utils::getHostListFromProperties(); - std::string& host = hostList.at(placeId); - return host; -} - -std::vector PlacesToNodeMapper::getInstancePortsList(long placeId) { - int numberOfWorkersPerHost; - int numberOfWorkers = 0; - int hostListModeNWorkers; - std::vector portList; - std::vector hostList = Utils::getHostListFromProperties(); - std::string nWorkers = Utils::getJasmineGraphProperty("org.jasminegraph.server.nworkers"); - int workerPort = Conts::JASMINEGRAPH_INSTANCE_PORT; - if (Utils::is_number(nWorkers)) { - numberOfWorkers = atoi(nWorkers.c_str()); - } else { - node_logger.log("Number of Workers specified in the properties is not an integer value.", "error"); - return portList; - } - - if (numberOfWorkers > 0 && hostList.size() > 0) { - numberOfWorkersPerHost = numberOfWorkers / hostList.size(); - hostListModeNWorkers = numberOfWorkers % hostList.size(); - } - - if (placeId > 0 && (placeId + 1) <= hostListModeNWorkers) { - workerPort = workerPort + (placeId * numberOfWorkersPerHost * 2) + placeId * 2 + 2; - } else if (placeId > 0 && (placeId + 1) > hostListModeNWorkers) { - workerPort = workerPort + (placeId * numberOfWorkersPerHost * 2) + hostListModeNWorkers * 2 + 2; - } - - for (int i = 0; i < numberOfWorkersPerHost; i++) { - workerPort = Conts::JASMINEGRAPH_INSTANCE_PORT + i * 2; - portList.push_back(workerPort); - } - - if ((placeId + 1) <= hostListModeNWorkers) { - workerPort = workerPort + 2; - portList.push_back(workerPort); - } - - return portList; -} - -std::vector PlacesToNodeMapper::getFileTransferServicePort(long placeId) { - int numberOfWorkersPerHost; - int numberOfWorkers = 0; - int hostListModeNWorkers; - std::vector portList; - std::vector hostList = Utils::getHostListFromProperties(); - std::string nWorkers = Utils::getJasmineGraphProperty("org.jasminegraph.server.nworkers"); - int workerPort = Conts::JASMINEGRAPH_INSTANCE_PORT; - int workerDataPort = Conts::JASMINEGRAPH_INSTANCE_DATA_PORT; - if (Utils::is_number(nWorkers)) { - numberOfWorkers = atoi(nWorkers.c_str()); - } else { - node_logger.log("Number of Workers is not specified", "error"); - numberOfWorkers = 0; - } - - if (numberOfWorkers > 0 && hostList.size() > 0) { - numberOfWorkersPerHost = numberOfWorkers / hostList.size(); - hostListModeNWorkers = numberOfWorkers % hostList.size(); - } - - if (placeId > 0 && (placeId + 1) <= hostListModeNWorkers) { - workerDataPort = workerDataPort + (placeId * numberOfWorkersPerHost * 2) + placeId * 2 + 2; - } else if (placeId > 0 && (placeId + 1) > hostListModeNWorkers) { - workerDataPort = workerDataPort + (placeId * numberOfWorkersPerHost * 2) + hostListModeNWorkers * 2 + 2; - } - - for (int i = 0; i < numberOfWorkersPerHost; i++) { - workerDataPort = workerDataPort + i * 2; - portList.push_back(workerDataPort); - } - - if ((placeId + 1) <= hostListModeNWorkers) { - workerDataPort = workerDataPort + 2; - portList.push_back(workerDataPort); - } - - return portList; -} diff --git a/src/util/Utils.cpp b/src/util/Utils.cpp index c3b534c7..7cddf831 100644 --- a/src/util/Utils.cpp +++ b/src/util/Utils.cpp @@ -13,9 +13,11 @@ limitations under the License. #include "Utils.h" +#include #include +#include +#include #include -#include #include #include @@ -25,13 +27,20 @@ limitations under the License. #include #include +#include "../../globals.h" +#include "../k8s/K8sInterface.h" #include "../server/JasmineGraphInstanceProtocol.h" +#include "../server/JasmineGraphServer.h" #include "Conts.h" #include "logger/Logger.h" using namespace std; Logger util_logger; +#ifdef UNIT_TEST +int jasminegraph_profile = PROFILE_K8S; +#endif + unordered_map Utils::propertiesMap; std::vector Utils::split(const std::string &s, char delimiter) { @@ -48,16 +57,15 @@ std::vector Utils::getFileContent(std::string file) { ifstream in(file); std::string str; - vector *vec = new vector(); + vector vec; + if (!in.is_open()) return vec; while (std::getline(in, str)) { // now we loop back and get the next line in 'str' - if (str.length() > 0) { - vec->insert(vec->begin(), str); + vec.push_back(str); } } - - return *vec; + return vec; }; std::string Utils::getFileContentAsString(std::string file) { @@ -90,23 +98,27 @@ void Utils::writeFileContent(const std::string &filePath, const std::string &con out.close(); } +static std::mutex propertiesMapMutex; +static bool propertiesMapInitialized = false; std::string Utils::getJasmineGraphProperty(std::string key) { - if (Utils::propertiesMap.empty()) { - std::vector::iterator it; - vector vec = Utils::getFileContent(ROOT_DIR "conf/jasminegraph-server.properties"); - it = vec.begin(); - - for (it = vec.begin(); it < vec.end(); it++) { - std::string item = *it; - if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { - std::vector vec2 = split(item, '='); - if (vec2.size() == 2) { - Utils::propertiesMap[vec2.at(0)] = vec2.at(1); - } else { - Utils::propertiesMap[vec2.at(0)] = string(" "); + if (!propertiesMapInitialized) { + propertiesMapMutex.lock(); + if (!propertiesMapInitialized) { // double-checking lock + const vector &vec = Utils::getFileContent(ROOT_DIR "conf/jasminegraph-server.properties"); + for (auto it = vec.begin(); it < vec.end(); it++) { + std::string item = *it; + if (item.length() > 0 && !(item.rfind("#", 0) == 0)) { + const std::vector &vec2 = split(item, '='); + if (vec2.size() == 2) { + Utils::propertiesMap[vec2.at(0)] = vec2.at(1); + } else { + Utils::propertiesMap[vec2.at(0)] = string(" "); + } } } } + propertiesMapInitialized = true; + propertiesMapMutex.unlock(); } auto it = Utils::propertiesMap.find(key); if (it != Utils::propertiesMap.end()) { @@ -190,10 +202,18 @@ bool Utils::fileExists(std::string fileName) { return access(fileName.c_str(), F * This method creates a new directory if it does not exist * @param dirName */ -void Utils::createDirectory(const std::string dirName) { - // TODO: check if directory exists before creating - // TODO: check return value - mkdir(dirName.c_str(), 0777); +int Utils::createDirectory(const std::string dirName) { + if (dirName.empty()) { + util_logger.error("Cannot mkdir empty dirName"); + } + string command = "mkdir -p " + dirName; + int status = system(command.c_str()); + if (status != 0) { + util_logger.warn("Command failed: " + command + " trying again"); + sleep(1); + status = system(command.c_str()); + } + return status; } std::vector Utils::getListOfFilesInDirectory(std::string dirName) { @@ -249,7 +269,7 @@ bool Utils::is_number(const std::string &compareString) { * @return */ std::string Utils::getFileName(std::string filePath) { - std::string filename = filePath.substr(filePath.find_last_of("/\\") + 1); + std::string filename = filePath.substr(filePath.find_last_of("/") + 1); return filename; } @@ -262,7 +282,7 @@ std::string Utils::getJasmineGraphHome() { jasminegraph_home = std::string(temp); } if (jasminegraph_home.empty()) { - util_logger.log("Returning empty value for " + Conts::JASMINEGRAPH_HOME, "warn"); + util_logger.warn("Returning empty value for " + Conts::JASMINEGRAPH_HOME); } return jasminegraph_home; } @@ -284,10 +304,10 @@ std::string Utils::getHomeDir() { * @param filePath */ int Utils::copyFile(const std::string sourceFilePath, const std::string destinationFilePath) { - util_logger.log("Starting file copy source: " + sourceFilePath + " destination: " + destinationFilePath, "info"); + util_logger.info("Starting file copy source: " + sourceFilePath + " destination: " + destinationFilePath); std::string command = "cp " + sourceFilePath + " " + destinationFilePath; int status = system(command.c_str()); - if (status != 0) util_logger.warn("Copy failed with exit code " + std::to_string(status)); + if (status != 0) util_logger.error("Copy failed with exit code " + std::to_string(status)); return status; } @@ -333,7 +353,7 @@ int Utils::compressFile(const std::string filePath, std::string mode) { int Utils::unzipFile(std::string filePath, std::string mode) { if (mode == "pigz") { if (access("/usr/bin/pigz", X_OK) != 0) { - util_logger.info("pigz not found. Compressing using gzip"); + util_logger.info("pigz not found. Decompressing using gzip"); mode = "gzip"; } } @@ -400,6 +420,7 @@ int Utils::unzipDirectory(std::string filePath) { } void Utils::assignPartitionsToWorkers(int numberOfWorkers, SQLiteDBInterface *sqlite) { + // Not used in K8s mode sqlite->runUpdate("DELETE FROM worker_has_partition"); std::vector>> v = @@ -474,15 +495,22 @@ void Utils::updateSLAInformation(PerformanceSQLiteDBInterface *perfSqlite, std:: perfSqlite->runInsert(insertQuery); } } else { - util_logger.log("Invalid SLA " + category + " for " + command + " command", "error"); + util_logger.error("Invalid SLA " + category + " for " + command + " command"); } } -int Utils::copyToDirectory(std::string currentPath, std::string copyPath) { - std::string command = "mkdir -p " + copyPath + "&& " + "cp " + currentPath + " " + copyPath; - int status = system(command.c_str()); +int Utils::copyToDirectory(std::string currentPath, std::string destinationDir) { + if (access(destinationDir.c_str(), F_OK)) { + std::string createDirCommand = "mkdir -p " + destinationDir; + if (system(createDirCommand.c_str())) { + util_logger.error("Creating directory " + destinationDir + " failed"); + return -1; + } + } + std::string copyCommand = "cp " + currentPath + " " + destinationDir; + int status = system(copyCommand.c_str()); if (status != 0) { - util_logger.error("Copying " + currentPath + " to directory " + copyPath + " failed with code " + + util_logger.error("Copying " + currentPath + " to directory " + destinationDir + " failed with code " + std::to_string(status)); } return status; @@ -567,7 +595,7 @@ std::string Utils::read_str_wrapper(int connFd, char *buf, size_t len, bool allo } std::string Utils::read_str_trim_wrapper(int connFd, char *buf, size_t len) { - string str = read_str_wrapper(connFd, buf, len, false); + string str = read_str_wrapper(connFd, buf, len); if (!str.empty()) str = trim_copy(str); return str; } @@ -665,6 +693,7 @@ std::string Utils::getJsonStringFromYamlFile(const std::string &yamlFile) { YAML::Node yamlNode = YAML::LoadFile(yamlFile); return to_string(yaml2json(yamlNode)); } + int Utils::createDatabaseFromDDL(const char *dbLocation, const char *ddlFileLocation) { if (!Utils::fileExists(ddlFileLocation)) { util_logger.error("DDL file not found: " + string(ddlFileLocation)); @@ -695,10 +724,115 @@ int Utils::createDatabaseFromDDL(const char *dbLocation, const char *ddlFileLoca return 0; } -bool Utils::fileExistsWithReadPermission(const string &path) { - return access(path.c_str(), R_OK) == 0; +static size_t write_callback(void *contents, size_t size, size_t nmemb, std::string *output) { + size_t totalSize = size * nmemb; + output->append(static_cast(contents), totalSize); + return totalSize; +} +std::string Utils::send_job(std::string job_group_name, std::string metric_name, std::string metric_value) { + CURL *curl; + CURLcode res; + std::string pushGatewayJobAddr; + if (jasminegraph_profile == PROFILE_K8S) { + std::unique_ptr interface(new K8sInterface()); + pushGatewayJobAddr = interface->getJasmineGraphConfig("pushgateway_address"); + } else { + pushGatewayJobAddr = getJasmineGraphProperty("org.jasminegraph.collector.pushgateway"); + } + + pushGatewayJobAddr += "metrics/job/"; + + std::string response_string; + curl = curl_easy_init(); + if (curl) { + std::string hostPGAddr; + const char *hostAddress = getenv("HOST_NAME"); + const char *port = getenv("PORT"); + + std::string uniqueWorkerID; + if (hostAddress) { + uniqueWorkerID = std::string(hostAddress) + ":" + std::string(port); + } else { + uniqueWorkerID = "Master"; + } + hostPGAddr = pushGatewayJobAddr + uniqueWorkerID; + curl_easy_setopt(curl, CURLOPT_URL, hostPGAddr.c_str()); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); + std::string job_data = metric_name + " " + metric_value + "\n"; + const char *data = job_data.c_str(); + + curl_slist *headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/x-prometheus-remote-write-v1"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data); + curl_easy_setopt(curl, CURLOPT_POST, 1); + + res = curl_easy_perform(curl); + long code = -1; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + if (res != CURLE_OK || code != 200) { + util_logger.error("curl failed: " + std::string(curl_easy_strerror(res)) + "| url: " + hostPGAddr + + "| data: " + job_data); + } + + curl_easy_cleanup(curl); + } + return response_string; +} + +std::map Utils::getMetricMap(std::string metricName) { + std::map map; + CURL *curl; + CURLcode res; + std::string response_cpu_usages; + + std::string prometheusAddr; + if (jasminegraph_profile == PROFILE_K8S) { + std::unique_ptr interface(new K8sInterface()); + prometheusAddr = interface->getJasmineGraphConfig("prometheus_address"); + } else { + prometheusAddr = getJasmineGraphProperty("org.jasminegraph.collector.prometheus"); + } + std::string prometheusQueryAddr = prometheusAddr + "api/v1/query?query=" + metricName; + + curl = curl_easy_init(); + if (curl) { + curl_easy_setopt(curl, CURLOPT_URL, prometheusQueryAddr.c_str()); + + // Set the callback function to handle the response data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_cpu_usages); + curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0); + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); + curl_easy_setopt(curl, CURLOPT_FORBID_REUSE, 1L); + + res = curl_easy_perform(curl); + if (res != CURLE_OK) { + util_logger.error("cURL failed: " + string(curl_easy_strerror(res))); + } else { + util_logger.info(response_cpu_usages); + Json::Value root; + Json::Reader reader; + reader.parse(response_cpu_usages, root); + const Json::Value results = root["data"]["result"]; + Json::Value currentExportedJobName; + for (int i = 0; i < results.size(); i++) { + currentExportedJobName = results[i]["metric"]["exported_job"]; + map[(currentExportedJobName.asString().c_str())] = (results[i]["value"][1]).asString(); + } + curl_easy_cleanup(curl); + } + } + + return map; } +bool Utils::fileExistsWithReadPermission(const string &path) { return access(path.c_str(), R_OK) == 0; } + std::fstream *Utils::openFile(const string &path, std::ios_base::openmode mode) { if (!fileExistsWithReadPermission(path)) { // Create the file if it doesn't exist @@ -708,3 +842,270 @@ std::fstream *Utils::openFile(const string &path, std::ios_base::openmode mode) // Now open the file in the desired mode return new std::fstream(path, mode | std::ios::binary); } + +bool Utils::uploadFileToWorker(std::string host, int port, int dataPort, int graphID, std::string filePath, + std::string masterIP, std::string uploadType) { + util_logger.info("Host:" + host + " Port:" + to_string(port) + " DPort:" + to_string(dataPort)); + bool result = true; + int sockfd; + char data[FED_DATA_LENGTH + 1]; + struct sockaddr_in serv_addr; + struct hostent *server; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + if (sockfd < 0) { + util_logger.error("Cannot create socket"); + return false; + } + + if (host.find('@') != std::string::npos) { + host = Utils::split(host, '@')[1]; + } + + server = gethostbyname(host.c_str()); + if (server == NULL) { + util_logger.error("ERROR, no host named " + host); + return false; + } + + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(port); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return false; + } + + if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, masterIP)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, uploadType, JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + std::string fileName = Utils::getFileName(filePath); + int fileSize = Utils::getFileSize(filePath); + std::string fileLength = to_string(fileSize); + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, std::to_string(graphID), + JasmineGraphInstanceProtocol::SEND_FILE_NAME)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, + JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileLength, + JasmineGraphInstanceProtocol::SEND_FILE_CONT)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + util_logger.info("Going to send file" + filePath + "/" + fileName + "through file transfer service to worker"); + Utils::sendFileThroughService(host, dataPort, fileName, filePath); + + string response; + int count = 0; + while (true) { + if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::FILE_RECV_CHK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + util_logger.info("Sent: " + JasmineGraphInstanceProtocol::FILE_RECV_CHK); + + util_logger.info("Checking if file is received"); + response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::FILE_RECV_WAIT) == 0) { + util_logger.info("Received: " + JasmineGraphInstanceProtocol::FILE_RECV_WAIT); + util_logger.info("Checking file status : " + to_string(count)); + count++; + sleep(1); + continue; + } else if (response.compare(JasmineGraphInstanceProtocol::FILE_ACK) == 0) { + util_logger.info("Received: " + JasmineGraphInstanceProtocol::FILE_ACK); + util_logger.info("File transfer completed for file : " + filePath); + break; + } + } + // Next we wait till the batch upload completes + while (true) { + if (!Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + util_logger.info("Sent: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_CHK); + + response = Utils::read_str_trim_wrapper(sockfd, data, FED_DATA_LENGTH); + if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT) == 0) { + util_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_WAIT); + sleep(1); + continue; + } else if (response.compare(JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK) == 0) { + util_logger.info("Received: " + JasmineGraphInstanceProtocol::BATCH_UPLOAD_ACK); + util_logger.info("Batch upload completed: " + fileName); + break; + } + } + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return true; +} + +bool Utils::sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath) { + int sockfd; + char data[FED_DATA_LENGTH + 1]; + socklen_t len; + struct sockaddr_in serv_addr; + struct hostent *server; + + util_logger.info("Sending file " + filePath + " through port " + std::to_string(dataPort)); + FILE *fp = fopen(filePath.c_str(), "r"); + if (fp == NULL) { + return false; + } + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + util_logger.error("Cannot create socket"); + return false; + } + + server = gethostbyname(host.c_str()); + if (server == NULL) { + util_logger.error("ERROR, no host named " + host); + return false; + } + + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(dataPort); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, fileName, + JasmineGraphInstanceProtocol::SEND_FILE_LEN)) { + close(sockfd); + return false; + } + + int fsize = Utils::getFileSize(filePath); + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, to_string(fsize), + JasmineGraphInstanceProtocol::SEND_FILE)) { + close(sockfd); + return false; + } + + bool status = true; + while (true) { + unsigned char buff[1024]; + int nread = fread(buff, 1, sizeof(buff), fp); + + /* If read was success, send data. */ + if (nread > 0) { + write(sockfd, buff, nread); + } else { + if (feof(fp)) util_logger.info("End of file"); + if (ferror(fp)) { + status = false; + util_logger.error("Error reading file: " + filePath); + } + break; + } + } + + fclose(fp); + close(sockfd); + return status; +} + +/* + * Function to transfer a partition from one worker to another + * Caller should ensure that the partition exists in the source worker + * and the destination worker is ready to accept the partition. + * */ +bool Utils::transferPartition(std::string sourceWorker, int sourceWorkerPort, std::string destinationWorker, + int destinationWorkerDataPort, std::string graphID, std::string partitionID, + std::string workerID, SQLiteDBInterface *sqlite) { + util_logger.info("### Transferring partition " + partitionID + " of graph " + graphID + " from " + sourceWorker + + " to " + destinationWorker); + + int sockfd; + char data[FED_DATA_LENGTH + 1]; + struct sockaddr_in serv_addr; + struct hostent *server; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + util_logger.error("Cannot create socket"); + return false; + } + + server = gethostbyname(sourceWorker.c_str()); + if (server == NULL) { + util_logger.error("ERROR, no host named " + sourceWorker); + return false; + } + + bzero((char *)&serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + bcopy((char *)server->h_addr, (char *)&serv_addr.sin_addr.s_addr, server->h_length); + serv_addr.sin_port = htons(sourceWorkerPort); + if (Utils::connect_wrapper(sockfd, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + return false; + } + + if (!Utils::performHandshake(sockfd, data, FED_DATA_LENGTH, destinationWorker)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, JasmineGraphInstanceProtocol::PUSH_PARTITION, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, + destinationWorker + ":" + to_string(destinationWorkerDataPort), + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + if (!Utils::sendExpectResponse(sockfd, data, INSTANCE_DATA_LENGTH, graphID + "," + partitionID, + JasmineGraphInstanceProtocol::OK)) { + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return false; + } + + util_logger.info("### Transfer partition completed"); + sqlite->runInsert( + "INSERT INTO worker_has_partition " + "(partition_idpartition, partition_graph_idgraph, worker_idworker) VALUES ('" + + partitionID + "', '" + graphID + "', '" + workerID + "')"); + + Utils::send_str_wrapper(sockfd, JasmineGraphInstanceProtocol::CLOSE); + close(sockfd); + return true; +} diff --git a/src/util/Utils.h b/src/util/Utils.h index 59a2e825..f8b60091 100644 --- a/src/util/Utils.h +++ b/src/util/Utils.h @@ -68,15 +68,15 @@ class Utils { static bool fileExists(std::string fileName); - static bool fileExistsWithReadPermission(const std::string& path); + static bool fileExistsWithReadPermission(const std::string &path); - static std::fstream* openFile(const std::string &path, std::ios_base::openmode mode); + static std::fstream *openFile(const std::string &path, std::ios_base::openmode mode); static int compressFile(const std::string filePath, std::string mode = "pigz"); static bool is_number(const std::string &compareString); - static void createDirectory(const std::string dirName); + static int createDirectory(const std::string dirName); static std::vector getListOfFilesInDirectory(std::string dirName); @@ -127,7 +127,7 @@ class Utils { * @return The string read or "" on error. Logs error if recv failed. Also logs error if allowEmpty is false and * read length 0 string. */ - static std::string read_str_wrapper(int connFd, char *buf, size_t len, bool allowEmpty); + static std::string read_str_wrapper(int connFd, char *buf, size_t len, bool allowEmpty = false); /** * Wrapper to recv(2) to read a string and trim it. @@ -169,6 +169,19 @@ class Utils { static std::string getJsonStringFromYamlFile(const std::string &yamlFile); static int createDatabaseFromDDL(const char *dbLocation, const char *ddlFileLocation); + + static std::string send_job(std::string job_group_name, std::string metric_name, std::string metric_value); + + static map getMetricMap(string metricName); + + static bool uploadFileToWorker(std::string host, int port, int dataPort, int graphID, std::string filePath, + std::string masterIP, std::string uploadType); + + static bool sendFileThroughService(std::string host, int dataPort, std::string fileName, std::string filePath); + + static bool transferPartition(std::string sourceWorker, int sourceWorkerPort, std::string destinationWorker, + int destinationWorkerDataPort, std::string graphID, std::string partitionID, + std::string workerID, SQLiteDBInterface *sqlite); }; #endif // JASMINEGRAPH_UTILS_H diff --git a/src/util/logger/Logger.cpp b/src/util/logger/Logger.cpp index 149c3ffa..bc1a6dba 100644 --- a/src/util/logger/Logger.cpp +++ b/src/util/logger/Logger.cpp @@ -18,6 +18,8 @@ limitations under the License. #include #include #include +#include +#include #include @@ -39,7 +41,16 @@ static string get_worker_name() { void Logger::log(std::string message, const std::string log_type) { pthread_t tid = pthread_self(); - message = "[" + worker_name + " : " + to_string(tid) + "] " + message; + + struct timeval tv; + gettimeofday(&tv, NULL); + long millis = tv.tv_sec * 1000 + tv.tv_usec / 1000; + // TODO: This temporarily fixes spdlog hanging after forking. This will prevent using the actual logger and simulate + // the behavior using cout instead. But it will not write logs to the log file. + cout << " [" << millis << "] [" << log_type << "] [" << worker_name << " : " << getpid() << ":" << tid << "] " + << message << endl; + return; + if (log_type.compare("info") == 0) { daily_logger->info(message); logger->info(message); diff --git a/src/util/scheduler/SchedulerService.cpp b/src/util/scheduler/SchedulerService.cpp index 1b9e92b6..70645d5f 100644 --- a/src/util/scheduler/SchedulerService.cpp +++ b/src/util/scheduler/SchedulerService.cpp @@ -23,7 +23,8 @@ void SchedulerService::startScheduler() { std::string schedulerEnabled = Utils::getJasmineGraphProperty("org.jasminegraph.scheduler.enabled"); if (schedulerEnabled == "true") { - schedulerservice_logger.log("#######SCHEDULER ENABLED#####", "info"); + sleep(3); + schedulerservice_logger.info("#######SCHEDULER ENABLED#####"); startPerformanceScheduler(); } diff --git a/src_python/.keep b/src_python/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/start-k8s.sh b/start-k8s.sh index a6af383c..b601b0f3 100755 --- a/start-k8s.sh +++ b/start-k8s.sh @@ -4,12 +4,22 @@ set -e TIMEOUT_SECONDS=60 +if [ $1 == "clean" ]; then + kubectl delete deployments -l application=jasminegraph + kubectl delete services -l application=jasminegraph + kubectl delete pvc -l application=jasminegraph + kubectl delete pv -l application=jasminegraph + exit 0 +fi + META_DB_PATH=${META_DB_PATH} PERFORMANCE_DB_PATH=${PERFORMANCE_DB_PATH} DATA_PATH=${DATA_PATH} LOG_PATH=${LOG_PATH} +AGGREGATE_PATH=${AGGREGATE_PATH} NO_OF_WORKERS=${NO_OF_WORKERS} ENABLE_NMON=${ENABLE_NMON} +MAX_COUNT=${MAX_COUNT} while [ $# -gt 0 ]; do @@ -50,12 +60,58 @@ if [ -z "$ENABLE_NMON" ]; then ENABLE_NMON="false" fi +if [ -z "$MAX_COUNT" ]; then + MAX_COUNT="4" +fi + kubectl apply -f ./k8s/rbac.yaml +kubectl apply -f ./k8s/pushgateway.yaml + +# wait until pushgateway starts listening +cur_timestamp="$(date +%s)" +end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" +spin="/-\|" +i=0 +while true; do + if [ "$(date +%s)" -gt "$end_timestamp" ]; then + echo "Timeout" + exit 1 + fi + pushgatewayIP="$(kubectl get services |& grep pushgateway | tr '\t' ' ' | tr -s ' ' | cut -d ' ' -f 3)" + if [ ! -z "$pushgatewayIP" ]; then + break + fi + printf "Waiting pushgateway to start [%c] \r" "${spin:i++%${#spin}:1}" + sleep .2 +done + +pushgateway_address="${pushgatewayIP}:9091" envsubst <"./k8s/prometheus.yaml" | kubectl apply -f - + +cur_timestamp="$(date +%s)" +end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" +while true; do + if [ "$(date +%s)" -gt "$end_timestamp" ]; then + echo "Timeout" + exit 1 + fi + prometheusIP="$(kubectl get services |& grep prometheus | tr '\t' ' ' | tr -s ' ' | cut -d ' ' -f 3)" + if [ ! -z "$prometheusIP" ]; then + break + fi + printf "Waiting prometheus to start [%c] \r" "${spin:i++%${#spin}:1}" + sleep .2 +done + +pushgateway_address="${pushgatewayIP}:9091/" \ + prometheus_address="${prometheusIP}:9090/" \ + max_worker_count="${MAX_COUNT}" \ + envsubst <"./k8s/configs.yaml" | kubectl apply -f - metadb_path="${META_DB_PATH}" \ performancedb_path="${PERFORMANCE_DB_PATH}" \ data_path="${DATA_PATH}" \ log_path="${LOG_PATH}" \ + aggregate_path="${AGGREGATE_PATH}" \ envsubst <"./k8s/volumes.yaml" | kubectl apply -f - no_of_workers="${NO_OF_WORKERS}" \ @@ -65,8 +121,6 @@ no_of_workers="${NO_OF_WORKERS}" \ # wait until master starts listening cur_timestamp="$(date +%s)" end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" -spin="/-\|" -i=0 while true; do if [ "$(date +%s)" -gt "$end_timestamp" ]; then echo "Timeout" diff --git a/test-docker.sh b/test-docker.sh index 389be44b..1abd2635 100755 --- a/test-docker.sh +++ b/test-docker.sh @@ -34,13 +34,13 @@ force_remove() { } stop_and_remove_containers() { - if [ "$(docker ps -a -q)" ]; then + if [ ! -z "$(docker ps -a -q)" ]; then docker ps -a -q | xargs docker rm -f &>/dev/null else echo "No containers to stop and remove." fi docker run -v '/tmp/jasminegraph:/tmp/jasminegraph' --entrypoint /bin/bash jasminegraph:test -c 'rm -rf /tmp/jasminegraph/*' || echo 'Not removing existing tmp logs' - if [ "$(docker ps -a -q)" ]; then + if [ ! -z "$(docker ps -a -q)" ]; then docker ps -a -q | xargs docker rm -f &>/dev/null fi } @@ -78,7 +78,7 @@ stop_tests_on_failure() { break fi fi - sleep 1 + sleep 5 done } @@ -97,7 +97,7 @@ while ! nc -zvn 127.0.0.1 7777 &>/dev/null; do echo "JasmineGraph is not listening" echo "Build log:" cat "$BUILD_LOG" - echo "Build log:" + echo -e '\n\e[33;1mMASTER LOG:\e[0m' cat "$RUN_LOG" force_remove "${TEST_ROOT}/env" stop_and_remove_containers @@ -106,6 +106,7 @@ while ! nc -zvn 127.0.0.1 7777 &>/dev/null; do sleep .5 done +sleep 2 stop_tests_on_failure & timeout "$TIMEOUT_SECONDS" python3 -u "${TEST_ROOT}/test.py" |& tee "$TEST_LOG" diff --git a/test-k8s.sh b/test-k8s.sh index 38fd4114..cb925fd8 100755 --- a/test-k8s.sh +++ b/test-k8s.sh @@ -26,6 +26,13 @@ WORKER_LOG_DIR="/tmp/jasminegraph" rm -rf "${WORKER_LOG_DIR}" mkdir -p "${WORKER_LOG_DIR}" +force_remove() { + local files=("$@") + for f in "${files[@]}"; do + rm -rf "$f" &>/dev/null || sudo rm -rf "$f" + done +} + build_and_run_on_k8s() { cd "$PROJECT_ROOT" docker build -t jasminegraph . |& tee "$BUILD_LOG" @@ -46,24 +53,50 @@ build_and_run_on_k8s() { --PERFORMANCE_DB_PATH performancedb_path "${TEST_ROOT}/env/databases/performancedb" \ --DATA_PATH "${TEST_ROOT}/env/data" \ --LOG_PATH "${LOG_DIR}" \ + --AGGREGATE_PATH "${TEST_ROOT}/env/aggregate" \ --NO_OF_WORKERS 2 \ --ENABLE_NMON false } clear_resources() { - kubectl delete deployments jasminegraph-master-deployment jasminegraph-worker1-deployment \ - jasminegraph-worker0-deployment - kubectl delete services jasminegraph-master-service jasminegraph-worker0-service jasminegraph-worker1-service - kubectl delete -f "${PROJECT_ROOT}/k8s/volumes.yaml" + ./start-k8s.sh clean } cd "$TEST_ROOT" -sudo rm -rf env +force_remove env cp -r env_init env cd "$PROJECT_ROOT" build_and_run_on_k8s +# Wait till all pods are running +cur_timestamp="$(date +%s)" +end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" +while true; do + if [ "$(date +%s)" -gt "$end_timestamp" ]; then + echo "Pods are not running" + echo "Build log:" + cat "$BUILD_LOG" + echo "Run log:" + cat "$RUN_LOG" + force_remove "${TEST_ROOT}/env" + clear_resources + exit 1 + fi + + set +e + pods_status="$(kubectl get pods | grep -v 'STATUS' | grep -v 'Running')" + set -e + if [ -z "$pods_status" ]; then + echo "All pods are running" + break + fi + + echo "Waiting for pods to be running" + echo "$pods_status" + sleep 5 +done + # Wait till JasmineGraph server start listening cur_timestamp="$(date +%s)" end_timestamp="$((cur_timestamp + TIMEOUT_SECONDS))" @@ -76,7 +109,7 @@ while ! nc -zvn -w 1 "$masterIP" 7777 &>/dev/null; do cat "$BUILD_LOG" echo "Run log:" cat "$RUN_LOG" - sudo rm -rf "${TEST_ROOT}/env" + force_remove "${TEST_ROOT}/env" clear_resources exit 1 fi @@ -84,9 +117,10 @@ while ! nc -zvn -w 1 "$masterIP" 7777 &>/dev/null; do sleep .5 done -echo +echo '-------------------- pods -------------------------------------' kubectl get pods -o wide echo +echo '------------------ services -----------------------------------' kubectl get services -o wide echo @@ -116,5 +150,5 @@ set +e clear_resources >/dev/null 2>&1 set -e -sudo rm -rf "${TEST_ROOT}/env" "${WORKER_LOG_DIR}" +force_remove "${TEST_ROOT}/env" "${WORKER_LOG_DIR}" exit "$exit_code" diff --git a/tests/unit/Dockerfile b/tests/unit/Dockerfile index 95dbfbe9..80597e4e 100644 --- a/tests/unit/Dockerfile +++ b/tests/unit/Dockerfile @@ -1,8 +1,11 @@ FROM miyurud/jasminegraph-prerequisites:20240101T095619 +RUN apt-get update +RUN apt-get install libcurl4-openssl-dev -y +RUN apt-get install -y sysstat +RUN apt-get install -y nmon + RUN ln -sf /usr/bin/python3.8 /usr/bin/python3 -RUN ln -sf /usr/include/python3.8 /usr/include/python3 -RUN ln -sf /usr/lib/x86_64-linux-gnu/libpython3.8.so /usr/lib/x86_64-linux-gnu/libpython3.so RUN python3 -m pip install --no-cache-dir gcovr==6.0 @@ -17,8 +20,10 @@ COPY ./build.sh ./build.sh COPY ./CMakeLists.txt ./CMakeLists.txt COPY ./src_python ./src_python COPY ./src ./src +COPY ./ddl ./ddl COPY ./main.h ./main.h COPY ./main.cpp ./main.cpp +COPY ./globals.h ./globals.h COPY ./conf ./conf COPY ./k8s ./k8s diff --git a/tests/unit/k8s/K8sInterface_test.cpp b/tests/unit/k8s/K8sInterface_test.cpp index ead841d9..513bd675 100644 --- a/tests/unit/k8s/K8sInterface_test.cpp +++ b/tests/unit/k8s/K8sInterface_test.cpp @@ -27,9 +27,7 @@ class K8sInterfaceTest : public ::testing::Test { const int HTTP_OK = 200; const int HTTP_CREATED = 201; -TEST_F(K8sInterfaceTest, TestConstructor) { - ASSERT_NE(interface->apiClient, nullptr); -} +TEST_F(K8sInterfaceTest, TestConstructor) { ASSERT_NE(interface->apiClient, nullptr); } TEST_F(K8sInterfaceTest, TestGetNodes) { v1_node_list_t *nodes = interface->getNodes(); @@ -47,9 +45,7 @@ TEST_F(K8sInterfaceTest, TestGetServiceList) { } TEST_F(K8sInterfaceTest, TestCreateJasmineGraphWorkerDeployment) { - v1_node_list_t *nodeList = interface->getNodes(); - auto nodeName = static_cast(nodeList->items->firstEntry->data)->metadata->name; - v1_deployment_t *deployment = interface->createJasmineGraphWorkerDeployment(1, "10.43.0.1", nodeName); + v1_deployment_t *deployment = interface->createJasmineGraphWorkerDeployment(1, "10.43.0.2", "10.43.0.1"); ASSERT_STREQ(deployment->metadata->name, "jasminegraph-worker1-deployment"); ASSERT_EQ(interface->apiClient->response_code, HTTP_CREATED); } @@ -95,3 +91,27 @@ TEST_F(K8sInterfaceTest, TestDeleteJasmineGraphMasterService) { ASSERT_EQ(interface->apiClient->response_code, HTTP_OK); ASSERT_EQ(interface->getMasterIp(), ""); } + +TEST_F(K8sInterfaceTest, createJasmineGraphPersistentVolume) { + auto result = interface->createJasmineGraphPersistentVolume(1); + ASSERT_EQ(interface->apiClient->response_code, HTTP_CREATED); + ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data"); +} + +TEST_F(K8sInterfaceTest, createJasmineGraphPersistentVolumeClaim) { + auto result = interface->createJasmineGraphPersistentVolumeClaim(1); + ASSERT_EQ(interface->apiClient->response_code, HTTP_CREATED); + ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data-claim"); +} + +TEST_F(K8sInterfaceTest, deleteJasmineGraphPersistentVolume) { + auto result = interface->deleteJasmineGraphPersistentVolume(1); + ASSERT_EQ(interface->apiClient->response_code, HTTP_OK); + ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data"); +} + +TEST_F(K8sInterfaceTest, deleteJasmineGraphPersistentVolumeClaim) { + auto result = interface->deleteJasmineGraphPersistentVolumeClaim(1); + ASSERT_EQ(interface->apiClient->response_code, HTTP_OK); + ASSERT_STREQ(result->metadata->name, "jasminegraph-worker1-data-claim"); +} diff --git a/tests/unit/k8s/K8sWorkerController_test.cpp b/tests/unit/k8s/K8sWorkerController_test.cpp index 35f75be7..b11ea139 100644 --- a/tests/unit/k8s/K8sWorkerController_test.cpp +++ b/tests/unit/k8s/K8sWorkerController_test.cpp @@ -19,19 +19,19 @@ limitations under the License. #include "gtest/gtest.h" class K8sWorkerControllerTest : public ::testing::Test { - protected: - K8sWorkerController *controller{}; - SQLiteDBInterface *metadb = NULL; - K8sInterface *interface {}; + public: + static K8sWorkerController *controller; + static SQLiteDBInterface *metadb; + static K8sInterface *interface; - void SetUp() override { + static void SetUpTestSuite() { metadb = new SQLiteDBInterface(TEST_RESOURCE_DIR "temp/jasminegraph_meta.db"); metadb->init(); - controller = new K8sWorkerController("10.43.0.1", 2, metadb); + controller = K8sWorkerController::getInstance("10.43.0.1", 2, metadb); interface = new K8sInterface(); } - void TearDown() override { + static void TearDownTestSuite() { delete controller; delete metadb; delete interface; @@ -39,6 +39,10 @@ class K8sWorkerControllerTest : public ::testing::Test { } }; +K8sWorkerController *K8sWorkerControllerTest::controller = nullptr; +SQLiteDBInterface *K8sWorkerControllerTest::metadb = nullptr; +K8sInterface *K8sWorkerControllerTest::interface = nullptr; + TEST_F(K8sWorkerControllerTest, TestConstructor) { ASSERT_EQ(controller->getMasterIp(), "10.43.0.1"); ASSERT_EQ(controller->getNumberOfWorkers(), 2); @@ -51,24 +55,20 @@ TEST_F(K8sWorkerControllerTest, TestConstructor) { ASSERT_EQ(service_list->items->count, 2); } -TEST_F(K8sWorkerControllerTest, TestScalingUpAndDown) { - controller->setNumberOfWorkers(4); - ASSERT_EQ(controller->getNumberOfWorkers(), 4); - auto result = metadb->runSelect("SELECT idworker FROM worker"); - ASSERT_EQ(result.size(), 4); - +TEST_F(K8sWorkerControllerTest, TestScaleUp) { + auto result = controller->scaleUp(2); + ASSERT_EQ(result.size(), 2); v1_deployment_list_t *deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); ASSERT_EQ(deployment_list->items->count, 4); v1_service_list_t *service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); ASSERT_EQ(service_list->items->count, 4); +} - controller->setNumberOfWorkers(0); - ASSERT_EQ(controller->getNumberOfWorkers(), 0); - result = metadb->runSelect("SELECT idworker FROM worker"); +TEST_F(K8sWorkerControllerTest, TestScaleUpBeyondLimit) { + auto result = controller->scaleUp(1); ASSERT_EQ(result.size(), 0); - - deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); - ASSERT_EQ(deployment_list->items->count, 0); - service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); - ASSERT_EQ(service_list->items->count, 0); + v1_deployment_list_t *deployment_list = interface->getDeploymentList(strdup("deployment=jasminegraph-worker")); + ASSERT_EQ(deployment_list->items->count, 4); + v1_service_list_t *service_list = interface->getServiceList(strdup("service=jasminegraph-worker")); + ASSERT_EQ(service_list->items->count, 4); } diff --git a/tests/unit/metadb/SQLiteDBInterface_test.cpp b/tests/unit/metadb/SQLiteDBInterface_test.cpp index 2c5a6cae..a8936213 100644 --- a/tests/unit/metadb/SQLiteDBInterface_test.cpp +++ b/tests/unit/metadb/SQLiteDBInterface_test.cpp @@ -66,4 +66,3 @@ TEST_F(SQLiteDBInterfaceTest, TestRunInsertAndUpdate) { ASSERT_EQ(data[0][2].second, "127.0.0.1"); ASSERT_EQ(data[0][3].second, "false"); } -