cherry-picked [master a69e22c]: Allow to run docker-compose without f…

…eefeed docker (#1726) * refactor: abstract steps in circle ci to make file for perf * chore: seperate docker-compose file for feefeed to allow public contributor to run without feefeed docker * fix: make start-services command with feefeed * refactor: move mix format for api generated code back to CI * debug: add debug CI workflow * style: simplify docker-compose file * debug: add log service command to perf * do not set env var in makefile * back to run whole workflow * refactor: COMPOSE_FULL_SERVICES in makefile * chore: spaces instead of tab in .gitmodules
omgnetwork · Sep 15, 2020 · 2652b46 · 2652b46
1 parent b254d36
commit 2652b46
Show file tree

Hide file tree

Showing 6 changed files with 171 additions and 85 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -55,6 +55,48 @@ commands:
           name: Attach workspace
           at: .
 
+  docker_login:
+    description: login to dockerhub for private repo access
+    steps:
+      - run: printf "%s\\n" "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
+
+  make_docker_images:
+    description: Builds docker images
+    steps:
+      - run: make docker-child_chain
+      - run: make docker-watcher
+      - run: make docker-watcher_info
+
+  install_elixir_and_check_docker_status:
+    description: Installs elixir and checks if docker is healthy
+    steps:
+      - run:
+          name: Print docker states
+          command: |
+            docker image ls
+            docker-compose ps
+      - restore_cache:
+          key: v2-asdf-install
+      - run:
+          name: Install Erlang and Elixir
+          command: |
+            [ -d ~/.asdf-vm ] || git clone https://github.com/asdf-vm/asdf.git ~/.asdf-vm --branch v0.7.4
+            echo 'source ~/.asdf-vm/asdf.sh' >> $BASH_ENV
+            source $BASH_ENV
+            asdf plugin-add erlang || asdf plugin-update erlang
+            asdf plugin-add elixir || asdf plugin-update elixir
+            asdf install
+          no_output_timeout: 2400
+      - save_cache:
+          key: v2-asdf-install
+          paths:
+            - ~/.asdf
+            - ~/.asdf-vm
+      - run: make install-hex-rebar
+      - run: sh .circleci/status.sh
+      - restore_cache:
+          key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }}
+
 jobs:
   barebuild:
     executor: metal
@@ -446,40 +488,14 @@ jobs:
           name: Setup data dir
           command: |
             [ -d data ] || mkdir data && chmod 777 data
-      - run: make docker-child_chain
-      - run: make docker-watcher
-      - run: make docker-watcher_info
+      - docker_login
+      - make_docker_images
       - run:
           name: Start daemon services
           command: |
             cd priv/cabbage
             make start_daemon_services-2 || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
-      - run:
-          name: Print docker states
-          command: |
-            docker image ls
-            docker-compose ps
-      - restore_cache:
-          key: v2-asdf-install
-      - run:
-          name: Install Erlang and Elixir
-          command: |
-            [ -d ~/.asdf-vm ] || git clone https://github.com/asdf-vm/asdf.git ~/.asdf-vm --branch v0.7.4
-            echo 'source ~/.asdf-vm/asdf.sh' >> $BASH_ENV
-            source $BASH_ENV
-            asdf plugin-add erlang || asdf plugin-update erlang
-            asdf plugin-add elixir || asdf plugin-update elixir
-            asdf install
-          no_output_timeout: 2400
-      - save_cache:
-          key: v2-asdf-install
-          paths:
-            - ~/.asdf
-            - ~/.asdf-vm
-      - run: make install-hex-rebar
-      - run: sh .circleci/status.sh
-      - restore_cache:
-          key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }}
+      - install_elixir_and_check_docker_status
       - run:
           name: Run specs
           command: |
@@ -488,12 +504,6 @@ jobs:
             make generate_api_code
             mix deps.get
             mix test
-      - run:
-          name: Run load test
-          command: |
-            cd priv/perf
-            make init
-            make test
       - run:
           name: (Cabbage) Format generated code and check for warnings
           command: |
@@ -504,24 +514,60 @@ jobs:
             mix format apps/watcher_info_api/lib/watcher_info_api/model/*.ex
             mix format apps/watcher_security_critical_api/lib/watcher_security_critical_api/model/*.ex
             MIX_ENV=test mix do compile --warnings-as-errors --ignore-module-conflict --force, test --exclude test
+      - save_cache:
+          key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }}
+          paths:
+            - "priv/cabbage/deps"
+      - run:
+          name: (Cabbage) Credo and formatting
+          command: |
+            cd priv/cabbage
+            mix do credo, format --check-formatted --dry-run
+
+  test_docker_compose_performance:
+    machine:
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - run:
+          name: Setup data dir
+          command: |
+            [ -d data ] || mkdir data && chmod 777 data
+      - docker_login
+      - make_docker_images
+      - run:
+          name: Start daemon services
+          command: |
+            cd priv/perf
+            make start-services
+      - run:
+          name: docker services logs
+          background: true
+          command: |
+            cd priv/perf
+            make log-services
+      - install_elixir_and_check_docker_status
+      - run:
+          name: Run load test
+          command: |
+            cd priv/perf
+            make init
+            export $(cat ../../localchain_contract_addresses.env | xargs)
+            make test
       - run:
           name: (Perf) Format generated code and check for warnings
           command: |
             cd priv/perf
             # run format ONLY on formatted code so that it cleans up quoted atoms because
             # we cannot exclude folders to --warnings-as-errors
             mix format apps/*_api/lib/*_api/model/*.ex
+            export $(cat ../../localchain_contract_addresses.env | xargs)
             make format-code-check-warnings
       - save_cache:
           key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }}
           paths:
-            - "priv/cabbage/deps"
             - "priv/perf/deps"
-      - run:
-          name: (Cabbage) Credo and formatting
-          command: |
-            cd priv/cabbage
-            mix do credo, format --check-formatted --dry-run
+
       - run:
           name: (Perf) Credo and formatting
           command: |
@@ -546,40 +592,14 @@ jobs:
             [ -d data1 ] || mkdir data1 && chmod 777 data1
             [ -d data2 ] || mkdir data2 && chmod 777 data2
             [ -d data ] || mkdir data && chmod 777 data
-      - run: make docker-child_chain
-      - run: make docker-watcher
-      - run: make docker-watcher_info
+      - docker_login
+      - make_docker_images
       - run:
           name: Start daemon services
           command: |
             cd priv/cabbage
             make start_daemon_services_reorg-2 || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;)
-      - run:
-          name: Print docker states
-          command: |
-            docker image ls
-            docker-compose ps
-      - restore_cache:
-          key: v2-asdf-install
-      - run:
-          name: Install Erlang and Elixir
-          command: |
-            [ -d ~/.asdf-vm ] || git clone https://github.com/asdf-vm/asdf.git ~/.asdf-vm --branch v0.7.4
-            echo 'source ~/.asdf-vm/asdf.sh' >> $BASH_ENV
-            source $BASH_ENV
-            asdf plugin-add erlang || asdf plugin-update erlang
-            asdf plugin-add elixir || asdf plugin-update elixir
-            asdf install
-          no_output_timeout: 2400
-      - save_cache:
-          key: v2-asdf-install
-          paths:
-            - ~/.asdf
-            - ~/.asdf-vm
-      - run: make install-hex-rebar
-      - run: sh .circleci/status.sh
-      - restore_cache:
-          key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }}
+      - install_elixir_and_check_docker_status
       - run:
           name: Print watcher logs
           command: make cabbage-reorg-watcher-logs
@@ -623,8 +643,9 @@ jobs:
             git submodule init
             git submodule update --remote
       - run: echo 'export PATH=~/.cargo/bin:$PATH' >> $BASH_ENV
+      - docker_login
       - run:
-          name: Start geth, postgres and pull in blockchain snapshot
+          name: Start geth, postgres, feefeed and pull in blockchain snapshot
           command: make start-services
           background: true
       - run: echo 'export PATH=~/.cargo/bin:$PATH' >> $BASH_ENV
@@ -714,12 +735,6 @@ jobs:
           command: |
             cd priv/cabbage
             mix test
-      - run:
-          name: Run load test
-          command: |
-            cd priv/perf
-            make init
-            make test
 
   publish_child_chain:
     machine:
@@ -920,6 +935,9 @@ workflows:
       - test_docker_compose_release:
           requires: [build]
           filters: *all_branches_and_tags
+      - test_docker_compose_performance:
+          requires: [build]
+          filters: *all_branches_and_tags
       - test_docker_compose_reorg:
           requires: [build]
           filters: *all_branches_and_tags

diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +0,0 @@
-[submodule "priv/cabbage"]
-	path = priv/cabbage
-	url = [email protected]:omgnetwork/specs.git
-        branch = master

diff --git a/Makefile b/Makefile
@@ -427,7 +427,7 @@ docker-remote-childchain:
 ###
 start-services:
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \
-	docker-compose up geth nginx postgres
+	docker-compose -f ./docker-compose.yml -f ./docker-compose.feefeed.yml up postgres_feefeed feefeed geth nginx postgres
 
 start-child_chain:
 	. ${OVERRIDING_VARIABLES} && \

diff --git a/docker-compose.feefeed.yml b/docker-compose.feefeed.yml
@@ -0,0 +1,64 @@
+version: "2.3"
+services:
+  childchain:
+    environment:
+      - FEE_ADAPTER=feed
+      - FEE_FEED_URL=http://172.27.0.107:4000/api/v1
+    depends_on:
+      feefeed:
+        condition: service_healthy
+
+  postgres_feefeed:
+    image: postgres:12.2-alpine
+    ports:
+      - "5433:5432"
+    restart: always
+    healthcheck:
+      test: pg_isready -U feefeed
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    environment:
+      POSTGRES_USER: feefeed
+      POSTGRES_PASSWORD: feefeed
+      POSTGRES_DB: feefeed
+    networks:
+      chain_net:
+        ipv4_address: 172.27.0.106
+
+  feefeed:
+    image: "omisego/feefeed:latest"
+    command: "start"
+    container_name: feefeed
+    environment:
+      - GITHUB_TOKEN=""
+      - GITHUB_ORGANISATION=omgnetwork
+      - GITHUB_REPO=fee-rules-public
+      - GITHUB_BRANCH=master
+      - RULES_FETCH_INTERVAL=20
+      - RATES_FETCH_INTERVAL=20
+      - GITHUB_FILENAME=fee_rules
+      - DATABASE_URL=postgresql://feefeed:[email protected]:5432/feefeed
+      - SECRET_KEY_BASE="Y8naENMR8b+vbPHILjwNtEfWFrnbGi2k+UYWm75VnKHfsavmyGLtTmmeJxAGK+zJ"
+      - DATADOG_DISABLED=true
+      - DATADOG_HOST="localhost"
+      - ETHEREUM_NODE_URL=http://172.27.0.102:80
+    ports:
+      - "4000:4000"
+    expose:
+      - "4000"
+    depends_on:
+      postgres_feefeed:
+        condition: service_healthy
+      nginx:
+        condition: service_healthy
+    restart: always
+    healthcheck:
+      test: curl localhost:4000
+      interval: 30s
+      timeout: 1s
+      retries: 5
+      start_period: 30s
+    networks:
+      chain_net:
+        ipv4_address: 172.27.0.107
diff --git a/priv/cabbage b/priv/cabbage
diff --git a/priv/perf/Makefile b/priv/perf/Makefile
@@ -1,15 +1,24 @@
 .PHONY: list
+
+COMPOSE_FULL_SERVICES=-f ../../docker-compose.yml -f ../../docker-compose.feefeed.yml
+
 list:
 	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$'
 
+clean:
+	docker-compose $(COMPOSE_FULL_SERVICES) down && docker volume prune --force
+
 start-services:
 	cd ../../ && \
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init-contracts && \
 	cd priv/perf/ && \
-	docker-compose -f ../../docker-compose.yml up -d
+	docker-compose $(COMPOSE_FULL_SERVICES) up -d
 
 stop-services:
-	docker-compose down
+	docker-compose $(COMPOSE_FULL_SERVICES) down
+
+log-services:
+	docker-compose $(COMPOSE_FULL_SERVICES) logs -f childchain feefeed watcher watcher_info geth
 
 init:
 	. scripts/generate_api_client.sh