diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e36b810dc..c3e1ae4928 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,6 +55,48 @@ commands: name: Attach workspace at: . + docker_login: + description: login to dockerhub for private repo access + steps: + - run: printf "%s\\n" "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin + + make_docker_images: + description: Builds docker images + steps: + - run: make docker-child_chain + - run: make docker-watcher + - run: make docker-watcher_info + + install_elixir_and_check_docker_status: + description: Installs elixir and checks if docker is healthy + steps: + - run: + name: Print docker states + command: | + docker image ls + docker-compose ps + - restore_cache: + key: v2-asdf-install + - run: + name: Install Erlang and Elixir + command: | + [ -d ~/.asdf-vm ] || git clone https://github.com/asdf-vm/asdf.git ~/.asdf-vm --branch v0.7.4 + echo 'source ~/.asdf-vm/asdf.sh' >> $BASH_ENV + source $BASH_ENV + asdf plugin-add erlang || asdf plugin-update erlang + asdf plugin-add elixir || asdf plugin-update elixir + asdf install + no_output_timeout: 2400 + - save_cache: + key: v2-asdf-install + paths: + - ~/.asdf + - ~/.asdf-vm + - run: make install-hex-rebar + - run: sh .circleci/status.sh + - restore_cache: + key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }} + jobs: barebuild: executor: metal @@ -446,40 +488,14 @@ jobs: name: Setup data dir command: | [ -d data ] || mkdir data && chmod 777 data - - run: make docker-child_chain - - run: make docker-watcher - - run: make docker-watcher_info + - docker_login + - make_docker_images - run: name: Start daemon services command: | cd priv/cabbage make start_daemon_services-2 || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;) - - run: - name: Print docker states - command: | - docker image ls - docker-compose ps - - restore_cache: - key: v2-asdf-install - - run: - name: Install Erlang and Elixir - command: | - [ -d ~/.asdf-vm ] || git clone https://github.com/asdf-vm/asdf.git ~/.asdf-vm --branch v0.7.4 - echo 'source ~/.asdf-vm/asdf.sh' >> $BASH_ENV - source $BASH_ENV - asdf plugin-add erlang || asdf plugin-update erlang - asdf plugin-add elixir || asdf plugin-update elixir - asdf install - no_output_timeout: 2400 - - save_cache: - key: v2-asdf-install - paths: - - ~/.asdf - - ~/.asdf-vm - - run: make install-hex-rebar - - run: sh .circleci/status.sh - - restore_cache: - key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }} + - install_elixir_and_check_docker_status - run: name: Run specs command: | @@ -488,12 +504,6 @@ jobs: make generate_api_code mix deps.get mix test - - run: - name: Run load test - command: | - cd priv/perf - make init - make test - run: name: (Cabbage) Format generated code and check for warnings command: | @@ -504,6 +514,46 @@ jobs: mix format apps/watcher_info_api/lib/watcher_info_api/model/*.ex mix format apps/watcher_security_critical_api/lib/watcher_security_critical_api/model/*.ex MIX_ENV=test mix do compile --warnings-as-errors --ignore-module-conflict --force, test --exclude test + - save_cache: + key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }} + paths: + - "priv/cabbage/deps" + - run: + name: (Cabbage) Credo and formatting + command: | + cd priv/cabbage + mix do credo, format --check-formatted --dry-run + + test_docker_compose_performance: + machine: + image: ubuntu-1604:201903-01 + steps: + - checkout + - run: + name: Setup data dir + command: | + [ -d data ] || mkdir data && chmod 777 data + - docker_login + - make_docker_images + - run: + name: Start daemon services + command: | + cd priv/perf + make start-services + - run: + name: docker services logs + background: true + command: | + cd priv/perf + make log-services + - install_elixir_and_check_docker_status + - run: + name: Run load test + command: | + cd priv/perf + make init + export $(cat ../../localchain_contract_addresses.env | xargs) + make test - run: name: (Perf) Format generated code and check for warnings command: | @@ -511,17 +561,13 @@ jobs: # run format ONLY on formatted code so that it cleans up quoted atoms because # we cannot exclude folders to --warnings-as-errors mix format apps/*_api/lib/*_api/model/*.ex + export $(cat ../../localchain_contract_addresses.env | xargs) make format-code-check-warnings - save_cache: key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }} paths: - - "priv/cabbage/deps" - "priv/perf/deps" - - run: - name: (Cabbage) Credo and formatting - command: | - cd priv/cabbage - mix do credo, format --check-formatted --dry-run + - run: name: (Perf) Credo and formatting command: | @@ -546,40 +592,14 @@ jobs: [ -d data1 ] || mkdir data1 && chmod 777 data1 [ -d data2 ] || mkdir data2 && chmod 777 data2 [ -d data ] || mkdir data && chmod 777 data - - run: make docker-child_chain - - run: make docker-watcher - - run: make docker-watcher_info + - docker_login + - make_docker_images - run: name: Start daemon services command: | cd priv/cabbage make start_daemon_services_reorg-2 || (START_RESULT=$?; docker-compose logs; exit $START_RESULT;) - - run: - name: Print docker states - command: | - docker image ls - docker-compose ps - - restore_cache: - key: v2-asdf-install - - run: - name: Install Erlang and Elixir - command: | - [ -d ~/.asdf-vm ] || git clone https://github.com/asdf-vm/asdf.git ~/.asdf-vm --branch v0.7.4 - echo 'source ~/.asdf-vm/asdf.sh' >> $BASH_ENV - source $BASH_ENV - asdf plugin-add erlang || asdf plugin-update erlang - asdf plugin-add elixir || asdf plugin-update elixir - asdf install - no_output_timeout: 2400 - - save_cache: - key: v2-asdf-install - paths: - - ~/.asdf - - ~/.asdf-vm - - run: make install-hex-rebar - - run: sh .circleci/status.sh - - restore_cache: - key: v2-mix-specs-cache-{{ .Branch }}-{{ checksum "mix.lock" }} + - install_elixir_and_check_docker_status - run: name: Print watcher logs command: make cabbage-reorg-watcher-logs @@ -623,8 +643,9 @@ jobs: git submodule init git submodule update --remote - run: echo 'export PATH=~/.cargo/bin:$PATH' >> $BASH_ENV + - docker_login - run: - name: Start geth, postgres and pull in blockchain snapshot + name: Start geth, postgres, feefeed and pull in blockchain snapshot command: make start-services background: true - run: echo 'export PATH=~/.cargo/bin:$PATH' >> $BASH_ENV @@ -714,12 +735,6 @@ jobs: command: | cd priv/cabbage mix test - - run: - name: Run load test - command: | - cd priv/perf - make init - make test publish_child_chain: machine: @@ -920,6 +935,9 @@ workflows: - test_docker_compose_release: requires: [build] filters: *all_branches_and_tags + - test_docker_compose_performance: + requires: [build] + filters: *all_branches_and_tags - test_docker_compose_reorg: requires: [build] filters: *all_branches_and_tags diff --git a/.gitmodules b/.gitmodules index a343bb6c6e..e69de29bb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +0,0 @@ -[submodule "priv/cabbage"] - path = priv/cabbage - url = git@github.com:omgnetwork/specs.git - branch = master \ No newline at end of file diff --git a/Makefile b/Makefile index ff3d96425c..b1b0af08c5 100644 --- a/Makefile +++ b/Makefile @@ -427,7 +427,7 @@ docker-remote-childchain: ### start-services: SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \ - docker-compose up geth nginx postgres + docker-compose -f ./docker-compose.yml -f ./docker-compose.feefeed.yml up postgres_feefeed feefeed geth nginx postgres start-child_chain: . ${OVERRIDING_VARIABLES} && \ diff --git a/docker-compose.feefeed.yml b/docker-compose.feefeed.yml new file mode 100644 index 0000000000..efd94bc431 --- /dev/null +++ b/docker-compose.feefeed.yml @@ -0,0 +1,64 @@ +version: "2.3" +services: + childchain: + environment: + - FEE_ADAPTER=feed + - FEE_FEED_URL=http://172.27.0.107:4000/api/v1 + depends_on: + feefeed: + condition: service_healthy + + postgres_feefeed: + image: postgres:12.2-alpine + ports: + - "5433:5432" + restart: always + healthcheck: + test: pg_isready -U feefeed + interval: 5s + timeout: 3s + retries: 5 + environment: + POSTGRES_USER: feefeed + POSTGRES_PASSWORD: feefeed + POSTGRES_DB: feefeed + networks: + chain_net: + ipv4_address: 172.27.0.106 + + feefeed: + image: "omisego/feefeed:latest" + command: "start" + container_name: feefeed + environment: + - GITHUB_TOKEN="" + - GITHUB_ORGANISATION=omgnetwork + - GITHUB_REPO=fee-rules-public + - GITHUB_BRANCH=master + - RULES_FETCH_INTERVAL=20 + - RATES_FETCH_INTERVAL=20 + - GITHUB_FILENAME=fee_rules + - DATABASE_URL=postgresql://feefeed:feefeed@172.27.0.106:5432/feefeed + - SECRET_KEY_BASE="Y8naENMR8b+vbPHILjwNtEfWFrnbGi2k+UYWm75VnKHfsavmyGLtTmmeJxAGK+zJ" + - DATADOG_DISABLED=true + - DATADOG_HOST="localhost" + - ETHEREUM_NODE_URL=http://172.27.0.102:80 + ports: + - "4000:4000" + expose: + - "4000" + depends_on: + postgres_feefeed: + condition: service_healthy + nginx: + condition: service_healthy + restart: always + healthcheck: + test: curl localhost:4000 + interval: 30s + timeout: 1s + retries: 5 + start_period: 30s + networks: + chain_net: + ipv4_address: 172.27.0.107 diff --git a/priv/cabbage b/priv/cabbage deleted file mode 160000 index 96d67f4e06..0000000000 --- a/priv/cabbage +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 96d67f4e065aad020dfde111419e95e397f443e8 diff --git a/priv/perf/Makefile b/priv/perf/Makefile index 1cb86b38b8..3d556783dc 100644 --- a/priv/perf/Makefile +++ b/priv/perf/Makefile @@ -1,15 +1,24 @@ .PHONY: list + +COMPOSE_FULL_SERVICES=-f ../../docker-compose.yml -f ../../docker-compose.feefeed.yml + list: @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' +clean: + docker-compose $(COMPOSE_FULL_SERVICES) down && docker volume prune --force + start-services: cd ../../ && \ SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init-contracts && \ cd priv/perf/ && \ - docker-compose -f ../../docker-compose.yml up -d + docker-compose $(COMPOSE_FULL_SERVICES) up -d stop-services: - docker-compose down + docker-compose $(COMPOSE_FULL_SERVICES) down + +log-services: + docker-compose $(COMPOSE_FULL_SERVICES) logs -f childchain feefeed watcher watcher_info geth init: . scripts/generate_api_client.sh