chore: add feefeed docker to elixir-omg setup (#1700)

* add feefeed adapter

* change ip

* add feefeed url

* fix test barebone release

* fix port

* remove override variable from barebone config

* chore: use https instead of gits to avoid permission issue

* test: update to cabbage using fee api

* fix: git submodule without --remote flag

So it would not always pull the head branch from remote.

* fix: update cabbage to fix reorg

* fix: circle ci mis-comment

* chore: add healthcheck and depends on for services

and mock off other tests aside from reorg

* chore: sync with cabbage update

* chore: add space before network

* chore: ignore geth folder

* debug: comment out make docker steps

* chore: update priv/perf for debug docker-compose pull

* debug: use feefeed-dev with more logging

* debug: use newer feefeed docker that fixes a bug

* chore: update cabbage commit

* chore: uncomment the commented CI jobs

* chore: update latest cabbage

* style: docker file space

* fix: all docker images command have dockerhub authenticate

* chore: update cabbage for code review

* fix: circle ci login to dockerhub

and update cabbage commit

* fix: fee for perf tests

- set in the test config to use the new fee amount 75
- for LoadTest.Common, pass in the fee_amount and use the value from config

* chore: back to feefeed:latest image

* style: mix format

* refactor: move docker_login out from make_docker_images

Logic-wise it does not make much sense actually to be together.

* chore: update cabbage to latest

* style: fix lint

* chore: update cabbage commit

* chore: use master from cabbage

* chore: bring back submodule udpate --remote

This is just better. change the branch in submodele file instead for testing.

Co-authored-by: jarindr <[email protected]>

.circleci/config.yml

@@ -55,6 +55,11 @@ commands:
           name: Attach workspace
           at: .
 
+  docker_login:
+    description: login to dockerhub for private repo access
+    steps:
+      - run: printf "%s\\n" "$DOCKER_PASS" | docker login -u "$DOCKER_USER" --password-stdin
+
   make_docker_images:
     description: Builds docker images
     steps:
@@ -483,6 +488,7 @@ jobs:
           name: Setup data dir
           command: |
             [ -d data ] || mkdir data && chmod 777 data
+      - docker_login
       - make_docker_images
       - run:
           name: Start daemon services
@@ -527,6 +533,7 @@ jobs:
           name: Setup data dir
           command: |
             [ -d data ] || mkdir data && chmod 777 data
+      - docker_login
       - make_docker_images
       - run:
           name: Start daemon services
@@ -578,6 +585,7 @@ jobs:
             [ -d data1 ] || mkdir data1 && chmod 777 data1
             [ -d data2 ] || mkdir data2 && chmod 777 data2
             [ -d data ] || mkdir data && chmod 777 data
+      - docker_login
       - make_docker_images
       - run:
           name: Start daemon services
@@ -628,8 +636,9 @@ jobs:
             git submodule init
             git submodule update --remote
       - run: echo 'export PATH=~/.cargo/bin:$PATH' >> $BASH_ENV
+      - docker_login
       - run:
-          name: Start geth, postgres and pull in blockchain snapshot
+          name: Start geth, postgres, feefeed and pull in blockchain snapshot
           command: make start-services
           background: true
       - run: echo 'export PATH=~/.cargo/bin:$PATH' >> $BASH_ENV

.gitignore

@@ -66,6 +66,8 @@ priv/apps/watcher_security_critical_api
 
 # where Geth gets it's snapshot data
 data/
+data1/
+data2/
 # local test setup
 localchain_contract_addresses.env
 

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "priv/cabbage"]
 	path = priv/cabbage
-	url = git@github.com:omgnetwork/specs.git
-        branch = master
+	url = https://github.com/omgnetwork/specs.git
+        branch = master

Makefile

@@ -427,7 +427,7 @@ docker-remote-childchain:
 ###
 start-services:
 	SNAPSHOT=SNAPSHOT_MIX_EXIT_PERIOD_SECONDS_120 make init_test && \
-	docker-compose up geth nginx postgres
+	docker-compose up postgres_feefeed feefeed geth nginx postgres
 
 start-child_chain:
 	. ${OVERRIDING_VARIABLES} && \

bin/variables

@@ -12,7 +12,10 @@ export ETHEREUM_EVENTS_CHECK_INTERVAL_MS=800
 export ETHEREUM_STALLED_SYNC_THRESHOLD_MS=20000
 export EXIT_PROCESSOR_SLA_MARGIN=5520
 export FEE_CLAIMER_ADDRESS=0x3b9f4c1dd26e0be593373b1d36cee2008cbeb837
-export FEE_ADAPTER=file
+export FEE_ADAPTER=feed
+export FEE_FEED_URL=http://127.0.0.1:4000/api/v1
+export STORED_FEE_UPDATE_INTERVAL_MINUTES=1
+export FEE_CHANGE_TOLERANCE_PERCENT=1
 export FEE_SPECS_FILE_PATH=./priv/dev-artifacts/fee_specs.dev.json
 export DD_HOSTNAME=localhost
 export DD_DISABLED=true

bin/variables_test_barebone

@@ -19,7 +19,10 @@ export ETHEREUM_HEIGHT_CHECK_INTERVAL_MS=800
 export ETHEREUM_EVENTS_CHECK_INTERVAL_MS=800
 export ETHEREUM_STALLED_SYNC_THRESHOLD_MS=20000
 export FEE_CLAIMER_ADDRESS=0x3b9f4c1dd26e0be593373b1d36cee2008cbeb837
-export FEE_ADAPTER=file
+export FEE_ADAPTER=feed
+export FEE_FEED_URL=http://127.0.0.1:4000/api/v1
+export STORED_FEE_UPDATE_INTERVAL_MINUTES=1
+export FEE_CHANGE_TOLERANCE_PERCENT=1
 # Fee specs file path needs to be an absolute path as the childchain will start deep in the _build subdirectory
 export FEE_SPECS_FILE_PATH=$(pwd)/priv/dev-artifacts/fee_specs.dev.json
 export DD_HOSTNAME=localhost

docker-compose.yml

@@ -16,7 +16,7 @@ services:
       retries: 5
     networks:
       chain_net:
-        ipv4_address: 172.27.0.106
+        ipv4_address: 172.27.0.100
 
   geth:
     image: ethereum/client-go:v1.9.12
@@ -99,7 +99,8 @@ services:
       - ETHEREUM_EVENTS_CHECK_INTERVAL_MS=800
       - ETHEREUM_STALLED_SYNC_THRESHOLD_MS=20000
       - FEE_CLAIMER_ADDRESS=0x3b9f4c1dd26e0be593373b1d36cee2008cbeb837
-      - FEE_ADAPTER=file
+      - FEE_ADAPTER=feed
+      - FEE_FEED_URL=http://172.27.0.107:4000/api/v1
       - STORED_FEE_UPDATE_INTERVAL_MINUTES=1
       - FEE_CHANGE_TOLERANCE_PERCENT=1
       - FEE_SPECS_FILE_PATH=/dev-artifacts/fee_specs.dev.json
@@ -124,6 +125,8 @@ services:
     depends_on:
       nginx:
         condition: service_healthy
+      feefeed:
+        condition: service_healthy
     networks:
       chain_net:
         ipv4_address: 172.27.0.103
@@ -181,7 +184,7 @@ services:
       - ETHEREUM_NETWORK=LOCALCHAIN
       - ETHEREUM_RPC_URL=http://172.27.0.102:80
       - CHILD_CHAIN_URL=http://172.27.0.103:9656
-      - DATABASE_URL=postgresql://omisego_dev:[email protected].106:5432/omisego_dev
+      - DATABASE_URL=postgresql://omisego_dev:[email protected].100:5432/omisego_dev
       - PORT=7534
       - APP_ENV=local_docker_development
       - DD_HOSTNAME=datadog
@@ -216,6 +219,61 @@ services:
       chain_net:
         ipv4_address: 172.27.0.105
 
+  postgres_feefeed:
+    image: postgres:12.2-alpine
+    ports:
+      - "5433:5432"
+    restart: always
+    healthcheck:
+      test: pg_isready -U feefeed
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    environment:
+      POSTGRES_USER: feefeed
+      POSTGRES_PASSWORD: feefeed
+      POSTGRES_DB: feefeed
+    networks:
+      chain_net:
+        ipv4_address: 172.27.0.106
+
+  feefeed:
+    image: "omisego/feefeed:latest"
+    command: "start"
+    container_name: feefeed
+    environment:
+      - GITHUB_TOKEN=""
+      - GITHUB_ORGANISATION=omgnetwork
+      - GITHUB_REPO=fee-rules-public
+      - GITHUB_BRANCH=master
+      - RULES_FETCH_INTERVAL=20
+      - RATES_FETCH_INTERVAL=20
+      - GITHUB_FILENAME=fee_rules
+      - DATABASE_URL=postgresql://feefeed:[email protected]:5432/feefeed
+      - SECRET_KEY_BASE="Y8naENMR8b+vbPHILjwNtEfWFrnbGi2k+UYWm75VnKHfsavmyGLtTmmeJxAGK+zJ"
+      - DATADOG_DISABLED=true
+      - DATADOG_HOST="localhost"
+      - ETHEREUM_NODE_URL=http://172.27.0.102:80
+    ports:
+      - "4000:4000"
+    expose:
+      - "4000"
+    depends_on:
+      postgres_feefeed:
+        condition: service_healthy
+      nginx:
+        condition: service_healthy
+    restart: always
+    healthcheck:
+      test: curl localhost:4000
+      interval: 30s
+      timeout: 1s
+      retries: 5
+      start_period: 30s
+    networks:
+      chain_net:
+        ipv4_address: 172.27.0.107
+
 networks:
   chain_net:
     driver: bridge

priv/perf/apps/load_test/lib/common/byzantine_events.ex

@@ -30,7 +30,7 @@ defmodule LoadTest.Common.ByzantineEvents do
 
   You probably want to prefill the child chain with transactions, see `OMG.Performance.ExtendedPerftest` or just:
   ```
-  LoadTest.Common.ExtendedPerftest.start(10_000, 16, randomized: false)
+  LoadTest.Common.ExtendedPerftest.start(10_000, 16, 75, randomized: false)
   ```
   (`randomized: false` is useful to test massive honest-standard-exiting, since it will create many unspent UTXOs for
   each of the spenders)

priv/perf/apps/load_test/lib/common/extended_perftest.ex

@@ -39,7 +39,7 @@ defmodule LoadTest.Common.ExtendedPerftest do
 
   Performance.init()
   spenders = Generators.generate_users(2)
-  LoadTest.Common.ExtendedPerftest.start(100, spenders, destdir: destdir)
+  LoadTest.Common.ExtendedPerftest.start(100, spenders, fee_amount, destdir: destdir)
   ```
 
   The results are going to be waiting for you in a file within `destdir` and will be logged.
@@ -49,8 +49,8 @@ defmodule LoadTest.Common.ExtendedPerftest do
     - :randomized - whether the non-change outputs of the txs sent out will be random or equal to sender (if `false`),
       defaults to `true`
   """
-  @spec start(pos_integer(), list(map()), keyword()) :: :ok
-  def start(ntx_to_send, spenders, opts \\ []) do
+  @spec start(pos_integer(), list(map()), pos_integer(), keyword()) :: :ok
+  def start(ntx_to_send, spenders, fee_amount, opts \\ []) do
     _ =
       Logger.info(
         "Number of spenders: #{inspect(length(spenders))}, number of tx to send per spender: #{inspect(ntx_to_send)}" <>
@@ -61,18 +61,18 @@ defmodule LoadTest.Common.ExtendedPerftest do
 
     opts = Keyword.merge(defaults, opts)
 
-    utxos = create_deposits(spenders, ntx_to_send)
+    utxos = create_deposits(spenders, ntx_to_send, fee_amount)
 
-    result = LoadTest.Common.Runner.run(ntx_to_send, utxos, opts, false)
+    result = LoadTest.Common.Runner.run(ntx_to_send, utxos, fee_amount, opts, false)
 
     Process.sleep(20_000)
 
     result
   end
 
-  @spec create_deposits(list(map()), pos_integer()) :: list()
-  defp create_deposits(spenders, ntx_to_send) do
-    Enum.map(make_deposits(ntx_to_send * 2, spenders), fn {:ok, owner, blknum, amount} ->
+  @spec create_deposits(list(map()), pos_integer(), pos_integer()) :: list()
+  defp create_deposits(spenders, ntx_to_send, fee_amount) do
+    Enum.map(make_deposits(ntx_to_send * 2 * fee_amount, spenders), fn {:ok, owner, blknum, amount} ->
       utxo_pos = ExPlasma.Utxo.pos(%{blknum: blknum, txindex: 0, oindex: 0})
       %{owner: owner, utxo_pos: utxo_pos, amount: amount}
     end)

priv/perf/apps/load_test/lib/common/runner.ex

@@ -24,15 +24,21 @@ defmodule LoadTest.Common.Runner do
 
   Foreach user runs n submit_transaction requests to the chain server. Requests are done sequentially for every user
   """
-  @spec run(pos_integer(), list(), keyword(), profile :: boolean()) :: :ok
-  def run(ntx_to_send, utxos, opts, true), do: do_profiled_run(ntx_to_send, utxos, opts)
-  def run(ntx_to_send, utxos, opts, false), do: do_run(ntx_to_send, utxos, opts)
+  @spec run(pos_integer(), list(), pos_integer(), keyword(), profile :: boolean()) :: :ok
+  def run(ntx_to_send, utxos, fee_amount, opts, true), do: do_profiled_run(ntx_to_send, utxos, fee_amount, opts)
+  def run(ntx_to_send, utxos, fee_amount, opts, false), do: do_run(ntx_to_send, utxos, fee_amount, opts)
 
-  defp do_run(ntx_to_send, utxos, opts) do
+  defp do_run(ntx_to_send, utxos, fee_amount, opts) do
     {duration, _result} =
       :timer.tc(fn ->
         # fire async transaction senders
-        manager = LoadTest.Common.SenderManager.start_link_all_senders(ntx_to_send, utxos, opts)
+        manager =
+          LoadTest.Common.SenderManager.start_link_all_senders(
+            ntx_to_send,
+            utxos,
+            fee_amount,
+            opts
+          )
 
         # Wait all senders do their job, checker will stop when it happens and stops itself
         wait_for(manager)
@@ -43,8 +49,8 @@ defmodule LoadTest.Common.Runner do
     :ok
   end
 
-  defp do_profiled_run(ntx_to_send, utxos, opts) do
-    :fprof.apply(&do_run/3, [ntx_to_send, utxos, opts], procs: [:all])
+  defp do_profiled_run(ntx_to_send, utxos, fee_amount, opts) do
+    :fprof.apply(&do_run/4, [ntx_to_send, utxos, fee_amount, opts], procs: [:all])
     :fprof.profile()
 
     destfile = Path.join(opts[:destdir], "perf_result_profiling_#{:os.system_time(:seconds)}")

priv/perf/apps/load_test/lib/common/sender_manager.ex

@@ -32,25 +32,25 @@ defmodule LoadTest.Common.SenderManager do
   @doc """
   Starts the sender's manager process
   """
-  @spec start_link_all_senders(pos_integer(), list(), keyword()) :: pid
-  def start_link_all_senders(ntx_to_send, utxos, opts) do
-    {:ok, mypid} = GenServer.start_link(__MODULE__, {ntx_to_send, utxos, opts}, name: __MODULE__)
+  @spec start_link_all_senders(pos_integer(), list(), pos_integer(), keyword()) :: pid
+  def start_link_all_senders(ntx_to_send, utxos, fee_amount, opts) do
+    {:ok, mypid} = GenServer.start_link(__MODULE__, {ntx_to_send, utxos, fee_amount, opts}, name: __MODULE__)
     mypid
   end
 
   @doc """
   Starts sender processes
   """
-  @spec init({pos_integer(), list(), keyword()}) :: {:ok, map()}
-  def init({ntx_to_send, utxos, opts}) do
+  @spec init({pos_integer(), list(), pos_integer(), keyword()}) :: {:ok, map()}
+  def init({ntx_to_send, utxos, fee_amount, opts}) do
     Process.flag(:trap_exit, true)
     _ = Logger.debug("init called with utxos: #{inspect(length(utxos))}, ntx_to_send: #{inspect(ntx_to_send)}")
 
     senders =
       utxos
       |> Enum.with_index(1)
       |> Enum.map(fn {utxo, seqnum} ->
-        {:ok, pid} = LoadTest.Common.SenderServer.start_link({seqnum, utxo, ntx_to_send, opts})
+        {:ok, pid} = LoadTest.Common.SenderServer.start_link({seqnum, utxo, ntx_to_send, fee_amount, opts})
         {seqnum, pid}
       end)
 
@@ -63,6 +63,7 @@ defmodule LoadTest.Common.SenderManager do
     {:ok,
      %{
        senders: senders,
+       fee_amount: fee_amount,
        events: [],
        block_times: [],
        goal: ntx_to_send,

priv/perf/apps/load_test/lib/common/sender_server.ex

@@ -19,7 +19,6 @@ defmodule LoadTest.Common.SenderServer do
 
   # Waiting time (in milliseconds) before unsuccessful Tx submission is retried.
   @tx_retry_waiting_time_ms 333
-  @fees_amount 1
 
   use GenServer
 
@@ -46,6 +45,7 @@ defmodule LoadTest.Common.SenderServer do
     :seqnum,
     :ntx_to_send,
     :spender,
+    :fee_amount,
     # {blknum, txindex, oindex, amount}, @see %LastTx above
     :last_tx,
     :child_chain_url,
@@ -80,7 +80,7 @@ defmodule LoadTest.Common.SenderServer do
       defaults to `true`
   """
   @spec init({pos_integer(), map(), pos_integer(), keyword()}) :: {:ok, state()}
-  def init({seqnum, utxo, ntx_to_send, opts}) do
+  def init({seqnum, utxo, ntx_to_send, fee_amount, opts}) do
     defaults = [randomized: true]
     opts = Keyword.merge(defaults, opts)
 
@@ -90,7 +90,7 @@ defmodule LoadTest.Common.SenderServer do
       )
 
     send(self(), :do)
-    {:ok, init_state(seqnum, utxo, ntx_to_send, opts)}
+    {:ok, init_state(seqnum, utxo, ntx_to_send, fee_amount, opts)}
   end
 
   @doc """
@@ -120,7 +120,7 @@ defmodule LoadTest.Common.SenderServer do
 
   defp prepare_and_submit_tx(state) do
     to_spend = 1
-    new_amount = state.last_tx.amount - to_spend - @fees_amount
+    new_amount = state.last_tx.amount - to_spend - state.fee_amount
 
     recipient =
       if state.randomized do
@@ -186,14 +186,16 @@ defmodule LoadTest.Common.SenderServer do
   end
 
   #   Generates module's initial state
-  @spec init_state(pos_integer(), map(), pos_integer(), keyword()) :: __MODULE__.state()
-  defp init_state(seqnum, %{owner: spender, utxo_pos: utxo_pos, amount: amount}, ntx_to_send, opts) do
+  @spec init_state(pos_integer(), map(), pos_integer(), pos_integer(), keyword()) :: __MODULE__.state()
+  defp init_state(seqnum, utxo, ntx_to_send, fee_amount, opts) do
+    %{owner: spender, utxo_pos: utxo_pos, amount: amount} = utxo
     {:ok, %ExPlasma.Utxo{blknum: blknum, txindex: txindex, oindex: oindex}} = ExPlasma.Utxo.new(utxo_pos)
 
     %__MODULE__{
       seqnum: seqnum,
       ntx_to_send: ntx_to_send,
       spender: spender,
+      fee_amount: fee_amount,
       last_tx: %LastTx{
         # initial state takes deposited value, put there on :init
         blknum: blknum,