diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000..98705754c5e5e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +# Ignored packages +packages/docs +packages/gas-profiler +packages/integration-test-utils +packages/test-rollup-workflow + +# Ignored dirs +docker +db + diff --git a/.github/workflows/dev-ecr-deploy.yml b/.github/workflows/dev-ecr-deploy.yml index 169137f427f99..a90614003fa24 100644 --- a/.github/workflows/dev-ecr-deploy.yml +++ b/.github/workflows/dev-ecr-deploy.yml @@ -15,14 +15,6 @@ jobs: - name: Setup node uses: actions/setup-node@v1 - - name: Install Dependencies - run: yarn install - - - name: Build - run: | - yarn clean - yarn build - - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -40,7 +32,7 @@ jobs: ECR_REPOSITORY: optimism/rollup-microservices IMAGE_TAG: latest run: | - docker build -f Dockerfile.microservices -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker build -f Dockerfile -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - name: Stop existing dev-rollup-microservices ECS task to auto-start task with new image diff --git a/.github/workflows/synthetix-prod-ecr-deploy.yml b/.github/workflows/synthetix-prod-ecr-deploy.yml deleted file mode 100644 index 72955d6b2e21c..0000000000000 --- a/.github/workflows/synthetix-prod-ecr-deploy.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Build & Tag Container, Push to ECR, Deploy to prod - -on: - push: - branches: - - synthetix-prod - -jobs: - build: - name: Build, Tag & push to ECR, Deploy to PROD - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Setup node - uses: actions/setup-node@v1 - - - name: Install Dependencies - run: yarn install - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_CI_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_CI_USER_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - - name: Build, tag, and push Geth to ECR - env: - AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_CI_AWS_ACCOUNT_ID }} - run: ./docker/publish-geth-container.sh synthetix-prod - - - name: Build, tag, and push Full Node to ECR - env: - AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_CI_AWS_ACCOUNT_ID }} - run: ./docker/publish-rollup-fullnode-container.sh synthetix-prod - - - name: Logout of Amazon ECR - if: always() - run: docker logout ${{ steps.login-ecr.outputs.registry }} diff --git a/.github/workflows/synthetix-uat-ecr-deploy.yml b/.github/workflows/synthetix-uat-ecr-deploy.yml deleted file mode 100644 index e7cbabcb42e55..0000000000000 --- a/.github/workflows/synthetix-uat-ecr-deploy.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Build & Tag Container, Push to ECR, Deploy to UAT - -on: - push: - branches: - - synthetix-uat - -jobs: - build: - name: Build, Tag & push to ECR, Deploy to UAT - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Setup node - uses: actions/setup-node@v1 - - - name: Install Dependencies - run: yarn install - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_CI_USER_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_CI_USER_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - - name: Build, tag, and push Geth to ECR - env: - AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_CI_AWS_ACCOUNT_ID }} - run: ./docker/publish-geth-container.sh synthetix-uat - - - name: Build, tag, and push Full Node to ECR - env: - AWS_ACCOUNT_NUMBER: ${{ secrets.AWS_CI_AWS_ACCOUNT_ID }} - run: ./docker/publish-rollup-fullnode-container.sh synthetix-uat - - - - name: Logout of Amazon ECR - if: always() - run: docker logout ${{ steps.login-ecr.outputs.registry }} diff --git a/.gitignore b/.gitignore index ad21c18e1a7c3..c9c313e114f13 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,5 @@ npm-debug.log /packages/docs/src/core/env/ # Local Dockerfiles -docker-compose.*.mine.yml \ No newline at end of file +*.local.yml +*.mine.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6d7e0dadf0d60..efb321809435e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,18 @@ FROM node:11 WORKDIR /server COPY . /server + RUN yarn +RUN yarn clean +RUN yarn build # Copy live env config updates file to /server so that it may be updated while running. COPY ./packages/rollup-core/config/env_var_updates.config /server -WORKDIR /server/packages/rollup-full-node +WORKDIR /server/packages/rollup-services + +# This is required for the wait_for_postgres script +RUN apt-get update +RUN apt-get install -y postgresql-client -EXPOSE 8545 -ENTRYPOINT [ "bash", "./exec/wait-for-nodes.sh", "yarn", "run", "server:fullnode:debug" ] +ENTRYPOINT [ "bash", "./exec/wait_for_postgres_and_geth.sh", "yarn", "run", "services" ] diff --git a/Dockerfile.dev b/Dockerfile.dev index 67f857340d8d6..ece5eaf77fd20 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,5 +1,11 @@ FROM node:11 -WORKDIR /mnt/full-node/packages/rollup-full-node + +# Used to mount the code so image isn't re-built every time code changes +WORKDIR /mnt/monorepo/packages/rollup-services + +# This is required for the wait_for_postgres script +RUN apt-get update +RUN apt-get install -y postgresql-client EXPOSE 8545 -CMD [ "bash", "./exec/wait-for-nodes.sh", "yarn", "run", "server:fullnode:debug" ] +ENTRYPOINT [ "bash", "./exec/build_and_run.sh", "yarn", "run", "services" ] diff --git a/Dockerfile.microservices b/Dockerfile.microservices deleted file mode 100644 index efd3b4d343d5f..0000000000000 --- a/Dockerfile.microservices +++ /dev/null @@ -1,16 +0,0 @@ -FROM node:11 - -WORKDIR /server -COPY . /server -RUN yarn - -# Copy live env config updates file to /server so that it may be updated while running. -COPY ./packages/rollup-core/config/env_var_updates.config /server - -WORKDIR /server/packages/rollup-services - -# This is required for the wait_for_postgres script -RUN apt-get update -RUN apt-get install -y postgresql-client - -ENTRYPOINT [ "bash", "./exec/wait_for_postgres_and_geth.sh", "yarn", "run", "services" ] diff --git a/aws/README.md b/aws/README.md deleted file mode 100644 index 160fb148f3f07..0000000000000 --- a/aws/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# AWS -This documentation contains all of the necessary information to build and deploy new images to an existing environment or to create an entirely new environment in AWS. - -# Building and Registering Containers -There are a few different docker containers in this repo that get built and published to AWS's registry. -As of right now they are defined by the Dockerfiles at: -* `docker/geth/Dockerfile` published (here) - * Manually built and deployed -* `Dockerfile` - * Automatically built and deployed to shared dev environment on merge to `master` - * Note: - * This is for the rollup-full-node but at the base directory for easy dependency inclusion - * Currently all source code must be built _outside_ of Docker, ahead of doing a docker build. - -Here are the steps for publishing new version to ECR: -* `docker/geth/Dockerfile` click on "View push commands" [here](https://us-east-2.console.aws.amazon.com/ecr/repositories/optimism/geth/?region=us-east-2) -* `Dockerfile` click on "View push commands" [here](https://us-east-2.console.aws.amazon.com/ecr/repositories/optimism/rollup-full-node/?region=us-east-2) - - -# Creating an AWS ECS Environment -The contents of this directory can be used to deploy a fully-functional Full Node to AWS. - -Below are some instructions on how to do so. For more info, the instructions below loosely follow [this tutorial](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-cli-tutorial-ec2.html). - -## Prerequisites -AWS: -* Set up an AWS Account, Access Key & Secret, and keypair -* Install the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) -* Install the [AWS ECS CLI](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ECS_CLI_installation.html) - -Other: -* Install [Docker](https://docs.docker.com/docker-for-mac/install/) -## Steps - -### 1) Configure the Amazon ECS CLI -1. Create a cluster configuration: - ``` - ecs-cli configure --cluster dev-full-node --default-launch-type EC2 --config-name dev-full-node-config --region us-east-2 - ``` - Note: choose an appropriate cluster name and have the config name derived from it. - -2. Create a profile to use to create the environment - ``` - ecs-cli configure profile --access-key --secret-key --profile-name dev-full-node-profile - ``` - -### 2) Create the Cluster -``` -ecs-cli up --keypair --capability-iam --size 1 --instance-type t3.micro --cluster-config dev-full-node-config --ecs-profile dev-full-node-profile --port 8545 --security-group --vpc --subnets -``` -Note: -* `size` is the number of ECS instances to create -* `instance-type` can be bumped up for a more powerful environment. More info [here](https://aws.amazon.com/ec2/instance-types/) -* If you haven't already, you'll need to create a Security Group. This is not individual to you, so see if your organization already has a suitable one. - - -This may take a few minutes to finish. The result will be a fully provisioned EC2 instance on which your service/task will be deployed. - -### 3) Choose the appropriate `docker-compose.yml` and `ecs-params.yml` -For the rest of the commands, you'll need to be in a directory with a `docker-compose.yml` and an `ecs-params.yml`. These will define the service(s) you are going to create. Change to the appropriate directory or create one of your own basing them off of an existing one. - -### 4) Deploy Tasks to Cluster -``` -ecs-cli compose up --create-log-groups --cluster-config dev-full-node-config --ecs-profile dev-full-node-profile -``` - -This will just start up the task(s) under the appropriate cluster. Ultimately we want a service to manage our task(s), but we don't want to do that until we know our tasks work. Make sure your tasks are functioning properly by checking their status and possibly even logging into CloudWatch and looking at the logs. - -To check the status of your task(s), run: -``` -ecs-cli ps --cluster-config dev-full-node-config --ecs-profile dev-full-node-profile -``` - -### 5) Create the ECS Service -First, kill the PoC tasks: -``` -ecs-cli compose down --cluster-config dev-full-node-config --ecs-profile dev-full-node-profile -``` - -Now create the service: -``` -ecs-cli compose service up --cluster-config dev-full-node-config --ecs-profile dev-full-node-profile -``` - -## Volumes -Right now volumes needed by the various containers are configured to be locally stored on the EC2 instance on which the containers are run and automatically created if not present. Eventually we will want to move to a more redundant form of storage (like EBS mounts), but this is fine for now. - -For now, if you want to modify/delete the data in an environment you will need to -* `ssh` into the EC2 instance running the containers -* Find the volume(s) you would like to modify/delete (they are located at `/var/lib/docker/volumes`) -* Modify/Delete them as necessary - * IMPORTANT: This will likely mess up any running tasks, so make sure to kill the task before - * Also note that the Service will auto-replace your tasks, so if you can't do this quickly, disable that feature during your maintenance. \ No newline at end of file diff --git a/aws/dev/full-node/docker-compose.yml b/aws/dev/full-node/docker-compose.yml deleted file mode 100644 index ce5f8c6254c1c..0000000000000 --- a/aws/dev/full-node/docker-compose.yml +++ /dev/null @@ -1,62 +0,0 @@ -version: "3" -services: - rollup-full-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:latest - volumes: - - full-node-data:/mnt/full-node:rw - - l1-node-data:/mnt/l1-node:rw - - l2-node-data:/mnt/l2-node:rw - ports: - - 8545:8545 - environment: - - OPCODE_WHITELIST_MASK - - L1_SEQUENCER_MNEMONIC - - L2_TO_L1_MESSAGE_RECEIVER_CONTRACT_ADDRESS - - FINALITY_DELAY_IN_BLOCKS - - L2_RPC_SERVER_HOST - - L2_RPC_SERVER_PORT - - L2_WALLET_MNEMONIC - - L2_WALLET_PRIVATE_KEY_PATH=/mnt/l2-node/private_key.txt - - LOCAL_L1_NODE_PORT - - LOCAL_L1_NODE_PERSISTENT_DB_PATH=/mnt/l1-node - - L2_NODE_WEB3_URL=http://0.0.0.0:9545/ - - logging: - driver: awslogs - options: - awslogs-group: rollup-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-rpc-server - - geth_l2: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/geth:latest - volumes: - - l2-node-data:/mnt/l2-node/l2:rw - environment: - - VOLUME_PATH=/mnt/l2-node/l2 - - HOSTNAME=0.0.0.0 - - PORT=9545 - - NETWORK_ID=108 - - KEYSTORE_PATH_SUFFIX=/keystore - - SEALER_PRIVATE_KEY_PATH_SUFFIX=/sealer_private_key.txt - - PRIVATE_KEY_PATH_SUFFIX=/private_key.txt - - ADDRESS_PATH_SUFFIX=/address.txt - - SEALER_ADDRESS_PATH_SUFFIX=/sealer_address.txt - - INITIAL_BALANCE=0x200000000000000000000000000000000000000000000000000000000000000 - - GENISIS_PATH=etc/rollup-fullnode.json - - SETUP_RUN_PATH_SUFFIX=/setup_run.txt - ports: - - 9545:9545 - - logging: - driver: awslogs - options: - awslogs-group: rollup-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-node - -volumes: - full-node-data: - l1-node-data: - l2-node-data: - diff --git a/aws/dev/full-node/ecs-params.yml b/aws/dev/full-node/ecs-params.yml deleted file mode 100644 index d02645a1f1b18..0000000000000 --- a/aws/dev/full-node/ecs-params.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: 1 -task_definition: - services: - rollup-full-node: - cpu_shares: 25 - mem_limit: 524288000 - geth_l2: - cpu_shares: 75 - mem_limit: 1523288000 - # This is all local for now -- eventually will change - ecs_network_mode: host - docker_volumes: - - name: l1-node-data - scope: shared - autoprovision: true - driver: 'local' - - name: l2-node-data - scope: shared - autoprovision: true - driver: 'local' - - name: full-node-data - scope: shared - autoprovision: true - driver: 'local' - diff --git a/aws/synthetix/dev/README.md b/aws/synthetix/dev/README.md deleted file mode 100644 index 8e261a4978590..0000000000000 --- a/aws/synthetix/dev/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Deploying Web RPC Server to Synthetix dev - -## Prerequisites -See prerequisites from parent AWS directory. - -## Steps - -### 1) Configure the Amazon ECS CLI -1. Create a cluster configuration: - ``` - ecs-cli configure --cluster synthetix-dev --default-launch-type EC2 --config-name synthetix-dev-config --region us-east-2 - ``` - -2. Create a profile to use to create the environment - ``` - ecs-cli configure profile --access-key --secret-key --profile-name synthetix-dev-profile - ``` - -### 2) Create the Cluster - ``` - ecs-cli up --keypair optimism-dev --capability-iam --size 1 --instance-type t2.medium --cluster-config synthetix-dev-config --ecs-profile synthetix-dev-profile --port 8545 --security-group --vpc --subnets - ``` - -This may take a few minutes to finish. The result will be a fully provisioned EC2 instance on which your service/task will be deployed. - -### 3) Choose the appropriate `docker-compose.yml` and `ecs-params.yml` -For the rest of the commands, you'll need to be in this directory to use the `docker-compose.yml` and an `ecs-params.yml`. -Make any necessary changes now. - -### 4) Deploy Service & Task to Cluster - ``` - ecs-cli compose --project-name synthetix-dev service up --vpc --cluster-config synthetix-dev-config --ecs-profile synthetix-dev-profile --create-log-groups - ``` - - -# Redeploying after first deploy -Repeat step #4 above diff --git a/aws/synthetix/dev/full-node/docker-compose.yml b/aws/synthetix/dev/full-node/docker-compose.yml deleted file mode 100644 index 525d039d28a90..0000000000000 --- a/aws/synthetix/dev/full-node/docker-compose.yml +++ /dev/null @@ -1,165 +0,0 @@ -version: "3" -services: - router: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:latest - ports: - - 8545:8545 - environment: - - IS_ROUTING_SERVER=1 - - STARTUP_WAIT_TIMEOUT=30 - - L2_RPC_SERVER_HOST - - L2_RPC_SERVER_PORT=8545 - - TRANSACTION_NODE_URL=http://0.0.0.0:8546 - - READ_ONLY_NODE_URL=http://0.0.0.0:8547 - - REQUEST_LIMIT_PERIOD_MILLIS=1000 - - MAX_NON_TRANSACTION_REQUESTS_PER_UNIT_TIME=10 - - MAX_TRANSACTIONS_PER_UNIT_TIME=5 -# These are just so it waits for the downstream nodes to start up - - L2_NODE_WEB3_URL=http://0.0.0.0:8546 - - L1_NODE_WEB3_URL=http://0.0.0.0:8547 - logging: - driver: awslogs - options: - awslogs-group: synthetix-dev-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-router - - transaction-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:latest - volumes: - - full-node-data:/mnt/full-node:rw - - l1-node-data:/mnt/l1-node:rw - - l2-node-data:/mnt/l2-node:rw - ports: - - 8546:8546 - environment: - - IS_TRANSACTION_NODE=1 - - STARTUP_WAIT_TIMEOUT=30 - - CLEAR_DATA_KEY - - OPCODE_WHITELIST_MASK - - L1_SEQUENCER_MNEMONIC - - L2_TO_L1_MESSAGE_RECEIVER_CONTRACT_ADDRESS - - FINALITY_DELAY_IN_BLOCKS - - L2_RPC_SERVER_HOST=0.0.0.0 - - L2_RPC_SERVER_PORT=8546 - - L2_RPC_SERVER_PERSISTENT_DB_PATH=/mnt/full-node/level - - L2_WALLET_PRIVATE_KEY=0x29f3edee0ad3abf8e2699402e0e28cd6492c9be7eaab00d732a791c33552f797 - - LOCAL_L1_NODE_PORT - - LOCAL_L1_NODE_PERSISTENT_DB_PATH=/mnt/l1-node - - L2_NODE_WEB3_URL=http://0.0.0.0:9545/ - - logging: - driver: awslogs - options: - awslogs-group: synthetix-dev-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-transaction-node - - read-only-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:latest - ports: - - 8547:8547 - environment: - - IS_READ_ONLY_NODE=1 - - STARTUP_WAIT_TIMEOUT=30 - - L2_RPC_SERVER_HOST=0.0.0.0 - - L2_RPC_SERVER_PORT=8547 - - L2_EXECUTION_MANAGER_ADDRESS=0x6454c9d69a4721feba60e26a367bd4d56196ee7c - - L2_WALLET_PRIVATE_KEY=0x29f3edee0ad3abf8e2699402e0e28cd6492c9be7eaab00d732a791c33552f797 - - L2_NODE_WEB3_URL=http://0.0.0.0:9545/ - logging: - driver: awslogs - options: - awslogs-group: synthetix-dev-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-read-only-node - - l2-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/geth:latest - volumes: - - l2-node-data:/mnt/l2-node/l2:rw - environment: - - CLEAR_DATA_KEY - - VOLUME_PATH=/mnt/l2-node/l2 - - HOSTNAME=0.0.0.0 - - PORT=9545 - - NETWORK_ID=108 - ports: - - 9545:9545 - - logging: - driver: awslogs - options: - awslogs-group: synthetix-dev-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-node - - graph-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/the-graph:latest - ports: - - '8000:8000' - - '8001:8001' - - '8020:8020' - - '8030:8030' - - '8040:8040' - environment: - postgres_host: 0.0.0.0:5432 - postgres_user: graph-node - postgres_pass: let-me-in - postgres_db: graph-node - ipfs: '0.0.0.0:5001' - ethereum: 'ovm:http://0.0.0.0:8545' - RUST_LOG: info - STARTUP_WAIT_TIMEOUT: 30 - OVM_URL_WITH_PORT: 'http://0.0.0.0:8545' - volumes: - - postgres-data:/data/postgres - - ipfs-data:/data/ipfs - - logging: - driver: awslogs - options: - awslogs-group: synthetix-dev-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph - - ipfs: - image: ipfs/go-ipfs:v0.4.23 - ports: - - '5001:5001' - volumes: - - ipfs-data:/data/ipfs - - logging: - driver: awslogs - options: - awslogs-group: synthetix-dev-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph-ipfs - - postgres: - image: postgres - ports: - - '5432:5432' - command: ["postgres", "-cshared_preload_libraries=pg_stat_statements"] - environment: - POSTGRES_USER: graph-node - POSTGRES_PASSWORD: let-me-in - POSTGRES_DB: graph-node - volumes: - - postgres-data:/var/lib/postgresql/data - - logging: - driver: awslogs - options: - awslogs-group: synthetix-dev-full-node - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph-postgres - -volumes: - full-node-data: - l1-node-data: - l2-node-data: - postgres-data: - ipfs-data: - diff --git a/aws/synthetix/dev/full-node/ecs-params.yml b/aws/synthetix/dev/full-node/ecs-params.yml deleted file mode 100644 index 43976e0aecd59..0000000000000 --- a/aws/synthetix/dev/full-node/ecs-params.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: 1 -task_definition: - services: - router: - cpu_shares: 204 - mem_limit: 262144000 - transaction-node: - cpu_shares: 408 - mem_limit: 524288000 - read-only-node: - cpu_shares: 204 - mem_limit: 262144000 - l2-node: - cpu_shares: 820 - mem_limit: 1597847999 - graph-node: - cpu_shares: 204 - mem_limit: 262144000 - ipfs: - cpu_shares: 102 - mem_limit: 262144000 - postgres: - cpu_shares: 102 - mem_limit: 262144000 - - - # This is all local for now -- eventually will change - ecs_network_mode: host - docker_volumes: - - name: l1-node-data - scope: shared - autoprovision: true - driver: 'local' - - name: l2-node-data - scope: shared - autoprovision: true - driver: 'local' - - name: full-node-data - scope: shared - autoprovision: true - driver: 'local' - - name: postgres-data - scope: shared - autoprovision: true - driver: 'local' - - name: ipfs-data - scope: shared - autoprovision: true - driver: 'local' diff --git a/aws/synthetix/prod/geth/README.md b/aws/synthetix/prod/geth/README.md deleted file mode 100644 index ba2a3273eaf0c..0000000000000 --- a/aws/synthetix/prod/geth/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Deploying L2 Geth node to Synthetix PROD - -## Prerequisites -See prerequisites from parent AWS directory. - -## Steps - -### 1) Configure the Amazon ECS CLI -1. Create a cluster configuration: - ``` - ecs-cli configure --cluster synthetix-prod-geth --default-launch-type EC2 --config-name synthetix-prod-geth-config --region us-east-2 - ``` - -2. Create a profile to use to create the environment - ``` - ecs-cli configure profile --access-key --secret-key --profile-name synthetix-prod-geth-profile - ``` - -### 2) Create the Cluster - ``` - ecs-cli up --keypair synthetix-prod --capability-iam --size 1 --instance-type c5.4xlarge --cluster-config synthetix-prod-geth-config --ecs-profile synthetix-prod-geth-profile --port 9545 --security-group --vpc --subnets - ``` - -This may take a few minutes to finish. The result will be a fully provisioned EC2 instance on which your service/task will be deployed. - -### 3) Choose the appropriate `docker-compose.yml` and `ecs-params.yml` -For the rest of the commands, you'll need to be in this directory to use the `docker-compose.yml` and an `ecs-params.yml`. -Make any necessary changes now. - -### 4) Deploy Service & Task to Cluster & register service discovery. - ``` - ecs-cli compose --project-name synthetix-prod-geth service up --private-dns-namespace synthetix-prod --vpc --enable-service-discovery --cluster-config synthetix-prod-geth-config --ecs-profile synthetix-prod-geth-profile --create-log-groups - ``` diff --git a/aws/synthetix/prod/geth/docker-compose.yml b/aws/synthetix/prod/geth/docker-compose.yml deleted file mode 100644 index 7ba000d3614fe..0000000000000 --- a/aws/synthetix/prod/geth/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: "3" -services: - geth_l2: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/geth:synthetix-prod - volumes: - - l2-node-data:/mnt/l2-node/l2:rw - environment: - - CLEAR_DATA_KEY - - VOLUME_PATH=/mnt/l2-node/l2 - - HOSTNAME=0.0.0.0 - - PORT=9545 - - NETWORK_ID=108 - ports: - - 9545:9545 - - logging: - driver: awslogs - options: - awslogs-group: sythetix-prod-geth - awslogs-region: us-east-2 - awslogs-stream-prefix: geth - -volumes: - l2-node-data: - diff --git a/aws/synthetix/prod/geth/ecs-params.yml b/aws/synthetix/prod/geth/ecs-params.yml deleted file mode 100644 index 937c9db37ca36..0000000000000 --- a/aws/synthetix/prod/geth/ecs-params.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 1 -task_definition: - services: - geth_l2: - essential: true -# healthcheck: -# test: ["CMD-SHELL", "curl -f -H \"Content-Type: application/json\" -d '{\"jsonrpc\": \"2.0\", \"id\": 9999999, \"method\": \"net_version\"}' http://localhost:9545/ || exit 1"] -# interval: 10s -# timeout: 5s -# retries: 3 -# start_period: 10s - mem_limit: 32653700000 - cpu_shares: 16384 - - - ecs_network_mode: awsvpc - docker_volumes: - - name: l2-node-data - scope: shared - autoprovision: true - driver: 'local' - -run_params: - network_configuration: - awsvpc_configuration: - subnets: - - - security_groups: - - - service_discovery_service: - name: 'geth' - description: 'Synthetix PROD Geth Service Discovery' - dns_config: - type: 'A' - ttl: 60 diff --git a/aws/synthetix/prod/web/README.md b/aws/synthetix/prod/web/README.md deleted file mode 100644 index bd45f3461b8ac..0000000000000 --- a/aws/synthetix/prod/web/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Deploying Web RPC Server to Synthetix PROD - -## Prerequisites -See prerequisites from parent AWS directory. - -## Steps - -### 1) Configure the Amazon ECS CLI -1. Create a cluster configuration: - ``` - ecs-cli configure --cluster synthetix-prod-web --default-launch-type EC2 --config-name synthetix-prod-web-config --region us-east-2 - ``` - -2. Create a profile to use to create the environment - ``` - ecs-cli configure profile --access-key --secret-key --profile-name synthetix-prod-web-profile - ``` - -### 2) Create the Cluster - ``` - ecs-cli up --keypair synthetix-prod --capability-iam --size 1 --instance-type c5.xlarge --cluster-config synthetix-prod-web-config --ecs-profile synthetix-prod-web-profile --port 8545 --security-group --vpc --subnets - ``` - -This may take a few minutes to finish. The result will be a fully provisioned EC2 instance on which your service/task will be deployed. - -### 3) Choose the appropriate `docker-compose.yml` and `ecs-params.yml` -For the rest of the commands, you'll need to be in this directory to use the `docker-compose.yml` and an `ecs-params.yml`. -Make any necessary changes now. - -### 4) Deploy Service & Task to Cluster - ``` - ecs-cli compose --project-name synthetix-prod-web service up --vpc --cluster-config synthetix-prod-web-config --ecs-profile synthetix-prod-web-profile --create-log-groups - ``` - - -# Redeploying after first deploy -Repeat step #4 above \ No newline at end of file diff --git a/aws/synthetix/prod/web/docker-compose.yml b/aws/synthetix/prod/web/docker-compose.yml deleted file mode 100644 index 4d15fb11e1439..0000000000000 --- a/aws/synthetix/prod/web/docker-compose.yml +++ /dev/null @@ -1,136 +0,0 @@ -version: "3" -services: - router: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:synthetix-prod - ports: - - 8545:8545 - environment: - - IS_ROUTING_SERVER=1 - - STARTUP_WAIT_TIMEOUT=60 - - L2_RPC_SERVER_HOST - - L2_RPC_SERVER_PORT=8545 - - TRANSACTION_NODE_URL=http://0.0.0.0:8546 - - READ_ONLY_NODE_URL=http://0.0.0.0:8547 - - REQUEST_LIMIT_PERIOD_MILLIS=1000 - - MAX_NON_TRANSACTION_REQUESTS_PER_UNIT_TIME=50 - - MAX_TRANSACTIONS_PER_UNIT_TIME=10 -# These are just so it waits for the downstream nodes to start up - - L2_NODE_WEB3_URL=http://0.0.0.0:8546 - - L1_NODE_WEB3_URL=http://0.0.0.0:8547 - - logging: - driver: awslogs - options: - awslogs-group: synthetix-prod-web - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-router - - transaction-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:synthetix-prod - volumes: - - full-node-data:/mnt/full-node:rw - ports: - - 8546:8546 - environment: - - IS_TRANSACTION_NODE=1 - - CLEAR_DATA_KEY - - STARTUP_WAIT_TIMEOUT=60 - - NO_L1_NODE=1 - - L2_RPC_SERVER_PERSISTENT_DB_PATH=/mnt/full-node/level - - L2_WALLET_PRIVATE_KEY=0x29f3edee0ad3abf8e2699402e0e28cd6492c9be7eaab00d732a791c33552f797 - - L2_NODE_WEB3_URL=http://synthetix-prod-geth.synthetix-prod:9545 - - L2_RPC_SERVER_HOST=0.0.0.0 - - L2_RPC_SERVER_PORT=8546 - - logging: - driver: awslogs - options: - awslogs-group: synthetix-prod-web - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-transaction-node - - read-only-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:synthetix-prod - ports: - - 8547:8547 - environment: - - IS_READ_ONLY_NODE=1 - - STARTUP_WAIT_TIMEOUT=30 - - L2_RPC_SERVER_HOST=0.0.0.0 - - L2_RPC_SERVER_PORT=8547 - - L2_EXECUTION_MANAGER_ADDRESS=0x6454c9d69a4721feba60e26a367bd4d56196ee7c - - L2_WALLET_PRIVATE_KEY=0x29f3edee0ad3abf8e2699402e0e28cd6492c9be7eaab00d732a791c33552f797 - - L2_NODE_WEB3_URL=http://synthetix-prod-geth.synthetix-prod:9545 - logging: - driver: awslogs - options: - awslogs-group: synthetix-prod-web - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-read-only-node - - graph-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/the-graph:latest - ports: - - '8000:8000' - - '8001:8001' - - '8020:8020' - - '8030:8030' - - '8040:8040' - environment: - postgres_host: 0.0.0.0:5432 - postgres_user: graph-node - postgres_pass: let-me-in - postgres_db: graph-node - ipfs: '0.0.0.0:5001' - ethereum: 'ovm:http://0.0.0.0:8546' - RUST_LOG: info - STARTUP_WAIT_TIMEOUT: 30 - OVM_URL_WITH_PORT: 'http://0.0.0.0:8546' - volumes: - - postgres-data:/data/postgres - - ipfs-data:/data/ipfs - - logging: - driver: awslogs - options: - awslogs-group: synthetix-prod-web - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph - - ipfs: - image: ipfs/go-ipfs:v0.4.23 - ports: - - '5001:5001' - volumes: - - ipfs-data:/data/ipfs - - logging: - driver: awslogs - options: - awslogs-group: synthetix-prod-web - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph-ipfs - - postgres: - image: postgres - ports: - - '5432:5432' - command: ["postgres", "-cshared_preload_libraries=pg_stat_statements"] - environment: - POSTGRES_USER: graph-node - POSTGRES_PASSWORD: let-me-in - POSTGRES_DB: graph-node - volumes: - - postgres-data:/var/lib/postgresql/data - - logging: - driver: awslogs - options: - awslogs-group: synthetix-prod-web - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph-postgres - -volumes: - full-node-data: - postgres-data: - ipfs-data: \ No newline at end of file diff --git a/aws/synthetix/prod/web/ecs-params.yml b/aws/synthetix/prod/web/ecs-params.yml deleted file mode 100644 index 76b05bcd57e73..0000000000000 --- a/aws/synthetix/prod/web/ecs-params.yml +++ /dev/null @@ -1,64 +0,0 @@ -version: 1 -task_definition: - services: - router: - essential: true - healthcheck: - test: ["CMD-SHELL", "curl -f -H \"Content-Type: application/json\" -d '{\"jsonrpc\": \"2.0\", \"id\": 9999999, \"method\": \"net_version\"}' http://localhost:8545/ || exit 1"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 5s - # 15% - cpu_shares: 409 - mem_limit: 1.125GB - transaction-node: - essential: true - healthcheck: - test: ["CMD-SHELL", "curl -f -H \"Content-Type: application/json\" -d '{\"jsonrpc\": \"2.0\", \"id\": 9999999, \"method\": \"net_version\"}' http://localhost:8546/ || exit 1"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 5s - # 40% - cpu_shares: 1638 - mem_limit: 2GB # Was 3GB but ECS says not enough memory - read-only-node: - essential: true - healthcheck: - test: ["CMD-SHELL", "curl -f -H \"Content-Type: application/json\" -d '{\"jsonrpc\": \"2.0\", \"id\": 9999999, \"method\": \"net_version\"}' http://localhost:8547/ || exit 1"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 5s - # 15% - cpu_shares: 818 - mem_limit: 1.125GB - graph-node: - # 15% - cpu_shares: 614 - mem_limit: 1.125GB - ipfs: - # 5% - cpu_shares: 204 - mem_limit: 0.375GB - postgres: - # 10% - cpu_shares: 409 - mem_limit: 0.75GB - - - ecs_network_mode: host - docker_volumes: - - name: full-node-data - scope: shared - autoprovision: true - driver: 'local' - - name: postgres-data - scope: shared - autoprovision: true - driver: 'local' - - name: ipfs-data - scope: shared - autoprovision: true - driver: 'local' diff --git a/aws/synthetix/uat/geth/README.md b/aws/synthetix/uat/geth/README.md deleted file mode 100644 index af158f6f14984..0000000000000 --- a/aws/synthetix/uat/geth/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Deploying L2 Geth node to Synthetix UAT - -## Prerequisites -See prerequisites from parent AWS directory. - -## Steps - -### 1) Configure the Amazon ECS CLI -1. Create a cluster configuration: - ``` - ecs-cli configure --cluster synthetix-uat-geth --default-launch-type EC2 --config-name synthetix-uat-geth-config --region us-east-2 - ``` - -2. Create a profile to use to create the environment - ``` - ecs-cli configure profile --access-key --secret-key --profile-name synthetix-uat-geth-profile - ``` - -### 2) Create the Cluster - ``` - ecs-cli up --keypair synthetix-uat --capability-iam --size 1 --instance-type c5.4xlarge --cluster-config synthetix-uat-geth-config --ecs-profile synthetix-uat-geth-profile --port 9545 --security-group --vpc --subnets - ``` - -This may take a few minutes to finish. The result will be a fully provisioned EC2 instance on which your service/task will be deployed. - -### 3) Choose the appropriate `docker-compose.yml` and `ecs-params.yml` -For the rest of the commands, you'll need to be in this directory to use the `docker-compose.yml` and an `ecs-params.yml`. -Make any necessary changes now. - -### 4) Deploy Service & Task to Cluster & register service discovery. - ``` - ecs-cli compose --project-name synthetix-uat-geth service up --private-dns-namespace synthetix-uat --vpc --enable-service-discovery --cluster-config synthetix-uat-geth-config --ecs-profile synthetix-uat-geth-profile --create-log-groups - ``` diff --git a/aws/synthetix/uat/geth/docker-compose.yml b/aws/synthetix/uat/geth/docker-compose.yml deleted file mode 100644 index a9c398055126b..0000000000000 --- a/aws/synthetix/uat/geth/docker-compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -version: "3" -services: - geth_l2: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/geth:synthetix-uat - volumes: - - l2-node-data:/mnt/l2-node/l2:rw - environment: - - CLEAR_DATA_KEY - - VOLUME_PATH=/mnt/l2-node/l2 - - HOSTNAME=0.0.0.0 - - PORT=9545 - - NETWORK_ID=108 - ports: - - 9545:9545 - - logging: - driver: awslogs - options: - awslogs-group: sythetix-uat-geth - awslogs-region: us-east-2 - awslogs-stream-prefix: geth - -volumes: - l2-node-data: - diff --git a/aws/synthetix/uat/geth/ecs-params.yml b/aws/synthetix/uat/geth/ecs-params.yml deleted file mode 100644 index 6bc92d817fbbf..0000000000000 --- a/aws/synthetix/uat/geth/ecs-params.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 1 -task_definition: - services: - geth_l2: - essential: true -# healthcheck: -# test: ["CMD-SHELL", "curl -f -H \"Content-Type: application/json\" -d '{\"jsonrpc\": \"2.0\", \"id\": 9999999, \"method\": \"net_version\"}' http://localhost:9545/ || exit 1"] -# interval: 10s -# timeout: 5s -# retries: 3 -# start_period: 10s - mem_limit: 32653700000 - cpu_shares: 16384 - - - ecs_network_mode: awsvpc - docker_volumes: - - name: l2-node-data - scope: shared - autoprovision: true - driver: 'local' - -run_params: - network_configuration: - awsvpc_configuration: - subnets: - - - security_groups: - - - service_discovery_service: - name: 'geth' - description: 'Synthetix UAT Geth Service Discovery' - dns_config: - type: 'A' - ttl: 60 diff --git a/aws/synthetix/uat/web/README.md b/aws/synthetix/uat/web/README.md deleted file mode 100644 index 97e22aeaf31b6..0000000000000 --- a/aws/synthetix/uat/web/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# Deploying Web RPC Server to Synthetix UAT - -## Prerequisites -See prerequisites from parent AWS directory. - -## Steps - -### 1) Configure the Amazon ECS CLI -1. Create a cluster configuration: - ``` - ecs-cli configure --cluster synthetix-uat-web --default-launch-type EC2 --config-name synthetix-uat-web-config --region us-east-2 - ``` - -2. Create a profile to use to create the environment - ``` - ecs-cli configure profile --access-key --secret-key --profile-name synthetix-uat-web-profile - ``` - -### 2) Create the Cluster - ``` - ecs-cli up --keypair synthetix-uat --capability-iam --size 1 --instance-type c5.xlarge --cluster-config synthetix-uat-web-config --ecs-profile synthetix-uat-web-profile --port 8545 --security-group --vpc --subnets - ``` - -This may take a few minutes to finish. The result will be a fully provisioned EC2 instance on which your service/task will be deployed. - -### 3) Choose the appropriate `docker-compose.yml` and `ecs-params.yml` -For the rest of the commands, you'll need to be in this directory to use the `docker-compose.yml` and an `ecs-params.yml`. -Make any necessary changes now. - -### 4) Deploy Service & Task to Cluster - ``` - ecs-cli compose --project-name synthetix-uat-web service up --vpc --cluster-config synthetix-uat-web-config --ecs-profile synthetix-uat-web-profile --create-log-groups - ``` - - -# Redeploying after first deploy -Repeat step #4 above \ No newline at end of file diff --git a/aws/synthetix/uat/web/docker-compose.yml b/aws/synthetix/uat/web/docker-compose.yml deleted file mode 100644 index b05dfc6ac2579..0000000000000 --- a/aws/synthetix/uat/web/docker-compose.yml +++ /dev/null @@ -1,136 +0,0 @@ -version: "3" -services: - router: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:synthetix-uat - ports: - - 8545:8545 - environment: - - IS_ROUTING_SERVER=1 - - STARTUP_WAIT_TIMEOUT=60 - - L2_RPC_SERVER_HOST - - L2_RPC_SERVER_PORT=8545 - - TRANSACTION_NODE_URL=http://0.0.0.0:8546 - - READ_ONLY_NODE_URL=http://0.0.0.0:8547 - - REQUEST_LIMIT_PERIOD_MILLIS=1000 - - MAX_NON_TRANSACTION_REQUESTS_PER_UNIT_TIME=50 - - MAX_TRANSACTIONS_PER_UNIT_TIME=10 - # These are just so it waits for the downstream nodes to start up - - L2_NODE_WEB3_URL=http://0.0.0.0:8546 - - L1_NODE_WEB3_URL=http://0.0.0.0:8547 - - logging: - driver: awslogs - options: - awslogs-group: synthetix-uat-web - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-router - - transaction-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:synthetix-uat - volumes: - - full-node-data:/mnt/full-node:rw - ports: - - 8546:8546 - environment: - - IS_TRANSACTION_NODE=1 - - CLEAR_DATA_KEY - - STARTUP_WAIT_TIMEOUT=60 - - NO_L1_NODE=1 - - L2_RPC_SERVER_PERSISTENT_DB_PATH=/mnt/full-node/level - - L2_WALLET_PRIVATE_KEY=0x29f3edee0ad3abf8e2699402e0e28cd6492c9be7eaab00d732a791c33552f797 - - L2_NODE_WEB3_URL=http://synthetix-uat-geth.synthetix-uat:9545 - - L2_RPC_SERVER_HOST=0.0.0.0 - - L2_RPC_SERVER_PORT=8546 - - logging: - driver: awslogs - options: - awslogs-group: synthetix-uat-web - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-transaction-node - - read-only-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:synthetix-uat - ports: - - 8547:8547 - environment: - - IS_READ_ONLY_NODE=1 - - STARTUP_WAIT_TIMEOUT=30 - - L2_RPC_SERVER_HOST=0.0.0.0 - - L2_RPC_SERVER_PORT=8547 - - L2_EXECUTION_MANAGER_ADDRESS=0x6454c9d69a4721feba60e26a367bd4d56196ee7c - - L2_WALLET_PRIVATE_KEY=0x29f3edee0ad3abf8e2699402e0e28cd6492c9be7eaab00d732a791c33552f797 - - L2_NODE_WEB3_URL=http://synthetix-uat-geth.synthetix-uat:9545 - logging: - driver: awslogs - options: - awslogs-group: synthetix-uat-web - awslogs-region: us-east-2 - awslogs-stream-prefix: l2-read-only-node - - graph-node: - image: .dkr.ecr.us-east-2.amazonaws.com/optimism/the-graph:latest - ports: - - '8000:8000' - - '8001:8001' - - '8020:8020' - - '8030:8030' - - '8040:8040' - environment: - postgres_host: 0.0.0.0:5432 - postgres_user: graph-node - postgres_pass: let-me-in - postgres_db: graph-node - ipfs: '0.0.0.0:5001' - ethereum: 'ovm:http://0.0.0.0:8546' - RUST_LOG: info - STARTUP_WAIT_TIMEOUT: 30 - OVM_URL_WITH_PORT: 'http://0.0.0.0:8546' - volumes: - - postgres-data:/data/postgres - - ipfs-data:/data/ipfs - - logging: - driver: awslogs - options: - awslogs-group: synthetix-uat-web - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph - - ipfs: - image: ipfs/go-ipfs:v0.4.23 - ports: - - '5001:5001' - volumes: - - ipfs-data:/data/ipfs - - logging: - driver: awslogs - options: - awslogs-group: synthetix-uat-web - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph-ipfs - - postgres: - image: postgres - ports: - - '5432:5432' - command: ["postgres", "-cshared_preload_libraries=pg_stat_statements"] - environment: - POSTGRES_USER: graph-node - POSTGRES_PASSWORD: let-me-in - POSTGRES_DB: graph-node - volumes: - - postgres-data:/var/lib/postgresql/data - - logging: - driver: awslogs - options: - awslogs-group: synthetix-uat-web - awslogs-region: us-east-2 - awslogs-stream-prefix: the-graph-postgres - -volumes: - full-node-data: - postgres-data: - ipfs-data: \ No newline at end of file diff --git a/aws/synthetix/uat/web/ecs-params.yml b/aws/synthetix/uat/web/ecs-params.yml deleted file mode 100644 index 76b05bcd57e73..0000000000000 --- a/aws/synthetix/uat/web/ecs-params.yml +++ /dev/null @@ -1,64 +0,0 @@ -version: 1 -task_definition: - services: - router: - essential: true - healthcheck: - test: ["CMD-SHELL", "curl -f -H \"Content-Type: application/json\" -d '{\"jsonrpc\": \"2.0\", \"id\": 9999999, \"method\": \"net_version\"}' http://localhost:8545/ || exit 1"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 5s - # 15% - cpu_shares: 409 - mem_limit: 1.125GB - transaction-node: - essential: true - healthcheck: - test: ["CMD-SHELL", "curl -f -H \"Content-Type: application/json\" -d '{\"jsonrpc\": \"2.0\", \"id\": 9999999, \"method\": \"net_version\"}' http://localhost:8546/ || exit 1"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 5s - # 40% - cpu_shares: 1638 - mem_limit: 2GB # Was 3GB but ECS says not enough memory - read-only-node: - essential: true - healthcheck: - test: ["CMD-SHELL", "curl -f -H \"Content-Type: application/json\" -d '{\"jsonrpc\": \"2.0\", \"id\": 9999999, \"method\": \"net_version\"}' http://localhost:8547/ || exit 1"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 5s - # 15% - cpu_shares: 818 - mem_limit: 1.125GB - graph-node: - # 15% - cpu_shares: 614 - mem_limit: 1.125GB - ipfs: - # 5% - cpu_shares: 204 - mem_limit: 0.375GB - postgres: - # 10% - cpu_shares: 409 - mem_limit: 0.75GB - - - ecs_network_mode: host - docker_volumes: - - name: full-node-data - scope: shared - autoprovision: true - driver: 'local' - - name: postgres-data - scope: shared - autoprovision: true - driver: 'local' - - name: ipfs-data - scope: shared - autoprovision: true - driver: 'local' diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml deleted file mode 100644 index 9dc42c4d70a7c..0000000000000 --- a/docker-compose.dev.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3" -services: - - rollup-full-node: - build: - context: . - dockerfile: Dockerfile.dev - volumes: - - .:/mnt/full-node:rw diff --git a/docker-compose.microservices.example.yml b/docker-compose.example.yml similarity index 95% rename from docker-compose.microservices.example.yml rename to docker-compose.example.yml index 4a78d932c1398..0c3f273ca9ea3 100644 --- a/docker-compose.microservices.example.yml +++ b/docker-compose.example.yml @@ -1,4 +1,4 @@ -# TODO: COPY THIS OVER TO docker-compose.microservices.mine.yml FOR YOUR OWN LOCAL CONFIG +# TODO: COPY THIS OVER TO docker-compose.local.yml FOR YOUR OWN LOCAL CONFIG version: "3" services: @@ -8,11 +8,14 @@ services: volumes: - l1-node-data:/mnt/l1-node:rw - l2-node-data:/mnt/l2-node:rw + - ./:/mnt/monorepo # Mounts the source build: context: . - dockerfile: Dockerfile.microservices + dockerfile: Dockerfile.dev # Relies on the code to be mounted via mount above environment: # LOCAL ONLY! + - REBUILD=1 # Set this to 1 if you want to rebuild on startup. Leave unset if you don't want to! + - FETCH_DEPS= # Set this if you want to run `yarn` on startup to fetch dependencies. Leave unset otherwise! - LOG_NEW_LINES=1 # Logs new lines instead of <\n> # DATA CLEARING - CLEAR_DATA_KEY=123 # Change this if you want to delete filesystem data on startup. diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 16e292596e325..0000000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,69 +0,0 @@ -version: "3" - -services: - - # Look at packages/rollup-full-node/README.md for info on configuration - rollup-full-node: - volumes: - - full-node-data:/mnt/full-node:rw - - l1-node-data:/mnt/l1-node:rw - - l2-node-data:/mnt/l2-node:rw - build: - context: . - dockerfile: Dockerfile - ports: - - 8545:8545 - environment: - - IS_TRANSACTION_NODE=1 - - CLEAR_DATA_KEY=35 - - STARTUP_WAIT_TIMEOUT=30 - - OPCODE_WHITELIST_MASK - - LOCAL_L1_NODE_PORT - - LOCAL_L1_NODE_PERSISTENT_DB_PATH=/mnt/l1-node - - L1_NODE_INFURA_NETWORK - - L1_NODE_INFURA_PROJECT_ID - - L1_NODE_WEB3_URL - - L1_SEQUENCER_PRIVATE_KEY - - L1_SEQUENCER_MNEMONIC - - L1_EARLIEST_BLOCK - - L1_TO_L2_TRANSACTION_PASSER_CONTRACT_ADDRESS - - L2_TO_L1_MESSAGE_RECEIVER_CONTRACT_ADDRESS - - FINALITY_DELAY_IN_BLOCKS - - L2_RPC_SERVER_HOST - - L2_RPC_SERVER_PORT=8545 - - L2_RPC_SERVER_PERSISTENT_DB_PATH=/mnt/full-node/level - - L2_WALLET_PRIVATE_KEY=0x29f3edee0ad3abf8e2699402e0e28cd6492c9be7eaab00d732a791c33552f797 - - L2_NODE_WEB3_URL=http://geth_l2:9545/ - # - LOCAL_L2_NODE_PERSISTENT_DB_PATH=/mnt/l2-node/ - - # This service is optional -- the rollup-full-node can be configured to run its own ganache instance. - # Geth is more performant, but that may not matter for your use case. - geth_l2: - volumes: - - l2-node-data:/mnt/l2-node/l2:rw - build: - context: docker/geth - dockerfile: Dockerfile - environment: - - CLEAR_DATA_KEY=35 - - VOLUME_PATH=/mnt/l2-node/l2 - - HOSTNAME=geth_l2 - - PORT=9545 - - NETWORK_ID=108 - expose: - - "9545" - ports: - - 9545:9545 - -volumes: - full-node-data: - l1-node-data: - l2-node-data: - - - - - - - - diff --git a/docker/geth/Dockerfile b/docker/geth/Dockerfile deleted file mode 100644 index 7f63efe3fcd37..0000000000000 --- a/docker/geth/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -## Begin ethereum/client-go -# Build Geth in a stock Go builder container -FROM golang:1.13-alpine as builder - -RUN apk add --no-cache make gcc musl-dev linux-headers git - -RUN mkdir /go-ethereum -RUN git clone https://github.com/ethereum-optimism/go-ethereum.git /go-ethereum -RUN cd /go-ethereum && make all - -# Pull all binaries into a second stage deploy alpine container -FROM alpine:latest - -RUN apk add --no-cache ca-certificates -COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/ - -EXPOSE 8545 8546 8547 30303 30303/udp - -## end ethereum/client-go - -RUN apk add --no-cache openssl jq -COPY entrypoint.sh /bin -RUN chmod +x /bin/entrypoint.sh - -EXPOSE 9545 -ENTRYPOINT ["sh", "/bin/entrypoint.sh"] diff --git a/docker/geth/README.md b/docker/geth/README.md deleted file mode 100644 index 09fa36057c96b..0000000000000 --- a/docker/geth/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Optimism Geth Container -This is a Docker container that builds on the default Geth image, adding configuration specific to the uses within Optimstic Rollup. - -# Building -You can build this image by running -`docker build -t optimism/geth .` - -# Running -You can run this image by running the base `docker-compose.yml` from the base dir -``` -cd ../.. # Go to root of this repo -docker compose up --build # If no changes since last build, can omit `--build` -``` - -# Configuration -Config is handled entirely through environment variables. Below are some config variable names, whether or not they're optional, and what they do: - -Data: -* `CLEAR_DATA_KEY` - (optional) Set to clear all persisted data in the geth node. Data is only cleared on startup when this variable is set and is different from last startup (e.g. last start up it wasn't set, this time it is or last start up it was set to a different value than it is this start up). -* `VOLUME_PATH` - (required) The base filesystem path to use to store persisted data from geth - -Node: -* `HOSTNAME` - (required) The hostname to use for this geth instance. It should almost always be `0.0.0.0`. -* `PORT` - (required) The port to expose this node through. This should be set to 9545 if there is not a good reason to set it to something else. -* `NETWORK_ID` - (required) The network ID for this node. This should be 108 for Optimistic Rollup L2 Node. - -# Publishing to AWS ECR: -Make sure the [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) is installed and [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration) - -1. Make sure you're authenticated: - ``` - aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin .dkr.ecr.us-east-2.amazonaws.com/optimism/geth - ``` -2. Change the working directory to `docker/geth`: - ``` - cd docker/geth - ``` -3. Build and tag latest: - ``` - docker build -t optimism/geth . - ``` -4. Tag the build: - ``` - docker tag optimism/geth:latest .dkr.ecr.us-east-2.amazonaws.com/optimism/geth:latest - ``` -5. Push tag to ECR: - ``` - docker push .dkr.ecr.us-east-2.amazonaws.com/optimism/geth:latest - ``` diff --git a/docker/geth/entrypoint.sh b/docker/geth/entrypoint.sh deleted file mode 100644 index ac415d0300c10..0000000000000 --- a/docker/geth/entrypoint.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh - -## Passed in from environment variables: -# HOSTNAME= -# PORT=8545 -# NETWORK_ID=108 -CLEAR_DATA_FILE_PATH="${VOLUME_PATH}/.clear_data_key_${CLEAR_DATA_KEY}" - -if [[ -n "$CLEAR_DATA_KEY" && ! -f "$CLEAR_DATA_FILE_PATH" ]]; then - echo "Detected change in CLEAR_DATA_KEY. Purging data." - rm -rf ${VOLUME_PATH}/* - rm -rf ${VOLUME_PATH}/.clear_data_key_* - echo "Local data cleared from '${VOLUME_PATH}/*'" - echo "Contents of volume dir: $(ls -alh $VOLUME_PATH)" - touch $CLEAR_DATA_FILE_PATH -fi - -echo "Starting Geth..." -## Command to kick off geth -geth --dev --datadir $VOLUME_PATH --rpc --rpcaddr $HOSTNAME --rpcvhosts=* --rpcport $PORT --networkid $NETWORK_ID --rpcapi 'eth,net' --gasprice '0' --targetgaslimit '4294967295' --nousb --gcmode=archive diff --git a/docker/publish-geth-container.sh b/docker/publish-geth-container.sh deleted file mode 100755 index 44c9815c0d401..0000000000000 --- a/docker/publish-geth-container.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# Publish Geth Container -# Optional 1st argument is tag name. Default is 'latest'. - -set -e - -if [ $# -eq 1 ]; then - GETH_TAG=$1 - echo "Found tag '$GETH_TAG'. Using this as the container tag." -fi - -BASE_DIR=$(dirname $0) -TAG=${GETH_TAG:-latest} - -if [ -z "$AWS_ACCOUNT_NUMBER" ]; then - echo "No AWS_ACCOUNT_NUMBER env variable is set. Please set it to use this script." - exit 1 -fi - -echo "\nAuthenticating within ECR...\n" -aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/geth" - -echo "\nBuilding Geth container...\n" -docker build -t "optimism/geth:$TAG" "$BASE_DIR/geth/." - -echo "\nTagging Geth container as $TAG...\n" -docker tag "optimism/geth:$TAG" "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/geth:$TAG" - -echo "\nPushing Geth container to ECR...\n" -docker push "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/geth:$TAG" - -echo "\nPublish complete!" diff --git a/docker/publish-rollup-fullnode-container.sh b/docker/publish-rollup-fullnode-container.sh deleted file mode 100755 index 507c616eaad38..0000000000000 --- a/docker/publish-rollup-fullnode-container.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Publish Rollup Fullnode Container -# Optional 1st argument is tag name. Default is 'latest'. - -set -e - -if [ $# -eq 1 ]; then - FULLNODE_TAG=$1 - echo "Found tag '$FULLNODE_TAG'. Using this as the container tag." -fi - -SCRIPT_DIR=$(dirname $0) -ROOT_DIR=$SCRIPT_DIR/.. - -TAG=${FULLNODE_TAG:-latest} - -if [ -z "$AWS_ACCOUNT_NUMBER" ]; then - echo "No AWS_ACCOUNT_NUMBER env variable is set. Please set it to use this script." - exit 1 -fi - -# Make sure we build so the container is using the current source -yarn --cwd $ROOT_DIR clean && yarn --cwd $ROOT_DIR build - -echo "\nAuthenticating within ECR...\n" -aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node" - -echo "\nBuilding fullnode container...\n" -docker build -t "optimism/rollup-full-node:$TAG" "$ROOT_DIR" - -echo "\nTagging fullnode container as $TAG in ECR...\n" -docker tag "optimism/rollup-full-node:$TAG" "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:$TAG" - -echo "\nPushing fullnode container to ECR...\n" -docker push "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-full-node:$TAG" - -echo "\nPublish complete!" diff --git a/docker/publish-rollup-microservices-container.sh b/docker/publish-rollup-microservices-container.sh index 4ede9f59b357f..cf83ffff6db88 100755 --- a/docker/publish-rollup-microservices-container.sh +++ b/docker/publish-rollup-microservices-container.sh @@ -26,7 +26,7 @@ echo "\nAuthenticating within ECR...\n" aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-microservices" echo "\nBuilding Microservices container...\n" -docker build -f "$ROOT_DIR/Dockerfile.microservices" -t "optimism/rollup-microservices:$TAG" "$ROOT_DIR" +docker build -f "$ROOT_DIR/Dockerfile" -t "optimism/rollup-microservices:$TAG" "$ROOT_DIR" echo "\nTagging Microservices container as $TAG in ECR...\n" docker tag "optimism/rollup-microservices:$TAG" "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/rollup-microservices:$TAG" diff --git a/docker/publish-the-graph-container.sh b/docker/publish-the-graph-container.sh deleted file mode 100755 index deff64ec3a191..0000000000000 --- a/docker/publish-the-graph-container.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# Publish The Graph Container -# Optional 1st argument is tag name. Default is 'latest'. - -set -e - -if [ $# -eq 1 ]; then - THE_GRAPH_TAG=$1 - echo "Found tag '$THE_GRAPH_TAG'. Using this as the container tag." -fi - -BASE_DIR=$(dirname $0) -TAG=${THE_GRAPH_TAG:-latest} - -if [ -z "$AWS_ACCOUNT_NUMBER" ]; then - echo "No AWS_ACCOUNT_NUMBER env variable is set. Please set it to use this script." - exit 1 -fi - -echo "\nAuthenticating within ECR...\n" -aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/the-graph" - -echo "\nBuilding The Graph container...\n" -docker build -t "optimism/the-graph:$TAG" "$BASE_DIR/the-graph/." - -echo "\nTagging The Graph container as $TAG in ECR...\n" -docker tag "optimism/the-graph:$TAG" "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/the-graph:$TAG" - -echo "\nPushing The Graph container to ECR...\n" -docker push "$AWS_ACCOUNT_NUMBER.dkr.ecr.us-east-2.amazonaws.com/optimism/the-graph:$TAG" - -echo "\nPublish complete!" diff --git a/docker/the-graph/Dockerfile b/docker/the-graph/Dockerfile deleted file mode 100644 index 1b91ead619130..0000000000000 --- a/docker/the-graph/Dockerfile +++ /dev/null @@ -1,67 +0,0 @@ -FROM rust:latest - -# Replace this with the graph-node branch you want to build the image from; -# Note: Docker Hub substitutes this automatically using our hooks/post_checkout script. -ENV SOURCE_BRANCH "master" - -# Install clang (required for dependencies) -RUN apt-get update \ - && apt-get install -y clang libclang-dev - -# Clone and build the graph-node repository -RUN git clone https://github.com/ethereum-optimism/graph-node.git \ - && cd graph-node \ - && git checkout "$SOURCE_BRANCH" \ - && cargo install --locked --path node \ - && cp docker/start-node /usr/local/bin \ - && cd .. \ - && rm -rf graph-node - -# Clone and install wait-for-it -RUN git clone https://github.com/vishnubob/wait-for-it.git \ - && cp wait-for-it/wait-for-it.sh /usr/local/bin \ - && chmod +x /usr/local/bin/wait-for-it.sh \ - && rm -rf wait-for-it - -ENV RUST_LOG "" -ENV GRAPH_LOG "" -ENV EARLY_LOG_CHUNK_SIZE "" -ENV ETHEREUM_RPC_PARALLEL_REQUESTS "" -ENV ETHEREUM_BLOCK_CHUNK_SIZE "" - -ENV postgres_host "" -ENV postgres_user "" -ENV postgres_pass "" -ENV postgres_db "" -ENV ipfs "" -ENV ethereum "" - -# HTTP port -EXPOSE 8000 - -# WebSocket port -EXPOSE 8001 - -# JSON-RPC port -EXPOSE 8020 - -RUN apt-get install gawk - -COPY wait-for-ovm.sh /usr/local/bin - -# Wait for IPFS and Postgres to start up. -# -# The awk commands below take the IPFS and Postgres and extract -# hostname:port from them. The IPFS port defaults to 443 for HTTPS -# and 80 for HTTP. The Postgres port defaults to 5432. -CMD wait-for-ovm.sh \ - && wait-for-it.sh \ - $(echo $ipfs | \ - gawk 'match($0, /^([a-z]+:\/\/)?([^\/:]+)(:([0-9]+))?.*$/, m) { print m[2]":"(m[4] ? m[4] : (m[1] == "https://" ? 443 : 80)) }') \ - -t 30 \ - && wait-for-it.sh \ - $(echo $postgres_host | \ - gawk 'match($0, /^([a-z]+:\/\/)?([^\/:]+)(:([0-9]+))?.*$/, m) { print m[2]":"(m[4] ? m[4] : 5432) }') \ - -t 30 \ - && sleep 5 \ - && start-node \ No newline at end of file diff --git a/docker/the-graph/wait-for-ovm.sh b/docker/the-graph/wait-for-ovm.sh deleted file mode 100755 index 836fff2213e54..0000000000000 --- a/docker/the-graph/wait-for-ovm.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh -# wait-for-ovm.sh -# NOTE: set the CLEAR_DATA_KEY environment variable to clear the $POSTGRES_DIR and $IPFS_DIR on startup. -# Directory will only be cleared if CLEAR_DATA_KEY is set AND different from last start. - -set -e - -if [ -z "$OVM_URL_WITH_PORT" ]; then - echo "Must set environment variable OVM_URL_WITH_PORT" - exit 1 -fi - -STARTUP_WAIT_TIMEOUT=${STARTUP_WAIT_TIMEOUT:-20} - -wait_for_server_to_be_reachable() -{ - if [ -n "$1" ]; then - COUNT=1 - until $(curl --output /dev/null --silent --fail -H "Content-Type: application/json" -d '{"jsonrpc": "2.0", "id": 9999999, "method": "net_version"}' $1); do - sleep 1 - echo "Slept $COUNT times for $1 to be up..." - - if [ "$COUNT" -ge "$STARTUP_WAIT_TIMEOUT" ]; then - echo "Timeout waiting for server at $1" - exit 1 - fi - COUNT=$(($COUNT+1)) - done - fi - - -} - -clear_data_if_necessary() -{ - POSTGRES_DIR=${POSTGRES_DIR:-/data/postgres} - IPFS_DIR=${IPFS_DIR:-/data/ipfs} - CLEAR_DATA_FILE_PATH="${IPFS_DIR}/.clear_data_key_${CLEAR_DATA_KEY}" - - if [ -n "$CLEAR_DATA_KEY" -a ! -f "$CLEAR_DATA_FILE_PATH" ]; then - echo "Detected change in CLEAR_DATA_KEY. Purging data." - rm -rf ${IPFS_DIR}/* - rm -rf ${IPFS_DIR}/.clear_data_key_* - echo "Local data cleared from '${IPFS_DIR}/*'" - echo "Contents of ipfs dir: $(ls -alh $IPFS_DIR)" - - rm -rf ${POSTGRES_DIR}/* - echo "Local data cleared from '${POSTGRES_DIR}/*'" - echo "Contents of postgres dir: $(ls -alh $POSTGRES_DIR)" - touch $CLEAR_DATA_FILE_PATH - else - echo "No change detected in CLEAR_DATA_KEY not deleting data." - fi -} - -clear_data_if_necessary - -wait_for_server_to_be_reachable $OVM_URL_WITH_PORT - ->&2 echo "OVM is up!" diff --git a/packages/rollup-services/exec/build_and_run.sh b/packages/rollup-services/exec/build_and_run.sh new file mode 100755 index 0000000000000..087ea86dfa2f7 --- /dev/null +++ b/packages/rollup-services/exec/build_and_run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# build_and_run.sh -- accepts a command to run after postgres connection succeeds +# conditionally builds based on REBUILD env var and then calls wait_for_postgres_and_geth.sh + +# Exits if any command fails +set -e + +cmd=$@ + +ROOT_DIR=../../.. + + +if [ -n "$REBUILD" ]; then + echo -e "\n\nREBUILD env var set, rebuilding...\n\n" + + if [ -n "$FETCH_DEPS" ]; then + echo -e "\nFetching dependencies (this will take forever the first time time)..." + yarn --cwd $ROOT_DIR --verbose + fi + + yarn --cwd $ROOT_DIR clean + yarn --cwd $ROOT_DIR build + echo -e "\n\nCode built proceeding with ./wait_for_postgres_and_geth.sh...\n\n" +else + echo -e "\n\nREBUILD env var not set, calling ./wait_for_postgres_and_geth.sh without building...\n\n" +fi + +exec $(dirname $0)/wait_for_postgres_and_geth.sh "$cmd"