From 523c6f33aa059cbf148fd694301149a0355765e2 Mon Sep 17 00:00:00 2001 From: Will Winder Date: Fri, 18 Nov 2022 16:35:38 -0500 Subject: [PATCH] General purpose docker container. --- Dockerfile | 75 ++++++++++ docker/README.md | 83 +++++++++++ docker/files/build/install.sh | 88 ++++++++++++ docker/files/build/kmd_config.json.example | 19 +++ docker/files/run/devmode_template.json | 46 ++++++ docker/files/run/future_template.json | 54 +++++++ docker/files/run/run.sh | 155 +++++++++++++++++++++ docker/files/run/template.json | 52 +++++++ 8 files changed, 572 insertions(+) create mode 100644 Dockerfile create mode 100644 docker/README.md create mode 100755 docker/files/build/install.sh create mode 100644 docker/files/build/kmd_config.json.example create mode 100644 docker/files/run/devmode_template.json create mode 100644 docker/files/run/future_template.json create mode 100755 docker/files/run/run.sh create mode 100644 docker/files/run/template.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..56c0a8b79b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,75 @@ +ARG GO_VERSION=1.17.5 +FROM golang:$GO_VERSION-bullseye as builder + +ARG CHANNEL=nightly +ARG URL= +ARG BRANCH= +ARG SHA= + +# Basic dependencies. +ENV HOME /node +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && \ + apt-get install -y \ + apt-utils \ + bsdmainutils \ + curl \ + git \ + git-core \ + python3 + +COPY ./docker/files/ /node/files +COPY ./installer/genesis /node/files/run/genesis +COPY ./cmd/updater/update.sh /node/files/build/update.sh +COPY ./installer/config.json.example /node/files/build/config.json + +RUN find /node/files + +# Install algod binaries. +RUN /node/files/build/install.sh \ + -p "/node/bin" \ + -d "/node/data" \ + -c "${CHANNEL}" \ + -u "${URL}" \ + -b "${BRANCH}" \ + -s "${SHA}" + +# Copy binaries into a clean image +# TODO: We don't need most of the binaries. +# Should we delete everything except goal/algod/algocfg/tealdbg? +FROM debian:bullseye-slim as final +COPY --from=builder "/node/bin/" "/node/bin" +COPY --from=builder "/node/data/" "/node/dataTemplate" +COPY --from=builder "/node/files/run" "/node/run" + +ENV BIN_DIR="/node/bin" +ENV PATH="$BIN_DIR:${PATH}" +ENV ALGOD_PORT=8080 +ENV ALGORAND_DATA="/algod/data" +RUN mkdir -p "$ALGORAND_DATA" +WORKDIR /node/data + +# curl is needed to lookup the fast catchup url +RUN apt-get update && apt-get install -y \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# TODO: This works fine, but causes problems when mounting a volume +# Use algorand user instead of root +#RUN groupadd -r algorand && \ +# useradd --no-log-init -r -g algorand algorand && \ +# chown -R algorand.algorand /node && \ +# chown -R algorand.algorand /algod +#USER algorand + +# Algod REST API +EXPOSE $ALGOD_PORT + +# Algod Gossip Port +EXPOSE 4160 + +# Prometheus Metrics +EXPOSE 9100 + +CMD ["/node/run/run.sh"] +#CMD ["/bin/bash"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..c40a21ed28 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,83 @@ +# Algod Container + +General purpose algod docker container. + + +# Image Configuration + +There are a number of special files and environment variables used to control how a container is started. + +## Default Configuration + +By default the following config.json overrides are applied: + +| Setting | Value | +| ------- | ----- | +| GossipFanout | 1 | +| EndpointAddress | 0.0.0.0:8080 | +| IncomingConnectionsLimit | 0 | +| Archival | false | +| IsIndexerActive | false | +| EnableDeveloperAPI | true | + +## Environment Variables + +The following environment variables can be supplied. Except when noted, it is possible to reconfigure deployments even after the data directory has been initialized. + +| Variable | Description | +| -------- | ----------- | +| NETWORK | Leave blank for a private network, otherwise specify one of mainnet, betanet, testnet, or devnet. Only used during a data directory initialization. | +| FAST_CATCHUP | If set on a public network, attempt to start fast-catchup during initial config. | +| TELEMETRY_NAME| If set on a public network, telemetry is reported with this name. | +| DEV_MODE | If set on a private network, enable dev mode. Only used during data directory initialization. | +| NUM_ROUNDS | If set on a private network, override default of 30000 participation keys. | +| TOKEN | If set, overrides the REST API token. | +| ADMIN_TOKEN | If set, overrides the REST API admin token. | + + +## Special Files + +Configuration can be modified by specifying certian files. These can be changed each time you start the container if the data directory is a mounted volume. + +| File | Description | +| ---- | ----------- | +| /etc/config.json | Override default configurations by providing your own file. | +| /etc/algod.token | Override default randomized REST API token. | +| /etc/algod.admin.token | Override default randomized REST API admin token. | + +TODO: `/etc/template.json` for overriding the private network topology. + +# Example Configuration + +The following command launches a container configured with one of the public networks: +``` +docker run --rm -it \ + -p 4190:8080 \ + -e NETWORK=mainnet \ + -e FAST_CATCHUP=1 \ + -e TELEMETRY_NAME=name \ + -e TOKEN=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ + -v ${PWD}/data:/algod/data/ \ + --name mainnet-container \ + algorand/algod:latest +``` + +Explanation of parts: +* `-p 4190:8080` maps the internal algod REST API to local port 4190 +* `-e NETWORK=` can be set to any of the supported public networks. +* `-e FAST_CATCHUP=` causes fast catchup to start shortly after launching the network. +* `-e TELEMETRY_NAME=` enables telemetry reporting to Algorand for network health analysis. +* `-e TOKEN=` sets the REST API token to use. +* `-v ${PWD}/data:/algod/data/` mounts a local volume to the data directory, which can be used to restart and upgrad the deployment. + + +# Mounting the Data Directory + +The data directory located at `/algod/data`. Mounting a volume at that location will allow you to shutdown and resume the node. + +## Private Network + +Private networks work a little bit differently. They are configured with, potentially, several data directories. The default topology supplied with this container is installed to `/algod/`, and has a single node named `data`. This means the private network has a data directory at `/algod/data`, matching the production configuration. + +Because the root directory contains some metadata, if persistence of the private network is required, you should mount the volume `/algod/` instead of `/algod/data`. This will ensure the extra metadata is included when changing images. + diff --git a/docker/files/build/install.sh b/docker/files/build/install.sh new file mode 100755 index 0000000000..20d5766e47 --- /dev/null +++ b/docker/files/build/install.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# Script to install algod in all sorts of different ways. +# +# Parameters: +# -d : Location where binaries will be installed. +# -c : Channel to install. Mutually exclusive with source options. +# -u : Git repository URL. Mutually exclusive with -c. +# -b : Git branch. Mutually exclusive with -c. +# -s : (optional) Git Commit SHA hash. Mutually exclusive with -c. + +set -e + +rootdir=$(dirname "$0") +pushd "$rootdir" + +BINDIR="" +CHANNEL="" +URL="" +BRANCH="" +SHA="" + +while getopts "p:d:c:u:b:s:" opt; do + case "$opt" in + p) BINDIR=$OPTARG; ;; + d) ALGORAND_DATA=$OPTARG; ;; + c) CHANNEL=$OPTARG; ;; + u) URL=$OPTARG; ;; + b) BRANCH=$OPTARG; ;; + s) SHA=$OPTARG; ;; + *) echo "unknown flag"; exit 1;; + esac +done + +if [ -z "$BINDIR" ]; then + echo "-d is required." + exit 1 +fi + +if [ -n "$CHANNEL" ] && [ -n "$BRANCH" ]; then + echo "Set only one of -c or -b " + exit 1 +fi + +if [ -n "$BRANCH" ] && [ -z "$URL" ]; then + echo "If using -b , must also set -u " + exit 1 +fi + +echo "Installing algod with options:" +echo " BINDIR = ${BINDIR}" +echo " DATADIR = ${ALGORAND_DATA}" +echo " CHANNEL = ${CHANNEL}" +echo " URL = ${URL}" +echo " BRANCH = ${BRANCH}" +echo " SHA = ${SHA}" + +if [ -n "$CHANNEL" ] && [ -n "$BRANCH" ]; then + echo "Do not provide CHANNEL and BRANCH." + exit 1 +fi + +# Deploy from release channel. +if [ -n "$CHANNEL" ]; then + ./update.sh -i -c "$CHANNEL" -p "$BINDIR" -d "${ALGORAND_DATA}" -n + exit 0 +fi + +# Build from source. +if [ -n "$BRANCH" ]; then + git clone --single-branch --branch "${BRANCH}" "${URL}" +else + git clone "${URL}" +fi + +cd go-algorand +if [ "${SHA}" != "" ]; then + echo "Checking out ${SHA}" + git checkout "${SHA}" +fi + +git log -n 5 + +./scripts/configure_dev.sh +make build +./scripts/dev_install.sh -p "${BINDIR}" -d "${ALGORAND_DATA}" + +"$BINDIR"/algod -v diff --git a/docker/files/build/kmd_config.json.example b/docker/files/build/kmd_config.json.example new file mode 100644 index 0000000000..0165dd0a63 --- /dev/null +++ b/docker/files/build/kmd_config.json.example @@ -0,0 +1,19 @@ +{ + "drivers": { + "sqlite": { + "wallets_dir": "", + "allow_unsafe_scrypt": false, + "scrypt": { + "scrypt_n": 65536, + "scrypt_r": 1, + "scrypt_p": 32 + } + }, + "ledger": { + "disable": false + } + }, + "session_lifetime_secs": 60, + "address": "", + "allowed_origins": null +} diff --git a/docker/files/run/devmode_template.json b/docker/files/run/devmode_template.json new file mode 100644 index 0000000000..8e756502b5 --- /dev/null +++ b/docker/files/run/devmode_template.json @@ -0,0 +1,46 @@ +{ + "Genesis": { + "ConsensusProtocol": "future", + "NetworkName": "devmodenet", + "FirstPartKeyRound": 0, + "LastPartKeyRound": NUM_ROUNDS, + "Wallets": [ + { + "Name": "Wallet1", + "Stake": 40, + "Online": true + }, + { + "Name": "Wallet2", + "Stake": 40, + "Online": true + }, + { + "Name": "Wallet3", + "Stake": 20, + "Online": true + } + ], + "DevMode": true + }, + "Nodes": [ + { + "Name": "data", + "IsRelay": false, + "Wallets": [ + { + "Name": "Wallet1", + "ParticipationOnly": false + }, + { + "Name": "Wallet2", + "ParticipationOnly": false + }, + { + "Name": "Wallet3", + "ParticipationOnly": false + } + ] + } + ] +} diff --git a/docker/files/run/future_template.json b/docker/files/run/future_template.json new file mode 100644 index 0000000000..d80128c580 --- /dev/null +++ b/docker/files/run/future_template.json @@ -0,0 +1,54 @@ +{ + "Genesis": { + "ConsensusProtocol": "future", + "NetworkName": "", + "FirstPartKeyRound": 0, + "LastPartKeyRound": NUM_ROUNDS, + "Wallets": [ + { + "Name": "Wallet1", + "Stake": 10, + "Online": true + }, + { + "Name": "Wallet2", + "Stake": 40, + "Online": true + }, + { + "Name": "Wallet3", + "Stake": 40, + "Online": false + }, + { + "Name": "Wallet4", + "Stake": 10, + "Online": false + } + ] + }, + "Nodes": [ + { + "Name": "data", + "IsRelay": true, + "Wallets": [ + { + "Name": "Wallet1", + "ParticipationOnly": false + }, + { + "Name": "Wallet2", + "ParticipationOnly": false + }, + { + "Name": "Wallet3", + "ParticipationOnly": false + }, + { + "Name": "Wallet4", + "ParticipationOnly": false + } + ] + } + ] +} diff --git a/docker/files/run/run.sh b/docker/files/run/run.sh new file mode 100755 index 0000000000..027c0cc7fe --- /dev/null +++ b/docker/files/run/run.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +set -ex + +# Script to configure or resume a network. Based on environment settings the +# node will be setup with a private network or connect to a public network. + +#################### +# Helper functions # +#################### + +function apply_configuration() { + cd "$ALGORAND_DATA" + + # check for config file overrides. + if [ -f "/etc/config.json" ]; then + cp /etc/config.json config.json + fi + if [ -f "/etc/algod.token" ]; then + cp /etc/algod.token algod.token + fi + if [ -f "/etc/algod.admin.token" ]; then + cp /etc/algod.admin.token algod.admin.token + fi + + # check for environment variable overrides. + if [ "$TOKEN" != "" ]; then + echo "$TOKEN" > algod.token + fi + if [ "$ADMIN_TOKEN" != "" ]; then + echo "$ADMIN_TOKEN" > algod.admin.token + fi + + # configure telemetry + if [ "$TELEMETRY_NAME" != "" ]; then + diagcfg telemetry name -n "$TELEMETRY_NAME" -d "$ALGORAND_DATA" + diagcfg telemetry enable -d "$ALGORAND_DATA" + else + diagcfg telemetry disable + fi +} + +function catchup() { + local FAST_CATCHUP_URL="https://algorand-catchpoints.s3.us-east-2.amazonaws.com/channel/CHANNEL/latest.catchpoint" + local CATCHPOINT=$(curl -s ${FAST_CATCHUP_URL/CHANNEL/$NETWORK}) + if [[ "$(echo $CATCHPOINT | wc -l | tr -d ' ')" != "1" ]]; then + echo "Problem starting fast catchup." + exit 1 + fi + + sleep 5 + goal node catchup "$CATCHPOINT" +} + +function start_public_network() { + cd "$ALGORAND_DATA" + + apply_configuration + + if [ $FAST_CATCHUP ]; then + catchup& + fi + # redirect output to stdout + algod -o +} + +function configure_data_dir() { + cd "$ALGORAND_DATA" + algocfg -d . set -p GossipFanout -v 1 + algocfg -d . set -p EndpointAddress -v "0.0.0.0:${ALGOD_PORT}" + algocfg -d . set -p IncomingConnectionsLimit -v 0 + algocfg -d . set -p Archival -v false + algocfg -d . set -p IsIndexerActive -v false + algocfg -d . set -p EnableDeveloperAPI -v true +} + +function start_new_public_network() { + cd /node + if [ ! -d "run/genesis/$NETWORK" ]; then + echo "No genesis file for '$NETWORK' is available." + exit 1 + fi + + mkdir -p "$ALGORAND_DATA" + mv dataTemplate/* "$ALGORAND_DATA" + rm -rf dataTemplate + + cp "run/genesis/$NETWORK/genesis.json" "$ALGORAND_DATA/genesis.json" + cd "$ALGORAND_DATA" + + mv config.json.example config.json + configure_data_dir + + local ID + case $NETWORK in + mainnet) ID=".algorand.network";; + testnet) ID=".algorand.network";; + betanet) ID=".algodev.network";; + alphanet) ID=".algodev.network";; + devnet) ID=".algodev.network";; + *) echo "Unknown network"; exit 1;; + esac + set -p DNSBootstrapID -v "$ID" + + start_public_network +} + +function start_private_network() { + apply_configuration + + # TODO: Is there a way to properly exec a private network? + goal network start -r "$ALGORAND_DATA/.." + tail -f "$ALGORAND_DATA/node.log" +} + +function start_new_private_network() { + cd /node + local TEMPLATE="template.json" + if [ "$DEV_MODE" ]; then + TEMPLATE="devmode_template.json" + fi + sed -i "s/NUM_ROUNDS/${NUM_ROUNDS:-30000}/" "run/$TEMPLATE" + goal network create -n dockernet -r "$ALGORAND_DATA/.." -t "run/$TEMPLATE" + configure_data_dir + start_private_network +} + +############## +# Entrypoint # +############## + +echo "Starting Algod Docker Container" +echo " ALGORAND_DATA: $ALGORAND_DATA" +echo " NETWORK: $NETWORK" +echo " ALGOD_PORT: $ALGOD_PORT" +echo " FAST_CATCHUP: $FAST_CATCHUP" +echo " DEV_MODE: $DEV_MODE" +echo " TOKEN: $TOKEN" +echo " TELEMETRY_NAME $TELEMETRY_NAME" + +# If data directory is initialized, start existing environment. +if [ -f "$ALGORAND_DATA/../network.json" ]; then + start_private_network + exit 1 +elif [ -f "$ALGORAND_DATA/genesis.json" ]; then + start_public_network + exit 1 +fi + +# Initialize and start network. +if [ "$NETWORK" == "" ]; then + start_new_private_network +else + start_new_public_network +fi diff --git a/docker/files/run/template.json b/docker/files/run/template.json new file mode 100644 index 0000000000..6e3f3cfe9c --- /dev/null +++ b/docker/files/run/template.json @@ -0,0 +1,52 @@ +{ + "Genesis": { + "NetworkName": "", + "FirstPartKeyRound": 0, + "LastPartKeyRound": NUM_ROUNDS, + "Wallets": [ + { + "Name": "Wallet1", + "Stake": 10, + "Online": true + }, + { + "Name": "Wallet2", + "Stake": 40, + "Online": true + }, + { + "Name": "Wallet3", + "Stake": 40, + "Online": false + }, + { + "Name": "Wallet4", + "Stake": 10, + "Online": false + } + ] + }, + "Nodes": [ + { + "Name": "data", + "Wallets": [ + { + "Name": "Wallet1", + "ParticipationOnly": false + }, + { + "Name": "Wallet2", + "ParticipationOnly": false + }, + { + "Name": "Wallet3", + "ParticipationOnly": false + }, + { + "Name": "Wallet4", + "ParticipationOnly": false + } + ] + } + ] +}