From bd1ef547d560b4e310ca445a0ac87632c8243b38 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 09:47:45 -0600 Subject: [PATCH 01/34] actions: add container workflow --- .github/workflows/container.yml | 47 +++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/container.yml diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 0000000000..60aa222f91 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,47 @@ +name: container + +on: + push: + branches: + - master + - rel/* + - feature/* + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3.0.2 + + - name: Generate Container Metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: | + docker.io/${{ github.repository_owner }}/algod + tags: | + type=sha + type=ref,event=tag + type=ref,event=branch + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and Push + uses: docker/build-push-action@v3 + with: + context: ./ + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + platforms: linux/amd64,linux/arm64,linux/arm/v7 + build-args: | + SHA=${{ github.sha }} + URL=${{ github.repositoryUrl }} From a46f191d94a6b96fbcce506bf4b262baf58fd463 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 09:57:14 -0600 Subject: [PATCH 02/34] container: use go 1.17.13 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 56c0a8b79b..a336589e94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG GO_VERSION=1.17.5 +ARG GO_VERSION=1.17.13 FROM golang:$GO_VERSION-bullseye as builder ARG CHANNEL=nightly From 9f44eb32dff6afc8f7e34da66b1c77fb4494f536 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:00:27 -0600 Subject: [PATCH 03/34] actions: setup qemu --- .github/workflows/container.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 60aa222f91..02bce98fb2 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -31,6 +31,9 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + - name: Setup Docker Buildx uses: docker/setup-buildx-action@v2 From fc4fd7cb4632881ba2fdec2f13b9e9b46b98385c Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:00:43 -0600 Subject: [PATCH 04/34] container: consolidate EXPOSE --- .github/workflows/container.yml | 2 +- Dockerfile | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 02bce98fb2..a9d7195d74 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -21,7 +21,7 @@ jobs: images: | docker.io/${{ github.repository_owner }}/algod tags: | - type=sha + type=sha,format=long,prefix= type=ref,event=tag type=ref,event=branch diff --git a/Dockerfile b/Dockerfile index a336589e94..c93d5c2b4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,14 +62,7 @@ RUN apt-get update && apt-get install -y \ # chown -R algorand.algorand /algod #USER algorand -# Algod REST API -EXPOSE $ALGOD_PORT - -# Algod Gossip Port -EXPOSE 4160 - -# Prometheus Metrics -EXPOSE 9100 +# Expose Algod REST API, Algod Gossip, and Prometheus Metrics ports +EXPOSE $ALGOD_PORT 4160 9100 CMD ["/node/run/run.sh"] -#CMD ["/bin/bash"] From f45538d9b85c8a3de8ec1150da7a3bac49791cb1 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:09:52 -0600 Subject: [PATCH 05/34] actions: execute on tag push --- .github/workflows/container.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index a9d7195d74..53551d2c35 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -6,6 +6,8 @@ on: - master - rel/* - feature/* + tags: + - "*" jobs: build: From 8f9b4472bf71f4c8287d338f4668d864c181b077 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:14:38 -0600 Subject: [PATCH 06/34] container: add --no-install-recommends --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c93d5c2b4c..50b6008c5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ARG SHA= ENV HOME /node ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ - apt-get install -y \ + apt-get install -y --no-install-recommends \ apt-utils \ bsdmainutils \ curl \ @@ -50,7 +50,7 @@ RUN mkdir -p "$ALGORAND_DATA" WORKDIR /node/data # curl is needed to lookup the fast catchup url -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* From 18296533cfe4f3596f20f538d49264e2b9823eaa Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:15:53 -0600 Subject: [PATCH 07/34] container: use algorand user --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 50b6008c5b..f737637ac8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,13 +54,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* -# TODO: This works fine, but causes problems when mounting a volume # Use algorand user instead of root -#RUN groupadd -r algorand && \ -# useradd --no-log-init -r -g algorand algorand && \ -# chown -R algorand.algorand /node && \ -# chown -R algorand.algorand /algod -#USER algorand +RUN groupadd --system algorand && \ + useradd --no-log-init --system --gid algorand algorand && \ + chown -R algorand.algorand /node && \ + chown -R algorand.algorand /algod + +USER algorand # Expose Algod REST API, Algod Gossip, and Prometheus Metrics ports EXPOSE $ALGOD_PORT 4160 9100 From 33f2b0951c66decf45fe97e56d5b9698ad12f268 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:20:36 -0600 Subject: [PATCH 08/34] container: consolidate ENV --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index f737637ac8..938ae7a139 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,11 +42,8 @@ COPY --from=builder "/node/bin/" "/node/bin" COPY --from=builder "/node/data/" "/node/dataTemplate" COPY --from=builder "/node/files/run" "/node/run" -ENV BIN_DIR="/node/bin" -ENV PATH="$BIN_DIR:${PATH}" -ENV ALGOD_PORT=8080 -ENV ALGORAND_DATA="/algod/data" -RUN mkdir -p "$ALGORAND_DATA" +ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" ALGORAND_DATA="/algod/data" + WORKDIR /node/data # curl is needed to lookup the fast catchup url From e7e9dd959c1ddf46bea3297244413ff709e03e95 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:21:53 -0600 Subject: [PATCH 09/34] container: remove debug statement --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 938ae7a139..fc000a5950 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,6 @@ COPY ./installer/genesis /node/files/run/genesis COPY ./cmd/updater/update.sh /node/files/build/update.sh COPY ./installer/config.json.example /node/files/build/config.json -RUN find /node/files - # Install algod binaries. RUN /node/files/build/install.sh \ -p "/node/bin" \ From c9652d3896e3c70ca3089b9bfb631b228e09909e Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:22:40 -0600 Subject: [PATCH 10/34] container: consolidate RUN --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index fc000a5950..eb9bb06e25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,10 +47,9 @@ WORKDIR /node/data # curl is needed to lookup the fast catchup url RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ - && rm -rf /var/lib/apt/lists/* - -# Use algorand user instead of root -RUN groupadd --system algorand && \ + && rm -rf /var/lib/apt/lists/* && \ + \ + groupadd --system algorand && \ useradd --no-log-init --system --gid algorand algorand && \ chown -R algorand.algorand /node && \ chown -R algorand.algorand /algod From 1a0654711fa19dafdd7723ccc0ea728226d957c5 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:25:57 -0600 Subject: [PATCH 11/34] container: move mkdir /algod/data --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index eb9bb06e25..3cbe52ee0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,9 +45,10 @@ ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" ALGORAND_DATA="/algod/data" WORKDIR /node/data # curl is needed to lookup the fast catchup url -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl \ - && rm -rf /var/lib/apt/lists/* && \ +RUN apt-get update && apt-get install -y --no-install-recommends curl && \ + rm -rf /var/lib/apt/lists/* && \ + \ + mkdir -p "$ALGORAND_DATA" && \ \ groupadd --system algorand && \ useradd --no-log-init --system --gid algorand algorand && \ From 84200f023ebbd76d74a4bb03ea661397411b53c0 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 10:32:41 -0600 Subject: [PATCH 12/34] container: working algorand user --- Dockerfile | 2 +- docker/files/run/run.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3cbe52ee0a..f1a4d6046d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl && \ mkdir -p "$ALGORAND_DATA" && \ \ groupadd --system algorand && \ - useradd --no-log-init --system --gid algorand algorand && \ + useradd --no-log-init --create-home --system --gid algorand algorand && \ chown -R algorand.algorand /node && \ chown -R algorand.algorand /algod diff --git a/docker/files/run/run.sh b/docker/files/run/run.sh index 027c0cc7fe..20c01ceeaa 100755 --- a/docker/files/run/run.sh +++ b/docker/files/run/run.sh @@ -22,6 +22,9 @@ function apply_configuration() { if [ -f "/etc/algod.admin.token" ]; then cp /etc/algod.admin.token algod.admin.token fi + if [ -f "/etc/logging.config" ]; then + cp /etc/logging.config logging.config + fi # check for environment variable overrides. if [ "$TOKEN" != "" ]; then From f2068620aa466aba982165c9c7daeec6c5f23f2a Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 11:15:17 -0600 Subject: [PATCH 13/34] container: do not set default CHANNEL --- Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1a4d6046d..499f8ec33a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,15 @@ -ARG GO_VERSION=1.17.13 +ARG GO_VERSION="1.17.13" + FROM golang:$GO_VERSION-bullseye as builder -ARG CHANNEL=nightly -ARG URL= -ARG BRANCH= -ARG SHA= +ARG CHANNEL +ARG URL +ARG BRANCH +ARG SHA # Basic dependencies. -ENV HOME /node -ENV DEBIAN_FRONTEND noninteractive +ENV HOME="/node" +ENV DEBIAN_FRONTEND="noninteractive" RUN apt-get update && \ apt-get install -y --no-install-recommends \ apt-utils \ From ba9c39ca859dbf582b2ce7e47cb471b62a518834 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 11:36:45 -0600 Subject: [PATCH 14/34] actions: use https git URL --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 53551d2c35..f148e3eda4 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -49,4 +49,4 @@ jobs: platforms: linux/amd64,linux/arm64,linux/arm/v7 build-args: | SHA=${{ github.sha }} - URL=${{ github.repositoryUrl }} + URL=${{ github.server_url }}/${{ github.repository }}.git From f2b27cfcf9fbae8cef254e9ac928459bcfb15909 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 14:27:29 -0600 Subject: [PATCH 15/34] container: tidy README --- docker/README.md | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/docker/README.md b/docker/README.md index c40a21ed28..b3027655e6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,13 +1,12 @@ # Algod Container -General purpose algod docker container. +General purpose algod container image. - -# Image Configuration +## Image Configuration There are a number of special files and environment variables used to control how a container is started. -## Default Configuration +### Default Configuration By default the following config.json overrides are applied: @@ -20,7 +19,7 @@ By default the following config.json overrides are applied: | IsIndexerActive | false | | EnableDeveloperAPI | true | -## Environment Variables +### Environment Variables The following environment variables can be supplied. Except when noted, it is possible to reconfigure deployments even after the data directory has been initialized. @@ -34,10 +33,9 @@ The following environment variables can be supplied. Except when noted, it is po | TOKEN | If set, overrides the REST API token. | | ADMIN_TOKEN | If set, overrides the REST API admin token. | +### Special Files -## Special Files - -Configuration can be modified by specifying certian files. These can be changed each time you start the container if the data directory is a mounted volume. +Configuration can be modified by specifying certain files. These can be changed each time you start the container if the data directory is a mounted volume. | File | Description | | ---- | ----------- | @@ -47,10 +45,11 @@ Configuration can be modified by specifying certian files. These can be changed TODO: `/etc/template.json` for overriding the private network topology. -# Example Configuration +## Example Configuration The following command launches a container configured with one of the public networks: -``` + +```bash docker run --rm -it \ -p 4190:8080 \ -e NETWORK=mainnet \ @@ -63,21 +62,20 @@ docker run --rm -it \ ``` Explanation of parts: + * `-p 4190:8080` maps the internal algod REST API to local port 4190 * `-e NETWORK=` can be set to any of the supported public networks. * `-e FAST_CATCHUP=` causes fast catchup to start shortly after launching the network. * `-e TELEMETRY_NAME=` enables telemetry reporting to Algorand for network health analysis. * `-e TOKEN=` sets the REST API token to use. -* `-v ${PWD}/data:/algod/data/` mounts a local volume to the data directory, which can be used to restart and upgrad the deployment. - +* `-v ${PWD}/data:/algod/data/` mounts a local volume to the data directory, which can be used to restart and upgrade the deployment. -# Mounting the Data Directory +## Mounting the Data Directory The data directory located at `/algod/data`. Mounting a volume at that location will allow you to shutdown and resume the node. -## Private Network +### Private Network Private networks work a little bit differently. They are configured with, potentially, several data directories. The default topology supplied with this container is installed to `/algod/`, and has a single node named `data`. This means the private network has a data directory at `/algod/data`, matching the production configuration. Because the root directory contains some metadata, if persistence of the private network is required, you should mount the volume `/algod/` instead of `/algod/data`. This will ensure the extra metadata is included when changing images. - From 92c704121ff1306f0eae9bc498c4cc433de8e3aa Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 15:53:41 -0600 Subject: [PATCH 16/34] container: consolidate ENV --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 499f8ec33a..19cd7f2ee6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,7 @@ ARG BRANCH ARG SHA # Basic dependencies. -ENV HOME="/node" -ENV DEBIAN_FRONTEND="noninteractive" +ENV HOME="/node" DEBIAN_FRONTEND="noninteractive" RUN apt-get update && \ apt-get install -y --no-install-recommends \ apt-utils \ From 9cbe9e1d325eeec681d7312d9d8f905b940245a1 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 16:19:22 -0600 Subject: [PATCH 17/34] container: remove unused packages in builder --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19cd7f2ee6..8485c6f129 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,9 +14,7 @@ RUN apt-get update && \ apt-utils \ bsdmainutils \ curl \ - git \ - git-core \ - python3 + git COPY ./docker/files/ /node/files COPY ./installer/genesis /node/files/run/genesis From b5f331c4a31c315bc36ce7de96fad3490659e0c9 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Mon, 19 Dec 2022 16:28:19 -0600 Subject: [PATCH 18/34] container: do not execute dev_install.sh --- .github/workflows/container.yml | 1 + Dockerfile | 8 ++++---- docker/files/build/install.sh | 1 - docker/files/run/run.sh | 7 +++---- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index f148e3eda4..c562a9f22b 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -50,3 +50,4 @@ jobs: build-args: | SHA=${{ github.sha }} URL=${{ github.server_url }}/${{ github.repository }}.git + BRANCH=${{ github.ref_name }} diff --git a/Dockerfile b/Dockerfile index 8485c6f129..ecd8255cd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ARG BRANCH ARG SHA # Basic dependencies. -ENV HOME="/node" DEBIAN_FRONTEND="noninteractive" +ENV HOME="/node" DEBIAN_FRONTEND="noninteractive" GOPATH="/node" RUN apt-get update && \ apt-get install -y --no-install-recommends \ apt-utils \ @@ -19,11 +19,11 @@ RUN apt-get update && \ COPY ./docker/files/ /node/files COPY ./installer/genesis /node/files/run/genesis COPY ./cmd/updater/update.sh /node/files/build/update.sh -COPY ./installer/config.json.example /node/files/build/config.json +COPY ./installer/config.json.example /node/files/run/config.json.example # Install algod binaries. RUN /node/files/build/install.sh \ - -p "/node/bin" \ + -p "${GOPATH}/bin" \ -d "/node/data" \ -c "${CHANNEL}" \ -u "${URL}" \ @@ -34,8 +34,8 @@ RUN /node/files/build/install.sh \ # TODO: We don't need most of the binaries. # Should we delete everything except goal/algod/algocfg/tealdbg? FROM debian:bullseye-slim as final + COPY --from=builder "/node/bin/" "/node/bin" -COPY --from=builder "/node/data/" "/node/dataTemplate" COPY --from=builder "/node/files/run" "/node/run" ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" ALGORAND_DATA="/algod/data" diff --git a/docker/files/build/install.sh b/docker/files/build/install.sh index 20d5766e47..ffec0931c7 100755 --- a/docker/files/build/install.sh +++ b/docker/files/build/install.sh @@ -83,6 +83,5 @@ git log -n 5 ./scripts/configure_dev.sh make build -./scripts/dev_install.sh -p "${BINDIR}" -d "${ALGORAND_DATA}" "$BINDIR"/algod -v diff --git a/docker/files/run/run.sh b/docker/files/run/run.sh index 20c01ceeaa..1afc24ba68 100755 --- a/docker/files/run/run.sh +++ b/docker/files/run/run.sh @@ -85,13 +85,12 @@ function start_new_public_network() { fi mkdir -p "$ALGORAND_DATA" - mv dataTemplate/* "$ALGORAND_DATA" - rm -rf dataTemplate - cp "run/genesis/$NETWORK/genesis.json" "$ALGORAND_DATA/genesis.json" cd "$ALGORAND_DATA" - mv config.json.example config.json + cp "/node/run/genesis/$NETWORK/genesis.json" genesis.json + cp /node/run/config.json.example config.json + configure_data_dir local ID From 639a72f495ecb58b30d62449becbfa54ba11d8b8 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 20 Dec 2022 10:10:07 -0600 Subject: [PATCH 19/34] container: only add specific binaries --- Dockerfile | 3 --- docker/files/build/install.sh | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ecd8255cd1..1477f2287d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,9 +30,6 @@ RUN /node/files/build/install.sh \ -b "${BRANCH}" \ -s "${SHA}" -# Copy binaries into a clean image -# TODO: We don't need most of the binaries. -# Should we delete everything except goal/algod/algocfg/tealdbg? FROM debian:bullseye-slim as final COPY --from=builder "/node/bin/" "/node/bin" diff --git a/docker/files/build/install.sh b/docker/files/build/install.sh index ffec0931c7..513ab781e8 100755 --- a/docker/files/build/install.sh +++ b/docker/files/build/install.sh @@ -84,4 +84,8 @@ git log -n 5 ./scripts/configure_dev.sh make build +find "${GOPATH}/bin" -type f -print0 | + xargs --null grep -E -Z -L "*(algocfg|algod|algoh|algokey|carpenter|catchupsrv|ddconfig.sh|diagcfg|find-nodes\.sh|goal|kmd|msgpacktool|node_exporter|tealcut|tealdbg|update.sh|updater|COPYING)$" | + xargs --null rm -v + "$BINDIR"/algod -v From ded50c97aa011cf89eb38aee11351462746eb560 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 20 Dec 2022 10:12:41 -0600 Subject: [PATCH 20/34] container: remove apt lists --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1477f2287d..7b7ae2f31a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,8 @@ RUN apt-get update && \ apt-utils \ bsdmainutils \ curl \ - git + git \ + && rm -rf /var/lib/apt/lists/* COPY ./docker/files/ /node/files COPY ./installer/genesis /node/files/run/genesis From 628e974d040ddde766f72e3212420f86c1265032 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 20 Dec 2022 10:31:22 -0600 Subject: [PATCH 21/34] container: move COPY after user creation --- Dockerfile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b7ae2f31a..3374489fd8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,13 +33,8 @@ RUN /node/files/build/install.sh \ FROM debian:bullseye-slim as final -COPY --from=builder "/node/bin/" "/node/bin" -COPY --from=builder "/node/files/run" "/node/run" - ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" ALGORAND_DATA="/algod/data" -WORKDIR /node/data - # curl is needed to lookup the fast catchup url RUN apt-get update && apt-get install -y --no-install-recommends curl && \ rm -rf /var/lib/apt/lists/* && \ @@ -48,11 +43,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl && \ \ groupadd --system algorand && \ useradd --no-log-init --create-home --system --gid algorand algorand && \ - chown -R algorand.algorand /node && \ - chown -R algorand.algorand /algod + chown -R algorand:algorand /algod USER algorand +COPY --chown=algorand:algorand --from=builder "/node/bin/" "/node/bin/" +COPY --chown=algorand:algorand --from=builder "/node/files/run/" "/node/run/" + # Expose Algod REST API, Algod Gossip, and Prometheus Metrics ports EXPOSE $ALGOD_PORT 4160 9100 From ccbf5fdbf748558357bc6ef8ca44b46307aec57a Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 20 Dec 2022 13:37:12 -0600 Subject: [PATCH 22/34] container: remove curl --- Dockerfile | 7 +------ docker/README.md | 3 +++ docker/files/run/run.sh | 7 ------- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3374489fd8..d816168a80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,12 +35,7 @@ FROM debian:bullseye-slim as final ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" ALGORAND_DATA="/algod/data" -# curl is needed to lookup the fast catchup url -RUN apt-get update && apt-get install -y --no-install-recommends curl && \ - rm -rf /var/lib/apt/lists/* && \ - \ - mkdir -p "$ALGORAND_DATA" && \ - \ +RUN mkdir -p "$ALGORAND_DATA" && \ groupadd --system algorand && \ useradd --no-log-init --create-home --system --gid algorand algorand && \ chown -R algorand:algorand /algod diff --git a/docker/README.md b/docker/README.md index b3027655e6..18cbdcf93e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -27,6 +27,7 @@ The following environment variables can be supplied. Except when noted, it is po | -------- | ----------- | | NETWORK | Leave blank for a private network, otherwise specify one of mainnet, betanet, testnet, or devnet. Only used during a data directory initialization. | | FAST_CATCHUP | If set on a public network, attempt to start fast-catchup during initial config. | +| CATCHPOINT | If set, use this specific catchpoint. | | TELEMETRY_NAME| If set on a public network, telemetry is reported with this name. | | DEV_MODE | If set on a private network, enable dev mode. Only used during data directory initialization. | | NUM_ROUNDS | If set on a private network, override default of 30000 participation keys. | @@ -54,6 +55,7 @@ docker run --rm -it \ -p 4190:8080 \ -e NETWORK=mainnet \ -e FAST_CATCHUP=1 \ + -e CATCHPOINT="25680000#FC44GULKJGKEP5TJEB4DTJEVJKDS5CBTUOOQ2XANGWNPHTLORQBA" \ -e TELEMETRY_NAME=name \ -e TOKEN=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ -v ${PWD}/data:/algod/data/ \ @@ -66,6 +68,7 @@ Explanation of parts: * `-p 4190:8080` maps the internal algod REST API to local port 4190 * `-e NETWORK=` can be set to any of the supported public networks. * `-e FAST_CATCHUP=` causes fast catchup to start shortly after launching the network. +* `-e CATCHPOINT=` causes fast catchup to use a specific catchpoint. * `-e TELEMETRY_NAME=` enables telemetry reporting to Algorand for network health analysis. * `-e TOKEN=` sets the REST API token to use. * `-v ${PWD}/data:/algod/data/` mounts a local volume to the data directory, which can be used to restart and upgrade the deployment. diff --git a/docker/files/run/run.sh b/docker/files/run/run.sh index 1afc24ba68..069ae56b26 100755 --- a/docker/files/run/run.sh +++ b/docker/files/run/run.sh @@ -44,13 +44,6 @@ function apply_configuration() { } function catchup() { - local FAST_CATCHUP_URL="https://algorand-catchpoints.s3.us-east-2.amazonaws.com/channel/CHANNEL/latest.catchpoint" - local CATCHPOINT=$(curl -s ${FAST_CATCHUP_URL/CHANNEL/$NETWORK}) - if [[ "$(echo $CATCHPOINT | wc -l | tr -d ' ')" != "1" ]]; then - echo "Problem starting fast catchup." - exit 1 - fi - sleep 5 goal node catchup "$CATCHPOINT" } From ab3e475e52063230398996e28d9b6f8ff14972d0 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 20 Dec 2022 13:37:41 -0600 Subject: [PATCH 23/34] container: properly remove unecessary binaries --- docker/files/build/install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/files/build/install.sh b/docker/files/build/install.sh index 513ab781e8..78b4ba552c 100755 --- a/docker/files/build/install.sh +++ b/docker/files/build/install.sh @@ -84,8 +84,8 @@ git log -n 5 ./scripts/configure_dev.sh make build -find "${GOPATH}/bin" -type f -print0 | - xargs --null grep -E -Z -L "*(algocfg|algod|algoh|algokey|carpenter|catchupsrv|ddconfig.sh|diagcfg|find-nodes\.sh|goal|kmd|msgpacktool|node_exporter|tealcut|tealdbg|update.sh|updater|COPYING)$" | - xargs --null rm -v +shopt -s extglob + +cd "$BINDIR" && rm -vrf !(algocfg|algod|algoh|algokey|carpenter|catchupsrv|ddconfig.sh|diagcfg|find-nodes.sh|goal|kmd|msgpacktool|node_exporter|tealcut|tealdbg|update.sh|updater|COPYING) "$BINDIR"/algod -v From 6bc69c4761ebfadf7b2541caf17e6b5ae40bd0de Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 20 Dec 2022 16:17:47 -0600 Subject: [PATCH 24/34] actions: add workflow_dispatch event trigger --- .github/workflows/container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index c562a9f22b..9eebf1da6c 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -1,6 +1,7 @@ name: container on: + workflow_dispatch: push: branches: - master From 0859fe0ec1f73d2c0e6d71556391d0f0fa694ada Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 20 Dec 2022 16:31:26 -0600 Subject: [PATCH 25/34] container: update docs --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 18cbdcf93e..bdb3c7e2ea 100644 --- a/docker/README.md +++ b/docker/README.md @@ -27,7 +27,7 @@ The following environment variables can be supplied. Except when noted, it is po | -------- | ----------- | | NETWORK | Leave blank for a private network, otherwise specify one of mainnet, betanet, testnet, or devnet. Only used during a data directory initialization. | | FAST_CATCHUP | If set on a public network, attempt to start fast-catchup during initial config. | -| CATCHPOINT | If set, use this specific catchpoint. | +| CATCHPOINT | If set, use this specific catchpoint, otherwise the latest one is used. | | TELEMETRY_NAME| If set on a public network, telemetry is reported with this name. | | DEV_MODE | If set on a private network, enable dev mode. Only used during data directory initialization. | | NUM_ROUNDS | If set on a private network, override default of 30000 participation keys. | From 65bce825811c945a5983d26849c3cd4588f28b93 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 20 Dec 2022 19:14:19 -0600 Subject: [PATCH 26/34] actions: add cache --- .github/workflows/container.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 9eebf1da6c..d48f15e65c 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -48,6 +48,8 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} platforms: linux/amd64,linux/arm64,linux/arm/v7 + cache-from: type=gha + cache-to: type=gha,mode=max build-args: | SHA=${{ github.sha }} URL=${{ github.server_url }}/${{ github.repository }}.git From 23b546af4908cb9492e1ab899ee048f9ca0bacbf Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Wed, 21 Dec 2022 08:32:21 -0600 Subject: [PATCH 27/34] container: readd curl --- Dockerfile | 5 ++++- docker/README.md | 3 --- docker/files/run/run.sh | 11 +++++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index d816168a80..8fc217dc6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,10 @@ FROM debian:bullseye-slim as final ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" ALGORAND_DATA="/algod/data" -RUN mkdir -p "$ALGORAND_DATA" && \ +# curl is needed to lookup the fast catchup url +RUN apt-get update && apt-get install -y --no-install-recommends curl && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p "$ALGORAND_DATA" && \ groupadd --system algorand && \ useradd --no-log-init --create-home --system --gid algorand algorand && \ chown -R algorand:algorand /algod diff --git a/docker/README.md b/docker/README.md index bdb3c7e2ea..b3027655e6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -27,7 +27,6 @@ The following environment variables can be supplied. Except when noted, it is po | -------- | ----------- | | NETWORK | Leave blank for a private network, otherwise specify one of mainnet, betanet, testnet, or devnet. Only used during a data directory initialization. | | FAST_CATCHUP | If set on a public network, attempt to start fast-catchup during initial config. | -| CATCHPOINT | If set, use this specific catchpoint, otherwise the latest one is used. | | TELEMETRY_NAME| If set on a public network, telemetry is reported with this name. | | DEV_MODE | If set on a private network, enable dev mode. Only used during data directory initialization. | | NUM_ROUNDS | If set on a private network, override default of 30000 participation keys. | @@ -55,7 +54,6 @@ docker run --rm -it \ -p 4190:8080 \ -e NETWORK=mainnet \ -e FAST_CATCHUP=1 \ - -e CATCHPOINT="25680000#FC44GULKJGKEP5TJEB4DTJEVJKDS5CBTUOOQ2XANGWNPHTLORQBA" \ -e TELEMETRY_NAME=name \ -e TOKEN=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ -v ${PWD}/data:/algod/data/ \ @@ -68,7 +66,6 @@ Explanation of parts: * `-p 4190:8080` maps the internal algod REST API to local port 4190 * `-e NETWORK=` can be set to any of the supported public networks. * `-e FAST_CATCHUP=` causes fast catchup to start shortly after launching the network. -* `-e CATCHPOINT=` causes fast catchup to use a specific catchpoint. * `-e TELEMETRY_NAME=` enables telemetry reporting to Algorand for network health analysis. * `-e TOKEN=` sets the REST API token to use. * `-v ${PWD}/data:/algod/data/` mounts a local volume to the data directory, which can be used to restart and upgrade the deployment. diff --git a/docker/files/run/run.sh b/docker/files/run/run.sh index 069ae56b26..a8237ce79f 100755 --- a/docker/files/run/run.sh +++ b/docker/files/run/run.sh @@ -44,6 +44,13 @@ function apply_configuration() { } function catchup() { + local FAST_CATCHUP_URL="https://algorand-catchpoints.s3.us-east-2.amazonaws.com/channel/CHANNEL/latest.catchpoint" + local CATCHPOINT=$(curl -s ${FAST_CATCHUP_URL/CHANNEL/$NETWORK}) + if [[ "$(echo $CATCHPOINT | wc -l | tr -d ' ')" != "1" ]]; then + echo "Problem starting fast catchup." + exit 1 + fi + sleep 5 goal node catchup "$CATCHPOINT" } @@ -53,8 +60,8 @@ function start_public_network() { apply_configuration - if [ $FAST_CATCHUP ]; then - catchup& + if [ "$FAST_CATCHUP" ]; then + catchup & fi # redirect output to stdout algod -o From 6804917580015910e2874536527eafac69dde1e2 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Wed, 21 Dec 2022 12:31:38 -0600 Subject: [PATCH 28/34] container: use ubuntu:18.04 as base --- .github/workflows/container.yml | 2 +- Dockerfile | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index d48f15e65c..358bc8149a 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -47,7 +47,7 @@ jobs: file: ./Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} - platforms: linux/amd64,linux/arm64,linux/arm/v7 + platforms: linux/amd64,linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max build-args: | diff --git a/Dockerfile b/Dockerfile index 8fc217dc6d..8a3b7beb8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,31 @@ -ARG GO_VERSION="1.17.13" +FROM ubuntu:18.04 as builder -FROM golang:$GO_VERSION-bullseye as builder +ARG GO_VERSION="1.17.13" ARG CHANNEL ARG URL ARG BRANCH ARG SHA +ARG TARGETARCH + +ADD https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz /go.tar.gz # Basic dependencies. ENV HOME="/node" DEBIAN_FRONTEND="noninteractive" GOPATH="/node" + RUN apt-get update && \ apt-get install -y --no-install-recommends \ + ca-certificates \ apt-utils \ bsdmainutils \ curl \ git \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* && \ + \ + tar -C /usr/local -xzf /go.tar.gz && \ + rm -rf /go.tar.gz + +ENV PATH="/usr/local/go/bin:${PATH}" COPY ./docker/files/ /node/files COPY ./installer/genesis /node/files/run/genesis From acc426d63059fb7058381093f36e6d9822e1eb9a Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Wed, 21 Dec 2022 15:16:28 -0600 Subject: [PATCH 29/34] container: calculate proper BUILD_NUMBER --- docker/files/build/install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/files/build/install.sh b/docker/files/build/install.sh index 78b4ba552c..2dbaccce6f 100755 --- a/docker/files/build/install.sh +++ b/docker/files/build/install.sh @@ -81,6 +81,8 @@ fi git log -n 5 +# make sure the makefile calculates BUILD_NUMBER +export BUILD_NUMBER="" ./scripts/configure_dev.sh make build From bc398db804b3cdacc3bf7c5eb228b2b589f52dbb Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Fri, 23 Dec 2022 10:20:03 -0600 Subject: [PATCH 30/34] container: fix fast catchup conditional --- docker/files/run/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/files/run/run.sh b/docker/files/run/run.sh index a8237ce79f..24e59ae145 100755 --- a/docker/files/run/run.sh +++ b/docker/files/run/run.sh @@ -60,7 +60,7 @@ function start_public_network() { apply_configuration - if [ "$FAST_CATCHUP" ]; then + if [[ $FAST_CATCHUP ]]; then catchup & fi # redirect output to stdout From 7ca067fac9d670459a9e3781b3379ac3c64b2889 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Fri, 23 Dec 2022 10:22:36 -0600 Subject: [PATCH 31/34] scripts: allow BRANCH to be overridden --- docker/files/build/install.sh | 5 ++--- scripts/compute_branch.sh | 29 ++++++++++++++++------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/docker/files/build/install.sh b/docker/files/build/install.sh index 2dbaccce6f..9cfef5e858 100755 --- a/docker/files/build/install.sh +++ b/docker/files/build/install.sh @@ -81,10 +81,9 @@ fi git log -n 5 -# make sure the makefile calculates BUILD_NUMBER -export BUILD_NUMBER="" ./scripts/configure_dev.sh -make build +# make sure the makefile uses specific values for BUILD_NUMBER and BRANCH +BUILD_NUMBER="" BRANCH="$BRANCH" make build shopt -s extglob diff --git a/scripts/compute_branch.sh b/scripts/compute_branch.sh index f0ff025d90..1d6e83fdca 100755 --- a/scripts/compute_branch.sh +++ b/scripts/compute_branch.sh @@ -1,18 +1,21 @@ #!/usr/bin/env bash -if [[ -n $(git status --porcelain) ]]; then - # If the branch isn't clean, default to HEAD to match old behavior. - BRANCH="HEAD" -elif [ -z "${TRAVIS_BRANCH}" ]; then - # if there is no travis branch, set based on tag or branch - case "$(git describe --tags)" in - *"beta") BRANCH="rel/beta" ;; - *"stable") BRANCH="rel/stable" ;; - *"nightly") BRANCH="rel/nightly" ;; - *) BRANCH=$(git rev-parse --abbrev-ref HEAD) - esac -else - BRANCH="${TRAVIS_BRANCH}" +BRANCH="${BRANCH:-}" +if [ -n "$BRANCH" ]; then + if [[ -n $(git status --porcelain) ]]; then + # If the branch isn't clean, default to HEAD to match old behavior. + BRANCH="HEAD" + elif [ -z "${TRAVIS_BRANCH}" ]; then + # if there is no travis branch, set based on tag or branch + case "$(git describe --tags)" in + *"beta") BRANCH="rel/beta" ;; + *"stable") BRANCH="rel/stable" ;; + *"nightly") BRANCH="rel/nightly" ;; + *) BRANCH=$(git rev-parse --abbrev-ref HEAD) ;; + esac + else + BRANCH="${TRAVIS_BRANCH}" + fi fi echo "${BRANCH}" From 96abc0bde13f1a3a285233a21b77ef21f0198390 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Tue, 3 Jan 2023 15:05:20 -0600 Subject: [PATCH 32/34] add missing ca-certificates --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8a3b7beb8f..89296cf6ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ FROM debian:bullseye-slim as final ENV PATH="/node/bin:${PATH}" ALGOD_PORT="8080" ALGORAND_DATA="/algod/data" # curl is needed to lookup the fast catchup url -RUN apt-get update && apt-get install -y --no-install-recommends curl && \ +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && \ rm -rf /var/lib/apt/lists/* && \ mkdir -p "$ALGORAND_DATA" && \ groupadd --system algorand && \ From 9d282b547debb2fd1e1d2d1661faeaebfee2f7f8 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Wed, 4 Jan 2023 09:06:14 -0600 Subject: [PATCH 33/34] actions: use latest tag when branch is rel/stable --- .github/workflows/container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 358bc8149a..2dd966b8b7 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -27,6 +27,7 @@ jobs: type=sha,format=long,prefix= type=ref,event=tag type=ref,event=branch + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'rel/stable') }} - name: Login to Docker Hub uses: docker/login-action@v2 From d49a8bb4a423643a31bf6834807df72b4fe87521 Mon Sep 17 00:00:00 2001 From: Lucky Baar Date: Wed, 4 Jan 2023 14:25:23 -0600 Subject: [PATCH 34/34] actions: pin checkout to v3 --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 2dd966b8b7..fa4b39c4cf 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Code - uses: actions/checkout@v3.0.2 + uses: actions/checkout@v3 - name: Generate Container Metadata id: meta