From e440575f3bc87ff15627d2a3fa245de7cf6d1dad Mon Sep 17 00:00:00 2001 From: Benjamin Toll Date: Thu, 12 Dec 2019 00:40:29 -0500 Subject: [PATCH 1/5] Bring `shellcheck` into the build process Let's use bitwise operations to determine package presence --- scripts/check_deps.sh | 132 ++++++++++++++++++++++------------ scripts/configure_dev-deps.sh | 22 +++++- 2 files changed, 107 insertions(+), 47 deletions(-) diff --git a/scripts/check_deps.sh b/scripts/check_deps.sh index afa820f873..76d6c33437 100755 --- a/scripts/check_deps.sh +++ b/scripts/check_deps.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# checkout_deps.sh - Quickly(!) checks the enlistment for any missing dependencies and attempts to resolve them. +# check_deps.sh - Quickly(!) checks the enlistment for any missing dependencies and attempts to resolve them. # Reports if any dependencies are still missing after attempts to resolve. # -# Syntax: checkout_deps.sh +# Syntax: check_deps.sh # # Outputs: status messages # @@ -11,77 +11,117 @@ # # Usage: Use before building to ensure dependencies are present (should be nop except after dependencies change) # -# Examples: scripts/checkout_deps.sh +# Examples: scripts/check_deps.sh -export GOPATH=$(go env GOPATH) -GOPATH1=$(echo $GOPATH | cut -d: -f1) +GREEN_FG=$(tput setaf 2) +RED_FG=$(tput setaf 1) +TEAL_FG=$(tput setaf 6) +YELLOW_FG=$(tput setaf 3) +END_FG_COLOR=$(tput sgr0) -ANY_MISSING=0 -# golint doesn't work with 'dep ensure' so we manually install it -GOLINT_MISSING=0 -STRINGER_MISSING=0 -SWAGGER_MISSING=0 +GOPATH=$(go env GOPATH) +export GOPATH +GOPATH1=$(echo "$GOPATH" | cut -d: -f1) +DEPS=0 -function check_deps() { - ANY_MISSING=0 - GOLINT_MISSING=0 +installing_dep() { + echo "$TEAL_FG[INSTALLING]$END_FG_COLOR $1..." +} - if [ ! -f "${GOPATH1}/bin/golint" ]; then - GOLINT_MISSING=1 - ANY_MISSING=1 - echo "... golint missing" +check_deps() { + if [ ! -f "${GOPATH1}/bin/golint" ] + then + DEPS=$(( "$DEPS" | 1 )) fi - if [ ! -f "${GOPATH1}/bin/stringer" ]; then - STRINGER_MISSING=1 - ANY_MISSING=1 - echo "... stringer missing" + if [ ! -f "${GOPATH1}/bin/stringer" ] + then + DEPS=$(( "$DEPS" | 2 )) fi - if [ ! -f "${GOPATH1}/bin/swagger" ]; then - SWAGGER_MISSING=1 - ANY_MISSING=1 - echo "... swagger missing" + if [ ! -f "${GOPATH1}/bin/swagger" ] + then + DEPS=$(( "$DEPS" | 4 )) fi - return ${ANY_MISSING} + # Don't print `shellcheck`s location. + if ! which shellcheck > /dev/null + then + DEPS=$(( "$DEPS" | 8 )) + fi } check_deps -if [ $? -eq 0 ]; then - echo Required dependencies already installed. + +if [ $DEPS -eq 0 ] +then + echo "$GREEN_FG[$0]$END_FG_COLOR Required dependencies already installed." exit 0 fi -if [ ${GOLINT_MISSING} -ne 0 ]; then - read -p "Install golint (using go get) (y/N): " OK - if [ "$OK" = "y" ]; then - echo "Installing golint..." +if [ $(( DEPS & 1 )) -ne 0 ] +then + read -rp "${YELLOW_FG}MISSING DEPENDENCY$END_FG_COLOR \`golint\`. Install? (using go get) (Y/n): " OK + if [[ "$OK" =~ ^""$|Y|y ]] + then + installing_dep golint GO111MODULE=off go get -u golang.org/x/lint/golint + DEPS=$(( "$DEPS" & ~1 )) fi fi -if [ ${STRINGER_MISSING} -ne 0 ]; then - read -p "Install stringer (using go get) (y/N): " OK - if [ "$OK" = "y" ]; then - echo "Installing stringer..." +if [ $(( DEPS & 2 )) -ne 0 ] +then + read -rp "${YELLOW_FG}MISSING DEPENDENCY$END_FG_COLOR \`stringer\`. Install? (using go get) (Y/n): " OK + if [[ "$OK" =~ ^""$|Y|y ]] + then + installing_dep stringer GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer + DEPS=$(( "$DEPS" & ~2 )) fi fi -if [ ${SWAGGER_MISSING} -ne 0 ]; then - read -p "Install swagger (using go get) (y/N): " OK - if [ "$OK" = "y" ]; then - echo "Installing swagger..." +if [ $(( DEPS & 4 )) -ne 0 ] +then + read -rp "${YELLOW_FG}MISSING DEPENDENCY$END_FG_COLOR \`swagger\`. Install? (using go get) (Y/n): " OK + if [[ "$OK" =~ ^""$|Y|y ]] + then + installing_dep swagger GO111MODULE=off go get -u github.com/go-swagger/go-swagger/cmd/swagger + DEPS=$(( "$DEPS" & ~4 )) fi fi -check_deps -if [ $? -eq 0 ]; then - echo Required dependencies have been installed - exit 0 +if [ $(( DEPS & 8 )) -ne 0 ] +then + + read -rp "${YELLOW_FG}MISSING DEPENDENCY$END_FG_COLOR \`shellcheck\`. Install? (using go get) (Y/n): " OK + if [[ "$OK" =~ ^""$|Y|y ]] + then + OS=$(uname) + + if [ "$OS" == "Linux" ] + then + if ! which sudo > /dev/null + then + apt-get install sudo -y + fi + + sudo apt-get install shellcheck -y + elif [ "$OS" == "Darwin" ] + then + brew install shellcheck + fi + + DEPS=$(( "$DEPS" & ~8 )) + fi +fi + +if [ $DEPS -eq 0 ] +then + echo -e "$GREEN_FG[$0]$END_FG_COLOR All required dependencies have been installed." else - echo Required dependencies still missing. Build will probably fail. - exit 0 + echo -e "$RED_FG[$0]$END_FG_COLOR Required dependencies still missing. Build will probably fail." + exit 1 fi + diff --git a/scripts/configure_dev-deps.sh b/scripts/configure_dev-deps.sh index 59737f15ca..6b270af1ef 100755 --- a/scripts/configure_dev-deps.sh +++ b/scripts/configure_dev-deps.sh @@ -4,7 +4,7 @@ set -ex function install_go_module { local OUTPUT - OUTPUT=$(GO111MODULE=off go get -u $1 2>&1) + OUTPUT=$(GO111MODULE=off go get -u "$1" 2>&1) if [ "${OUTPUT}" != "" ]; then echo "error: executing \"go get -u $1\" failed : ${OUTPUT}" exit 1 @@ -14,3 +14,23 @@ function install_go_module { install_go_module golang.org/x/lint/golint install_go_module golang.org/x/tools/cmd/stringer install_go_module github.com/go-swagger/go-swagger/cmd/swagger + +OS=$(uname) + +if [ "$OS" == "Linux" ] +then + # This script is called from multiple locations, so to be safe we + # will do this check and not assume it's already been done. + if ! which sudo > /dev/null + then + apt-get update + apt-get install sudo -y + fi + + sudo apt-get update + sudo apt-get install shellcheck -y +elif [ "$OS" == "Darwin" ] +then + brew install shellcheck +fi + From 8ec220dd5440d47255c915cf34474121b1100ca0 Mon Sep 17 00:00:00 2001 From: Benjamin Toll Date: Thu, 12 Dec 2019 13:33:43 -0500 Subject: [PATCH 2/5] Added `check_shell` target to Makefile --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 935b24e332..c4e43e7419 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,11 @@ vet: check_license: ./scripts/check_license.sh +# Turn on recursive globbing to find all shell scripts. +check_shell: + bash -c "shopt -s globstar" + shellcheck */**/*.sh + sanity: vet fix lint fmt check_license cover: @@ -243,4 +248,4 @@ dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS))) install: build scripts/dev_install.sh -p $(GOPATH1)/bin -.PHONY: default fmt vet lint check_license sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN +.PHONY: default fmt vet lint check_license check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN From 9a106e348cbdd3f7380f4326b2dfe68eeaeae7ae Mon Sep 17 00:00:00 2001 From: Benjamin Toll Date: Thu, 12 Dec 2019 17:00:17 -0500 Subject: [PATCH 3/5] Move install of shellcheck into `scripts/configure_dev.sh` Also, add shellcheck dependency to other dockerfiles. --- docker/build/Dockerfile | 2 +- docker/build/Dockerfile-deploy | 2 +- scripts/centos-build.Dockerfile | 2 +- scripts/configure_dev-deps.sh | 19 ------------------- scripts/configure_dev.sh | 11 +++++++---- 5 files changed, 10 insertions(+), 26 deletions(-) diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index f502a81ec6..d3d0e20e9b 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:16.04 ENV GOLANG_VERSION 1.12 -RUN apt update && apt install -y git libboost-all-dev wget sqlite3 autoconf build-essential +RUN apt-get update && apt-get install -y git libboost-all-dev wget sqlite3 autoconf build-essential shellcheck WORKDIR /root RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local ENV GOROOT=/usr/local/go \ diff --git a/docker/build/Dockerfile-deploy b/docker/build/Dockerfile-deploy index 560f7d0279..69fdbd3145 100644 --- a/docker/build/Dockerfile-deploy +++ b/docker/build/Dockerfile-deploy @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ENV GOLANG_VERSION 1.12 -RUN apt update && apt install -y git libboost-all-dev wget sqlite3 autoconf jq bsdmainutils +RUN apt-get update && apt-get install -y git libboost-all-dev wget sqlite3 autoconf jq bsdmainutils shellcheck WORKDIR /root RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local ENV GOROOT=/usr/local/go \ diff --git a/scripts/centos-build.Dockerfile b/scripts/centos-build.Dockerfile index c484ab2935..2ea453c261 100644 --- a/scripts/centos-build.Dockerfile +++ b/scripts/centos-build.Dockerfile @@ -1,6 +1,6 @@ FROM centos:7 WORKDIR /root RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -RUN yum install -y autoconf awscli git gnupg2 nfs-utils python36 sqlite3 boost-devel expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 +RUN yum install -y autoconf awscli git gnupg2 nfs-utils python36 sqlite3 boost-devel expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 shellcheck ENTRYPOINT ["/bin/bash"] diff --git a/scripts/configure_dev-deps.sh b/scripts/configure_dev-deps.sh index 6b270af1ef..6b4dad873e 100755 --- a/scripts/configure_dev-deps.sh +++ b/scripts/configure_dev-deps.sh @@ -15,22 +15,3 @@ install_go_module golang.org/x/lint/golint install_go_module golang.org/x/tools/cmd/stringer install_go_module github.com/go-swagger/go-swagger/cmd/swagger -OS=$(uname) - -if [ "$OS" == "Linux" ] -then - # This script is called from multiple locations, so to be safe we - # will do this check and not assume it's already been done. - if ! which sudo > /dev/null - then - apt-get update - apt-get install sudo -y - fi - - sudo apt-get update - sudo apt-get install shellcheck -y -elif [ "$OS" == "Darwin" ] -then - brew install shellcheck -fi - diff --git a/scripts/configure_dev.sh b/scripts/configure_dev.sh index d70150028e..1d4721bc9a 100755 --- a/scripts/configure_dev.sh +++ b/scripts/configure_dev.sh @@ -3,7 +3,7 @@ set -e SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -OS=$(${SCRIPTPATH}/ostype.sh) +OS=$("$SCRIPTPATH"/ostype.sh) function install_or_upgrade { if brew ls --versions "$1" >/dev/null; then @@ -14,13 +14,14 @@ function install_or_upgrade { } if [ "${OS}" = "linux" ]; then - if [ -z "$(dpkg -l sudo 2>/dev/null | grep ^ii)" ] ; then + if ! which sudo > /dev/null + then apt-get update apt-get -y install sudo fi sudo apt-get update - sudo apt-get -y install libboost-all-dev expect jq autoconf + sudo apt-get install -y libboost-all-dev expect jq autoconf shellcheck elif [ "${OS}" = "darwin" ]; then brew update brew tap homebrew/cask @@ -30,6 +31,8 @@ elif [ "${OS}" = "darwin" ]; then install_or_upgrade libtool install_or_upgrade autoconf install_or_upgrade automake + install_or_upgrade shellcheck fi -${SCRIPTPATH}/configure_dev-deps.sh +"$SCRIPTPATH"/configure_dev-deps.sh + From a88758545752f224e5336328619e06b8aab53169 Mon Sep 17 00:00:00 2001 From: Benjamin Toll Date: Thu, 12 Dec 2019 17:10:33 -0500 Subject: [PATCH 4/5] Use `find` command in make target instead of recursive globbing What's up with the `exec +` syntax? From the man page: ``` -exec command {} + This variant of the -exec action runs the specified command on the selected files, but the command line is built by appending each selected file name at the end; the total number of invocations of the command will be much less than the number of matched files. The command line is built in much the same way that xargs builds its command lines. Only one instance of `{}' is allowed within the command, and (when find is being invoked from a shell) it should be quoted (for example, '{}') to protect it from interpretation by shells. The command is executed in the starting directory. If any invocation returns a non-zero value as exit status, then find returns a non-zero exit status. If find encounters an error, this can sometimes cause an immediate exit, so some pending commands may not be run at all. This variant of -exec always returns true. ``` --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c4e43e7419..3f0f4036e7 100644 --- a/Makefile +++ b/Makefile @@ -56,10 +56,8 @@ vet: check_license: ./scripts/check_license.sh -# Turn on recursive globbing to find all shell scripts. check_shell: - bash -c "shopt -s globstar" - shellcheck */**/*.sh + find . -type f -name *.sh -exec shellcheck {} + sanity: vet fix lint fmt check_license From a95adabcead2cbcc7bd82f7cda744d2f66214d5e Mon Sep 17 00:00:00 2001 From: Benjamin Toll Date: Thu, 12 Dec 2019 18:17:38 -0500 Subject: [PATCH 5/5] Only check for missing dependencies List any that are missing and the echo the script to run to install. --- Makefile | 2 +- scripts/check_deps.sh | 110 +++++++++--------------------------------- 2 files changed, 25 insertions(+), 87 deletions(-) diff --git a/Makefile b/Makefile index 3f0f4036e7..940b175b61 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ check_license: ./scripts/check_license.sh check_shell: - find . -type f -name *.sh -exec shellcheck {} + + find . -type f -name "*.sh" -exec shellcheck {} + sanity: vet fix lint fmt check_license diff --git a/scripts/check_deps.sh b/scripts/check_deps.sh index 76d6c33437..8a57510f33 100755 --- a/scripts/check_deps.sh +++ b/scripts/check_deps.sh @@ -21,107 +21,45 @@ END_FG_COLOR=$(tput sgr0) GOPATH=$(go env GOPATH) export GOPATH -GOPATH1=$(echo "$GOPATH" | cut -d: -f1) -DEPS=0 +GO_BIN="$(echo "$GOPATH" | cut -d: -f1)/bin" +MISSING=0 -installing_dep() { - echo "$TEAL_FG[INSTALLING]$END_FG_COLOR $1..." +missing_dep() { + echo "$YELLOW_FG[WARNING]$END_FG_COLOR Mising dependency \`$TEAL_FG${1}$END_FG_COLOR\`." + MISSING=1 } -check_deps() { - if [ ! -f "${GOPATH1}/bin/golint" ] - then - DEPS=$(( "$DEPS" | 1 )) - fi - - if [ ! -f "${GOPATH1}/bin/stringer" ] - then - DEPS=$(( "$DEPS" | 2 )) - fi +GO_DEPS=( + "$GO_BIN/golint" + "$GO_BIN/stringer" + "$GO_BIN/swagger" +) - if [ ! -f "${GOPATH1}/bin/swagger" ] - then - DEPS=$(( "$DEPS" | 4 )) - fi +check_deps() { + for path in ${GO_DEPS[*]} + do + if [ ! -f "$path" ] + then + # Parameter expansion is faster than invoking another process. + # https://www.linuxjournal.com/content/bash-parameter-expansion + missing_dep "${path##*/}" + fi + done # Don't print `shellcheck`s location. if ! which shellcheck > /dev/null then - DEPS=$(( "$DEPS" | 8 )) + missing_dep shellcheck fi } check_deps -if [ $DEPS -eq 0 ] -then - echo "$GREEN_FG[$0]$END_FG_COLOR Required dependencies already installed." - exit 0 -fi - -if [ $(( DEPS & 1 )) -ne 0 ] -then - read -rp "${YELLOW_FG}MISSING DEPENDENCY$END_FG_COLOR \`golint\`. Install? (using go get) (Y/n): " OK - if [[ "$OK" =~ ^""$|Y|y ]] - then - installing_dep golint - GO111MODULE=off go get -u golang.org/x/lint/golint - DEPS=$(( "$DEPS" & ~1 )) - fi -fi - -if [ $(( DEPS & 2 )) -ne 0 ] -then - read -rp "${YELLOW_FG}MISSING DEPENDENCY$END_FG_COLOR \`stringer\`. Install? (using go get) (Y/n): " OK - if [[ "$OK" =~ ^""$|Y|y ]] - then - installing_dep stringer - GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer - DEPS=$(( "$DEPS" & ~2 )) - fi -fi - -if [ $(( DEPS & 4 )) -ne 0 ] -then - read -rp "${YELLOW_FG}MISSING DEPENDENCY$END_FG_COLOR \`swagger\`. Install? (using go get) (Y/n): " OK - if [[ "$OK" =~ ^""$|Y|y ]] - then - installing_dep swagger - GO111MODULE=off go get -u github.com/go-swagger/go-swagger/cmd/swagger - DEPS=$(( "$DEPS" & ~4 )) - fi -fi - -if [ $(( DEPS & 8 )) -ne 0 ] -then - - read -rp "${YELLOW_FG}MISSING DEPENDENCY$END_FG_COLOR \`shellcheck\`. Install? (using go get) (Y/n): " OK - if [[ "$OK" =~ ^""$|Y|y ]] - then - OS=$(uname) - - if [ "$OS" == "Linux" ] - then - if ! which sudo > /dev/null - then - apt-get install sudo -y - fi - - sudo apt-get install shellcheck -y - elif [ "$OS" == "Darwin" ] - then - brew install shellcheck - fi - - DEPS=$(( "$DEPS" & ~8 )) - fi -fi - -if [ $DEPS -eq 0 ] +if [ $MISSING -eq 0 ] then - echo -e "$GREEN_FG[$0]$END_FG_COLOR All required dependencies have been installed." + echo "$GREEN_FG[$0]$END_FG_COLOR Required dependencies installed." else - echo -e "$RED_FG[$0]$END_FG_COLOR Required dependencies still missing. Build will probably fail." + echo -e "$RED_FG[$0]$END_FG_COLOR Required dependencies missing. Run \`${TEAL_FG}./scripts/configure-dev.sh$END_FG_COLOR\` to install." exit 1 fi