diff --git a/README.md b/README.md index 94c3d57..54d088d 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,15 @@ -[1]: https://github.com/opsgang/fetch # libs -... reusable scripts - retrieve specific versions with [opsgang/fetch][1] :) +... reusable script bundles to make automation easy and consistent. -[![Run Status](https://api.shippable.com/projects/5a588d01e0a7bb07007efbd7/badge?branch=master)](https://app.shippable.com/github/opsgang/libs) - -**You need to have GNU coreutils installed for these scripts to work.** - -Mac users: We use `realpath`, and GNU versions of `sed`, `awk`, `sort`. - -BSD flavours will not work, so `homebrew` some GNU-ey goodness if you plan to use these scripts locally. - -## USAGE - -Typically we source all scripts under bash/habitual/ in our scripts - -To grab these to a local dir e.g. ./lib, you can use fetch: - -```bash - -# retrieve latest v 1.x (but < 2.x) of files under bash/habitual -fetch --repo="https://github.com/opsgang/libs" --tag="~> 1.0" --source-path=/bash/habitual ./lib/habitual - -# Now source all habitual libs in your scripts with something like: -for lib in $(find ./habitual -type f | grep -v 'README' | grep -vP '\.(awk|md|markdown|txt)$'); do - ! . $lib && echo "ERROR $0: ... could not source $lib" && return 1 -done - -``` - -## DOCUMENTATION - -Generated from simple inline markup. - -To generate it yourself: +Built and tested with [shippable](https://shippable.com). -```bash -cd bash -libs=$(find ./ -path './t' -prune -o -name '*.functions' -print) -for lib in $libs; do awk -f bashdoc-to-md.awk $lib > $lib.md ; done -``` - -## BUILDS / PKGS - -Currently these run tests against the bash functions. - -TODO: - -On a git tag push event, create bundles of related scripts and upload -as binary assets to a github release. e.g. all utility scripts for running terraform, -or building an AMI _opsgang_ style. - -These could then be retrieved with [opsgang/fetch][1]. - -## TESTS - -```bash - -cd ./bash # all tests must be run from this dir. - -# to run all tests for a particular script e.g. habitual/std.functions: -t/habitual/std.functions - -# to run individual tests for a script e.g. for habitual/functions -t/habitual/std.functions t_can_source_multiple_files t_check_var_defined - -# to run all tests for all scripts under ./habitual -t() { - local suite="$1" rc=0 - [[ "$(basename $(realpath .))" != "bash" ]] && echo 'ERROR: run from ./bash dir' && return 1 - [[ -z "$suite" ]] && echo 'ERROR: pass suite name' >&2 && return 1 - - libs=$(find $suite -path './t' -prune -o -name '*.functions' -print) - for lib in $libs; do - rc=0 - f="t/${lib#./}" - [[ ! -x "$f" ]] && echo "no tests for $lib" && continue - $f || rc=1 - done +[![Run Status](https://api.shippable.com/projects/5a588d01e0a7bb07007efbd7/badge?branch=master)](https://app.shippable.com/github/opsgang/libs) - return $rc -} +--- -# ... then you can run things like ... -t ./ || echo "FAILURES" # run tests for all functions. -t habitual || echo "FAILURES" # run tests for all functions under habitual +## AVAILABLE LIBS -``` +* bash: see [this README](bash/README.md) +--- diff --git a/bash/README.md b/bash/README.md new file mode 100644 index 0000000..4d2b363 --- /dev/null +++ b/bash/README.md @@ -0,0 +1,167 @@ +[1]: https://github.com/opsgang/fetch#tag-constraint-expressions "OG fetch tag constraints" +[2]: https://github.com/opsgang/fetch/releases "OG fetch releases" +[3]: bashdoc-to-md.awk.md +[4]: https://github.com/settings/tokens +# libs/bash + +_Make your life easier and consistent when scripting automation, by sourcing these libs._ + +Tested bundles are available on the [releases page](https://github.com/opsgang/libs/releases). + +> **You need to have GNU coreutils, and realpath installed for these scripts to work.** +> +> They also rely on GNU versions of `grep`, `sed`, `awk`, `sort` and `find`. +> +> BSD flavours will not work, so Mac users should `homebrew` some GNU-ey goodness +> before using these scripts locally. + +* [BUNDLES](#bundles) + * [HOWTO: USE ONE](#howto-use-one) + * [HOWTO: GET ONE](#howto-get-one) + +* [TESTS](#tests) + +* [DOCS - bashdoc](#docs) + +--- + +--- + +## BUNDLES + +On a release, we create attached bundles (_read_ .tgz) that contains all libs required to perform +a specific automation goal. e.g to run [terraform](https://terraform.io) in a consistent and +non-interactive way. + +The bundles all contain an entrypoint script that does all of the sourcing of the +other libs for you. + +--- + +### HOWTO: USE ONE + +Just source the entrypoint script which is always called `opsgang.sourcelibs`. + +e.g. If you've installed the bundle under /my/dir ... + +```bash +# In my calling script, use the opsgang libs: +. /my/dir/opsgang.sourcelibs || exit 1 +``` + +--- + +### HOWTO: GET ONE + +#### a quick aside - the github access token + +> Github will rate-limit your downloads a lot more if you don't send +> an authorization token with the request. +> +> Just create a github [personal access token][4] for the user that will +> perform the downloads, and add this to the curl commands in the examples. + +e.g. + +* add to the curl `-H 'Authorization: token {{mytoken}}'`, where {{mytoken}} + is replaced with your shiny github token. + +#### ... I want an exact version or the latest + +> You can download and untgz a version (or latest) in one line if you want. + +e.g. to download _latest_ release of `terraform_run.tgz` bundle to `/my/dir`: + +```bash +curl -sSL -H 'Accept: application/octet-stream' \ + https://github.com/opsgang/libs/releases/download/latest/terraform_run.tgz \ + | tar -xvz -C /my/dir +``` + +... or replace /latest in url with desired tag e.g. /0.0.1 + +#### ... I want to specify a version constraint like '~>1.0' + +> You can use the repo's download helper to install the version of a bundle that +> meets your criteria. + +e.g to download and untgz the latest 1.x version of terraform\_run.gz to `/my/dir/`: + +```bash +curl --retry 3 -sSL + https://raw.githubusercontent.com/opsgang/libs/master/bash/bundles/dl_release.sh \ +| GITHUB_TOKEN=$token bash -s -- terraform_run.gz '~>1.0' /my/dir +``` + +> GITHUB\_TOKEN can be omitted, but your downloads will be rate-limited. +> See +> +> The local download dir will be created if needed. +> +> See the [opsgang/fetch README][1] for a description of version constraints. +> +> If you already have [opsgang/fetch][2] in your \$PATH, dl\_release.sh will run quicker. + +--- + +--- + +## TESTS + +Each lib file has a corresponding test script that is run as part of CI. + +Specific Tests in a test script can be run (or all by default). + +A convenience function is in the example below, if you want to run all available test scripts. + +```bash + +cd ./bash # all tests must be run from this dir. + +# to run all tests for a particular script e.g. habitual/std.functions: +t/habitual/std.functions + +# to run individual tests for a script e.g. for habitual/functions +t/habitual/std.functions t_can_source_multiple_files t_check_var_defined + +# to run all tests for all scripts under ./habitual +t() { + local suite="$1" rc=0 + [[ "$(basename $(realpath .))" != "bash" ]] && echo 'ERROR: run from ./bash dir' && return 1 + [[ -z "$suite" ]] && echo 'ERROR: pass suite name' >&2 && return 1 + + libs=$(find $suite -path './t' -prune -o -name '*.functions' -print) + for lib in $libs; do + rc=0 + f="t/${lib#./}" + [[ ! -x "$f" ]] && echo "no tests for $lib" && continue + $f || rc=1 + done + + return $rc +} + +# ... then you can run things like ... +t ./ || echo "FAILURES" # run tests for all functions. +t habitual || echo "FAILURES" # run tests for all functions under habitual + +``` + +--- + +--- + +## DOCS + +> Each lib file has its own .md documentation that sits alongside it in this repo. + +The markdown is generated from a simple inline markup ([see here for more info][3]). + +To generate it yourself for all lib files. + +```bash +cd bash +libs=$(find ./ -path './t' -prune -o -name '*.functions' -print) +for lib in $libs; do awk -f bashdoc-to-md.awk $lib > $lib.md ; done +``` + diff --git a/bash/bundles/dl_release.sh b/bash/bundles/dl_release.sh new file mode 100755 index 0000000..63b11db --- /dev/null +++ b/bash/bundles/dl_release.sh @@ -0,0 +1,135 @@ +#!/bin/bash +GHREPO=https://github.com/opsgang/libs + +FETCH_TAG='v0.1.1' +FETCH_URL="https://github.com/opsgang/fetch/releases/download/$FETCH_TAG/fetch.tgz" +FETCH_DL='/tmp/ghfetch' + +usage() { + cat < '' + + e.g. + $_SN terraform_run.tgz '~>1.0' /home/me/project/ + + OR provide your github token for preferential rate-limit. + GITHUB_TOKEN=\$token $_SN terraform_run.tgz '~>1.0' /home/me/project/ + + ... will create /home/me/project if it does not exist. + + ... will also download opsgang/fetch binary to /tmp/ghfetch if not + already in your \$PATH. This will be deleted on success. +EOF +} + +retry_fetch() { + local rc=1 retries=3 delay=3 + + while [[ $(( retries-- )) -gt 0 ]]; do + $FETCH \ + --repo="$GHREPO" \ + --tag="$CONSTRAINT" \ + --release-asset="$BUNDLE" \ + $DOWNLOAD_DIR + + [[ $? -eq 0 ]] && rc=0 && break + + echo "INFO: $_SN: ... failed fetching. Retries left: $retries" + done + + [[ $? -ne 0 ]] && echo "ERROR $_SN: ... giving up. Is github.com down?" + return $rc +} + +discover_fetch() { + local rc=1 + local valid_names="fetch ghfetch /tmp/ghfetch" + local rx='^ *fetch \[global options\] $' + + for name in $valid_names; do + if type -P $name >/dev/null && $name --help 2>/dev/null | grep "$rx" >/dev/null + then + type -P $name + rc=0 + break + fi + done + return $rc +} + +fetch_me_fetch() { + echo "INFO $_SN: downloading opsgang 'fetch' binary to $FETCH_DL" + echo "INFO $_SN: ... we use this to retrieve the release binary" + echo "INFO $_SN: version that meets the constraint $CONSTRAINT." + + curl \ + -sS -L --retry 3 \ + -H 'Accept: application/octet-stream' \ + $FETCH_URL \ + | tar -xzv -C /tmp \ + && mv /tmp/fetch /tmp/ghfetch + + if [[ $? -eq 0 ]]; then + echo "INFO $_SN: fetch downloaded to as $FETCH_DL" + echo "INFO $_SN: ... learn more about what fetch can do at" + echo "INFO $_SN: https://github.com/opsgang/fetch" + return 0 + else + echo "ERROR $_SN: could not download opsgang fetch ($FETCH_URL)" + return 1 + fi +} + +_auth_header() { + if [[ ! -z "$GITHUB_TOKEN" ]]; then + echo "-H 'Authorization: token $GITHUB_TOKEN'" + fi +} + +_SN=dl_release.sh + +BUNDLE="$1" +CONSTRAINT="$2" +DOWNLOAD_DIR="$3" + +RC=0 + +echo "INFO $_SN: Running." + +for var in BUNDLE CONSTRAINT DOWNLOAD_DIR; do + [[ -z "${!var}" ]] && RC=1 +done + +[[ $RC -ne 0 ]] && echo "ERROR $_SN requires 3 args." && usage && exit 1 + +if [[ -z "$GITHUB_TOKEN" ]]; then + echo "INFO $_SN: \$GITHUB_TOKEN not in env:" + echo "INFO $_SN: ... you will be strictly rate limited by github." +else + export GITHUB_OAUTH_TOKEN="$GITHUB_TOKEN" +fi + +if ! FETCH=$(discover_fetch); then + ! fetch_me_fetch && exit 1 + FETCH="$FETCH_DL" +else + echo "INFO $_SN: Found a suitable 'fetch': $FETCH" +fi + +exit 0 + +if [[ -e "$DOWNLOAD_DIR" ]]; then + if [[ ! -d "$DOWNLOAD_DIR" ]]; then + echo "ERROR $_SN: $DOWNLOAD_DIR exists but not a directory" + echo "ERROR $_SN: ... won't be able to download to there!" + exit 1 + fi +else + echo "INFO $_SN: creating $DOWNLOAD_DIR if needed." + mkdir -p $DOWNLOAD_DIR && [[ ! -d "$DOWNLOAD_DIR" ]] && exit 1 +fi +retry_fetch || exit 1 + + diff --git a/bash/bundles/release b/bash/bundles/release index e91cbb1..65d96f5 100755 --- a/bash/bundles/release +++ b/bash/bundles/release @@ -71,6 +71,8 @@ body_text() { local tag="$1" commit="$2" local bt='```' # don't want to deal with backticks being interpolated echo " +[1]: https://github.com/opsgang/libs/tree/$tag/bash/README.md#howto-get-one +[2]: https://github.com/opsgang/libs/tree/$tag/bash/README.md#howto-use-one ## $tag _Built from sha1 ${commit}_. @@ -81,13 +83,10 @@ _Built from sha1 ${commit}_. * terraform_run.tgz: all bash libs to run terraform consistently regardless of version. -> Download one of these, untgz and source the opsgang.sourcelibs file in your bash code. -> -> e.g in your own bash code add the line: +For how to get one of these bundles, see [here][1] - or just `curl` and untar it from here. + +For how to use a bundle, see [here][2]. -${bt}bash -. /path/to/opsgang.sourcelibs || exit 1 -${bt} " } diff --git a/bash/bundles/sourcelibs.tmpl b/bash/bundles/sourcelibs.tmpl index e90ed85..040bf04 100644 --- a/bash/bundles/sourcelibs.tmpl +++ b/bash/bundles/sourcelibs.tmpl @@ -2,7 +2,7 @@ # # Source this file in your script and all necessary libs # will also be consumed. -# +# __OPSGANG_LIBS=" " if [[ -z "$BASH_SOURCE" ]] || [[ "$BASH_SOURCE" == "$0" ]]; then @@ -10,8 +10,21 @@ if [[ -z "$BASH_SOURCE" ]] || [[ "$BASH_SOURCE" == "$0" ]]; then exit 1 fi RC=0 + +OPSGANG_add_source() { + export OPSGANG_SOURCED="$OPSGANG_SOURCED $*" +} + +OPSGANG_is_sourced() { + OPSGANG_list_sourced | grep -P "^$1$" >/dev/null +} + +OPSGANG_list_sourced() { + printf "%s\n" $OPSGANG_SOURCED +} + __source() { - local file="" file_abs="" rc=0 + local file="" file_abs="" rc=0 local this="$BASH_SOURCE" local libdir="$(cd "$(dirname "$this")"; pwd)" @@ -19,7 +32,12 @@ __source() { for file in $libs; do file_abs=$libdir/$file [[ ! -z $DEBUG ]] && echo "DEBUG $this: ... sourcing $file_abs" >&2 - ! . $file_abs && echo "ERROR $this: could not source $file_abs" >&2 && rc=1 + if . $file_abs + then + OPSGANG_add_source "$file_abs" + else + echo "ERROR $this: could not source $file_abs" >&2 && rc=1 + fi done return $rc } diff --git a/bash/terraform/terraform_run.functions b/bash/terraform/terraform_run.functions index e4690ac..d1fb92d 100644 --- a/bash/terraform/terraform_run.functions +++ b/bash/terraform/terraform_run.functions @@ -244,7 +244,7 @@ terraform_init() { # terraform_apply() { i "... applying terraform" - local opts="$TERRAFORM_APPLY_OPTS" # opts to pass apply subcmd + local opts="$TERRAFORM_APPLY_OPTS -input=false" # opts to pass apply subcmd # ... for terraform 0.11.0 or later semver_a_ge_b "$(terraform_version)" "0.11.0" && opts="$opts -auto-approve" @@ -268,7 +268,9 @@ terraform_apply() { # # Will use `remote cfg` and `get` instead of `init` for older terraform versions. # -# Will run `apply -auto-approve` for newer versions of terraform. +# Will always run with `-input=false`. These scripts are for automation after all! +# +# Will also append `-auto-approve` for terraform v0.11+. # # See [terraform_init](#terraform_init) and [terraform_apply](#terraform_apply). # @@ -363,7 +365,7 @@ terraform_run() { __show_env # if DEVMODE, will print out exported vars. - $TERRAFORM plan || exit 1 + $TERRAFORM plan -input=false || exit 1 no_unpushed_changes || exit 1 diff --git a/bash/terraform/terraform_run.functions.md b/bash/terraform/terraform_run.functions.md index 0187c68..1f36d00 100644 --- a/bash/terraform/terraform_run.functions.md +++ b/bash/terraform/terraform_run.functions.md @@ -184,7 +184,9 @@ Arg 1: (optional) path to dir containing your terraform. Defaults to current dir Will use `remote cfg` and `get` instead of `init` for older terraform versions. -Will run `apply -auto-approve` for newer versions of terraform. +Will always run with `-input=false`. These scripts are for automation after all! + +Will also append `-auto-approve` for terraform v0.11+. See [terraform_init](#terraform_init) and [terraform_apply](#terraform_apply).