bootstrap

#!/bin/bash
##
# A quick/simple script to get masterless salt deployed and configured.
# See help text (./bootstrap -h) and README.rst for requirements.
#
# skeys.gpg, generated with:
#   gpg --homedir /etc/salt/gpgkeys --export-secret-keys --export-options export-backup <KEY-ID> \
#       | gpg --symmetric --cipher-algo TWOFISH -o skeys.gpg
#
# BS_*: This script will dump all "BS_*" environment variables into a configuration
#       file for salt-minion; this provides a way to mangle teckhost.conf.
#
# Critical Order:
#   1. Prepare apt
#   2. Install apt/salt dependencies,
#   3. Add salt repo
#   4. Install salt
#   5. Chown salt
##
set -x

main() {
	parse_options "$@"

	# Pre-flight
	safety_checks
	lock acquire "$0" || die 'Unable to acquire lock'
	pristine_apt || die 'Failed to set pristine apt configuration'
	mkdir -p /etc/salt/minion.d
	echo 'master: invalid.tld' >/etc/salt/minion.d/teckhost.conf

	# Hack: IPv6 is often incorrectly implemented and first boot is very touchy
	echo 'precedence ::ffff:0:0/96  100' >>/etc/gai.conf
	echo 'Acquire::ForceIPv4 "true";' >/etc/apt/apt.conf.d/99force-ipv4

	# Install and configure salt-minion (solo)
	install_salt || die 'Failed to install salt-minion service'
	configure_minion || die 'Failed to configure salt-minion'
	deploy_gpgkeys || die 'Failed to unpack GPG keys'
	chown -R root:root /etc/salt/gpgkeys

	# Run Highstate and Configure System
	run_highstate || die 'Provisioning process (highstate) failed'

	# Cleanup
	lock destroy "$0"
}

show_usage() {
	t="$(printf '\t')"
	cat <<-EOF
	Deploy masterless salt on a host managed by MTecknology.

	Usage: $0 [-h] <options>

	Options:
	  -k [http]${t}Location of encrypted blob containing Salt GPG keys (can be file)
	  -p [path]${t}File with password used to decrypt gpg blob
	  -l X${t}Log level (0=Debug, 1=Info, 2=Warn, 3=Error)
	  -h${t}${t}Print this help text

	Environment Variables (Defaults):
	  TH_SALTGPG${t}https://raw.githubusercontent.com/MTecknology/teckhost/master/pillar/skeys.gpg
	  TH_SALTPWF${t}/tmp/gpgpassphrase
	  LOG_LEVEL${t}1  (info)
	  keydir${t}/etc/salt/gpgkeys
	  BS_*${t}${t}<none>
	EOF
}

# Parse options, prioritizing args > env > defaults
parse_options() {
	# Default values
	# These can be overridden by setting environment variables
	export TH_SALTGPG="${TH_SALTGPG:-https://raw.githubusercontent.com/MTecknology/teckhost/master/pillar/skeys.gpg}"
	export TH_SALTPWF="${TH_SALTPWF:-/tmp/gpgpassphrase}"
	export LOG_LEVEL="${LOG_LEVEL:-1}"

	# Modify defaults
	while getopts 'k:p:l:h' opt; do
		case "$opt" in
			k) TH_SALTGPG="$OPTARG";;
			p) TH_SALTPWF="$OPTARG";;
			l) LOG_LEVEL="$OPTARG";;
			h) show_usage; exit 0;;
			*) die "Unexpected argument provided: '$OPT'";;
		esac
	done
}

# Verify all expected data is currently present or else die with reason
safety_checks() {
	command_present apt-get || die 'Must have apt-get available'
}

# Ensure a clean apt state prior to salt management
pristine_apt() {
	# Find $OSCODENAME
	# Seems excessive when only one path is likely, but who knows how this might get used
	if [[ -n "$OSCODENAME" ]]; then
		log "$DEBUG" "Found \$OSCODENAME=$OSCODENAME in environment"
	elif [[ -f /etc/os-release ]]; then
		# most likely path
		. /etc/os-release
		OSCODENAME="$VERSION_CODENAME"
	elif command_present 'lsb_release'; then
		OSCODENAME="$(lsb_release -cs)"
	else
		die 'Could not figure out OSCODENAME'
	fi

	# Force a pristine/known-good apt configuration
	rm -rf /etc/apt/sources.list*
	mkdir /etc/apt/sources.list.d
	cat >/etc/apt/sources.list <<-EOF
		deb http://deb.debian.org/debian $OSCODENAME main contrib non-free non-free-firmware
		deb http://security.debian.org/debian-security $OSCODENAME-security main contrib non-free non-free-firmware
		deb http://deb.debian.org/debian $OSCODENAME-updates main contrib non-free non-free-firmware

		# Newer Packages (use with extreme caution)
		deb http://deb.debian.org/debian testing main non-free non-free-firmware contrib
		deb http://deb.debian.org/debian sid main non-free non-free-firmware contrib
		EOF
	cat >/etc/apt/preferences.d/pinning <<-EOF
		Package: *
		Pin: release a=stable
		Pin-Priority: 700

		Package: *
		Pin: release a=stable-security
		Pin-Priority: 700

		Package: *
		Pin: release a=testing
		Pin-Priority: 400

		Package: *
		Pin: release a=unstable
		Pin-Priority: 300
		EOF

	# Update package cache
	apt-get update
}

# Install the salt master and get vendor-garbage configured correctly
install_salt() {
	# Dependencies
	apt-get update || return 1
	apt-get install -y debconf-utils wget python3-venv build-essential python3-dev git || return 1

	# Application
	python3 -m venv /opt/salt
        #/opt/salt/bin/pip3 install cryptography pygit2 salt==3007.1
	# https://github.com/saltstack/salt/issues/66590
        #/opt/salt/bin/pip3 install pygit2==1.14.1

	# https://github.com/saltstack/salt/issues/66590
	/opt/salt/bin/pip3 install salt==3007.1 pygit2==1.14.1

	# Directory structure
	mkdir -p /etc/salt/minion.d
	mkdir -p /etc/salt/pki/minion
}

# Run a highstate
run_highstate() {
	# Ensure /dev/shm is mounted
	if ! mountpoint -q /dev/shm; then
		test -d /dev/shm || mkdir /dev/shm
		mount -t tmpfs none /dev/shm
	fi

	# This is an ugly hack because of some networking hiccups during some deployments.
	if ! /opt/salt/bin/salt-call --local -l info state.highstate; then
		log "$WARN" 'FIRST HIGHSTATE FAILED; Sleeping before less verbose retry'
		sleep 120
		# Less verbosity to help with information gathering
		/opt/salt/bin/salt-call --local -l quiet --state-verbose=false state.highstate
	fi
}

# Create default minion configuration
configure_minion() {
	# Copy BS_* env vars to grains file
	# Provides persistence for information that is likely coming from grub for testing.
	echo -e 'grains:\n  bootstrap:' >/etc/salt/minion.d/bootstrap.conf
	for var in "${!BS_@}"; do
		log "$DEBUG" "Saving bootstrap var(${var#BS_}): ${!var}"
		echo "    ${var#BS_}: ${!var}" >>/etc/salt/minion.d/bootstrap.conf
	done

	# Write temporary config file; Salt will overwrite using (bootstrap.conf) grains.
	if [ -z "$BS_devdir" ]; then
		cat >/etc/salt/minion.d/teckhost.conf <<-EOF
			file_client: local
			top_file_merging_strategy: same
			fileserver_backend:
			  - gitfs
			gitfs_saltenv_whitelist: [base]
			gitfs_remotes:
			  - ${BS_gitrepo:-https://github.com/MTecknology/teckhost.git}:
			    - root: ${BS_states_root:-states}
			    - base: ${BS_gitfs_base:-cicd-release}
			ext_pillar:
			  - git:
			    - ${BS_gitfs_pillar_base:-cicd-release} ${BS_gitrepo:-https://github.com/MTecknology/teckhost.git}:
			      - root: ${BS_pillar_root:-pillar}
			      - env: base
			EOF
	else
		cat >/etc/salt/minion.d/teckhost.conf <<-EOF
			file_client: local
			file_roots:
			  base:
			    - ${BS_devdir}/${BS_states_root:-states}
			pillar_roots:
			  base:
			    - ${BS_devdir}/${BS_pillar_root:-pillar}
			EOF
	fi
	# NOTE: Salt will overwrite this, verify bootstrap (BS) grains are written.
}

# Download and unpack Salt GPG keys
# NOTE: This requires a password, shared among admins
deploy_gpgkeys() {
	# Get the encrypted keys
	gpgblob="$(mktemp)"
	keydir="${keydir:-/etc/salt/gpgkeys}"
	if [[ -f "$TH_SALTGPG" ]]; then
		log "$DEBUG" "Copying GPG blob from $TH_SALTGPG"
		cp "$TH_SALTGPG" "$gpgblob" || return 1
	else
		log "$DEBUG" "Downloading GPG blob from $TH_SALTGPG"
		wget "$TH_SALTGPG" -O "$gpgblob" || return 1
	fi

	# Deploy GPG keys
	mkdir "$keydir"; chmod 0700 "$keydir"
	unset pwfile; [[ -f "$TH_SALTPWF" ]] && pwfile=('--passphrase-file' "$TH_SALTPWF")
	gpg --batch "${pwfile[@]}" --decrypt "$gpgblob" | gpg --homedir "$keydir" --import

	# Verification
	gpg --homedir "$keydir" --list-secret-keys | grep -q 'salt' ||
		die 'Could not verify gpg keys were installed correctly'

	# Cleanup (leave broken for investigation)
	rm "$gpgblob"
}


##
# Copied from https://github.com/MTecknology/script-helpers
##

# Log Levels
DEBUG=0; INFO=1; WARN=2; ERROR=3
readonly DEBUG INFO WARN ERROR
export DEBUG INFO WARN ERROR

# Check if a command (or alias/function) is available.
# Usage: command_present bin
command_present() {
	command -v "$1" >/dev/null && return 0
	alias | grep -q "\s$1=" 2>/dev/null && return 0
	return 1
}

# Print a formatted (critical) message and exit with status.
# Usage: die [exit_status] message
die() {
	lock destroy "$0"

	# If first argument was an integer, use as exit_status
	case "$1" in
		(*[!0123456789]*) _exit_status=1;;
		(*) _exit_status="$1"; shift;;
	esac

	printf '*** CRITICAL: %s ***\n' "$1"
	exit "$_exit_status"
}

# Manage a lock file.
# Usage: lock operation [key]
lock() {
	_h="$(printf '%s' "${2:-$0}" | cksum | awk '{print $1}')"
	case "$1" in
		(acquire) _lock_acquire "/tmp/$_h.lock";;
		(destroy) rm -f "/tmp/$_h.lock";;
	esac
}

# Create a lock file and populate it with PID.
_lock_acquire() {
	# Check if running
	[ -e "$1" ] && kill -0 "$(cat "$1")" && return 1

	# make sure the lockfile is removed when we exit and then claim it
	# shellcheck disable=SC2064 #[we want this expanding now]
	trap "rm -f '$1'; exit" INT TERM EXIT
	echo $$ > "$1"

	return 0
}

# Print a formatted message if env[LOG_LEVEL] >= level.
log() {
	if [ "${LOG_LEVEL:-2}" -le "$1" ]; then
		case "$1" in
			(0) _lvl='DEBUG';;
			(1) _lvl='INFO';;
			(2) _lvl='WARN';;
			(3) _lvl='ERROR';;
			(*) _lvl='UNKNOWN';;
		esac

		printf '*** %s: %s ***\n' "$_lvl" "$2"
	fi
}


##
# Kick off the script
##

main "$@"