scripts/maintenance: Assorted cleanup (stderr output, drop --workspace-dir, etc.)

scripts/maintenance/clean-aws.sh

@@ -4,6 +4,7 @@ usage() {
   cat <<EOF
 
 $(basename "$0") deletes AWS resources tagged with tags specified in a tag file.
+Requires that 'docker' and 'jq' are installed.
 
 AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environmental variables must be set.
 
@@ -30,9 +31,6 @@ Options:
                     is replaced with either the following days' date or date-override.
                     Only use if --tag-file is not used.
 
-  --workspace-dir   (optional) Parent directory for a temporary directory. /tmp is
-                    used by default.
-
   --dry-run         (optional) If set, grafiti will only do a dry run, i.e. not
                     delete any resources.
 
@@ -45,7 +43,6 @@ region=
 config_file=
 tag_file=
 date_override=
-workspace=
 dry_run=
 
 while [ $# -gt 0 ]; do
@@ -77,55 +74,56 @@ while [ $# -gt 0 ]; do
       date_override="${2:-}"
       shift
     ;;
-    --workspace-dir)
-      workspace="${2:-}"
-      shift
-    ;;
     --dry-run)
       dry_run="$1"
     ;;
     *)
-      echo "Flag '$2' is not supported."
-      exit
+      echo "Flag '$1' is not supported." >&2
+      exit 1
     ;;
   esac
   shift
 done
 
+if ! command -V docker >/dev/null || ! command -V jq >/dev/null; then
+  echo "Missing required dependencies" >&2
+  exit 1
+fi
+
 if [ -n "$AWS_REGION" ]; then
   region="${AWS_REGION:-}"
 fi
 
 if [ -z "$version" ]; then
-  echo "Grafiti image version required."
+  echo "Grafiti image version required." >&2
   exit 1
 fi
 
 if [ -z "$region" ]; then
-  echo "Must provide an AWS region, set the AWS_REGION, or set a region in your ~/.aws/config}"
+  echo "Must provide an AWS region, set the AWS_REGION, or set a region in your ~/.aws/config" >&2
   exit 1
 fi
 
 if [ -n "$tag_file" ] && [ -n "$date_override" ]; then
-  echo "Cannot use both --tag-file and --date-override flags simultaneously."
+  echo "Cannot use both --tag-file and --date-override flags simultaneously." >&2
   exit 1
 fi
 
 set -e
 
-tmp_dir="/tmp/config"
-if [ -n "$workspace" ]; then
-  tmp_dir="$(readlink -m "${workspace}/config")"
-fi
+tmp_dir="$(readlink -m "$(mktemp -d clean-aws-XXXXXXXXXX)")"
 mkdir -p "$tmp_dir"
 trap 'rm -rf "$tmp_dir"; exit' EXIT
 
-if [ -z "$config_file" ]; then
-  config_file="$(mktemp -p "$tmp_dir" --suffix=.toml)"
-  echo "maxNumRequestRetries = 11" > "$config_file"
+if [ -n "$config_file" ]; then
+  cat "$config_file" >"$tmp_dir/config.toml"
+else
+  echo "maxNumRequestRetries = 11" >"$tmp_dir/config.toml"
 fi
 
-if [ -z "$tag_file" ]; then
+if [ -n "$tag_file" ]; then
+  cat "$tag_file" >"$tmp_dir/tag.json"
+else
   tag_file="$(mktemp -p "$tmp_dir")"
 
   date_string="$(date "+%Y-%m-%d" -d "-1 day")\",\"$(date "+%Y-%-m-%-d" -d "-1 day")\",
@@ -136,7 +134,7 @@ if [ -z "$tag_file" ]; then
   	date_string="$date_override"
   fi
 
-  cat <<EOF > "$tag_file"
+  cat <<EOF >"$tmp_dir/tag.json"
 {"TagFilters":[{"Key":"expirationDate","Values":["${date_string}"]}]}
 EOF
 fi
@@ -151,7 +149,7 @@ fi
 if [ ! $force ]; then
   read -rp "Proceed deleting these resources? [y/N]: " yn
   if [ "$yn" != "y" ]; then
-    echo "Aborting deletion and cleaning up."
+    echo "Aborting deletion and cleaning up." >&2
     exit 1
   fi
 fi
@@ -164,8 +162,8 @@ docker run -t --rm --name grafiti-deleter \
 	-e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
   -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \
   -e AWS_REGION="$region" \
-	-e CONFIG_FILE="/tmp/config/$(basename "$config_file")" \
-	-e TAG_FILE="/tmp/config/$(basename "$tag_file")" \
+	-e CONFIG_FILE="/tmp/config/config.toml" \
+	-e TAG_FILE="/tmp/config/tag.json" \
 	quay.io/coreos/grafiti:"${version}" \
 	bash -c "grafiti $dry_run --config \"\$CONFIG_FILE\" --ignore-errors delete --all-deps --delete-file \"\$TAG_FILE\""
 

scripts/maintenance/tag-aws.sh

@@ -5,8 +5,8 @@ usage() {
 
 $(basename "$0") tags AWS resources with 'expirationDate: some-date-string',
 defaulting to the following days' date, and excludes all resources tagged with
-tag keys/values specified in an 'exclude' file. Requires that both 'jq' and the
-AWS CLI are installed.
+tag keys/values specified in an 'exclude' file. Requires that 'docker' is
+installed.
 
 AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environmental variables must be set.
 
@@ -35,9 +35,6 @@ Options:
                     with 'expirationDate: some-date-string', where some-date-string
                     is replaced with either the following days' date or date-override.
 
-  --workspace-dir   (optional) Parent directory for a temporary directory. /tmp is
-                    used by default.
-
   --dry-run         (optional) If set, grafiti will only do a dry run, i.e. not tag
                     any resources.
 
@@ -50,7 +47,6 @@ region=
 config_file=
 exclude_file=
 date_override=
-workspace=
 start_hour=8
 end_hour=1
 dry_run=
@@ -92,37 +88,38 @@ while [ $# -gt 0 ]; do
       date_override="${2:-}"
       shift
     ;;
-    --workspace-dir)
-      workspace="${2:-}"
-      shift
-    ;;
     --dry-run)
       dry_run="$1"
     ;;
     *)
-      echo "Flag '$2' is not supported."
-      exit
+      echo "Flag '$1' is not supported." >&2
+      exit 1
     ;;
   esac
   shift
 done
 
+if ! command -V docker >/dev/null; then
+  echo "Missing required dependencies" >&2
+  exit 1
+fi
+
 if [ -n "$AWS_REGION" ]; then
   region="${AWS_REGION:-}"
 fi
 
 if [ -z "$version" ]; then
-  echo "Grafiti image version required."
+  echo "Grafiti image version required." >&2
   exit 1
 fi
 
 if [ -z "$start_hour" ] || [ -z "$end_hour" ]; then
-  echo "Start hour and end hour must be specified."
+  echo "Start hour and end hour must be specified." >&2
   exit 1
 fi
 
 if [ -z "$region" ]; then
-  echo "Must provide an AWS region, set the AWS_REGION, or set a region in your ~/.aws/config}"
+  echo "Must provide an AWS region, set the AWS_REGION, or set a region in your ~/.aws/config" >&2
   exit 1
 fi
 
@@ -131,11 +128,7 @@ set -e
 # Tag all resources present in CloudTrail over the specified time period with the
 # following day's date as default, or with the DATE_VALUE_OVERRIDE value.
 # Format YYYY-MM-DD.
-tmp_dir="/tmp/config"
-if [ -n "$workspace" ]; then
-  tmp_dir="$(readlink -m "${workspace}/config")"
-fi
-mkdir -p "$tmp_dir"
+tmp_dir="$(readlink -m "$(mktemp -d tag-aws-XXXXXXXXXX)")"
 trap 'rm -rf "$tmp_dir"; exit' EXIT
 
 date_string='now|strftime(\"%Y-%m-%d\")'
@@ -145,9 +138,10 @@ fi
 
 # Configure grafiti to tag all resources created between START_HOUR and END_HOUR's
 # ago
-if [ -z "$config_file" ]; then
-  config_file="$(mktemp -p "$tmp_dir" --suffix=.toml)"
-  cat <<EOF > "$config_file"
+if [ -n "$config_file" ]; then
+  cat "$config_file" >"$tmp_dir/config.toml"
+else
+  cat <<EOF >"$tmp_dir/config.toml"
 endHour = -${end_hour}
 startHour = -${start_hour}
 includeEvent = false
@@ -159,13 +153,14 @@ fi
 
 # Exclusion file prevents tagging of resources that already have tags with the key
 # "expirationDate"
-if [ -z "$exclude_file" ]; then
-  exclude_file="$(mktemp -p "$tmp_dir")"
-  echo '{"TagFilters":[{"Key":"expirationDate","Values":[]}]}' > "$exclude_file"
+if [ -n "$exclude_file" ]; then
+  cat "$exclude_file" >"$tmp_dir/exclude"
+else
+  echo '{"TagFilters":[{"Key":"expirationDate","Values":[]}]}' >"$tmp_dir/exclude"
 fi
 
 echo "Tagging resources with the following configuration:"
-cat "$config_file"
+cat "$tmp_dir/config.toml"
 
 if [ -n "$dry_run" ]; then
   echo "Dry run flag set. Not tagging any resources."
@@ -174,7 +169,7 @@ fi
 if [ ! $force ]; then
   read -rp "Proceed tagging these resources? [y/N]: " yn
   if [ "$yn" != "y" ]; then
-    echo "Aborting tagging and cleaning up."
+    echo "Aborting tagging and cleaning up." >&2
     exit 1
   fi
 fi
@@ -187,8 +182,8 @@ docker run -t --rm --name grafiti-tagger \
 	-e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
   -e AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \
   -e AWS_REGION="$region" \
-	-e CONFIG_FILE="/tmp/config/$(basename "$config_file")" \
-	-e TAG_FILE="/tmp/config/$(basename "$exclude_file")" \
+	-e CONFIG_FILE="/tmp/config/config.toml" \
+	-e TAG_FILE="/tmp/config/exclude" \
 	quay.io/coreos/grafiti:"${version}" \
   bash -c "grafiti --config \"\$CONFIG_FILE\" parse | \
 	grafiti --config \"\$CONFIG_FILE\" filter --ignore-file \"\$TAG_FILE\" | \

scripts/maintenance/tag-route53-hosted-zones.sh

@@ -38,15 +38,15 @@ while [ $# -gt 0 ]; do
       shift
     ;;
     *)
-      echo "Flag '$2' is not supported."
-      exit
+      echo "Flag '$1' is not supported." >&2
+      exit 1
     ;;
   esac
   shift
 done
 
-if ! command -v jq > /dev/null || ! command -v aws > /dev/null; then
-  "Dependencies not installed."
+if ! command -V jq >/dev/null || ! command -V aws >/dev/null; then
+  echo "Missing required dependencies" >&2
   exit 1
 fi
 
@@ -68,7 +68,7 @@ echo "$tags"
 if [ ! $force ]; then
   read -rp "Proceed tagging these resources? [y/N]: " yn
   if [ "$yn" != "y" ]; then
-    echo "Aborting tagging and cleaning up."
+    echo "Aborting tagging and cleaning up." >&2
     exit 1
   fi
 fi
@@ -77,7 +77,7 @@ private_zones=$(aws route53 list-hosted-zones | \
                 jq ".HostedZones[] | select(.Config.PrivateZone == true) | .Id" | \
                 sed "s@\"@@g")
 
-for key in $(echo -e "$tags" | jq ".[].Key"); do
+for key in $(echo "$tags" | jq ".[].Key"); do
   for zone in $private_zones; do
     zone="${zone##*/}"
     is_not_tagged=$(aws route53 list-tags-for-resource \
@@ -87,11 +87,11 @@ for key in $(echo -e "$tags" | jq ".[].Key"); do
     if [ -z "$is_not_tagged" ]; then
       if aws route53 change-tags-for-resource \
       --resource-type hostedzone \
-      --add-tags "$(echo -e "$tags")" \
+      --add-tags "$tags" \
       --resource-id "${zone##*/}"; then
         echo "Tagged hosted zone ${zone##*/}"
       else
-        echo "Error tagging hosted zone ${zone##*/}"
+        echo "Error tagging hosted zone ${zone##*/}" >&2
       fi
     fi
   done

tests/jenkins-jobs/maintenance/tag_clean_aws_grafiti_job.groovy

@@ -77,7 +77,6 @@ for region in "\${regions[@]}"; do
   \$SCRIPT_DIR/maintenance/\$TAG_CLEAN.sh \\
     --grafiti-version "\$GRAFITI_VERSION" \\
     --aws-region "\$region" \\
-    --workspace-dir "\$WORKSPACE" \\
     --force \\
     \$DATE_OVERRIDE_FLAG
 done