pull packaging/docker diffs from release-7.1

packaging/docker/Dockerfile

@@ -43,6 +43,7 @@ RUN yum install -y \
     telnet-0.17-66.el7 \
     traceroute-2.0.22-2.el7 \
     unzip-6.0-22.el7_9 \
+    openssl-1.0.2k-24.el7_9.x86_64 \
     vim-enhanced-7.4.629-8.el7_9 && \
     yum clean all && \
     rm -rf /var/cache/yum
@@ -177,13 +178,13 @@ RUN yum -y install \
     rm -rf /var/cache/yum
 
 WORKDIR /tmp
-RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \
-    echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a  kubectl" > kubectl.txt && \
+RUN curl -Ls https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl -o kubectl && \
+    echo "860c3d37a5979491895767e7332404d28dc0d7797c7673c33df30ca80e215a07  kubectl" > kubectl.txt && \
     sha256sum --quiet -c kubectl.txt && \
     mv kubectl /usr/local/bin/kubectl && \
     chmod 755 /usr/local/bin/kubectl && \
-    curl -Ls https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \
-    echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd  awscliv2.zip" > awscliv2.txt && \
+    curl -Ls https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.7.34.zip -o "awscliv2.zip" && \
+    echo "daf9253f0071b5cfee9532bc5220bedd7a5d29d4e0f92b42b9e3e4c496341e88  awscliv2.zip" > awscliv2.txt && \
     sha256sum --quiet -c awscliv2.txt && \
     unzip -qq awscliv2.zip && \
     ./aws/install && \

packaging/docker/Dockerfile.eks

@@ -53,6 +53,18 @@ RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd6
     mv tini /usr/bin/ && \
     rm -rf /tmp/*
 
+RUN curl -Ls https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl -o kubectl && \
+    echo "860c3d37a5979491895767e7332404d28dc0d7797c7673c33df30ca80e215a07  kubectl" > kubectl.txt && \
+    sha256sum --quiet -c kubectl.txt && \
+    mv kubectl /usr/local/bin/kubectl && \
+    chmod 755 /usr/local/bin/kubectl && \
+    curl -Ls https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m)-2.7.34.zip -o "awscliv2.zip" && \
+    echo "daf9253f0071b5cfee9532bc5220bedd7a5d29d4e0f92b42b9e3e4c496341e88  awscliv2.zip" > awscliv2.txt && \
+    sha256sum --quiet -c awscliv2.txt && \
+    unzip -qq awscliv2.zip && \
+    ./aws/install && \
+    rm -rf /tmp/*
+
 WORKDIR /
 
 FROM golang:1.16.7-bullseye AS go-build
@@ -173,19 +185,6 @@ RUN yum -y install \
     yum clean all && \
     rm -rf /var/cache/yum
 
-WORKDIR /tmp
-RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \
-    echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a  kubectl" > kubectl.txt && \
-    sha256sum --quiet -c kubectl.txt && \
-    mv kubectl /usr/local/bin/kubectl && \
-    chmod 755 /usr/local/bin/kubectl && \
-    curl -Ls https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \
-    echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd  awscliv2.zip" > awscliv2.txt && \
-    sha256sum --quiet -c awscliv2.txt && \
-    unzip -qq awscliv2.zip && \
-    ./aws/install && \
-    rm -rf /tmp/*
-
 # TODO: Log4J complains that it's eating the HTracer logs.  Even without it, we get per-operation
 # time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec).
 ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh

packaging/docker/build-images.sh

@@ -2,12 +2,17 @@
 set -Eeuo pipefail
 script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
 reset=$(tput sgr0)
+red=$(tput setaf 1)
 blue=$(tput setaf 4)
 
-function logg() {
+function logg () {
     printf "${blue}##### $(date +"%H:%M:%S") #  %-56.55s #####${reset}\n" "${1}"
 }
 
+ function loge () {
+     printf "${red}##### $(date +"%H:%M:%S") #  %-56.55s #####${reset}\n" "${1}"
+ }
+
 function pushd () {
     command pushd "$@" > /dev/null
 }
@@ -16,7 +21,19 @@ function popd () {
     command popd > /dev/null
 }
 
-function create_fake_website_directory() {
+function error_exit () {
+     echo "${red}################################################################################${reset}"
+     loge "${0} FAILED"
+     echo "${red}################################################################################${reset}"
+ }
+
+ trap error_exit ERR
+
+ function create_fake_website_directory () {
+     if [ ${#} -ne 1 ]; then
+         loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}"
+     fi
+     local stripped_binaries_and_from_where="${1}"
     fdb_binaries=( 'fdbbackup' 'fdbcli' 'fdbserver' 'fdbmonitor' )
     logg "PREPARING WEBSITE"
     website_directory="${script_dir}/website"
@@ -112,7 +129,7 @@ function create_fake_website_directory() {
     fdb_website="file:///tmp/website"
 }
 
-function compile_ycsb() {
+function compile_ycsb () {
     logg "COMPILING YCSB"
     if [ "${use_development_java_bindings}" == "true" ]; then
         logg "INSTALL JAVA BINDINGS"
@@ -150,7 +167,13 @@ function compile_ycsb() {
     popd || exit 128
 }
 
-function build_and_push_images(){
+function build_and_push_images () {
+     if [ ${#} -ne 3 ]; then
+         loge "INCORRECT NUMBER OF ARGS FOR ${FUNCNAME[0]}"
+     fi
+     local dockerfile_name="${1}"
+     local use_development_java_bindings="${2}"
+     local push_docker_images="${3}"
     declare -a tags_to_push=()
     for image in "${image_list[@]}"; do
         logg "BUILDING ${image}"
@@ -237,16 +260,12 @@ image_list=(
 )
 registry=""
 tag_base="foundationdb/"
-# THESE CONTROL THE PATH OF FUNCTIONS THAT ARE CALLED BELOW
-stripped_binaries_and_from_where="stripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" )
-dockerfile_name="Dockerfile"
-use_development_java_bindings="false"
-push_docker_images="false"
 
 if [ -n "${OKTETO_NAMESPACE+x}" ]; then
     logg "RUNNING IN OKTETO/AWS"
     # these are defaults for the Apple development environment
-    aws_region=$(curl -s "http://169.254.169.254/latest/meta-data/placement/region")
+    imdsv2_token=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+    aws_region=$(curl -H "X-aws-ec2-metadata-token: ${imdsv2_token}" "http://169.254.169.254/latest/meta-data/placement/region")
     aws_account_id=$(aws --output text sts get-caller-identity --query 'Account')
     build_output_directory="${HOME}/build_output"
     fdb_library_versions=( "${fdb_version}" )
@@ -257,20 +276,24 @@ if [ -n "${OKTETO_NAMESPACE+x}" ]; then
     else
         tag_postfix="${OKTETO_NAME:-dev}"
     fi
-    stripped_binaries_and_from_where="unstripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" )
-    dockerfile_name="Dockerfile.eks"
-    use_development_java_bindings="true"
-    push_docker_images="true"
+
+    # build regular images
+     create_fake_website_directory stripped_local
+     build_and_push_images Dockerfile true true
+
+     # build debug images
+     create_fake_website_directory unstripped_local
+     build_and_push_images Dockerfile.eks true true
 else
     echo "Dear ${USER}, you probably need to edit this file before running it. "
     echo "${0} has a very narrow set of situations where it will be successful,"
     echo "or even useful, when executed unedited"
     exit 1
+    # this set of options will creat standard images from a local build
+    # create_fake_website_directory stripped_local
+    # build_and_push_images Dockerfile false false
 fi
 
-create_fake_website_directory
-build_and_push_images
-
 echo "${blue}################################################################################${reset}"
 logg "COMPLETED ${0}"
 echo "${blue}################################################################################${reset}"

packaging/docker/fdb.bash

@@ -5,7 +5,7 @@
 #
 # This source file is part of the FoundationDB open source project
 #
-# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+# Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

packaging/docker/run_ycsb.sh

@@ -1,22 +1,44 @@
 #!/usr/bin/env bash
-set -Eeuxo pipefail
+set -Eeuo pipefail
+
+function logg () {
+    printf "##### $(date +'%Y-%m-%dT%H:%M:%SZ') #  %-56.55s #####\n" "${1}"
+}
+
+function error_exit () {
+    echo "################################################################################"
+    logg "${0} FAILED"
+    logg "RUN_ID: ${RUN_ID}"
+    logg "WORKLOAD: ${WORKLOAD}"
+    logg "ENVIRONMENT IS:"
+    env
+    echo "################################################################################"
+}
+
+trap error_exit ERR
 
 namespace=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
-POD_NUM=$(echo $POD_NAME | cut -d - -f3)
-KEY="ycsb_load_${POD_NUM}_of_${NUM_PODS}_complete"
-CLI=$(ls /var/dynamic-conf/bin/*/fdbcli | head -n1)
 
-echo "WAITING FOR ALL PODS TO COME UP"
-while [[ $(kubectl get pods -n ${namespace} -l name=ycsb,run=${RUN_ID} --field-selector=status.phase=Running | grep -cv NAME) -lt ${NUM_PODS} ]]; do
+logg "WAITING FOR ${NUM_PODS} PODS TO COME UP IN ${namespace}"
+while [[ $(kubectl get pods -n "${namespace}" -l name=ycsb,run="${RUN_ID}" --field-selector=status.phase=Running | grep -cv NAME) -lt ${NUM_PODS} ]]; do
     sleep 1
 done
-echo "ALL PODS ARE UP"
+logg "${NUM_PODS} PODS ARE UP IN ${namespace}"
 
-echo "RUNNING YCSB"
-./bin/ycsb.sh ${MODE} foundationdb -s -P workloads/${WORKLOAD} ${YCSB_ARGS}
-echo "YCSB FINISHED"
+logg "RUNNING YCSB ${WORKLOAD}"
+set -x
+./bin/ycsb.sh "${MODE}" foundationdb -s -P "workloads/${WORKLOAD}" "${YCSB_ARGS}"
+set +x
+logg "YCSB ${WORKLOAD} FINISHED"
 
-echo "COPYING HISTOGRAMS TO S3"
-aws s3 sync --sse aws:kms --exclude "*" --include "histogram.*" /tmp s3://${BUCKET}/ycsb_histograms/${namespace}/${POD_NAME}
-echo "COPYING HISTOGRAMS TO S3 FINISHED"
+logg "COPYING HISTOGRAMS TO S3"
+set -x
+aws s3 sync --sse aws:kms --exclude "*" --include "histogram.*" /tmp "s3://${BUCKET}/ycsb_histograms/${namespace}/${POD_NAME}"
+set +x
+logg "COPYING HISTOGRAMS TO S3 FINISHED"
 
+echo "################################################################################"
+logg "COMPLETED ${0}"
+logg "RUN_ID: ${RUN_ID}"
+logg "WORKLOAD: ${WORKLOAD}"
+echo "################################################################################"