From 2b025b17f9d58a6a50e83d8b0bcb612ecde43fb9 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 11 Nov 2019 16:58:33 -0800 Subject: [PATCH 1/8] Retry GitHub status updates --- hack/jenkins/common.sh | 42 ++++++++++++++++++++++++---- hack/jenkins/minikube_set_pending.sh | 39 ++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 61d9dfe90caa..8242d5588404 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -289,11 +289,41 @@ ${SUDO_PREFIX} rm -f "${KUBECONFIG}" || true rmdir "${TEST_HOME}" echo ">> ${TEST_HOME} completed at $(date)" -if [[ "${MINIKUBE_LOCATION}" != "master" ]]; then - readonly target_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" - curl -s "https://api.github.com/repos/kubernetes/minikube/statuses/${COMMIT}?access_token=$access_token" \ - -H "Content-Type: application/json" \ - -X POST \ - -d "{\"state\": \"$status\", \"description\": \"Jenkins\", \"target_url\": \"$target_url\", \"context\": \"${JOB_NAME}\"}" +if [[ "${MINIKUBE_LOCATION}" == "master" ]]; then + exit $result fi + +# retry_github_status provides reliable github status updates +function retry_github_status() { + local pr=$1 + local context=$2 + local state=$4 + local token=$5 + local target=$6 + + # Retry in case we hit our GitHub API quota or fail other ways. + local attempt=0 + local timeout=2 + local code=-1 + + while [[ "${attempt}" -lt 6 ]]; do + code=$(curl -s -i --write-out "%{http_code}" -L \ + "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "{\"state\": \"$state\", \"description\": \"Jenkins\", \"target_url\": \"$target\", \"context\": \"${context}\"}" || echo 999) + + # 2xx HTTP codes + if [[ "${code}" =~ ^2 ]]; then + break + fi + + echo "HTTP code ${code}! Retrying in ${timeout} .." + sleep "${timeout}" + attempt=$(( attempt + 1 )) + timeout=$(( timeout * 2 )) + done +} + +retry_github_status "${MINIKUBE_LOCATION}" "${JOB_NAME}" "${status}" "${access_token}" "https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" exit $result diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index a595773ff8bf..d31ac030a74b 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -42,11 +42,40 @@ jobs=( 'none_Linux' ) +# retry_github_status provides reliable github status updates +function retry_github_status() { + local pr=$1 + local context=$2 + local state=$4 + local token=$5 + local target=$6 + + # Retry in case we hit our GitHub API quota or fail other ways. + local attempt=0 + local timeout=2 + local code=-1 + + while [[ "${attempt}" -lt 6 ]]; do + code=$(curl -s -i --write-out "%{http_code}" -L \ + "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "{\"state\": \"$state\", \"description\": \"Jenkins\", \"target_url\": \"$target\", \"context\": \"${context}\"}" || echo 999) + + # 2xx HTTP codes + if [[ "${code}" =~ ^2 ]]; then + break + fi + + echo "HTTP code ${code}! Retrying in ${timeout} .." + sleep "${timeout}" + attempt=$(( attempt + 1 )) + timeout=$(( timeout * 2 )) + done +} + for j in ${jobs[@]}; do - target_url="https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt" - curl "https://api.github.com/repos/kubernetes/minikube/statuses/${ghprbActualCommit}?access_token=$access_token" \ - -H "Content-Type: application/json" \ - -X POST \ - -d "{\"state\": \"pending\", \"description\": \"Jenkins\", \"target_url\": \"${target_url}\", \"context\": \"${j}\"}" + update_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \ + "https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt" done From a1b8d571bd2f1f46be493baacbdd7f7930935fad Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 11 Nov 2019 17:14:22 -0800 Subject: [PATCH 2/8] Increase retry limit to 8, just in case --- hack/jenkins/common.sh | 2 +- hack/jenkins/minikube_set_pending.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 8242d5588404..5607c97e3793 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -306,7 +306,7 @@ function retry_github_status() { local timeout=2 local code=-1 - while [[ "${attempt}" -lt 6 ]]; do + while [[ "${attempt}" -lt 8 ]]; do code=$(curl -s -i --write-out "%{http_code}" -L \ "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ -H "Content-Type: application/json" \ diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index d31ac030a74b..1edad9466c58 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -55,7 +55,7 @@ function retry_github_status() { local timeout=2 local code=-1 - while [[ "${attempt}" -lt 6 ]]; do + while [[ "${attempt}" -lt 8 ]]; do code=$(curl -s -i --write-out "%{http_code}" -L \ "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ -H "Content-Type: application/json" \ From dac8f12e9fefdde5c5eb4985f006bd8fa25f2b98 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 11 Nov 2019 17:18:50 -0800 Subject: [PATCH 3/8] Update function name --- hack/jenkins/minikube_set_pending.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index 1edad9466c58..1d31e7682b7e 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -75,7 +75,7 @@ function retry_github_status() { } for j in ${jobs[@]}; do - update_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \ + retry_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \ "https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt" done From bed9b128a8016c84d555cb07bc43701c53004a6f Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 11 Nov 2019 17:58:11 -0800 Subject: [PATCH 4/8] Fix argument indexes --- hack/jenkins/common.sh | 6 +++--- hack/jenkins/minikube_set_pending.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 5607c97e3793..764c4b4324cc 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -297,9 +297,9 @@ fi function retry_github_status() { local pr=$1 local context=$2 - local state=$4 - local token=$5 - local target=$6 + local state=$3 + local token=$4 + local target=$5 # Retry in case we hit our GitHub API quota or fail other ways. local attempt=0 diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index 1d31e7682b7e..5bf6dfea6053 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -46,9 +46,9 @@ jobs=( function retry_github_status() { local pr=$1 local context=$2 - local state=$4 - local token=$5 - local target=$6 + local state=$3 + local token=$4 + local target=$5 # Retry in case we hit our GitHub API quota or fail other ways. local attempt=0 From e96059df2124b6f83a70446a7e6edda278956bd8 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 11 Nov 2019 18:04:28 -0800 Subject: [PATCH 5/8] Remove headers from output, as they corrupted --write-out --- hack/jenkins/common.sh | 3 ++- hack/jenkins/minikube_set_pending.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 764c4b4324cc..a360ab7b4b59 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -307,7 +307,7 @@ function retry_github_status() { local code=-1 while [[ "${attempt}" -lt 8 ]]; do - code=$(curl -s -i --write-out "%{http_code}" -L \ + code=$(curl -s --write-out "%{http_code}" -L \ "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ -H "Content-Type: application/json" \ -X POST \ @@ -325,5 +325,6 @@ function retry_github_status() { done } + retry_github_status "${MINIKUBE_LOCATION}" "${JOB_NAME}" "${status}" "${access_token}" "https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" exit $result diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index 5bf6dfea6053..ae7015d7aa1d 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -56,7 +56,7 @@ function retry_github_status() { local code=-1 while [[ "${attempt}" -lt 8 ]]; do - code=$(curl -s -i --write-out "%{http_code}" -L \ + code=$(curl -s --write-out "%{http_code}" -L \ "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ -H "Content-Type: application/json" \ -X POST \ From cfcb4984486c488a1f649b08ee85ff9abe2243f8 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 11 Nov 2019 18:14:41 -0800 Subject: [PATCH 6/8] Send output to /dev/null to avoid corrupting write_out --- hack/jenkins/common.sh | 3 +-- hack/jenkins/minikube_set_pending.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index a360ab7b4b59..ba1a1081eaf0 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -307,7 +307,7 @@ function retry_github_status() { local code=-1 while [[ "${attempt}" -lt 8 ]]; do - code=$(curl -s --write-out "%{http_code}" -L \ + code=$(curl -o /dev/null -s --write-out "%{http_code}" -L \ "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ -H "Content-Type: application/json" \ -X POST \ @@ -325,6 +325,5 @@ function retry_github_status() { done } - retry_github_status "${MINIKUBE_LOCATION}" "${JOB_NAME}" "${status}" "${access_token}" "https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" exit $result diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index ae7015d7aa1d..6703e7e42b22 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -56,7 +56,7 @@ function retry_github_status() { local code=-1 while [[ "${attempt}" -lt 8 ]]; do - code=$(curl -s --write-out "%{http_code}" -L \ + code=$(curl -o /dev/null -s --write-out "%{http_code}" -L \ "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ -H "Content-Type: application/json" \ -X POST \ From f69498ca992da1b940fd84ae16db8251590e8f64 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 11 Nov 2019 19:00:31 -0800 Subject: [PATCH 7/8] Include GitHub response to make debugging easier, escape strings --- hack/jenkins/common.sh | 10 +++++++--- hack/jenkins/minikube_set_pending.sh | 9 ++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index ba1a1081eaf0..641b1bf679c3 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -293,6 +293,8 @@ if [[ "${MINIKUBE_LOCATION}" == "master" ]]; then exit $result fi +set -x + # retry_github_status provides reliable github status updates function retry_github_status() { local pr=$1 @@ -307,17 +309,19 @@ function retry_github_status() { local code=-1 while [[ "${attempt}" -lt 8 ]]; do - code=$(curl -o /dev/null -s --write-out "%{http_code}" -L \ - "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ + local out=$(mktemp) + code=$(curl -o "${out}" -s --write-out "%{http_code}" -L \ + "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=${token}" \ -H "Content-Type: application/json" \ -X POST \ - -d "{\"state\": \"$state\", \"description\": \"Jenkins\", \"target_url\": \"$target\", \"context\": \"${context}\"}" || echo 999) + -d "{\"state\": \"${state}\", \"description\": \"Jenkins\", \"target_url\": \"${target}\", \"context\": \"${context}\"}" || echo 999) # 2xx HTTP codes if [[ "${code}" =~ ^2 ]]; then break fi + cat "${out}" && rm -f "${out}" echo "HTTP code ${code}! Retrying in ${timeout} .." sleep "${timeout}" attempt=$(( attempt + 1 )) diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index 6703e7e42b22..ce504c0746af 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -56,17 +56,19 @@ function retry_github_status() { local code=-1 while [[ "${attempt}" -lt 8 ]]; do - code=$(curl -o /dev/null -s --write-out "%{http_code}" -L \ - "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=$token" \ + local out=$(mktemp) + code=$(curl -o "${out}" -s --write-out "%{http_code}" -L \ + "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=${token}" \ -H "Content-Type: application/json" \ -X POST \ - -d "{\"state\": \"$state\", \"description\": \"Jenkins\", \"target_url\": \"$target\", \"context\": \"${context}\"}" || echo 999) + -d "{\"state\": \"${state}\", \"description\": \"Jenkins\", \"target_url\": \"${target}\", \"context\": \"${context}\"}" || echo 999) # 2xx HTTP codes if [[ "${code}" =~ ^2 ]]; then break fi + cat "${out}" && rm -f "${out}" echo "HTTP code ${code}! Retrying in ${timeout} .." sleep "${timeout}" attempt=$(( attempt + 1 )) @@ -74,6 +76,7 @@ function retry_github_status() { done } + for j in ${jobs[@]}; do retry_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \ "https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt" From 5efde8bf7de28bf25105e8584aa2a91219e38da3 Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Mon, 11 Nov 2019 19:21:46 -0800 Subject: [PATCH 8/8] The PR field is actually the commit field --- hack/jenkins/common.sh | 8 +++----- hack/jenkins/minikube_set_pending.sh | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/hack/jenkins/common.sh b/hack/jenkins/common.sh index 641b1bf679c3..4d32792d4bb7 100755 --- a/hack/jenkins/common.sh +++ b/hack/jenkins/common.sh @@ -293,11 +293,9 @@ if [[ "${MINIKUBE_LOCATION}" == "master" ]]; then exit $result fi -set -x - # retry_github_status provides reliable github status updates function retry_github_status() { - local pr=$1 + local commit=$1 local context=$2 local state=$3 local token=$4 @@ -311,7 +309,7 @@ function retry_github_status() { while [[ "${attempt}" -lt 8 ]]; do local out=$(mktemp) code=$(curl -o "${out}" -s --write-out "%{http_code}" -L \ - "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=${token}" \ + "https://api.github.com/repos/kubernetes/minikube/statuses/${commit}?access_token=${token}" \ -H "Content-Type: application/json" \ -X POST \ -d "{\"state\": \"${state}\", \"description\": \"Jenkins\", \"target_url\": \"${target}\", \"context\": \"${context}\"}" || echo 999) @@ -329,5 +327,5 @@ function retry_github_status() { done } -retry_github_status "${MINIKUBE_LOCATION}" "${JOB_NAME}" "${status}" "${access_token}" "https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" +retry_github_status "${COMMIT}" "${JOB_NAME}" "${status}" "${access_token}" "https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt" exit $result diff --git a/hack/jenkins/minikube_set_pending.sh b/hack/jenkins/minikube_set_pending.sh index ce504c0746af..75851e3e8fb1 100755 --- a/hack/jenkins/minikube_set_pending.sh +++ b/hack/jenkins/minikube_set_pending.sh @@ -44,7 +44,7 @@ jobs=( # retry_github_status provides reliable github status updates function retry_github_status() { - local pr=$1 + local commit=$1 local context=$2 local state=$3 local token=$4 @@ -58,7 +58,7 @@ function retry_github_status() { while [[ "${attempt}" -lt 8 ]]; do local out=$(mktemp) code=$(curl -o "${out}" -s --write-out "%{http_code}" -L \ - "https://api.github.com/repos/kubernetes/minikube/statuses/${pr}?access_token=${token}" \ + "https://api.github.com/repos/kubernetes/minikube/statuses/${commit}?access_token=${token}" \ -H "Content-Type: application/json" \ -X POST \ -d "{\"state\": \"${state}\", \"description\": \"Jenkins\", \"target_url\": \"${target}\", \"context\": \"${context}\"}" || echo 999) @@ -76,7 +76,6 @@ function retry_github_status() { done } - for j in ${jobs[@]}; do retry_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \ "https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${j}.txt"