Standardize curl flags with scurl (#7658)

Introduce `bin/scurl` which invokes `curl` with a standard set of flags to enforce best practices.
All `curl` invocations have been replaced with `scurl`.

Fixes #7593

Signed-off-by: Alex Leong <[email protected]>
(cherry picked from commit 2c6db25)
Signed-off-by: Oliver Gould <[email protected]>

.devcontainer/Dockerfile

@@ -28,7 +28,8 @@ RUN groupadd --gid=$USER_GID $USER \
 # Install a Docker client that uses the host's Docker daemon
 ARG USE_MOBY=false
 ENV DOCKER_BUILDKIT=1
-RUN curl --proto '=https' --tlsv1.3 -vsSfL https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/script-library/docker-debian.sh \
+COPY bin/scurl /usr/local/bin/scurl
+RUN scurl -v https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/script-library/docker-debian.sh \
     | bash -s --  true /var/run/docker-host.sock /var/run/docker.sock "${USER}" "${USE_MOBY}" latest
 
 RUN (echo "LC_ALL=en_US.UTF-8" \
@@ -40,24 +41,24 @@ ENV HOME=/home/$USER
 RUN mkdir -p $HOME/bin
 ENV PATH=$HOME/bin:$PATH
 
-RUN curl --proto '=https' --tlsv1.3 -vsSfLo $HOME/bin/kubectl "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+RUN scurl -vo $HOME/bin/kubectl "https://dl.k8s.io/release/$(scurl https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
     && chmod 755 $HOME/bin/kubectl
-RUN curl --proto '=https' --tlsv1.3 -vsSfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh \
+RUN scurl -v https://raw.githubusercontent.com/rancher/k3d/main/install.sh \
     | USE_SUDO=false K3D_INSTALL_DIR=$HOME/bin bash
 
 ARG YQ_VERSION=v4.2.0
-RUN curl --proto '=https' --tlsv1.3 -vsSfL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -o $HOME/bin/yq \
+RUN scurl -vo $HOME/bin/yq "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
     && chmod +x $HOME/bin/yq
 
-RUN curl --proto '=https' --tlsv1.3 -vsSfL https://sh.rustup.rs \
+RUN scurl -v https://sh.rustup.rs \
     | sh -s -- -y --default-toolchain 1.56.1 -c rustfmt -c clippy -c rls
 
 RUN mkdir /tmp/cargo-deny && cd /tmp/cargo-deny && \
-    curl --proto '=https' --tlsv1.3 -vsSfL https://github.com/EmbarkStudios/cargo-deny/releases/download/0.11.0/cargo-deny-0.11.0-x86_64-unknown-linux-musl.tar.gz | tar zxf - && \
+    scurl -v https://github.com/EmbarkStudios/cargo-deny/releases/download/0.11.0/cargo-deny-0.11.0-x86_64-unknown-linux-musl.tar.gz | tar zxf - && \
     mv cargo-deny-0.11.0-x86_64-unknown-linux-musl/cargo-deny $HOME/bin && \
     cd .. && rm -rf /tmp/cargo-deny
 
-RUN curl --proto '=https' --tlsv1.3 -vsSfL https://run.linkerd.io/install-edge | sh \
+RUN scurl -v https://run.linkerd.io/install-edge | sh \
     && ln -s $(readlink ~/.linkerd2/bin/linkerd) ~/bin/linkerd
 
 ENTRYPOINT ["/usr/local/share/docker-init.sh"]

.dockerignore

@@ -7,6 +7,7 @@
 **/node_modules
 bin
 !bin/fetch-proxy
+!bin/scurl
 !bin/install-deps
 !bin/web
 **/Dockerfile*

.github/workflows/codecov.yml

@@ -33,7 +33,7 @@ jobs:
     steps:
       - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
       - name: Yarn setup
-        run: curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.21.1 --network-concurrency 1
+        run: bin/scurl -o- https://yarnpkg.com/install.sh | bash -s -- --version 1.21.1 --network-concurrency 1
       - name: Unit tests
         run: |
           export PATH="$HOME/.yarn/bin:$PATH"
@@ -54,7 +54,7 @@ jobs:
       options: --security-opt seccomp=unconfined
     steps:
       - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
-      - run: mkdir -p target && cd target && curl -vsL https://github.com/xd009642/tarpaulin/releases/download/0.18.0/cargo-tarpaulin-0.18.0-travis.tar.gz | tar zxvf - && chmod 755 cargo-tarpaulin
+      - run: mkdir -p target && cd target && bin/scurl -v https://github.com/xd009642/tarpaulin/releases/download/0.18.0/cargo-tarpaulin-0.18.0-travis.tar.gz | tar zxvf - && chmod 755 cargo-tarpaulin
       - run: target/cargo-tarpaulin tarpaulin --workspace --out Xml
       - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b
         with:

.github/workflows/release.yml

@@ -301,7 +301,7 @@ jobs:
       run: echo "INSTALL=install-edge" >> $GITHUB_ENV
     - name: Check published version
       run: |
-        until RES=$(curl -sL https://run.linkerd.io/$INSTALL | grep "LINKERD2_VERSION=\${LINKERD2_VERSION:-$TAG}") \
+        until RES=$(bin/scurl https://run.linkerd.io/$INSTALL | grep "LINKERD2_VERSION=\${LINKERD2_VERSION:-$TAG}") \
           || (( count++ >= 10 ))
         do
           sleep 30

.github/workflows/unit_tests.yml

@@ -34,7 +34,7 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
     - name: Yarn setup
-      run: curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.21.1 --network-concurrency 1
+      run: bin/scurl --retry=2 https://yarnpkg.com/install.sh | bash -s -- --version 1.21.1 --network-concurrency 1
     - name: JS unit tests
       run: |
         export PATH="$HOME/.yarn/bin:$PATH"

Dockerfile-proxy

@@ -13,12 +13,13 @@ RUN ./bin/install-deps $TARGETARCH
 FROM --platform=$BUILDPLATFORM curlimages/curl:7.76.1 as fetch
 WORKDIR /build
 COPY bin/fetch-proxy bin/fetch-proxy
+COPY bin/scurl bin/scurl
 COPY .proxy-version proxy-version
 ARG TARGETARCH
 RUN (proxy=$(bin/fetch-proxy $(cat proxy-version) $TARGETARCH) && \
     mv "$proxy" linkerd2-proxy)
 ARG LINKERD_AWAIT_VERSION=v0.2.4
-RUN curl -fsSvLo linkerd-await https://github.com/linkerd/linkerd-await/releases/download/release%2F${LINKERD_AWAIT_VERSION}/linkerd-await-${LINKERD_AWAIT_VERSION}-${TARGETARCH} && chmod +x linkerd-await
+RUN bin/scurl -o linkerd-await https://github.com/linkerd/linkerd-await/releases/download/release%2F${LINKERD_AWAIT_VERSION}/linkerd-await-${LINKERD_AWAIT_VERSION}-${TARGETARCH} && chmod +x linkerd-await
 
 ## compile proxy-identity agent
 FROM go-deps as golang

bin/_test-helpers.sh

@@ -370,7 +370,7 @@ run_test(){
 # Returns the latest version for the release channel
 # $1: release channel to check
 latest_release_channel() {
-    curl -s https://versioncheck.linkerd.io/version.json | grep -o "$1-[0-9]*.[0-9]*.[0-9]*"
+    "$bindir"/scurl https://versioncheck.linkerd.io/version.json | grep -o "$1-[0-9]*.[0-9]*.[0-9]*"
 }
 
 # Install a specific Linkerd version.
@@ -382,7 +382,7 @@ install_version() {
     local install_url=$1
     local version=$2
 
-    curl -s "$install_url" | HOME=$tmp sh > /dev/null 2>&1
+    "$bindir"/scurl "$install_url" | HOME=$tmp sh > /dev/null 2>&1
 
     local linkerd_path=$tmp/.linkerd2/bin/linkerd
     local test_app_namespace=upgrade-test

bin/create-release-tag

@@ -3,6 +3,8 @@
 set -o errexit
 set -o nounset
 
+bindir=$( cd "${0%/*}" && pwd )
+
 # Assert at least one and at most two arguments were passed.
 if [[ $# -lt 1 || $# -gt 2 ]]; then
     echo "Error: ${0##*/} accepts 1 or 2 arguments" >&2
@@ -61,7 +63,7 @@ if [ $# -ne 1 ]; then
     url="https://run.linkerd.io/install-edge"
 
     # Get the current edge version.
-    current_edge=$(curl -sL $url | awk -v tag_format="$edge_tag_regex" '$0 ~ tag_format')
+    current_edge=$("$bindir"/scurl $url | awk -v tag_format="$edge_tag_regex" '$0 ~ tag_format')
 
     # Get the third and fourth groups of the regex; they are the month and
     # month minor values for the current edge version.

bin/docker

@@ -22,7 +22,7 @@ if [ ! -f "$dockerbin" ]; then
   mkdir -p "$targetbin"
   (
       cd "$tmp"
-      curl -Lsf -o ./docker.tar.gz "$url"
+      "$bindir"/scurl -o ./docker.tar.gz "$url"
       tar zf ./docker.tar.gz -x docker/docker
   )
   mv "$tmp/docker/docker" "$dockerbin"

bin/fetch-proxy

@@ -13,7 +13,7 @@ builddir="$rootdir/target/proxy"
 
 version=${1:-latest}
 if [ "$version" = latest ]; then
-  version=$(curl -sL https://api.github.com/repos/linkerd/linkerd2-proxy/releases/latest |jq -r .tag_name | sed 's,^release/,,')
+  version=$("$bindir"/scurl https://api.github.com/repos/linkerd/linkerd2-proxy/releases/latest |jq -r .tag_name | sed 's,^release/,,')
 fi
 
 assetbase="https://github.com/linkerd/linkerd2-proxy/releases/download/release%2F${version}"
@@ -24,8 +24,8 @@ shafile="${pkgname}.txt"
 
 mkdir -p "$builddir"
 cd "$builddir"
-curl -sLO "$assetbase/$pkgfile"
-curl -sLO "$assetbase/$shafile"
+"$bindir"/scurl -O "$assetbase/$pkgfile"
+"$bindir"/scurl -O "$assetbase/$shafile"
 
 tar -zxvf "$pkgfile" >&2
 expected=$(awk '{print $1}' "$shafile")

bin/helm

@@ -24,7 +24,7 @@ if [ ! -f "$helmbin" ]; then
     mkdir -p "$targetbin"
     (
         cd "$tmp"
-        curl -Lsf -o "./helm.tar.gz" "$helmcurl"
+        "$bindir"/scurl -o "./helm.tar.gz" "$helmcurl"
         tar zf "./helm.tar.gz" -x "$targetdir"
         chmod +x "$targetdir/helm"
     )

bin/helm-docs

@@ -47,7 +47,7 @@ if [ ! -f "$helmdocsbin" ]; then
     mkdir -p "$targetbin"
     (
         cd "$tmp"
-        curl -Lsf -o "./helm-docs.tar.gz" "$helmdocscurl"
+        "$bindir"/scurl -o "./helm-docs.tar.gz" "$helmdocscurl"
         tar zf "./helm-docs.tar.gz" -x "helm-docs"
         chmod +x "helm-docs"
     )

bin/install-pr

@@ -74,20 +74,21 @@ fi
 linkerd2_pulls_url="https://api.github.com/repos/linkerd/linkerd2/pulls"
 linkerd2_integration_url="https://api.github.com/repos/linkerd/linkerd2/actions/workflows/integration_tests.yml"
 
+bindir=$( cd "${BASH_SOURCE[0]%/*}" && pwd )
+
 # Get the URL for downloading the artifacts archive
 auth="Authorization: token $GITHUB_TOKEN"
-branch=$(curl -sL -H "$auth" "$linkerd2_pulls_url/$pr" | jq -r '.head.ref')
-artifacts=$(curl -sL -H "$auth" "$linkerd2_integration_url/runs?branch=$branch" | jq -r '.workflow_runs[0].artifacts_url')
-archive=$(curl -sL -H "$auth" "$artifacts" | jq -r '.artifacts[0].archive_download_url')
+branch=$("$bindir"/scurl -H "$auth" "$linkerd2_pulls_url/$pr" | jq -r '.head.ref')
+artifacts=$("$bindir"/scurl -H "$auth" "$linkerd2_integration_url/runs?branch=$branch" | jq -r '.workflow_runs[0].artifacts_url')
+archive=$("$bindir"/scurl -H "$auth" "$artifacts" | jq -r '.artifacts[0].archive_download_url')
 
-bindir=$( cd "${BASH_SOURCE[0]%/*}" && pwd )
 dir=$(mktemp -d -t "linkerd-pr-$pr.XXXXXXXXXX")
 
 cd "$dir" || exit
 
 echo "### Downloading images ###"
 
-curl -L -o archive.zip -H "$auth" "$archive"
+"$bindir"/scurl -o archive.zip -H "$auth" "$archive"
 unzip -o archive.zip -d image-archives/
 
 echo "### Loading images into Docker ###"

bin/k3d

@@ -24,7 +24,13 @@ if [ ! -f "$k3dbin" ]; then
   fi
 
   mkdir -p "$targetbin"
+<<<<<<< HEAD
   curl -sfL -o "$k3dbin" https://github.com/rancher/k3d/releases/download/$k3dversion/k3d-$os-$arch
+||||||| parent of 2c6db25d (Standardize curl flags with scurl (#7658))
+  curl -sfL -o "$k3dbin" https://github.com/rancher/k3d/releases/latest/download/k3d-$os-$arch
+=======
+  "$bindir"/scurl -o "$k3dbin" https://github.com/rancher/k3d/releases/latest/download/k3d-$os-$arch
+>>>>>>> 2c6db25d (Standardize curl flags with scurl (#7658))
   chmod +x "$k3dbin"
 fi
 

bin/kind

@@ -24,7 +24,7 @@ if [ ! -f "$kindbin" ]; then
   fi
 
   mkdir -p "$targetbin"
-  curl -sfL -o "$kindbin" https://github.com/kubernetes-sigs/kind/releases/download/$kindversion/kind-$os-$arch
+  "$bindir"/scurl -o "$kindbin" https://github.com/kubernetes-sigs/kind/releases/download/$kindversion/kind-$os-$arch
   chmod +x "$kindbin"
 fi
 

bin/kubectl

@@ -26,7 +26,7 @@ if [ ! -f "$kubectlbin" ]; then
   fi
 
   mkdir -p "$targetbin"
-  curl -sfL -o "$kubectlbin" https://storage.googleapis.com/kubernetes-release/release/$kubectlversion/bin/$os/$arch/kubectl${exe}
+  "$bindir"/scurl -o "$kubectlbin" https://storage.googleapis.com/kubernetes-release/release/$kubectlversion/bin/$os/$arch/kubectl${exe}
   chmod +x "$kubectlbin"
 fi
 

bin/lint

@@ -24,7 +24,7 @@ lintbin=$targetbin/.golangci-lint-$lintversion$exe
 
 if [ ! -f "$lintbin" ]; then
   mkdir -p "$targetbin"
-  curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/v$lintversion/install.sh | sh -s -- -b . v$lintversion
+  "$bindir"/scurl https://raw.githubusercontent.com/golangci/golangci-lint/v$lintversion/install.sh | sh -s -- -b . v$lintversion
   mv ./golangci-lint$exe "$lintbin"
 fi
 

bin/protoc

@@ -29,7 +29,7 @@ if [ ! -f "$protocbin" ]; then
   mkdir -p "$targetbin"
   (
     cd "$tmp"
-    curl -L --silent --fail -o "./protoc.zip" "$protocurl"
+    "$bindir"/scurl -o "./protoc.zip" "$protocurl"
     unzip -q "./protoc.zip" bin/protoc
     chmod +x bin/protoc
   )

bin/scurl

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+exec curl --proto '=https' --tlsv1.2 -sSfL "$@"

bin/shellcheck

@@ -22,7 +22,7 @@ if [ ! -f "$scbin" ]; then
   fi
 
   mkdir -p "$targetbin"
-  curl -sLf "https://github.com/koalaman/shellcheck/releases/download/$scversion/shellcheck-${scversion?}.$file" | tar -OxJv "shellcheck-${scversion}/shellcheck" > "$scbin"
+  "$bindir"/scurl "https://github.com/koalaman/shellcheck/releases/download/$scversion/shellcheck-${scversion?}.$file" | tar -OxJv "shellcheck-${scversion}/shellcheck" > "$scbin"
   chmod +x "$scbin"
 fi
 

bin/test-scale

@@ -15,6 +15,8 @@ set -e
 NAMESPACES=5
 REPLICAS=5
 
+bindir=$( cd "${0%/*}" && pwd )
+
 # TODO: share these functions with test-run
 
 check_linkerd_binary(){
@@ -75,11 +77,11 @@ linkerd_namespace=${2:-l5d-scale}
 # Deploy Books
 #
 
-BOOKS_BACKEND=$(curl -s https://raw.githubusercontent.com/BuoyantIO/booksapp/main/k8s/mysql-backend.yml)
+BOOKS_BACKEND=$("$bindir"/scurl https://raw.githubusercontent.com/BuoyantIO/booksapp/main/k8s/mysql-backend.yml)
 
-AUTHORS_SP=$(curl -s https://run.linkerd.io/booksapp/authors.swagger)
-BOOKS_SP=$(curl -s https://run.linkerd.io/booksapp/books.swagger)
-WEBAPP_SP=$(curl -s https://run.linkerd.io/booksapp/webapp.swagger)
+AUTHORS_SP=$("$bindir"/scurl https://run.linkerd.io/booksapp/authors.swagger)
+BOOKS_SP=$("$bindir"/scurl https://run.linkerd.io/booksapp/books.swagger)
+WEBAPP_SP=$("$bindir"/scurl https://run.linkerd.io/booksapp/webapp.swagger)
 
 # deploy books backend and service profiles to N namespaces
 for ((i=1; i <= NAMESPACES; i++)); do
@@ -92,7 +94,7 @@ for ((i=1; i <= NAMESPACES; i++)); do
     echo "$WEBAPP_SP"  | bin/linkerd profile -n "$booksns" webapp  --open-api - | kubectl apply -f -
 done
 
-BOOKS_APP=$(curl -s https://raw.githubusercontent.com/BuoyantIO/booksapp/main/k8s/mysql-app.yml)
+BOOKS_APP=$("$bindir"/scurl https://raw.githubusercontent.com/BuoyantIO/booksapp/main/k8s/mysql-app.yml)
 
 # add "-sleep=10ms" param to the traffic app (~100rps)
 traffic_param="        - \"webapp:7000\""
@@ -120,7 +122,7 @@ done
 # Deploy Emojivoto
 #
 
-EMOJIVOTO=$(curl -s https://run.linkerd.io/emojivoto.yml)
+EMOJIVOTO=$("$bindir"/scurl https://run.linkerd.io/emojivoto.yml)
 
 # delete namespace
 EMOJIVOTO=$(echo "$EMOJIVOTO" | tail -n +6)
@@ -146,7 +148,7 @@ done
 # Lifecycle / bb
 #
 
-LIFECYCLE=$(curl -s https://raw.githubusercontent.com/linkerd/linkerd-examples/master/lifecycle/lifecycle.yml)
+LIFECYCLE=$("$bindir"/scurl https://raw.githubusercontent.com/linkerd/linkerd-examples/master/lifecycle/lifecycle.yml)
 
 # inject
 LIFECYCLE=$(echo "$LIFECYCLE" | $linkerd_path -l "$linkerd_namespace" inject -)

web/Dockerfile

@@ -10,8 +10,8 @@ ARG TARGETARCH
 RUN ./bin/install-deps $TARGETARCH
 
 ## bundle web assets
-FROM --platform=$BUILDPLATFORM node:15-buster as webpack-bundle
-RUN curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.22.10 --network-concurrency 1
+FROM --platform=$BUILDPLATFORM node:16-bullseye as webpack-bundle
+RUN bin/scurl --retry=2 https://yarnpkg.com/install.sh | bash -s -- --version 1.22.10 --network-concurrency 1
 
 ENV PATH /root/.yarn/bin:$PATH
 ENV ROOT /linkerd-build