From 61c3515b4ce91811b7d5098c7463cb3e32c1a236 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Thu, 24 Nov 2022 09:34:34 -0800 Subject: [PATCH 1/5] [single-machine-performance] Push agent containers to SMP ECR This commit is an attempt to introduce pushing containers from Agent CI for single-machine-performance's Regression Detector in our isolated infrastructure. Much like we have done for vectordotdev/vector we intend to run the Regression Detector on Agent changes, giving a reasonable statistical guarantee that a change does or does not modify Agent performance by more than random chance. In order for the Regression Detector to run jobs it must have access to a 'baseline' and 'comparison' target. Baseline in this project would be a container built from current `main` branch, comparison would be a container built from the tip of a PR. The main thing demonstrated here is that the team credentials SMP has created for Agent are functional and are able to push up a containers. This PR is, however, a temporary thing and the method in #14438 is preferred once we have all the correct cross-account permissions in place. This PR will unblock single-machine-performance team meanwhile. Contrary to the approach taken in PR #14438 I have duplicate the agent7 build job and adjusted the ECR/tag location, logging in with credentials we have stored in project secrets. This is not aesthetically pleasing nor resource efficient but we expect this situation to last until at most early December. In a later commit we will introduce job submission and will rely on being able to compute the tag of a previous pipeline's container from available Gitlab metadata, specificall `CI_COMMIT_SHA` for the comparison container and whatever metadata maps to the base branch's current SHA, `CI_MERGE_REQUEST_SOURCE_BRANCH_SHA`? REF SMP-208 Signed-off-by: Brian L. Troutwine --- .gitlab/container_build/docker_linux.yml | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index 4492d4c66750d0..eda92384f46be0 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -127,6 +127,42 @@ docker_build_agent7: TEST_IMG: "true" BUILD_ARG: --target release --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb +# NOTE this is a temporary workaround and the method taken in #14438 is +# preferred in the long-term. +docker_build_agent7_single_machine_performance: + rules: + !reference [.on_a7] + needs: + - job: agent_deb-x64-a7 + artifacts: false + script: + - aws s3 sync --only-show-errors $S3_ARTIFACTS_URI Dockerfiles/agent + # Setup AWS credentials for single-machine-performance AWS account + - SMP_ACCOUNT_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-account-id --with-decryption --query "Parameter.Value" --out text) + - SMP_ECR_URL=${SMP_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com + - SMP_AGENT_TEAM_ID=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-agent-team-id --with-decryption --query "Parameter.Value" --out text) + - aws configure set aws_access_key_id $(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-bot-access-key-id --with-decryption --query "Parameter.Value" --out text) --profile single-machine-performance + - aws configure set aws_secret_access_key $(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.single-machine-performance-bot-access-key --with-decryption --query "Parameter.Value" --out text) --profile single-machine-performance + - aws configure set region us-west-2 --profile single-machine-performance + # Login to Single Machine Performance ECR + - aws ecr get-login-password --profile single-machine-performance | docker login --username "AWS" --password-stdin "$SMP_ECR_URL" + # Calculate SMP tag, note that it must be deterministic and able to be + # computed across pipeline executions. + - TARGET_TAG=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:${CI_COMMIT_SHA}-7-amd64 + # DockerHub login for build to limit rate limit when pulling base images + - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text) + - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text | docker login --username "$DOCKER_REGISTRY_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" + # Pull base images + - inv -e docker.pull-base-images Dockerfiles/agent/amd64/Dockerfile + # Build image + - docker build --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} --target release --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb --file $BUILD_CONTEXT/$ARCH/Dockerfile --tag ${TARGET_TAG}-unsquashed $BUILD_CONTEXT + # Squash image, test, and push to ECR + - docker-squash ${TARGET_TAG}-unsquashed -t ${TARGET_TAG} + - test "true" && docker run -v `pwd`/Dockerfiles/agent:/tmp/build ${TARGET_TAG} python /tmp/build/test_image_contents.py + - docker push $TARGET_TAG + # Workaround for temporary network failures + retry: 2 + docker_build_agent7_arm64: extends: .docker_build_job_definition_arm64 rules: From dbd866f0dd2e6a620d11f3ece2ec27664ad49c42 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Mon, 28 Nov 2022 07:45:42 -0800 Subject: [PATCH 2/5] add stage, image, tag to job to job Signed-off-by: Brian L. Troutwine --- .gitlab/container_build/docker_linux.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index eda92384f46be0..05baa79d130983 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -130,6 +130,9 @@ docker_build_agent7: # NOTE this is a temporary workaround and the method taken in #14438 is # preferred in the long-term. docker_build_agent7_single_machine_performance: + stage: container_build + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64:$DATADOG_AGENT_BUILDIMAGES + tags: ["runner:docker"] rules: !reference [.on_a7] needs: From 83501a2af0a32254969bfe3c48c8ef4735645968 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Mon, 28 Nov 2022 10:40:39 -0800 Subject: [PATCH 3/5] variable fixes Signed-off-by: Brian L. Troutwine --- .gitlab/container_build/docker_linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index 05baa79d130983..9df9bede94a933 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -158,7 +158,7 @@ docker_build_agent7_single_machine_performance: # Pull base images - inv -e docker.pull-base-images Dockerfiles/agent/amd64/Dockerfile # Build image - - docker build --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} --target release --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb --file $BUILD_CONTEXT/$ARCH/Dockerfile --tag ${TARGET_TAG}-unsquashed $BUILD_CONTEXT + - docker build --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} --target release --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb --file Dockerifles/agent/amd64/Dockerfile --tag ${TARGET_TAG}-unsquashed Dockerfiles/agent # Squash image, test, and push to ECR - docker-squash ${TARGET_TAG}-unsquashed -t ${TARGET_TAG} - test "true" && docker run -v `pwd`/Dockerfiles/agent:/tmp/build ${TARGET_TAG} python /tmp/build/test_image_contents.py From aec5bd1e45577af76c31691047fe9c4cc20cb841 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Mon, 28 Nov 2022 11:29:52 -0800 Subject: [PATCH 4/5] re-introduce variables Signed-off-by: Brian L. Troutwine --- .gitlab/container_build/docker_linux.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index 9df9bede94a933..13430f7366dd9e 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -138,6 +138,12 @@ docker_build_agent7_single_machine_performance: needs: - job: agent_deb-x64-a7 artifacts: false + variables: + BUILD_CONTEXT: Dockerfiles/agent + TAG_SUFFIX: -7 + ARCH: amd64 + TEST_IMG: "true" + BUILD_ARG: --target release --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb script: - aws s3 sync --only-show-errors $S3_ARTIFACTS_URI Dockerfiles/agent # Setup AWS credentials for single-machine-performance AWS account @@ -151,17 +157,17 @@ docker_build_agent7_single_machine_performance: - aws ecr get-login-password --profile single-machine-performance | docker login --username "AWS" --password-stdin "$SMP_ECR_URL" # Calculate SMP tag, note that it must be deterministic and able to be # computed across pipeline executions. - - TARGET_TAG=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:${CI_COMMIT_SHA}-7-amd64 + - TARGET_TAG=${SMP_ECR_URL}/${SMP_AGENT_TEAM_ID}-agent:${CI_COMMIT_SHA}${TAG_SUFFIX}-${ARCH} # DockerHub login for build to limit rate limit when pulling base images - DOCKER_REGISTRY_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_LOGIN_SSM_KEY --with-decryption --query "Parameter.Value" --out text) - aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.$DOCKER_REGISTRY_PWD_SSM_KEY --with-decryption --query "Parameter.Value" --out text | docker login --username "$DOCKER_REGISTRY_LOGIN" --password-stdin "$DOCKER_REGISTRY_URL" # Pull base images - - inv -e docker.pull-base-images Dockerfiles/agent/amd64/Dockerfile + - inv -e docker.pull-base-images $BUILD_CONTEXT/$ARCH/Dockerfile # Build image - - docker build --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} --target release --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb --file Dockerifles/agent/amd64/Dockerfile --tag ${TARGET_TAG}-unsquashed Dockerfiles/agent + - docker build --build-arg CIBUILD=true --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} $BUILD_ARG --file $BUILD_CONTEXT/$ARCH/Dockerfile --tag ${TARGET_TAG}-unsquashed $BUILD_CONTEXT # Squash image, test, and push to ECR - docker-squash ${TARGET_TAG}-unsquashed -t ${TARGET_TAG} - - test "true" && docker run -v `pwd`/Dockerfiles/agent:/tmp/build ${TARGET_TAG} python /tmp/build/test_image_contents.py + - test "$TEST_IMG" && docker run -v `pwd`/$BUILD_CONTEXT:/tmp/build ${TARGET_TAG} python /tmp/build/test_image_contents.py - docker push $TARGET_TAG # Workaround for temporary network failures retry: 2 From 72a679ba340090845f0e369920a342a7dc542ab8 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Tue, 29 Nov 2022 08:21:22 -0800 Subject: [PATCH 5/5] Trigger Build