feat(ci.aztec-labs.com): CI cost and metrics tracking

.github/workflows/ci3.yml

@@ -88,6 +88,7 @@ jobs:
           PR_COMMITS: ${{ github.event.pull_request.commits }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
           GITHUB_REF_NAME: ${{ github.ref_name }}
+          GITHUB_ACTOR: ${{ github.actor }}
         # NOTE: $CI_MODE is set in the Determine CI Mode step.
         run: ./.github/ci3.sh $CI_MODE
 

ci3/aws_request_instance_type

@@ -86,6 +86,21 @@ if [ -z "${iid:-}" -o "${iid:-}" == "None" ]; then
   echo $iid > $iid_path
 fi
 
+tags="Key=Name,Value=$name Key=Group,Value=build-instance"
+[ -n "${GITHUB_ACTOR:-}" ] && tags+=" Key=GithubActor,Value=$GITHUB_ACTOR"
+[ -n "${CI_MODE:-}" ] && tags+=" Key=CICommand,Value=$CI_MODE"
+[ -n "${CI_DASHBOARD:-}" ] && tags+=" Key=Dashboard,Value=$CI_DASHBOARD"
+if [ "${UNSAFE_AWS_KEEP_ALIVE:-0}" -eq 1 ]; then
+  echo_stderr "You have set UNSAFE_AWS_KEEP_ALIVE=1, so the instance will not be terminated after 1.5 hours by the reaper script. Make sure you shut the machine down when done."
+  tags+=" Key=Keep-Alive,Value=true"
+fi
+aws ec2 create-tags --resources $iid --tags $tags
+
+# Record the instance type so callers can pass it downstream (e.g. into Docker).
+echo $instance_type > $state_dir/instance_type
+# Record whether this is spot or on-demand.
+[ -f "$sir_path" ] && echo spot > $state_dir/spot || echo ondemand > $state_dir/spot
+
 while [ -z "${ip:-}" ]; do
   sleep 1
   ip=$(aws ec2 describe-instances \

ci3/bootstrap_ec2

@@ -89,6 +89,8 @@ if [[ -f "$state_dir/sir" ]]; then
   sir=$(cat $state_dir/sir)
 fi
 iid=$(cat $state_dir/iid)
+export EC2_INSTANCE_TYPE=$(cat $state_dir/instance_type 2>/dev/null || echo "unknown")
+export EC2_SPOT=$(cat $state_dir/spot 2>/dev/null || echo "unknown")
 
 # If AWS credentials are not set, try to load them from ~/.aws/build_instance_credentials.
 if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then
@@ -192,16 +194,6 @@ container_script=$(
       log_ci_run FAILED \$ci_log_id
       merge_train_failure_slack_notify \$ci_log_id
       release_canary_slack_notify \$ci_log_id
-      ci_failed_data=\$(jq -n \\
-        --arg status "failed" \\
-        --arg log_id "\$ci_log_id" \\
-        --arg ref_name "\${TARGET_BRANCH:-\$REF_NAME}" \\
-        --arg commit_hash "\$COMMIT_HASH" \\
-        --arg commit_author "\$COMMIT_AUTHOR" \\
-        --arg commit_msg "\$COMMIT_MSG" \\
-        --argjson exit_code "\$code" \\
-        '{status: \$status, log_id: \$log_id, ref_name: \$ref_name, commit_hash: \$commit_hash, commit_author: \$commit_author, commit_msg: \$commit_msg, exit_code: \$exit_code, timestamp: now | todate}')
-      redis_publish "ci:run:failed" "\$ci_failed_data"
       ;;
   esac
   exit \$code
@@ -331,6 +323,9 @@ function run {
         -e AWS_TOKEN=\$aws_token \
         -e NAMESPACE=${NAMESPACE:-} \
         -e NETWORK=${NETWORK:-} \
+        -e GITHUB_ACTOR=${GITHUB_ACTOR:-} \
+        -e EC2_INSTANCE_TYPE=${EC2_INSTANCE_TYPE:-unknown} \
+        -e EC2_SPOT=${EC2_SPOT:-unknown} \
         --pids-limit=65536 \
         --shm-size=2g \
         aztecprotocol/devbox:3.0 bash -c $(printf '%q' "$container_script")

ci3/ci-metrics/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.12
+
+RUN apt update && apt install -y jq redis-tools && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt gunicorn
+RUN git config --global --add safe.directory /aztec-packages
+COPY . .
+EXPOSE 8081
+CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:8081", "app:app"]