From 7e277fe6d1e20de052cec4846e4556465a9c66dd Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 13 Aug 2024 08:47:55 -0400 Subject: [PATCH 1/5] chore(ci): print detailed target timings At end of earthly-ci, use the debug manifest file to try and get a handle on what took long (hard to see from log) --- scripts/earthly-ci | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/earthly-ci b/scripts/earthly-ci index 0e7224077f5c..77df8126ad74 100755 --- a/scripts/earthly-ci +++ b/scripts/earthly-ci @@ -32,13 +32,34 @@ function wipe_non_cache_docker_state { sudo service docker restart } -EARTHLY_ARGS="--secret AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} --secret AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-}" +EARTHLY_RUN_STATS_JSON="earthly-run-stats.json" +EARTHLY_ARGS="--logstream-debug-manifest-file $EARTHLY_RUN_STATS_JSON --secret AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} --secret AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-}" + +function print_earthly_command_timings() { + # Use jq to extract target names, start times, and end times + jq -r ' + .targets[] | + .name as $name | + (.startedAtUnixNanos | tonumber) as $start | + (.endedAtUnixNanos | tonumber) as $end | + "\($name) \($start) \($end)" + ' $EARTHLY_RUN_STATS_JSON | while read name start end; do + # Calculate duration in seconds using pure bash + duration_ns=$((end - start)) + duration_s=$((duration_ns / 1000000000)) + duration_ms=$(( (duration_ns % 1000000000) / 1000000 )) + # Print target name and duration in seconds and milliseconds + printf "Target: %s - Duration: %d.%03d seconds\n" "$name" "$duration_s" "$duration_ms" + done +} # Handle earthly commands and retries while [ $ATTEMPT_COUNT -lt $MAX_ATTEMPTS ]; do if earthly $EARTHLY_ARGS $@ 2>&1 | tee $OUTPUT_FILE >&2 ; then + print_earthly_command_timings exit 0 # Success, exit the script else + print_earthly_command_timings # Increment attempt counter ATTEMPT_COUNT=$((ATTEMPT_COUNT + 1)) echo "Attempt #$ATTEMPT_COUNT failed." From ee2f60b1322d41ddb85fec71471c5088d2b1231b Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 13 Aug 2024 09:25:53 -0400 Subject: [PATCH 2/5] Update earthly-ci --- scripts/earthly-ci | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/earthly-ci b/scripts/earthly-ci index 77df8126ad74..bc123ce13e1e 100755 --- a/scripts/earthly-ci +++ b/scripts/earthly-ci @@ -36,6 +36,7 @@ EARTHLY_RUN_STATS_JSON="earthly-run-stats.json" EARTHLY_ARGS="--logstream-debug-manifest-file $EARTHLY_RUN_STATS_JSON --secret AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} --secret AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-}" function print_earthly_command_timings() { + jq --version || return # Use jq to extract target names, start times, and end times jq -r ' .targets[] | @@ -49,8 +50,8 @@ function print_earthly_command_timings() { duration_s=$((duration_ns / 1000000000)) duration_ms=$(( (duration_ns % 1000000000) / 1000000 )) # Print target name and duration in seconds and milliseconds - printf "Target: %s - Duration: %d.%03d seconds\n" "$name" "$duration_s" "$duration_ms" - done + printf "(%d.%03ds) %s\n" "$duration_s" "$duration_ms" "$name" + done | sort -n } # Handle earthly commands and retries From c2a87c211af35dfe79cede72138a7e5af46b2baf Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 13 Aug 2024 09:37:16 -0400 Subject: [PATCH 3/5] Update earthly-ci --- scripts/earthly-ci | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/earthly-ci b/scripts/earthly-ci index bc123ce13e1e..bb83f5f4172b 100755 --- a/scripts/earthly-ci +++ b/scripts/earthly-ci @@ -38,6 +38,7 @@ EARTHLY_ARGS="--logstream-debug-manifest-file $EARTHLY_RUN_STATS_JSON --secret A function print_earthly_command_timings() { jq --version || return # Use jq to extract target names, start times, and end times + echo "TARGET TIMES" jq -r ' .targets[] | .name as $name | From fe6cdc5c1035803bee07fb878ea60e61fc9c8f20 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 13 Aug 2024 09:39:01 -0400 Subject: [PATCH 4/5] Update earthly-ci --- scripts/earthly-ci | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/earthly-ci b/scripts/earthly-ci index bb83f5f4172b..a8834cf1976a 100755 --- a/scripts/earthly-ci +++ b/scripts/earthly-ci @@ -58,10 +58,10 @@ function print_earthly_command_timings() { # Handle earthly commands and retries while [ $ATTEMPT_COUNT -lt $MAX_ATTEMPTS ]; do if earthly $EARTHLY_ARGS $@ 2>&1 | tee $OUTPUT_FILE >&2 ; then - print_earthly_command_timings + print_earthly_command_timings || true exit 0 # Success, exit the script else - print_earthly_command_timings + print_earthly_command_timings || true # Increment attempt counter ATTEMPT_COUNT=$((ATTEMPT_COUNT + 1)) echo "Attempt #$ATTEMPT_COUNT failed." From 0c2ea6cf764a9be228ebfa6f13da2775a8e82702 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 13 Aug 2024 09:57:15 -0400 Subject: [PATCH 5/5] Update earthly-ci --- scripts/earthly-ci | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/earthly-ci b/scripts/earthly-ci index a8834cf1976a..d7e3a230e674 100755 --- a/scripts/earthly-ci +++ b/scripts/earthly-ci @@ -36,7 +36,7 @@ EARTHLY_RUN_STATS_JSON="earthly-run-stats.json" EARTHLY_ARGS="--logstream-debug-manifest-file $EARTHLY_RUN_STATS_JSON --secret AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} --secret AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-}" function print_earthly_command_timings() { - jq --version || return + jq --version >/dev/null || return # Use jq to extract target names, start times, and end times echo "TARGET TIMES" jq -r ' @@ -51,7 +51,7 @@ function print_earthly_command_timings() { duration_s=$((duration_ns / 1000000000)) duration_ms=$(( (duration_ns % 1000000000) / 1000000 )) # Print target name and duration in seconds and milliseconds - printf "(%d.%03ds) %s\n" "$duration_s" "$duration_ms" "$name" + printf "%d.%03ds - %s\n" "$duration_s" "$duration_ms" "$name" done | sort -n }