From 83d3eb53b44b6e3e7c8464a609bec9aa8e89c354 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:03:32 +0000 Subject: [PATCH 1/9] take averages for compilation and execution report for small programs for more accurate reporting --- .github/workflows/reports.yml | 34 ++++++++++++++++++++++------- test_programs/compilation_report.sh | 27 +++++++++++++++++++---- test_programs/execution_report.sh | 26 ++++++++++++++++++---- 3 files changed, 71 insertions(+), 16 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 37864dd9b68..6afe193efb7 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -252,13 +252,13 @@ jobs: - name: Generate Compilation report working-directory: ./test_programs run: | - ./compilation_report.sh + ./compilation_report.sh 0 1 mv compilation_report.json ../compilation_report.json - name: Generate Execution report working-directory: ./test_programs run: | - ./execution_report.sh + ./execution_report.sh 0 1 mv execution_report.json ../execution_report.json - name: Upload compilation report @@ -286,10 +286,10 @@ jobs: matrix: include: - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, is_library: true } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root, take_average: true } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner, take_average: true } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail, take_average: true } + - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset, take_average: true } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private } - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public } @@ -330,20 +330,38 @@ jobs: path: test-repo ref: ${{ matrix.project.ref }} - - name: Generate compilation report + - name: Generate compilation report without averages working-directory: ./test-repo/${{ matrix.project.path }} + if: ${{ !matrix.project.take_average }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh chmod +x ./compilation_report.sh ./compilation_report.sh 1 + + - name: Generate compilation report with averages + working-directory: ./test-repo/${{ matrix.project.path }} + if: ${{ matrix.project.take_average }} + run: | + mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh + chmod +x ./compilation_report.sh + ./compilation_report.sh 1 1 - - name: Generate execution report + - name: Generate execution report without averages working-directory: ./test-repo/${{ matrix.project.path }} if: ${{ !matrix.project.is_library }} + if: ${{ !matrix.project.take_average }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh ./execution_report.sh 1 + - name: Generate execution report with averages + working-directory: ./test-repo/${{ matrix.project.path }} + if: ${{ !matrix.project.is_library }} + if: ${{ matrix.project.take_average }} + run: | + mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh + ./execution_report.sh 1 1 + - name: Move compilation report id: compilation_report shell: bash diff --git a/test_programs/compilation_report.sh b/test_programs/compilation_report.sh index d050e7c9c34..ac76c7c7e66 100755 --- a/test_programs/compilation_report.sh +++ b/test_programs/compilation_report.sh @@ -10,7 +10,7 @@ tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression") echo "{\"compilation_reports\": [ " > $current_dir/compilation_report.json # If there is an argument that means we want to generate a report for only the current directory -if [ "$#" -ne 0 ]; then +if [ "$1" == "1" ]; then base_path="$current_dir" tests_to_profile=(".") fi @@ -32,12 +32,31 @@ for dir in ${tests_to_profile[@]}; do # The default package to run is the supplied list hardcoded at the top of the script PACKAGE_NAME=$dir # Otherwise default to the current directory as the package we want to run - if [ "$#" -ne 0 ]; then + if [ "$1" == "1" ]; then PACKAGE_NAME=$(basename $current_dir) fi - COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') - echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"$COMPILE_TIME\"" >> $current_dir/compilation_report.json + NUM_RUNS=1 + TOTAL_TIME=0 + + # Passing a second argument will take an average of five runs + # rather than + if [ "$2" == "1" ]; then + NUM_RUNS=5 + fi + + echo "num_runs: $NUM_RUNS" + + for ((i = 1; i <= NUM_RUNS; i++)); do + COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') + TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $COMPILE_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) + done + AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) + + echo $PACKAGE_NAME + echo $AVG_TIME + + echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\""$AVG_TIME"s\"" >> $current_dir/compilation_report.json if (($ITER == $NUM_ARTIFACTS)); then echo "}" >> $current_dir/compilation_report.json diff --git a/test_programs/execution_report.sh b/test_programs/execution_report.sh index bdb1500d19e..a07385204e2 100755 --- a/test_programs/execution_report.sh +++ b/test_programs/execution_report.sh @@ -10,7 +10,7 @@ tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression") echo "{\"execution_reports\": [ " > $current_dir/execution_report.json # If there is an argument that means we want to generate a report for only the current directory -if [ "$#" -ne 0 ]; then +if [ "$1" == "1" ]; then base_path="$current_dir" tests_to_profile=(".") fi @@ -32,7 +32,7 @@ for dir in ${tests_to_profile[@]}; do # The default package to run is the supplied list hardcoded at the top of the script PACKAGE_NAME=$dir # Otherwise default to the current directory as the package we want to run - if [ "$#" -ne 0 ]; then + if [ "$1" == "1" ]; then PACKAGE_NAME=$(basename $current_dir) fi @@ -44,8 +44,26 @@ for dir in ${tests_to_profile[@]}; do exit 1 fi - COMPILE_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') - echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"$COMPILE_TIME\"" >> $current_dir/execution_report.json + + NUM_RUNS=1 + TOTAL_TIME=0 + + if [ "$2" == "1" ]; then + NUM_RUNS=5 + fi + + echo "num_runs: $NUM_RUNS" + + for ((i = 1; i <= NUM_RUNS; i++)); do + EXECUTION_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') + TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $EXECUTION_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) + done + AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) + + echo $PACKAGE_NAME + echo $AVG_TIME + + echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\""$AVG_TIME"s\"" >> $current_dir/execution_report.json if (($ITER == $NUM_ARTIFACTS)); then echo "}" >> $current_dir/execution_report.json From 3b8986b82652bfe5f2e441d52797c02b988980c8 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:06:47 +0000 Subject: [PATCH 2/9] combine conditions for execution report --- .github/workflows/reports.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 6afe193efb7..24aff37e041 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -348,16 +348,14 @@ jobs: - name: Generate execution report without averages working-directory: ./test-repo/${{ matrix.project.path }} - if: ${{ !matrix.project.is_library }} - if: ${{ !matrix.project.take_average }} + if: ${{ !matrix.project.is_library }} && ${{ !matrix.project.take_average }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh ./execution_report.sh 1 - name: Generate execution report with averages working-directory: ./test-repo/${{ matrix.project.path }} - if: ${{ !matrix.project.is_library }} - if: ${{ matrix.project.take_average }} + if: ${{ !matrix.project.is_library }} && ${{ matrix.project.take_average }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh ./execution_report.sh 1 1 From aff2a8552585e40a15f5ac8615dbf89f0692e750 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:12:48 +0000 Subject: [PATCH 3/9] combinee exprs --- .github/workflows/reports.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 24aff37e041..f8750e5aaf5 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -348,14 +348,14 @@ jobs: - name: Generate execution report without averages working-directory: ./test-repo/${{ matrix.project.path }} - if: ${{ !matrix.project.is_library }} && ${{ !matrix.project.take_average }} + if: ${{ !matrix.project.is_library && !matrix.project.take_average }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh ./execution_report.sh 1 - name: Generate execution report with averages working-directory: ./test-repo/${{ matrix.project.path }} - if: ${{ !matrix.project.is_library }} && ${{ matrix.project.take_average }} + if: ${{ !matrix.project.is_library && matrix.project.take_average }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh ./execution_report.sh 1 1 From 7c3f3db43ad7b192dcc8687601b2ff83c8a4c8ba Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:21:03 +0000 Subject: [PATCH 4/9] add m prefix for now as to not update the action --- test_programs/compilation_report.sh | 7 ++++++- test_programs/execution_report.sh | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/test_programs/compilation_report.sh b/test_programs/compilation_report.sh index ac76c7c7e66..0204167e6f0 100755 --- a/test_programs/compilation_report.sh +++ b/test_programs/compilation_report.sh @@ -49,6 +49,7 @@ for dir in ${tests_to_profile[@]}; do for ((i = 1; i <= NUM_RUNS; i++)); do COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') + echo $COMPILE_TIME TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $COMPILE_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) done AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) @@ -56,7 +57,11 @@ for dir in ${tests_to_profile[@]}; do echo $PACKAGE_NAME echo $AVG_TIME - echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\""$AVG_TIME"s\"" >> $current_dir/compilation_report.json + # Keep only last three decimal points + AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME") + echo $AVG_TIME + + echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"0m"$AVG_TIME"s\"" >> $current_dir/compilation_report.json if (($ITER == $NUM_ARTIFACTS)); then echo "}" >> $current_dir/compilation_report.json diff --git a/test_programs/execution_report.sh b/test_programs/execution_report.sh index a07385204e2..a5676afff07 100755 --- a/test_programs/execution_report.sh +++ b/test_programs/execution_report.sh @@ -56,6 +56,7 @@ for dir in ${tests_to_profile[@]}; do for ((i = 1; i <= NUM_RUNS; i++)); do EXECUTION_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') + echo $EXECUTION_TIME TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $EXECUTION_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) done AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) @@ -63,7 +64,11 @@ for dir in ${tests_to_profile[@]}; do echo $PACKAGE_NAME echo $AVG_TIME - echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\""$AVG_TIME"s\"" >> $current_dir/execution_report.json + # Keep only last three decimal points + AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME") + echo $AVG_TIME + + echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"0m"$AVG_TIME"s\"" >> $current_dir/execution_report.json if (($ITER == $NUM_ARTIFACTS)); then echo "}" >> $current_dir/execution_report.json From 54f1d9bef0e12e99bbe0c0480374d4f33a5be236 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:29:59 +0000 Subject: [PATCH 5/9] remove echo from total_time --- test_programs/compilation_report.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_programs/compilation_report.sh b/test_programs/compilation_report.sh index 0204167e6f0..f8b7f482b9e 100755 --- a/test_programs/compilation_report.sh +++ b/test_programs/compilation_report.sh @@ -50,7 +50,7 @@ for dir in ${tests_to_profile[@]}; do for ((i = 1; i <= NUM_RUNS; i++)); do COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') echo $COMPILE_TIME - TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $COMPILE_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) + TOTAL_TIME=$($total_time + $(echo $time_str | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)) done AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) From 9448f5537efabb6ccca2e359a433577d68cc5be0 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:31:17 +0000 Subject: [PATCH 6/9] simplify execution total time calc --- test_programs/execution_report.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_programs/execution_report.sh b/test_programs/execution_report.sh index a5676afff07..ba2b24a5cc6 100755 --- a/test_programs/execution_report.sh +++ b/test_programs/execution_report.sh @@ -57,7 +57,8 @@ for dir in ${tests_to_profile[@]}; do for ((i = 1; i <= NUM_RUNS; i++)); do EXECUTION_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') echo $EXECUTION_TIME - TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $EXECUTION_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) + # Convert to seconds and add to total time + TOTAL_TIME=$($total_time + $(echo $EXECUTION_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)) done AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) From 65c99fe26062c426939981ab41acf0b0c62eb503 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:35:38 +0000 Subject: [PATCH 7/9] keep the parens --- test_programs/compilation_report.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test_programs/compilation_report.sh b/test_programs/compilation_report.sh index f8b7f482b9e..2c6551ddefc 100755 --- a/test_programs/compilation_report.sh +++ b/test_programs/compilation_report.sh @@ -50,16 +50,13 @@ for dir in ${tests_to_profile[@]}; do for ((i = 1; i <= NUM_RUNS; i++)); do COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') echo $COMPILE_TIME - TOTAL_TIME=$($total_time + $(echo $time_str | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)) + # Convert to seconds and add to total time + TOTAL_TIME=$(($total_time + $(echo $COMPILE_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc))) done - AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) - - echo $PACKAGE_NAME - echo $AVG_TIME + AVG_TIME=$(($TOTAL_TIME / $NUM_RUNS)) # Keep only last three decimal points AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME") - echo $AVG_TIME echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"0m"$AVG_TIME"s\"" >> $current_dir/compilation_report.json From f464558427c923db36f105a0a11b8520375cfd48 Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:47:32 +0000 Subject: [PATCH 8/9] remove debug stuff --- test_programs/compilation_report.sh | 9 +++------ test_programs/execution_report.sh | 9 ++------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/test_programs/compilation_report.sh b/test_programs/compilation_report.sh index 2c6551ddefc..9f2f8b96147 100755 --- a/test_programs/compilation_report.sh +++ b/test_programs/compilation_report.sh @@ -45,16 +45,13 @@ for dir in ${tests_to_profile[@]}; do NUM_RUNS=5 fi - echo "num_runs: $NUM_RUNS" - for ((i = 1; i <= NUM_RUNS; i++)); do COMPILE_TIME=$((time nargo compile --force --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') - echo $COMPILE_TIME - # Convert to seconds and add to total time - TOTAL_TIME=$(($total_time + $(echo $COMPILE_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc))) + # Convert time to seconds and add to total time + TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $COMPILE_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) done - AVG_TIME=$(($TOTAL_TIME / $NUM_RUNS)) + AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) # Keep only last three decimal points AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME") diff --git a/test_programs/execution_report.sh b/test_programs/execution_report.sh index ba2b24a5cc6..bd82ee209bf 100755 --- a/test_programs/execution_report.sh +++ b/test_programs/execution_report.sh @@ -56,18 +56,13 @@ for dir in ${tests_to_profile[@]}; do for ((i = 1; i <= NUM_RUNS; i++)); do EXECUTION_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') - echo $EXECUTION_TIME # Convert to seconds and add to total time - TOTAL_TIME=$($total_time + $(echo $EXECUTION_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)) + TOTAL_TIME=$(echo "$TOTAL_TIME + $(echo $EXECUTION_TIME | sed -E 's/([0-9]+)m([0-9.]+)s/\1 * 60 + \2/' | bc)" | bc) done - AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) - - echo $PACKAGE_NAME - echo $AVG_TIME + AVG_TIME=$(echo "$TOTAL_TIME / $NUM_RUNS" | bc -l) # Keep only last three decimal points AVG_TIME=$(awk '{printf "%.3f\n", $1}' <<< "$AVG_TIME") - echo $AVG_TIME echo -e " {\n \"artifact_name\":\"$PACKAGE_NAME\",\n \"time\":\"0m"$AVG_TIME"s\"" >> $current_dir/execution_report.json From 6eae0ac48433fd0fcd646ced9efffa1b5404df3f Mon Sep 17 00:00:00 2001 From: Maxim Vezenov Date: Thu, 19 Dec 2024 17:49:54 +0000 Subject: [PATCH 9/9] remove one more dbeug --- test_programs/execution_report.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_programs/execution_report.sh b/test_programs/execution_report.sh index bd82ee209bf..bb3adf143f0 100755 --- a/test_programs/execution_report.sh +++ b/test_programs/execution_report.sh @@ -51,9 +51,7 @@ for dir in ${tests_to_profile[@]}; do if [ "$2" == "1" ]; then NUM_RUNS=5 fi - - echo "num_runs: $NUM_RUNS" - + for ((i = 1; i <= NUM_RUNS; i++)); do EXECUTION_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s') # Convert to seconds and add to total time