diff --git a/.github/workflows/performance-benchmarks-go.yml b/.github/workflows/performance-benchmarks-go.yml new file mode 100644 index 000000000..15b7f9db9 --- /dev/null +++ b/.github/workflows/performance-benchmarks-go.yml @@ -0,0 +1,108 @@ +# This workflow performs benchmark testing in Go. +name: Performance Benchmarks Go + +on: + workflow_call: + inputs: + dafny: + description: "The Dafny version to run" + required: true + type: string + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + +jobs: + testGo: + strategy: + fail-fast: false + matrix: + library: [AwsEncryptionSDK] + benchmark-dir: [esdk-performance-testing] + go-version: ["1.23", "1.24", "1.25"] + os: [ + # Sed script doesn't work properly on windows + # windows-latest, + ubuntu-22.04, + # TODO: Re-enable macOS after https://t.corp.amazon.com/P205755286 + # macos-15, + ] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Support longpaths on Git checkout + run: | + git config --global core.longpaths true + - uses: actions/checkout@v5 + - name: Init Submodules + shell: bash + run: | + git submodule update --init libraries + git submodule update --init --recursive mpl + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2 + role-session-name: GoTests + + - name: Setup Dafny + uses: ./mpl/.github/actions/setup_dafny + with: + dafny-version: ${{ inputs.dafny }} + + - name: Install Smithy-Dafny codegen dependencies + uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + + - name: Setup NASM for Windows (aws-lc-sys) + if: matrix.os == 'windows-latest' + uses: ilammy/setup-nasm@v1 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version: ${{ matrix.go-version }} + + - name: Compile ${{ matrix.library }} implementation + shell: bash + working-directory: ${{ matrix.library }} + run: | + # This works because `node` is installed by default on GHA runners + CORES=$(node -e 'console.log(os.cpus().length)') + make transpile_go CORES=$CORES + + - name: Run Performance Benchmarks - Quick Mode + shell: bash + working-directory: ./${{matrix.benchmark-dir}}/benchmarks/go + run: | + go run . --config ../../config/test-scenarios.yaml --quick + + - name: Parse and Format Logs + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="go_results.json" + UPLOAD_FILE="cloudwatch_logs.json" + TIMESTAMP=$(date +%s%3N) + jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE + + - name: Upload logs to CloudWatch + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="cloudwatch_logs.json" + LOG_GROUP="aws-esdk-performance-benchmarks" + LOG_STREAM="go/${{matrix.go-version}}/quick_benchmarks/${{ github.workflow }}" + + # Create log stream (ignore if exists) + aws logs create-log-stream \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" 2>/dev/null || true + + aws logs put-log-events \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --log-events file://$LOG_FILE diff --git a/.github/workflows/performance-benchmarks-java.yml b/.github/workflows/performance-benchmarks-java.yml new file mode 100644 index 000000000..7b98182ae --- /dev/null +++ b/.github/workflows/performance-benchmarks-java.yml @@ -0,0 +1,79 @@ +# This workflow performs benchmark testing in Java. +name: Performance Benchmarks Go + +on: + workflow_call: + +jobs: + testJava: + strategy: + fail-fast: false + matrix: + benchmark-dir: [esdk-performance-testing] + java: [17] + os: [ + # Sed script doesn't work properly on windows + # windows-latest, + ubuntu-22.04, + # TODO: Re-enable macOS after https://t.corp.amazon.com/P205755286 + # macos-15, + ] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Support longpaths on Git checkout + run: | + git config --global core.longpaths true + - uses: actions/checkout@v5 + - name: Init Submodules + shell: bash + run: | + git submodule update --init libraries + git submodule update --init --recursive mpl + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2 + role-session-name: GoTests + + - name: Setup Java Corretto ${{matrix.java}} + uses: actions/setup-java@v5 + with: + distribution: "corretto" + java-version: ${{matrix.java}} + + - name: Run Performance Benchmarks - Quick Mode + shell: bash + working-directory: ./${{matrix.benchmark-dir}}/benchmarks/java + run: | + mvn clean compile + mvn exec:java -Dexec.mainClass="com.amazon.esdk.benchmark.Program" -Dexec.args="--quick" + + - name: Parse and Format Logs + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="java_results.json" + UPLOAD_FILE="cloudwatch_logs.json" + TIMESTAMP=$(date +%s%3N) + jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE + + - name: Upload logs to CloudWatch + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="cloudwatch_logs.json" + LOG_GROUP="aws-esdk-performance-benchmarks" + LOG_STREAM="java/corretto/${{matrix.java}}/quick_benchmarks/${{ github.workflow }}" + + # Create log stream (ignore if exists) + aws logs create-log-stream \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" 2>/dev/null || true + + aws logs put-log-events \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --log-events file://$LOG_FILE diff --git a/.github/workflows/performance-benchmarks-net.yml b/.github/workflows/performance-benchmarks-net.yml new file mode 100644 index 000000000..7547073a5 --- /dev/null +++ b/.github/workflows/performance-benchmarks-net.yml @@ -0,0 +1,118 @@ +# This workflow performs benchmark testing in Net. +name: Performance Benchmarks Net + +on: + workflow_call: + inputs: + dafny: + description: "The Dafny version to run" + required: true + type: string + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + +jobs: + testNet: + strategy: + fail-fast: false + matrix: + library: [AwsEncryptionSDK] + benchmark-dir: [esdk-performance-testing] + os: [ + ubuntu-22.04, + # TODO: Re-enable macOS after https://t.corp.amazon.com/P205755286 + # macos-15, + ] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + env: + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_NOLOGO: 1 + steps: + - name: Support longpaths on Git checkout + run: | + git config --global core.longpaths true + - uses: actions/checkout@v5 + - name: Init Submodules + shell: bash + run: | + git submodule update --init libraries + git submodule update --init --recursive mpl + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2 + role-session-name: NetTests + + - name: Setup .NET Core SDK 6 + uses: actions/setup-dotnet@v5 + with: + dotnet-version: "6.0.x" + + - name: Setup Dafny + uses: ./mpl/.github/actions/setup_dafny + with: + dafny-version: ${{ inputs.dafny }} + + - name: Install Smithy-Dafny codegen dependencies + uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ env.DAFNY_VERSION }} + library: ${{ matrix.library }} + diff-generated-code: false + update-and-regenerate-mpl: true + + - name: Download Dependencies + working-directory: ${{ matrix.library }} + run: make setup_net + + - name: Compile ${{ matrix.library }} implementation + shell: bash + working-directory: ${{ matrix.library }} + run: | + # This works because `node` is installed by default on GHA runners + CORES=$(node -e 'console.log(os.cpus().length)') + make transpile_net CORES=$CORES + + - name: Run Performance Benchmarks - Quick Mode + shell: bash + working-directory: ./${{matrix.benchmark-dir}}/benchmarks/net + run: | + dotnet build --configuration Release + dotnet run --configuration Release -- --quick + + - name: Parse and Format Logs + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="net_results.json" + UPLOAD_FILE="cloudwatch_logs.json" + TIMESTAMP=$(date +%s%3N) + jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE + + - name: Upload logs to CloudWatch + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="cloudwatch_logs.json" + LOG_GROUP="aws-esdk-performance-benchmarks" + LOG_STREAM="net/${{matrix.os}}/quick_benchmarks/${{ github.workflow }}" + + # Create log stream (ignore if exists) + aws logs create-log-stream \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" 2>/dev/null || true + + aws logs put-log-events \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --log-events file://$LOG_FILE diff --git a/.github/workflows/performance-benchmarks-python.yml b/.github/workflows/performance-benchmarks-python.yml new file mode 100644 index 000000000..e3c40cb73 --- /dev/null +++ b/.github/workflows/performance-benchmarks-python.yml @@ -0,0 +1,73 @@ +# This workflow performs benchmark testing in Java. +name: Performance Benchmarks Go + +on: + workflow_call: + +jobs: + testPython: + strategy: + fail-fast: false + matrix: + benchmark-dir: [esdk-performance-testing] + python-version: ["3.11", "3.13"] + os: [ubuntu-22.04, macos-14] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Support longpaths on Git checkout + run: | + git config --global core.longpaths true + - uses: actions/checkout@v5 + - name: Init Submodules + shell: bash + run: | + git submodule update --init libraries + git submodule update --init --recursive mpl + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2 + role-session-name: GoTests + + - name: Setup Python ${{ matrix.python-version }} for running tests + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + + - name: Run Performance Benchmarks - Quick Mode + shell: bash + working-directory: ./${{matrix.benchmark-dir}}/benchmarks/python + run: | + pip install -r requirements.txt + python esdk_benchmark.py --quick + + - name: Parse and Format Logs + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="python_results.json" + UPLOAD_FILE="cloudwatch_logs.json" + TIMESTAMP=$(date +%s%3N) + jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE + + - name: Upload logs to CloudWatch + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="cloudwatch_logs.json" + LOG_GROUP="aws-esdk-performance-benchmarks" + LOG_STREAM="python/${{matrix.python-version}}/quick_benchmarks/${{ github.workflow }}" + + # Create log stream (ignore if exists) + aws logs create-log-stream \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" 2>/dev/null || true + + aws logs put-log-events \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --log-events file://$LOG_FILE diff --git a/.github/workflows/performance-benchmarks-rust.yml b/.github/workflows/performance-benchmarks-rust.yml new file mode 100644 index 000000000..97f0c4560 --- /dev/null +++ b/.github/workflows/performance-benchmarks-rust.yml @@ -0,0 +1,126 @@ +# This workflow performs benchmark testing in Rust. +name: Performance Benchmarks Rust + +on: + workflow_call: + inputs: + dafny: + description: "The Dafny version to run" + required: true + type: string + regenerate-code: + description: "Regenerate code using smithy-dafny" + required: false + default: false + type: boolean + +jobs: + testRust: + strategy: + fail-fast: false + matrix: + library: [AwsEncryptionSDK] + benchmark-dir: [esdk-performance-testing] + os: [ + ubuntu-22.04, + # TODO: Re-enable macOS after https://t.corp.amazon.com/P205755286 + # macos-15, + ] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Support longpaths on Git checkout + run: | + git config --global core.longpaths true + - uses: actions/checkout@v5 + - name: Init Submodules + shell: bash + run: | + git submodule update --init libraries + git submodule update --init --recursive mpl + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Dafny-Role-us-west-2 + role-session-name: RustTests + + - name: Setup Rust Toolchain for GitHub CI + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + components: rustfmt + + - name: Setup Dafny + uses: ./mpl/.github/actions/setup_dafny + with: + dafny-version: 4.10.0 + + - name: Setup Java 17 for codegen + uses: actions/setup-java@v5 + with: + distribution: "corretto" + java-version: "17" + + - name: Setup NASM for Windows (aws-lc-sys) + if: matrix.os == 'windows-latest' + uses: ilammy/setup-nasm@v1 + + - name: Install Smithy-Dafny codegen dependencies + uses: ./.github/actions/install_smithy_dafny_codegen_dependencies + + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ env.DAFNY_VERSION }} + library: ${{ matrix.library }} + diff-generated-code: false + update-and-regenerate-mpl: true + + - name: Run make polymorph_rust + shell: bash + working-directory: ${{ matrix.library }} + run: | + make polymorph_rust + + - name: Compile ${{ matrix.library }} implementation + shell: bash + working-directory: ${{ matrix.library }} + run: | + # This works because `node` is installed by default on GHA runners + CORES=$(node -e 'console.log(os.cpus().length)') + make transpile_rust CORES=$CORES + + - name: Run Performance Benchmarks - Quick Mode + shell: bash + working-directory: ./${{matrix.benchmark-dir}}/benchmarks/rust + run: | + cargo run --release -- --quick + + - name: Parse and Format Logs + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="rust_results.json" + UPLOAD_FILE="cloudwatch_logs.json" + TIMESTAMP=$(date +%s%3N) + jq -c --arg ts "$(date +%s)000" '[.results[] as $result | .metadata as $meta | {timestamp: ($ts | tonumber), message: ({metadata: $meta, result: $result} | tostring)}]' $LOG_FILE > $UPLOAD_FILE + + - name: Upload logs to CloudWatch + working-directory: ./${{matrix.benchmark-dir}}/results/raw-data/ + run: | + LOG_FILE="cloudwatch_logs.json" + LOG_GROUP="aws-esdk-performance-benchmarks" + LOG_STREAM="rust/${{matrix.os}}/quick_benchmarks/${{ github.workflow }}" + + # Create log stream (ignore if exists) + aws logs create-log-stream \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" 2>/dev/null || true + + aws logs put-log-events \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --log-events file://$LOG_FILE diff --git a/.github/workflows/performance-benchmarks.yml b/.github/workflows/performance-benchmarks.yml new file mode 100644 index 000000000..a09705a0c --- /dev/null +++ b/.github/workflows/performance-benchmarks.yml @@ -0,0 +1,55 @@ +# This workflow runs every day 09:00 UTC (1AM PST) +name: Performance Benchmarks + +permissions: + id-token: write + contents: read + +on: + pull_request: + paths: + - ".github/workflows/performance-benchmarks.yml" + schedule: + - cron: "00 09 * * *" + +jobs: + getVersion: + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/dafny_version.yaml + performance-benchmarks-go: + needs: getVersion + uses: ./.github/workflows/performance-benchmarks-go.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + performance-benchmarks-java: + needs: getVersion + uses: ./.github/workflows/performance-benchmarks-java.yml + performance-benchmarks-net: + needs: getVersion + uses: ./.github/workflows/performance-benchmarks-net.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + performance-benchmarks-rust: + needs: getVersion + uses: ./.github/workflows/performance-benchmarks-rust.yml + with: + dafny: ${{needs.getVersion.outputs.version}} + performance-benchmarks-python: + needs: getVersion + uses: ./.github/workflows/performance-benchmarks-python.yml + notify: + needs: + [ + getVersion, + performance-benchmarks-go, + performance-benchmarks-java, + performance-benchmarks-net, + performance-benchmarks-rust, + performance-benchmarks-python, + ] + if: ${{ failure() }} + uses: aws/aws-cryptographic-material-providers-library/.github/workflows/slack-notification.yml@main + with: + message: "Performance Benchmarks failed on `${{ github.repository }}`. View run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_CI }}