diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index ed72dd025fd73..d15c979afbaaa 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -48,7 +48,7 @@ jobs: should-run: runs-on: ubuntu-latest timeout-minutes: 5 - if: github.event_name != 'pull_request' + #if: github.event_name != 'pull_request' outputs: source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }} comment_valid: ${{ steps.comment.outputs.isTeamMember }} @@ -121,17 +121,7 @@ jobs: comparison-sha: ${{ steps.pr-metadata-merge-queue.outputs.COMPARISON_SHA || steps.pr-metadata-comment.outputs.COMPARISON_SHA }} comparison-tag: ${{ steps.pr-metadata-merge-queue.outputs.COMPARISON_TAG || steps.pr-metadata-comment.outputs.COMPARISON_TAG }} - # below are used in the experiment/analyze jobs - cpus: ${{ steps.system.outputs.CPUS }} - memory: ${{ steps.system.outputs.MEMORY }} - vector-cpus: ${{ steps.system.outputs.VECTOR_CPUS }} - - replicas: ${{ steps.experimental-meta.outputs.REPLICAS }} - warmup-seconds: ${{ steps.experimental-meta.outputs.WARMUP_SECONDS }} - total-samples: ${{ steps.experimental-meta.outputs.TOTAL_SAMPLES }} - p-value: ${{ steps.experimental-meta.outputs.P_VALUE }} smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} - lading-version: ${{ steps.experimental-meta.outputs.LADING_VERSION }} steps: - uses: actions/checkout@v3 with: @@ -141,25 +131,29 @@ jobs: # But, we can retrieve this info from some commands. - name: Get PR metadata (issue_comment) id: pr-metadata-comment - if: github.event_name == 'issue_comment' + #if: github.event_name == 'issue_comment' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - export PR_NUMBER=${{ github.event.issue.number }} + export PR_NUMBER="20712" echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT - gh pr checkout ${{ github.event.issue.number }} + gh pr checkout 20712 - export BASELINE_SHA=$(git merge-base master HEAD) + #export BASELINE_SHA=$(git merge-base master HEAD) + export BASELINE_SHA="ea0ec6f4f871c6cfae354b4c00403b646ba144bb" echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT - export COMPARISON_SHA=$(git rev-parse HEAD) + #export COMPARISON_SHA=$(git rev-parse HEAD) + export COMPARISON_SHA="73da9bbf89d49ce7abc5ad8c8526c7679f289dae" echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT - export BASELINE_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${BASELINE_SHA}" + #export BASELINE_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${BASELINE_SHA}" + export BASELINE_TAG="0.39.0-distroless-libc" echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT - export COMPARISON_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${COMPARISON_SHA}" + #export COMPARISON_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${COMPARISON_SHA}" + export COMPARISON_TAG="0.38.0-distroless-libc" echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT echo "pr number is: ${PR_NUMBER}" @@ -189,7 +183,9 @@ jobs: # queue to 1, we can get the PR number by parsing the merge queue temp branch's ref. - name: Get PR metadata (merge queue) id: pr-metadata-merge-queue - if: github.event_name != 'issue_comment' + #if: github.event_name != 'issue_comment' + # Something known to be false + if: github.repository == 'octo-org/octo-repo-prod' run: | export PR_NUMBER=$(echo "${{ github.ref }}" | sed -n 's|^refs/heads/gh-readonly-queue/master/pr-\([0-9]*\)-.*$|\1|p') echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT @@ -233,40 +229,15 @@ jobs: id: experimental-meta run: | export WARMUP_SECONDS="45" - export REPLICAS="10" export TOTAL_SAMPLES="600" - export P_VALUE="0.1" - export SMP_CRATE_VERSION="0.11.0" - export LADING_VERSION="0.19.1" - - echo "warmup seconds: ${WARMUP_SECONDS}" - echo "replicas: ${REPLICAS}" - echo "total samples: ${TOTAL_SAMPLES}" - echo "regression p-value: ${P_VALUE}" + export SMP_CRATE_VERSION="0.15.1" + echo "smp crate version: ${SMP_CRATE_VERSION}" - echo "lading version: ${LADING_VERSION}" echo "WARMUP_SECONDS=${WARMUP_SECONDS}" >> $GITHUB_OUTPUT - echo "REPLICAS=${REPLICAS}" >> $GITHUB_OUTPUT echo "TOTAL_SAMPLES=${TOTAL_SAMPLES}" >> $GITHUB_OUTPUT - echo "P_VALUE=${P_VALUE}" >> $GITHUB_OUTPUT echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT - echo "LADING_VERSION=${LADING_VERSION}" >> $GITHUB_OUTPUT - - name: Setup system details - id: system - run: | - export CPUS="7" - export MEMORY="30g" - export VECTOR_CPUS="4" - - echo "cpus total: ${CPUS}" - echo "memory total: ${MEMORY}" - echo "vector cpus: ${VECTOR_CPUS}" - - echo "CPUS=${CPUS}" >> $GITHUB_OUTPUT - echo "MEMORY=${MEMORY}" >> $GITHUB_OUTPUT - echo "VECTOR_CPUS=${VECTOR_CPUS}" >> $GITHUB_OUTPUT - name: (PR comment) Get PR branch if: ${{ github.event_name == 'issue_comment' }} @@ -285,81 +256,81 @@ jobs: ## BUILD ## - build-baseline: - name: Build baseline Vector container - runs-on: ubuntu-20.04-4core - timeout-minutes: 30 - needs: - - compute-metadata - steps: - - uses: colpal/actions-clean@v1 - - - uses: actions/checkout@v3 - - - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-metadata.outputs.baseline-sha }} - path: baseline-vector - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3.3.0 - - - name: Build 'vector' target image - uses: docker/build-push-action@v5.4.0 - with: - context: baseline-vector/ - cache-from: type=gha - cache-to: type=gha,mode=max - file: regression/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar - tags: | - vector:${{ needs.compute-metadata.outputs.baseline-tag }} - - - name: Upload image as artifact - uses: actions/upload-artifact@v3 - with: - name: baseline-image - path: "${{ runner.temp }}/baseline-image.tar" - - build-comparison: - name: Build comparison Vector container - runs-on: ubuntu-20.04-4core - timeout-minutes: 30 - needs: - - compute-metadata - steps: - - uses: colpal/actions-clean@v1 - - - uses: actions/checkout@v3 - - - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-metadata.outputs.comparison-sha }} - path: comparison-vector - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3.3.0 - - - name: Build 'vector' target image - uses: docker/build-push-action@v5.4.0 - with: - context: comparison-vector/ - cache-from: type=gha - cache-to: type=gha,mode=max - file: regression/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar - tags: | - vector:${{ needs.compute-metadata.outputs.comparison-tag }} - - - name: Upload image as artifact - uses: actions/upload-artifact@v3 - with: - name: comparison-image - path: "${{ runner.temp }}/comparison-image.tar" + #build-baseline: + # name: Build baseline Vector container + # runs-on: ubuntu-20.04-4core + # timeout-minutes: 30 + # needs: + # - compute-metadata + # steps: + # - uses: colpal/actions-clean@v1 + + # - uses: actions/checkout@v3 + + # - uses: actions/checkout@v3 + # with: + # ref: ${{ needs.compute-metadata.outputs.baseline-sha }} + # path: baseline-vector + + # - name: Set up Docker Buildx + # id: buildx + # uses: docker/setup-buildx-action@v3.3.0 + + # - name: Build 'vector' target image + # uses: docker/build-push-action@v5.4.0 + # with: + # context: baseline-vector/ + # cache-from: type=gha + # cache-to: type=gha,mode=max + # file: regression/Dockerfile + # builder: ${{ steps.buildx.outputs.name }} + # outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar + # tags: | + # vector:${{ needs.compute-metadata.outputs.baseline-tag }} + + # - name: Upload image as artifact + # uses: actions/upload-artifact@v3 + # with: + # name: baseline-image + # path: "${{ runner.temp }}/baseline-image.tar" + + #build-comparison: + # name: Build comparison Vector container + # runs-on: ubuntu-20.04-4core + # timeout-minutes: 30 + # needs: + # - compute-metadata + # steps: + # - uses: colpal/actions-clean@v1 + + # - uses: actions/checkout@v3 + + # - uses: actions/checkout@v3 + # with: + # ref: ${{ needs.compute-metadata.outputs.comparison-sha }} + # path: comparison-vector + + # - name: Set up Docker Buildx + # id: buildx + # uses: docker/setup-buildx-action@v3.3.0 + + # - name: Build 'vector' target image + # uses: docker/build-push-action@v5.4.0 + # with: + # context: comparison-vector/ + # cache-from: type=gha + # cache-to: type=gha,mode=max + # file: regression/Dockerfile + # builder: ${{ steps.buildx.outputs.name }} + # outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar + # tags: | + # vector:${{ needs.compute-metadata.outputs.comparison-tag }} + + # - name: Upload image as artifact + # uses: actions/upload-artifact@v3 + # with: + # name: comparison-image + # path: "${{ runner.temp }}/comparison-image.tar" confirm-valid-credentials: name: Confirm AWS credentials are minimally valid @@ -390,16 +361,11 @@ jobs: needs: - compute-metadata - confirm-valid-credentials - - build-baseline steps: - - name: 'Download baseline image' - uses: actions/download-artifact@v3 - with: - name: baseline-image - name: Load baseline image run: | - docker load --input baseline-image.tar + docker pull timberio/vector:${{ needs.compute-metadata.outputs.baseline-tag }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4.0.2 @@ -419,7 +385,7 @@ jobs: - name: Tag & push baseline image run: | - docker tag vector:${{ needs.compute-metadata.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} + docker tag timberio/vector:${{ needs.compute-metadata.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} upload-comparison-image-to-ecr: @@ -429,16 +395,11 @@ jobs: needs: - compute-metadata - confirm-valid-credentials - - build-comparison steps: - - name: 'Download comparison image' - uses: actions/download-artifact@v3 - with: - name: comparison-image - name: Load comparison image run: | - docker load --input comparison-image.tar + docker pull timberio/vector:${{ needs.compute-metadata.outputs.comparison-tag }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4.0.2 @@ -458,7 +419,7 @@ jobs: - name: Tag & push comparison image run: | - docker tag vector:${{ needs.compute-metadata.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} + docker tag timberio/vector:${{ needs.compute-metadata.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} submit-job: @@ -484,8 +445,8 @@ jobs: -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-metadata.outputs.comparison-sha }} + #with: + # ref: ${{ needs.compute-metadata.outputs.comparison-sha }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4.0.2 @@ -509,20 +470,11 @@ jobs: chmod +x ${{ runner.temp }}/bin/smp ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \ - --lading-version ${{ needs.compute-metadata.outputs.lading-version }} \ - --total-samples ${{ needs.compute-metadata.outputs.total-samples }} \ - --warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \ - --replicas ${{ needs.compute-metadata.outputs.replicas }} \ --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} \ --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} \ --baseline-sha ${{ needs.compute-metadata.outputs.baseline-sha }} \ --comparison-sha ${{ needs.compute-metadata.outputs.comparison-sha }} \ - --target-command "/usr/local/bin/vector" \ --target-config-dir ${{ github.workspace }}/regression/ \ - --target-cpu-allotment "${{ needs.compute-metadata.outputs.cpus }}" \ - --target-memory-allotment "${{ needs.compute-metadata.outputs.memory }}" \ - --target-environment-variables "VECTOR_THREADS=${{ needs.compute-metadata.outputs.vector-cpus }},VECTOR_REQUIRE_HEALTHY=true" \ - --target-name vector \ --submission-metadata ${{ runner.temp }}/submission-metadata - uses: actions/upload-artifact@v3 @@ -541,7 +493,6 @@ jobs: --wait \ --wait-delay-seconds 60 \ --wait-timeout-minutes 90 \ - --use-consignor welch \ --submission-metadata ${{ runner.temp }}/submission-metadata - name: Handle cancellation if necessary @@ -632,7 +583,7 @@ jobs: chmod +x ${{ runner.temp }}/bin/smp ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \ - --submission-metadata ${{ runner.temp }}/submission-metadata --use-consignor welch + --submission-metadata ${{ runner.temp }}/submission-metadata - name: Check status, cancelled if: ${{ cancelled() }} @@ -725,7 +676,7 @@ jobs: ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \ --submission-metadata ${{ runner.temp }}/submission-metadata \ - --output-path "${{ runner.temp }}/outputs" --use-consignor welch + --output-path "${{ runner.temp }}/outputs" - name: Read regression report id: read-analysis @@ -796,8 +747,8 @@ jobs: if: always() needs: - compute-metadata - - build-baseline - - build-comparison + #- build-baseline + #- build-comparison - confirm-valid-credentials - upload-baseline-image-to-ecr - upload-comparison-image-to-ecr diff --git a/.github/workflows/workload_checks.yml b/.github/workflows/workload_checks.yml deleted file mode 100644 index b64f810ce652a..0000000000000 --- a/.github/workflows/workload_checks.yml +++ /dev/null @@ -1,210 +0,0 @@ -# Workload Checks Suite -# -# Runs Vector Workload Checks. -# -# Runs on: -# - scheduled UTC midnight Tues-Sat -# - on demand by a PR comment matching either of: -# - '/ci-run-regression' -# - '/ci-run-all' -# (the comment issuer must be a member of the Vector GH team) -# -# This workflow runs the collection of our workload checks, using the latest Vector nightly image, -# which depends on when the workflow is invoked. -# -# The goal is to establish a baseline of check results for a variety of cases -# and visualize trends for important Vector use cases. - -name: Workload Checks Suite - -on: - workflow_call: - workflow_dispatch: - schedule: - # At midnight UTC Tue-Sat - - cron: '0 0 * * 2-6' -env: - SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }} - -jobs: - compute-metadata: - name: Compute metadata - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - replicas: ${{ steps.experimental-meta.outputs.REPLICAS }} - warmup-seconds: ${{ steps.experimental-meta.outputs.WARMUP_SECONDS }} - total-samples: ${{ steps.experimental-meta.outputs.TOTAL_SAMPLES }} - smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} - lading-version: ${{ steps.experimental-meta.outputs.LADING_VERSION }} - - target-sha: ${{ steps.git-metadata.outputs.TARGET_SHA }} - - steps: - - uses: actions/checkout@v3 - - - name: Get git metadata - id: git-metadata - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - export TARGET_SHA=$(git rev-parse HEAD) - echo "TARGET_SHA=${TARGET_SHA}" >> $GITHUB_OUTPUT - - echo "target sha is: ${TARGET_SHA}" - - if [ "${TARGET_SHA}" = "" ] ; then - echo "TARGET_SHA not found, exiting." - exit 1 - fi - - - name: Setup experimental metadata - id: experimental-meta - run: | - export WARMUP_SECONDS="45" - export REPLICAS="10" - export TOTAL_SAMPLES="600" - export SMP_CRATE_VERSION="0.11.0" - export LADING_VERSION="0.19.1" - - echo "warmup seconds: ${WARMUP_SECONDS}" - echo "replicas: ${REPLICAS}" - echo "total samples: ${TOTAL_SAMPLES}" - echo "smp crate version: ${SMP_CRATE_VERSION}" - echo "lading version: ${LADING_VERSION}" - - echo "WARMUP_SECONDS=${WARMUP_SECONDS}" >> $GITHUB_OUTPUT - echo "REPLICAS=${REPLICAS}" >> $GITHUB_OUTPUT - echo "TOTAL_SAMPLES=${TOTAL_SAMPLES}" >> $GITHUB_OUTPUT - echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT - echo "LADING_VERSION=${LADING_VERSION}" >> $GITHUB_OUTPUT - - submit-job: - name: Submit workload checks job - runs-on: ubuntu-latest - timeout-minutes: 90 - needs: - - compute-metadata - steps: - - name: Check status, in-progress - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.target-sha }} \ - -f state='pending' \ - -f description='Experiments submitted to the Workload Checks cluster.' \ - -f context='Workload Checks Suite / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - uses: actions/checkout@v3 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4.0.2 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Download SMP binary - run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - - - name: Submit job - env: - RUST_LOG: info - run: | - CURRENT_DATE=$(date --utc '+%Y_%m_%d') - RUST_LOG_DEBUG="debug,aws_config::profile::credentials=error" - - chmod +x ${{ runner.temp }}/bin/smp - RUST_BACKTRACE=1 RUST_LOG="${RUST_LOG_DEBUG}" ${{ runner.temp }}/bin/smp \ - --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} \ - job submit-workload \ - --lading-version ${{ needs.compute-metadata.outputs.lading-version }} \ - --total-samples ${{ needs.compute-metadata.outputs.total-samples }} \ - --warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \ - --replicas ${{ needs.compute-metadata.outputs.replicas }} \ - --target-image timberio/vector:nightly-debian \ - --target-sha ${{ needs.compute-metadata.outputs.target-sha }} \ - --target-config-dir ${{ github.workspace }}/workload-checks \ - --target-name vector \ - --target-command "/usr/bin/vector" \ - --target-environment-variables "DD_HOSTNAME=smp-workload-checks,DD_DD_URL=http://127.0.0.1:9092,DD_API_KEY=00000001" \ - --tags smp_status=nightly,client_team="vector",tag_date="${CURRENT_DATE}" \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - uses: actions/upload-artifact@v3 - with: - name: vector-submission-metadata - path: ${{ runner.temp }}/submission-metadata - - - name: Await job - timeout-minutes: 120 - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} \ - job status \ - --wait \ - --wait-delay-seconds 60 \ - --wait-timeout-minutes 90 \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - name: Handle cancellation if necessary - if: ${{ cancelled() }} - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - name: Check status, cancelled - if: ${{ cancelled() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.target-sha }} \ - -f state='failure' \ - -f description='Experiments submitted to the Workload Checks cluster cancelled.' \ - -f context='Workload Checks Suite / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, success - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.target-sha }} \ - -f state='success' \ - -f description='Experiments submitted to the Workload Checks cluster successfully.' \ - -f context='Workload Checks Suite / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, failure - if: ${{ failure() }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.target-sha }} \ - -f state='success' \ - -f description='Experiments submitted to the Workload Checks Suite failed.' \ - -f context='Workload Checks Suite / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/regression/cases/datadog_agent_remap_blackhole/experiment.yaml b/regression/cases/datadog_agent_remap_blackhole/experiment.yaml new file mode 100644 index 0000000000000..d8616fe176d66 --- /dev/null +++ b/regression/cases/datadog_agent_remap_blackhole/experiment.yaml @@ -0,0 +1,12 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/datadog_agent_remap_blackhole_acks/experiment.yaml b/regression/cases/datadog_agent_remap_blackhole_acks/experiment.yaml new file mode 100644 index 0000000000000..d8616fe176d66 --- /dev/null +++ b/regression/cases/datadog_agent_remap_blackhole_acks/experiment.yaml @@ -0,0 +1,12 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/datadog_agent_remap_datadog_logs/experiment.yaml b/regression/cases/datadog_agent_remap_datadog_logs/experiment.yaml new file mode 100644 index 0000000000000..d8616fe176d66 --- /dev/null +++ b/regression/cases/datadog_agent_remap_datadog_logs/experiment.yaml @@ -0,0 +1,12 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/datadog_agent_remap_datadog_logs_acks/experiment.yaml b/regression/cases/datadog_agent_remap_datadog_logs_acks/experiment.yaml new file mode 100644 index 0000000000000..d8616fe176d66 --- /dev/null +++ b/regression/cases/datadog_agent_remap_datadog_logs_acks/experiment.yaml @@ -0,0 +1,12 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/file_to_blackhole/experiment.yaml b/regression/cases/file_to_blackhole/experiment.yaml index 9b6d2c53c5271..c72f0f63e8515 100644 --- a/regression/cases/file_to_blackhole/experiment.yaml +++ b/regression/cases/file_to_blackhole/experiment.yaml @@ -1 +1,12 @@ optimization_goal: egress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/fluent_elasticsearch/experiment.yaml b/regression/cases/fluent_elasticsearch/experiment.yaml new file mode 100644 index 0000000000000..d8616fe176d66 --- /dev/null +++ b/regression/cases/fluent_elasticsearch/experiment.yaml @@ -0,0 +1,12 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/http_elasticsearch/experiment.yaml b/regression/cases/http_elasticsearch/experiment.yaml new file mode 100644 index 0000000000000..d8616fe176d66 --- /dev/null +++ b/regression/cases/http_elasticsearch/experiment.yaml @@ -0,0 +1,12 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/http_text_to_http_json/experiment.yaml b/regression/cases/http_text_to_http_json/experiment.yaml new file mode 100644 index 0000000000000..d8616fe176d66 --- /dev/null +++ b/regression/cases/http_text_to_http_json/experiment.yaml @@ -0,0 +1,12 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/http_to_http_acks/experiment.yaml b/regression/cases/http_to_http_acks/experiment.yaml new file mode 100644 index 0000000000000..d8616fe176d66 --- /dev/null +++ b/regression/cases/http_to_http_acks/experiment.yaml @@ -0,0 +1,12 @@ +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/http_to_http_json/experiment.yaml b/regression/cases/http_to_http_json/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/http_to_http_json/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/http_to_http_noack/experiment.yaml b/regression/cases/http_to_http_noack/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/http_to_http_noack/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/http_to_s3/experiment.yaml b/regression/cases/http_to_s3/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/http_to_s3/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/otlp_grpc_to_blackhole/experiment.yaml b/regression/cases/otlp_grpc_to_blackhole/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/otlp_grpc_to_blackhole/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/otlp_http_to_blackhole/experiment.yaml b/regression/cases/otlp_http_to_blackhole/experiment.yaml index 88dfaecaa4a52..d8616fe176d66 100644 --- a/regression/cases/otlp_http_to_blackhole/experiment.yaml +++ b/regression/cases/otlp_http_to_blackhole/experiment.yaml @@ -1 +1,12 @@ optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/socket_to_socket_blackhole/experiment.yaml b/regression/cases/socket_to_socket_blackhole/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/socket_to_socket_blackhole/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/splunk_hec_indexer_ack_blackhole/experiment.yaml b/regression/cases/splunk_hec_indexer_ack_blackhole/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/splunk_hec_indexer_ack_blackhole/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/splunk_hec_route_s3/experiment.yaml b/regression/cases/splunk_hec_route_s3/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/splunk_hec_route_s3/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/splunk_hec_to_splunk_hec_logs_acks/experiment.yaml b/regression/cases/splunk_hec_to_splunk_hec_logs_acks/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/splunk_hec_to_splunk_hec_logs_acks/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/splunk_hec_to_splunk_hec_logs_noack/experiment.yaml b/regression/cases/splunk_hec_to_splunk_hec_logs_noack/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/splunk_hec_to_splunk_hec_logs_noack/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/syslog_humio_logs/experiment.yaml b/regression/cases/syslog_humio_logs/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/syslog_humio_logs/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/syslog_log2metric_humio_metrics/experiment.yaml b/regression/cases/syslog_log2metric_humio_metrics/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/syslog_log2metric_humio_metrics/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/syslog_log2metric_splunk_hec_metrics/experiment.yaml b/regression/cases/syslog_log2metric_splunk_hec_metrics/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/syslog_log2metric_splunk_hec_metrics/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/experiment.yaml b/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/experiment.yaml index 88dfaecaa4a52..d8616fe176d66 100644 --- a/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/experiment.yaml +++ b/regression/cases/syslog_log2metric_tag_cardinality_limit_blackhole/experiment.yaml @@ -1 +1,12 @@ optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/syslog_loki/experiment.yaml b/regression/cases/syslog_loki/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/syslog_loki/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/syslog_regex_logs2metric_ddmetrics/experiment.yaml b/regression/cases/syslog_regex_logs2metric_ddmetrics/experiment.yaml index 88dfaecaa4a52..d8616fe176d66 100644 --- a/regression/cases/syslog_regex_logs2metric_ddmetrics/experiment.yaml +++ b/regression/cases/syslog_regex_logs2metric_ddmetrics/experiment.yaml @@ -1 +1,12 @@ optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/cases/syslog_splunk_hec_logs/experiment.yaml b/regression/cases/syslog_splunk_hec_logs/experiment.yaml new file mode 100644 index 0000000000000..0109cf929c0e6 --- /dev/null +++ b/regression/cases/syslog_splunk_hec_logs/experiment.yaml @@ -0,0 +1,13 @@ + +optimization_goal: ingress_throughput + +target: + name: vector + command: /usr/bin/vector + cpu_allotment: 7 + memory_allotment: 30g + + environment: + VECTOR_THREADS: 4 + VECTOR_REQUIRE_HEALTHY: true + diff --git a/regression/config.yaml b/regression/config.yaml new file mode 100644 index 0000000000000..85f95930a6ac2 --- /dev/null +++ b/regression/config.yaml @@ -0,0 +1,37 @@ +lading: + version: 0.19.1 + +target: + + +# Link templates for reports. +# +# Values may be removed to disable corresponding links in reports. +# +# Links that may be set: +# - `metrics_dashboard`: link to a metrics dashboard +# - `profiles`: link to profiles +# - `per_experiment_logs`: link to logs, for each experiment +# +# Additionally, arbitrary links may be added to experiment configuration files. +# These should be added as a list of key-value pairs in a 'report_links' +# section. For example: +# ``` +# report_links: +# - text: Link Text +# link: "link template" +# ``` +# +# Available variables: +# - `job_id`: the job ID +# - `start_time_ms`: start time of the job's metrics data, in ms +# - `end_time_ms`: end time of the job's metrics data, in ms +# - `filter_start`: arbitrary time before this job's replicates were run, +# suitable for filtering logs and profiles by job ID. +# - `filter_end`: arbitrary time after this job's replicates were run and logs +# were uploaded, suitable for filtering logs and profiles by job ID. +# +# Additional variables for per-experiment links: +# - `experiment`: the name of the experiment +report: + metrics_dashboard: "https://app.datadoghq.com/dashboard/ykh-ua8-vcu/SMP-Regression-Detector-Metrics?fromUser=true&refresh_mode=paused&tpl_var_run-id%5B0%5D={{ job_id }}&view=spans&from_ts={{ start_time_ms }}&to_ts={{ end_time_ms }}&live=false"