diff --git a/.github/actions/run-service/action.yml b/.github/actions/run-service/action.yml new file mode 100644 index 0000000000..5267c7ce25 --- /dev/null +++ b/.github/actions/run-service/action.yml @@ -0,0 +1,34 @@ +name: Run service +description: Runs a Docker image from cache +inputs: + repository: + required: true + description: The repository of the image to warm up + tag: + required: true + description: The tag of the image to warm up (latest tag not supported) + flags: + required: true + description: The flags to pass to the service +runs: + using: "composite" + steps: + - name: Build cache filename + id: build_cache_filename + shell: bash + run: | + cache_filename=$(printf '%s_%s' "${{ inputs.repository }}" "${{ inputs.tag }}" | tr -cs '[:alnum:]_' '_') + echo "cache_filename=$cache_filename" >> $GITHUB_OUTPUT + - name: Restore image cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: ${{ steps.build_cache_filename.outputs.cache_filename }}.tar + key: ${{ inputs.repository }}:${{ inputs.tag }} + - name: Load image + shell: bash + run: | + docker load -i ${{ steps.build_cache_filename.outputs.cache_filename }}.tar + - name: Run service + shell: bash + run: | + docker run -d ${{ inputs.flags }} ${{ inputs.repository }}:${{ inputs.tag }} diff --git a/.github/actions/warm-up-service/action.yml b/.github/actions/warm-up-service/action.yml new file mode 100644 index 0000000000..cd45df5627 --- /dev/null +++ b/.github/actions/warm-up-service/action.yml @@ -0,0 +1,25 @@ +name: Warm up service +inputs: + repository: + required: true + description: The repository of the image to warm up + tag: + required: true + description: The tag of the image to warm up (latest tag not supported) +runs: + using: "composite" + steps: + - name: Pull Docker image + id: pull_image + shell: bash + run: | + image=${{ inputs.repository }}:${{ inputs.tag }} + docker pull $image + cache_filename=$(printf '%s_%s' "${{ inputs.repository }}" "${{ inputs.tag }}" | tr -cs '[:alnum:]_' '_') + echo "cache_filename=$cache_filename" >> $GITHUB_OUTPUT + docker save $image > $cache_filename.tar + - name: Cache Docker image + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: "${{ steps.pull_image.outputs.cache_filename }}.tar" + key: "${{ inputs.repository }}:${{ inputs.tag }}" diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index 78c7fc7129..26216b890c 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -8,4 +8,13 @@ clean: rm -rf test-apps.yml test-apps.yml: test-apps.cue - cat <(echo "# Code generated via \`make test-apps.yml\`; DO NOT EDIT.") <(cue export --out=yaml $<) > $@ \ No newline at end of file + cat <(echo "# Code generated via \`make test-apps.yml\`; DO NOT EDIT.") <(cue export --out=yaml $<) > $@ + +unit-integration-tests.yml: unit-integration-tests.cue services.cue go-versions.cue + cat <(echo "# Code generated via \`make unit-integration-tests.yml\`; DO NOT EDIT.") <(cue export services.cue go-versions.cue --out=yaml $<) > $@ + +pull-request.yml: pull-request.cue services.cue go-versions.cue + cat <(echo "# Code generated via \`make pull-request.yml\`; DO NOT EDIT.") <(cue export services.cue go-versions.cue --out=yaml $<) > $@ + +.PHONY: cue +cue: test-apps.yml unit-integration-tests.yml pull-request.yml diff --git a/.github/workflows/api-check.yml b/.github/workflows/api-check.yml index 1176b35c43..b4b592429d 100644 --- a/.github/workflows/api-check.yml +++ b/.github/workflows/api-check.yml @@ -2,6 +2,10 @@ name: API Stability Check on: pull_request: + types: + - opened + - synchronize + - reopened paths: - 'ddtrace/tracer/**' - 'scripts/apiextractor/**' diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 236d0333fe..a0880a9b69 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -10,19 +10,25 @@ on: schedule: # nightly - cron: "0 0 * * *" pull_request: # on pull requests touching appsec files + types: + - opened + - synchronize + - reopened paths: - '.github/workflows/appsec.yml' - 'internal/appsec/**' - 'appsec/**' - 'contrib/**/appsec.go' - '**/go.mod' - merge_group: push: - branches: release-v* + branches: + - release-v* tags-ignore: - 'contrib/**' - 'instrumentation/**' - + - 'internal/**' + - 'orchestrion/**' + - 'scripts/**' env: DD_APPSEC_WAF_TIMEOUT: 1m GOEXPERIMENT: synctest # TODO: remove once go1.25 is the minimum supported version diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 68bb519c96..68f870dfbc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -8,14 +8,13 @@ on: required: true type: string push: - branches: [ main, master ] - tags-ignore: - - 'contrib/**' - - 'instrumentation/**' + branches: + - mq-working-branch-** pull_request: - # The branches below must be a subset of the branches above - branches: [ main ] - merge_group: + types: + - opened + - synchronize + - reopened jobs: analyze: diff --git a/.github/workflows/ecosystems-label-pr.yml b/.github/workflows/ecosystems-label-pr.yml index 2945d18268..5978af88d3 100644 --- a/.github/workflows/ecosystems-label-pr.yml +++ b/.github/workflows/ecosystems-label-pr.yml @@ -1,12 +1,12 @@ name: Label APM Ecosystems Pull Requests on: pull_request: - paths: - - "contrib/**" types: - opened - reopened - edited + paths: + - "contrib/**" permissions: contents: read pull-requests: write diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml index 14f36525d9..728f70e040 100644 --- a/.github/workflows/generate.yml +++ b/.github/workflows/generate.yml @@ -3,12 +3,13 @@ name: Generate on: push: branches: - - main - release-* + - mq-working-branch-** pull_request: - branches: - - main - - release-* + types: + - opened + - synchronize + - reopened workflow_call: inputs: go-version: diff --git a/.github/workflows/go-versions.cue b/.github/workflows/go-versions.cue new file mode 100644 index 0000000000..5f9c57bb09 --- /dev/null +++ b/.github/workflows/go-versions.cue @@ -0,0 +1,8 @@ +package workflows + +// If you bump the supported versions here, you need to run `make cue` to +// update the YAML files. +_go_versions: [ + "1.24", + "1.25", +] diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index 4a7196c9f9..2d6904899c 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -8,11 +8,14 @@ on: type: string push: branches: - - main - release-v* + - mq-working-branch-** tags-ignore: - 'contrib/**' - 'instrumentation/**' + - 'internal/**' + - 'orchestrion/**' + - 'scripts/**' schedule: - cron: '00 00 * * *' workflow_dispatch: diff --git a/.github/workflows/main-branch-tests.yml b/.github/workflows/main-branch-tests.yml index 84470e14a9..d2af52c746 100644 --- a/.github/workflows/main-branch-tests.yml +++ b/.github/workflows/main-branch-tests.yml @@ -1,19 +1,16 @@ name: Main Branch and Release Tests on: - workflow_call: # allows to reuse this workflow - inputs: - ref: - description: 'The branch to run the workflow on' - required: true - type: string push: branches: - - main - release-v* + - mq-working-branch-** tags-ignore: - 'contrib/**' - 'instrumentation/**' + - 'internal/**' + - 'orchestrion/**' + - 'scripts/**' concurrency: group: ${{ github.ref }} @@ -21,24 +18,10 @@ concurrency: jobs: unit-integration-tests: - strategy: - matrix: - go-version: [ "1.25", "1.24" ] - fail-fast: false uses: ./.github/workflows/unit-integration-tests.yml + permissions: + contents: read + id-token: write + pull-requests: write with: - go-version: ${{ matrix.go-version }} - ref: ${{ inputs.ref || github.ref }} - secrets: inherit - multios-unit-tests: - strategy: - matrix: - runs-on: [ macos-latest, windows-latest, ubuntu-latest ] - go-version: [ "1.25", "1.24" ] - fail-fast: false - uses: ./.github/workflows/multios-unit-tests.yml - with: - go-version: ${{ matrix.go-version }} - runs-on: ${{ matrix.runs-on }} - ref: ${{ inputs.ref || github.ref }} - secrets: inherit + go-version: "1.25" # Should be the highest supported version of Go diff --git a/.github/workflows/multios-unit-tests.yml b/.github/workflows/multios-unit-tests.yml index 05fce9fbf5..a932b103ff 100644 --- a/.github/workflows/multios-unit-tests.yml +++ b/.github/workflows/multios-unit-tests.yml @@ -4,13 +4,11 @@ on: workflow_dispatch: # manually inputs: go-version: + description: The Go version to use required: true type: string runs-on: - required: true - type: string - ref: - description: 'The branch to run the workflow on' + description: The OS to run the tests on required: true type: string workflow_call: @@ -21,10 +19,6 @@ on: runs-on: required: true type: string - ref: - description: 'The branch to run the workflow on' - required: true - type: string env: DD_APPSEC_WAF_TIMEOUT: 1m # Increase time WAF time budget to reduce CI flakiness @@ -51,10 +45,16 @@ jobs: shell: pwsh run: | "normalized_workspace=${{ github.workspace }}" >> $env:GITHUB_ENV + - name: Restore repo cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: .git + key: gitdb-${{ github.repository_id }}-${{ github.sha }} - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v2.7.0 with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ github.sha }} + clean: false - name: Setup Go and development tools uses: ./.github/actions/setup-go with: @@ -64,7 +64,7 @@ jobs: - name: Mac OS Coreutils if: inputs.runs-on == 'macos-latest' run: brew install coreutils - - name: "Runner ${{ matrix.runner-index }}: Test Core and Contrib (No Integration Tests)" + - name: "Runner: Test Core and Contrib (No Integration Tests)" shell: bash run: | export PATH="${{ github.workspace }}/bin:${PATH}" diff --git a/.github/workflows/orchestrion.yml b/.github/workflows/orchestrion.yml index 7e880bd420..36286a1291 100644 --- a/.github/workflows/orchestrion.yml +++ b/.github/workflows/orchestrion.yml @@ -15,13 +15,19 @@ on: DD_API_KEY: required: false pull_request: - merge_group: + types: + - opened + - synchronize + - reopened push: branches: - release-v* tags-ignore: - 'contrib/**' - 'instrumentation/**' + - 'internal/**' + - 'orchestrion/**' + - 'scripts/**' permissions: read-all diff --git a/.github/workflows/parametric-tests.yml b/.github/workflows/parametric-tests.yml index a8a08e3eff..4fa69f69e1 100644 --- a/.github/workflows/parametric-tests.yml +++ b/.github/workflows/parametric-tests.yml @@ -9,15 +9,19 @@ on: type: string push: branches: - - main - release-v* + - mq-working-branch-** tags-ignore: - 'contrib/**' - 'instrumentation/**' + - 'internal/**' + - 'orchestrion/**' + - 'scripts/**' pull_request: - branches: - - "**" - merge_group: + types: + - opened + - synchronize + - reopened workflow_dispatch: inputs: ref: diff --git a/.github/workflows/pull-request-title-validation.yml b/.github/workflows/pull-request-title-validation.yml index 6015bc8eaf..449e5e8100 100644 --- a/.github/workflows/pull-request-title-validation.yml +++ b/.github/workflows/pull-request-title-validation.yml @@ -2,7 +2,10 @@ name: Validate PR Title on: pull_request: - types: [opened, edited,reopened,synchronize] + types: + - opened + - edited + - reopened jobs: check-title: diff --git a/.github/workflows/pull-request.cue b/.github/workflows/pull-request.cue new file mode 100644 index 0000000000..d09767c46b --- /dev/null +++ b/.github/workflows/pull-request.cue @@ -0,0 +1,131 @@ +package workflows + +"name": "Pull Request Tests" + +"on": "pull_request": "types": [ + "opened", + "synchronize", + "reopened", +] + +"concurrency": { + "group": "${{ github.ref }}" + "cancel-in-progress": true +} + +"jobs": { + "warm-repo-cache": { + "runs-on": "ubuntu-latest" + "steps": [ + { + "name": "Checkout" + "uses": "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + "with": { + "ref": "${{ github.event.pull_request.head.sha }}" + } + }, + { + "name": "Cache" + "uses": "actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809" + "with": { + "path": ".git" + "key": "gitdb-${{ github.repository_id }}-${{ github.sha }}" + } + }, + ] + } + "warm-services-cache": { + "runs-on": "ubuntu-latest" + "strategy": "matrix": "image": [#ToImage & {_svc: _datadog_agent_svc}] + "steps": [ + { + "name": "Restore repo cache" + "uses": "actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809" + "with": { + "path": ".git" + "key": "gitdb-${{ github.repository_id }}-${{ github.sha }}" + } + }, + { + "name": "Checkout" + "uses": "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + "with": { + "ref": "${{ github.sha }}" + "clean": false + } + }, + { + "name": "Warm up service" + "uses": "./.github/actions/warm-up-service" + "with": { + "repository": "${{ matrix.image.repository }}" + "tag": "${{ matrix.image.tag }}" + } + }, + ] + } + "unit-integration-tests": { + "name": "PR Unit and Integration Tests" + "needs": [ + "warm-repo-cache", + "warm-services-cache", + ] + "strategy": { + "matrix": { + "go-version": _go_versions + } + "fail-fast": false + } + "uses": "./.github/workflows/unit-integration-tests.yml" + "with": { + "go-version": "${{ matrix.go-version }}" + } + "secrets": "inherit" + } + "multios-unit-tests": { + "needs": [ + "warm-repo-cache", + "warm-services-cache", + ] + "strategy": { + "matrix": { + "runs-on": ["macos-latest", "windows-latest", "ubuntu-latest"] + "go-version": _go_versions + } + "fail-fast": false + } + "uses": "./.github/workflows/multios-unit-tests.yml" + "with": { + "go-version": "${{ matrix.go-version }}" + "runs-on": "${{ matrix.runs-on }}" + } + "secrets": "inherit" + } + // This is a simple join point to make it easy to set up branch protection rules in GitHub. + "pull-request-tests-done": { + "name": "PR Unit and Integration Tests / ${{ matrix.name }}" + "strategy": { + "matrix": { + "name": ["test-contrib", "test-core"] + } + } + "needs": [ + "unit-integration-tests", + "multios-unit-tests", + ] + "runs-on": "ubuntu-latest" + "if": "success() || failure()" + "steps": [ + { + "name": "Success" + "if": "needs.unit-integration-tests.result == 'success' && needs.multios-unit-tests.result == 'success'" + "run": "echo 'Success!'" + }, + { + "name": "Failure" + "if": "needs.unit-integration-tests.result != 'success' || needs.multios-unit-tests.result != 'success'" + "run": "echo 'Failure!' && exit 1" + }, + ] + } +} diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 0a3e4b18c1..2294ba3641 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -1,30 +1,100 @@ +# Code generated via `make pull-request.yml`; DO NOT EDIT. name: Pull Request Tests - -on: +"on": pull_request: - branches: - - "**" - merge_group: - push: - branches: - - 'mq-working-branch-**' - tags-ignore: - - 'contrib/**' - - 'instrumentation/**' - + types: + - opened + - synchronize + - reopened concurrency: group: ${{ github.ref }} cancel-in-progress: true - jobs: + warm-repo-cache: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 + with: + path: .git + key: gitdb-${{ github.repository_id }}-${{ github.sha }} + warm-services-cache: + runs-on: ubuntu-latest + strategy: + matrix: + image: + - repository: datadog/agent + tag: 7.69.2 + steps: + - name: Restore repo cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 + with: + path: .git + key: gitdb-${{ github.repository_id }}-${{ github.sha }} + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + ref: ${{ github.sha }} + clean: false + - name: Warm up service + uses: ./.github/actions/warm-up-service + with: + repository: ${{ matrix.image.repository }} + tag: ${{ matrix.image.tag }} unit-integration-tests: name: PR Unit and Integration Tests + needs: + - warm-repo-cache + - warm-services-cache + strategy: + matrix: + go-version: + - "1.24" + - "1.25" + fail-fast: false uses: ./.github/workflows/unit-integration-tests.yml - permissions: - contents: read - id-token: write - pull-requests: write with: - go-version: "1.24" - ref: ${{ github.ref }} + go-version: ${{ matrix.go-version }} secrets: inherit + multios-unit-tests: + needs: + - warm-repo-cache + - warm-services-cache + strategy: + matrix: + runs-on: + - macos-latest + - windows-latest + - ubuntu-latest + go-version: + - "1.24" + - "1.25" + fail-fast: false + uses: ./.github/workflows/multios-unit-tests.yml + with: + go-version: ${{ matrix.go-version }} + runs-on: ${{ matrix.runs-on }} + secrets: inherit + pull-request-tests-done: + name: PR Unit and Integration Tests / ${{ matrix.name }} + strategy: + matrix: + name: + - test-contrib + - test-core + needs: + - unit-integration-tests + - multios-unit-tests + runs-on: ubuntu-latest + if: success() || failure() + steps: + - name: Success + if: needs.unit-integration-tests.result == 'success' && needs.multios-unit-tests.result == 'success' + run: echo 'Success!' + - name: Failure + if: needs.unit-integration-tests.result != 'success' || needs.multios-unit-tests.result != 'success' + run: echo 'Failure!' && exit 1 diff --git a/.github/workflows/services.cue b/.github/workflows/services.cue new file mode 100644 index 0000000000..43e9068d53 --- /dev/null +++ b/.github/workflows/services.cue @@ -0,0 +1,284 @@ +package workflows + +import ( + "list" + "strings" +) + +#Service: { + image: string + env?: {[string]: string | number | bool} + options?: string + ports?: [...string] + volumes?: [...string] +} + +#Services: {[string]: #Service} + +#Services: { + _names: [...string] + + if len(_names) == 0 { + _services + } + if len(_names) > 0 { + for name in _names + if list.Contains(_names, name) { + "\(name)": _services[name] + } + } +} + +#Image: { + repository: string + tag?: string +} + +#ToImage: { + _svc: #Service + _parts: [...string] & strings.Split(_svc.image, ":") + + repository: _parts[0] + tag: _parts[1] +} & #Image + +_datadog_agent_svc: #Service & { + "image": "datadog/agent:7.69.2" + "env": { + "DD_HOSTNAME": "github-actions-worker" + "DD_APM_ENABLED": true + "DD_BIND_HOST": "0.0.0.0" + "DD_API_KEY": "invalid_key_but_this_is_fine" + "DD_TEST_AGENT_HOST": "localhost" + "DD_TEST_AGENT_PORT": 9126 + } + "options": """ + --health-cmd "bash -c '> "$GITHUB_OUTPUT" + - name: Cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: .git + key: gitdb-system-tests-${{ steps.pin.outputs.sha }} system-tests: if: github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'DataDog/dd-trace-go') # Note: Not using large runners because the jobs spawned by this pipeline # don't seem to get a noticable speedup from using larger runners. runs-on: ubuntu-latest + needs: + - warm-repo-cache strategy: matrix: weblog-variant: @@ -124,11 +149,17 @@ jobs: SYSTEM_TESTS_E2E_DD_APP_KEY: ${{ secrets.SYSTEM_TESTS_E2E_DD_APP_KEY }} name: Test (${{ matrix.weblog-variant }}, ${{ matrix.scenario }}) steps: - - name: Checkout system tests - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Restore repo cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: - repository: 'DataDog/system-tests' - ref: ${{ inputs.ref }} + path: .git + key: gitdb-system-tests-${{ needs.warm-repo-cache.outputs.sha }} + + - name: Checkout system tests + shell: bash + run: | + git config safe.directory "$GITHUB_WORKSPACE" + git checkout -f ${{ needs.warm-repo-cache.outputs.sha }} - name: Checkout dd-trace-go uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/unit-integration-tests.cue b/.github/workflows/unit-integration-tests.cue new file mode 100644 index 0000000000..746e7f9374 --- /dev/null +++ b/.github/workflows/unit-integration-tests.cue @@ -0,0 +1,260 @@ +"name": "Unit and Integration Tests" + +"on": "workflow_call": "inputs": "go-version": { + "required": true + "type": "string" +} + +"env": { + // Increase time WAF time budget to reduce CI flakiness + // Users may build our library with GOTOOLCHAIN=local. If they do, and our + // go.mod file specifies a newer Go version than their local toolchain, their + // build will break. Run our tests with GOTOOLCHAIN=local to ensure that + // our library builds with all of the Go versions we claim to support, + // without having to download a newer one. + "DD_APPSEC_WAF_TIMEOUT": "1m" + "GOTOOLCHAIN": "local" + "GODEBUG": "x509negativeserial=1" + "GOEXPERIMENT": "synctest" // TODO: remove once go1.25 is the minimum supported version + "TEST_RESULT_PATH": "/tmp/test-results" +} + +"permissions": "contents": "read" + +"jobs": { + "set-up": { + "runs-on": "ubuntu-latest" + "outputs": { + "matrix": "${{ steps.matrix.outputs.matrix }}" + } + "steps": [ + { + "name": "Restore repo cache" + "uses": "actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809" + "with": { + "path": ".git" + "key": "gitdb-${{ github.repository_id }}-${{ github.sha }}" + } + }, + { + "name": "Checkout" + "uses": "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + "with": { + "ref": "${{ github.sha }}" + "clean": false + } + }, + { + "name": "Compute Matrix" + "id": "matrix" + "run": """ + echo -n "matrix=" >> "${GITHUB_OUTPUT}" + go run ./scripts/ci_contrib_matrix.go >> "${GITHUB_OUTPUT}" + """ + }, + ] + } + "test-contrib-matrix": { + "needs": [ + "set-up", + ] + "runs-on": "group": "APM Larger Runners" + "env": { + "INTEGRATION": true + } + "strategy": { + "matrix": { + "chunk": "${{ fromJson(needs.set-up.outputs.matrix) }}" + } + } + "services": #Services + "steps": [ + { + "name": "Restore repo cache" + "uses": "actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809" + "with": { + "path": ".git" + "key": "gitdb-${{ github.repository_id }}-${{ github.sha }}" + } + }, + { + "name": "Checkout" + "uses": "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + "with": { + "ref": "${{ github.sha }}" + "clean": false + } + }, + { + "name": "Setup Go and development tools" + "uses": "./.github/actions/setup-go" + "with": { + "go-version": "${{ inputs.go-version }}" + "tools-dir": "${{ github.workspace }}/_tools" + "tools-bin": "${{ github.workspace }}/bin" + } + }, + { + "name": "Test Contrib" + "if": "always()" + "env": { + "TEST_RESULTS": "${{ env.TEST_RESULT_PATH }}" + } + "run": """ + export PATH="${{ github.workspace }}/bin:${PATH}" + ./scripts/ci_test_contrib.sh default ${{ toJson(matrix.chunk) }} + """ + }, + { + "name": "Upload the results to Datadog CI App" + "if": "always()" + "continue-on-error": true + "uses": "./.github/actions/dd-ci-upload" + "with": { + "dd-api-key": "${{ secrets.DD_CI_API_KEY }}" + "path": "${{ env.TEST_RESULT_PATH }}" + "tags": "go:${{ inputs.go-version }},arch:${{ runner.arch }},os:${{ runner.os }},distribution:${{ runner.distribution }}" + } + }, + { + "name": "Upload Coverage" + "if": "always()" + "continue-on-error": true + "shell": "bash" + "run": "bash <(curl -s https://codecov.io/bash) -t ${{ secrets.CODECOV_TOKEN }}" + }, + ] + } + "test-contrib": { + "needs": [ + "test-contrib-matrix", + ] + "runs-on": "group": "APM Larger Runners" + "if": "success() || failure()" + "continue-on-error": true + "steps": [ + { + "name": "Success" + "if": "needs.test-contrib-matrix.result == 'success'" + "run": "echo 'Success!'" + }, + { + "name": "Failure" + "if": "needs.test-contrib-matrix.result != 'success'" + "run": "echo 'Failure!' && exit 1" + }, + ] + } + "test-core": { + "runs-on": "group": "APM Larger Runners" + "env": { + "INTEGRATION": true + } + "steps": [ + { + "name": "Restore repo cache" + "uses": "actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809" + "with": { + "path": ".git" + "key": "gitdb-${{ github.repository_id }}-${{ github.sha }}" + } + }, + { + "name": "Checkout" + "uses": "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + "with": { + "ref": "${{ github.sha }}" + "clean": false + } + }, + { + "name": "Setup Go and development tools" + "uses": "./.github/actions/setup-go" + "with": { + "go-version": "${{ inputs.go-version }}" + "tools-dir": "${{ github.workspace }}/_tools" + "tools-bin": "${{ github.workspace }}/bin" + } + }, + { + "name": "Start datadog/agent" + "uses": "./.github/actions/run-service" + "with": { + #ToImage & {_svc: _datadog_agent_svc} + } + }, + { + "name": "Test Core" + "env": { + "DD_APPSEC_WAF_TIMEOUT": "1h" + "TEST_RESULTS": "${{ env.TEST_RESULT_PATH }}" + } + "run": """ + export PATH="${{ github.workspace }}/bin:${PATH}" + ls -al "${{ github.workspace }}/bin" + ./scripts/ci_test_core.sh + """ + }, + { + "name": "Upload the results to Datadog CI App" + "if": "always()" + "continue-on-error": true + "uses": "./.github/actions/dd-ci-upload" + "with": { + "dd-api-key": "${{ secrets.DD_CI_API_KEY }}" + "path": "${{ env.TEST_RESULT_PATH }}" + "tags": "go:${{ inputs.go-version }},arch:${{ runner.arch }},os:${{ runner.os }},distribution:${{ runner.distribution }}" + } + }, + { + "name": "Upload Coverage" + "if": "always()" + "continue-on-error": true + "shell": "bash" + "run": "bash <(curl -s https://codecov.io/bash) -t ${{ secrets.CODECOV_TOKEN }}" + }, + ] + } + "upload-test-results": { + "needs": [ + "test-contrib", + "test-core", + ] + "if": "always()" // Make sure this always runs, even if test-contrib or test-core fails + "runs-on": "group": "APM Larger Runners" + "services": #Services & {_names: ["datadog-agent", "testagent"]} + "steps": [ + { + "name": "Get Datadog APM Test Agent Logs" + "if": "always()" + "shell": "bash" + "run": "docker logs ${{ job.services.testagent.id }}" + }, + { + "name": "Get Datadog APM Test Agent Trace Check Summary Results" + "if": "always()" + "shell": "bash" + "run": """ + RESPONSE=$(curl -s -w "\\n%{http_code}" -o response.txt "http://127.0.0.1:9126/test/trace_check/failures?return_all=true") + RESPONSE_CODE=$(echo "$RESPONSE" | awk 'END {print $NF}') + SUMMARY_RESPONSE=$(curl -s -w "\\n%{http_code}" -o summary_response.txt "http://127.0.0.1:9126/test/trace_check/summary?return_all=true") + SUMMARY_RESPONSE_CODE=$(echo "$SUMMARY_RESPONSE" | awk 'END {print $NF}') + if [[ $RESPONSE_CODE -eq 200 ]]; then + echo " " + cat response.txt + echo " - All APM Test Agent Check Traces returned successful!" + echo "APM Test Agent Check Traces Summary Results:" + cat summary_response.txt | jq "." + else + echo "APM Test Agent Check Traces failed with response code: $RESPONSE_CODE" + echo "Failures:" + cat response.txt + echo "APM Test Agent Check Traces Summary Results:" + cat summary_response.txt | jq "." + exit 1 + fi + """ + }, + ] + } +} diff --git a/.github/workflows/unit-integration-tests.yml b/.github/workflows/unit-integration-tests.yml index e1e3b6a662..a4f7e47b6a 100644 --- a/.github/workflows/unit-integration-tests.yml +++ b/.github/workflows/unit-integration-tests.yml @@ -1,52 +1,45 @@ +# Code generated via `make unit-integration-tests.yml`; DO NOT EDIT. name: Unit and Integration Tests - -on: +"on": workflow_call: inputs: go-version: required: true type: string - ref: - description: 'The branch to run the workflow on' - required: true - type: string - env: - DD_APPSEC_WAF_TIMEOUT: 1m # Increase time WAF time budget to reduce CI flakiness - # Users may build our library with GOTOOLCHAIN=local. If they do, and our - # go.mod file specifies a newer Go version than their local toolchain, their - # build will break. Run our tests with GOTOOLCHAIN=local to ensure that - # our library builds with all of the Go versions we claim to support, - # without having to download a newer one. + DD_APPSEC_WAF_TIMEOUT: 1m GOTOOLCHAIN: local - GODEBUG: "x509negativeserial=1" - GOEXPERIMENT: synctest # TODO: remove once go1.25 is the minimum supported version + GODEBUG: x509negativeserial=1 + GOEXPERIMENT: synctest TEST_RESULT_PATH: /tmp/test-results - permissions: contents: read - jobs: set-up: runs-on: ubuntu-latest outputs: matrix: ${{ steps.matrix.outputs.matrix }} steps: + - name: Restore repo cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 + with: + path: .git + key: gitdb-${{ github.repository_id }}-${{ github.sha }} - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ inputs.ref || github.ref }} - + ref: ${{ github.sha }} + clean: false - name: Compute Matrix id: matrix - run: |- - echo -n "matrix=" >> "${GITHUB_OUTPUT}" - go run ./scripts/ci_contrib_matrix.go >> "${GITHUB_OUTPUT}" - + run: |2- + echo -n "matrix=" >> "${GITHUB_OUTPUT}" + go run ./scripts/ci_contrib_matrix.go >> "${GITHUB_OUTPUT}" test-contrib-matrix: - needs: set-up + needs: + - set-up runs-on: - group: "APM Larger Runners" + group: APM Larger Runners env: INTEGRATION: true strategy: @@ -54,26 +47,20 @@ jobs: chunk: ${{ fromJson(needs.set-up.outputs.matrix) }} services: datadog-agent: - image: datadog/agent:latest + image: datadog/agent:7.69.2 env: - DD_HOSTNAME: "github-actions-worker" + DD_HOSTNAME: github-actions-worker DD_APM_ENABLED: true - DD_BIND_HOST: "0.0.0.0" - DD_API_KEY: "invalid_key_but_this_is_fine" - DD_TEST_AGENT_HOST: "localhost" + DD_BIND_HOST: 0.0.0.0 + DD_API_KEY: invalid_key_but_this_is_fine + DD_TEST_AGENT_HOST: localhost DD_TEST_AGENT_PORT: 9126 - # We need to specify a custom health-check. By default, this container will remain "unhealthy" since - # we don't fully configure it with a valid API key (and possibly other reasons) - # This command just checks for our ability to connect to port 8126 - options: >- - --health-cmd "bash -c '- - --name "kafka" + - "9092:9092" + - "9093:9093" localstack: image: localstack/localstack:latest ports: - - 4566:4566 + - "4566:4566" steps: + - name: Restore repo cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 + with: + path: .git + key: gitdb-${{ github.repository_id }}-${{ github.sha }} - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ inputs.ref || github.ref }} - fetch-depth: $(( ${{ github.event_name == 'pull_request' && github.event.pull_request.commits || 0 }} + 1 )) - + ref: ${{ github.sha }} + clean: false - name: Setup Go and development tools uses: ./.github/actions/setup-go with: @@ -218,10 +210,9 @@ jobs: if: always() env: TEST_RESULTS: ${{ env.TEST_RESULT_PATH }} - run: | - export PATH="${{ github.workspace }}/bin:${PATH}" - ./scripts/ci_test_contrib.sh default ${{ toJson(matrix.chunk) }} - + run: |2- + export PATH="${{ github.workspace }}/bin:${PATH}" + ./scripts/ci_test_contrib.sh default ${{ toJson(matrix.chunk) }} - name: Upload the results to Datadog CI App if: always() continue-on-error: true @@ -230,69 +221,60 @@ jobs: dd-api-key: ${{ secrets.DD_CI_API_KEY }} path: ${{ env.TEST_RESULT_PATH }} tags: go:${{ inputs.go-version }},arch:${{ runner.arch }},os:${{ runner.os }},distribution:${{ runner.distribution }} - - name: Upload Coverage if: always() continue-on-error: true shell: bash run: bash <(curl -s https://codecov.io/bash) -t ${{ secrets.CODECOV_TOKEN }} - test-contrib: needs: - test-contrib-matrix runs-on: - group: "APM Larger Runners" + group: APM Larger Runners if: success() || failure() continue-on-error: true steps: - name: Success if: needs.test-contrib-matrix.result == 'success' - run: echo "Success!" + run: echo 'Success!' - name: Failure if: needs.test-contrib-matrix.result != 'success' - run: echo "Failure!" && exit 1 - + run: echo 'Failure!' && exit 1 test-core: runs-on: - group: "APM Larger Runners" + group: APM Larger Runners env: - INTEGRATION: true - services: - datadog-agent: - image: datadog/agent:latest - env: - DD_HOSTNAME: "github-actions-worker" - DD_APM_ENABLED: true - DD_BIND_HOST: "0.0.0.0" - DD_API_KEY: "invalid_key_but_this_is_fine" - # We need to specify a custom health-check. By default, this container will remain "unhealthy" since - # we don't fully configure it with a valid API key (and possibly other reasons) - # This command just checks for our ability to connect to port 8126 - options: >- - --health-cmd "bash -c '- - --health-cmd "bash -c '