From c0423b640380c2c09d961aca36c185c28b9df510 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 9 Apr 2025 11:22:07 +0200 Subject: [PATCH 1/5] Create several schedules with different steps via env vars --- .buildkite/pipeline.schedule-daily.yml | 43 ++++++++++++++++++++++++++ catalog-info.yaml | 25 ++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 0edfa787ff7..1a2b937c12f 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -25,6 +25,8 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_7_BRANCH') == "true" - label: "Check integrations local stacks - Stack Version v8.19" trigger: "integrations" @@ -37,6 +39,8 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_8_BRANCH') == "true" - label: "Check integrations local stacks - Stack Version v8.19 - LogsDB" trigger: "integrations" @@ -50,6 +54,37 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_8_BRANCH') == "true" + + # - label: "Check integrations local stacks and basic subscription and LogsDB" + # trigger: "integrations" + # build: + # env: + # SERVERLESS: "false" + # FORCE_CHECK_ALL: "true" + # PUBLISH_COVERAGE_REPORTS: "false" + # ELASTIC_SUBSCRIPTION: "basic" + # STACK_LOGSDB_ENABLED: "true" + # depends_on: + # - step: "check" + # allow_failure: false + # if: | + # build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" + + # - label: "Check integrations local stacks and basic subscription" + # trigger: "integrations" + # build: + # env: + # SERVERLESS: "false" + # FORCE_CHECK_ALL: "true" + # PUBLISH_COVERAGE_REPORTS: "false" + # ELASTIC_SUBSCRIPTION: "basic" + # depends_on: + # - step: "check" + # allow_failure: false + # if: | + # build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" - label: "Check integrations local stacks - Stack Version v9.1" trigger: "integrations" @@ -62,6 +97,8 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_9_BRANCH') == "true" - label: "Check integrations in serverless - project: Observability" key: "trigger-integrations-serverless-obs" @@ -72,6 +109,8 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_SERVERLESS') == "true" - label: "Check integrations in serverless - project: Security" key: "trigger-integrations-serverless-security" @@ -82,6 +121,8 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('TEST_PACKAGES_SERVERLESS') == "true" - label: ":package: Publish missing packages" key: "trigger-integrations-publish" @@ -89,3 +130,5 @@ steps: depends_on: - step: "check" allow_failure: false + if: | + build.env('REPUBLISH_PACKAGES') == "true" diff --git a/catalog-info.yaml b/catalog-info.yaml index 7de45547ca0..814e813a289 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -176,10 +176,33 @@ spec: spec: pipeline_file: ".buildkite/pipeline.schedule-daily.yml" schedules: - main_daily: + main_daily_others: branch: "main" cronline: "30 1 * * *" message: "Run the daily jobs" + env: + TEST_PACKAGES_7_BRANCH: "true" + TEST_PACKAGES_SERVERLESS: "true" + REPUBLISH_PACKAGES: "true" + main_daily_8_version: + branch: "main" + cronline: "30 3 * * *" + message: "Run the daily jobs for 8.x" + env: + TEST_PACKAGES_8_BRANCH: "true" + main_daily_9_version: + branch: "main" + cronline: "30 4 * * *" + message: "Run the daily jobs for 9.x" + env: + TEST_PACKAGES_9_BRANCH: "true" + # main_daily_basic_subscription: + # branch: "main" + # cronline: "30 5 * * *" + # message: "Run the daily jobs for basic subscription" + # env: + # TEST_PACKAGES_BASIC_SUBSCRIPTION: "true" + provider_settings: trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false From 9f4c5b4f66b5bc283b6c5c0eb8a27f54962ce5f2 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 9 Apr 2025 11:22:19 +0200 Subject: [PATCH 2/5] Test subject packages --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 1384cfb02bc..62d343af7fb 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -760,7 +760,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(nginx|apache|elastic_package_registry|bluecoat|snort)$' } check_package() { From 8f0306302f06a0ac389f7eb7f85d97a47244eb27 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 9 Apr 2025 11:31:10 +0200 Subject: [PATCH 3/5] Changes for debugging --- .buildkite/pipeline.schedule-daily.yml | 6 ++++++ .buildkite/pipeline.yml | 1 + 2 files changed, 7 insertions(+) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 1a2b937c12f..20cd14241ae 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -22,6 +22,7 @@ steps: SERVERLESS: "false" FORCE_CHECK_ALL: "true" STACK_VERSION: 7.17.28 + BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -36,6 +37,7 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 8.19.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "true" + BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -51,6 +53,7 @@ steps: STACK_VERSION: 8.19.0-SNAPSHOT STACK_LOGSDB_ENABLED: "true" PUBLISH_COVERAGE_REPORTS: "false" + BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -94,6 +97,7 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 9.1.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" + BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -106,6 +110,7 @@ steps: build: env: SERVERLESS_PROJECT: observability + BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -118,6 +123,7 @@ steps: build: env: SERVERLESS_PROJECT: security + BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3294a98ac1b..a931dfc0cf6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -104,6 +104,7 @@ steps: command: ".buildkite/scripts/report_issues.sh" env: CI_MAX_TESTS_REPORTED: 30 + DRY_RUN: "true" agents: image: "${LINUX_AGENT_IMAGE}" cpu: "8" From 9f50d8723894fe932f3d836b5a0b17f46f381fd5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 9 Apr 2025 11:36:02 +0200 Subject: [PATCH 4/5] Update cronline settings --- catalog-info.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 814e813a289..f3ccea38d8c 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -176,9 +176,9 @@ spec: spec: pipeline_file: ".buildkite/pipeline.schedule-daily.yml" schedules: - main_daily_others: + main_daily: branch: "main" - cronline: "30 1 * * *" + cronline: "30 0 * * *" message: "Run the daily jobs" env: TEST_PACKAGES_7_BRANCH: "true" @@ -186,19 +186,19 @@ spec: REPUBLISH_PACKAGES: "true" main_daily_8_version: branch: "main" - cronline: "30 3 * * *" + cronline: "30 1 * * *" message: "Run the daily jobs for 8.x" env: TEST_PACKAGES_8_BRANCH: "true" main_daily_9_version: branch: "main" - cronline: "30 4 * * *" + cronline: "30 3 * * *" message: "Run the daily jobs for 9.x" env: TEST_PACKAGES_9_BRANCH: "true" # main_daily_basic_subscription: # branch: "main" - # cronline: "30 5 * * *" + # cronline: "30 4 * * *" # message: "Run the daily jobs for basic subscription" # env: # TEST_PACKAGES_BASIC_SUBSCRIPTION: "true" From c493969f17105320bdcba15eb719ee380a775765 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 9 Apr 2025 12:06:55 +0200 Subject: [PATCH 5/5] Remove changes for debugging --- .buildkite/pipeline.schedule-daily.yml | 60 ++++++++++++-------------- .buildkite/pipeline.yml | 1 - .buildkite/scripts/common.sh | 2 +- catalog-info.yaml | 12 +++--- 4 files changed, 34 insertions(+), 41 deletions(-) diff --git a/.buildkite/pipeline.schedule-daily.yml b/.buildkite/pipeline.schedule-daily.yml index 20cd14241ae..fea43a0b41f 100644 --- a/.buildkite/pipeline.schedule-daily.yml +++ b/.buildkite/pipeline.schedule-daily.yml @@ -22,7 +22,6 @@ steps: SERVERLESS: "false" FORCE_CHECK_ALL: "true" STACK_VERSION: 7.17.28 - BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -37,7 +36,6 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 8.19.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "true" - BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -53,41 +51,40 @@ steps: STACK_VERSION: 8.19.0-SNAPSHOT STACK_LOGSDB_ENABLED: "true" PUBLISH_COVERAGE_REPORTS: "false" - BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false if: | build.env('TEST_PACKAGES_8_BRANCH') == "true" - # - label: "Check integrations local stacks and basic subscription and LogsDB" - # trigger: "integrations" - # build: - # env: - # SERVERLESS: "false" - # FORCE_CHECK_ALL: "true" - # PUBLISH_COVERAGE_REPORTS: "false" - # ELASTIC_SUBSCRIPTION: "basic" - # STACK_LOGSDB_ENABLED: "true" - # depends_on: - # - step: "check" - # allow_failure: false - # if: | - # build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" + - label: "Check integrations local stacks and basic subscription and LogsDB" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + PUBLISH_COVERAGE_REPORTS: "false" + ELASTIC_SUBSCRIPTION: "basic" + STACK_LOGSDB_ENABLED: "true" + depends_on: + - step: "check" + allow_failure: false + if: | + build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" - # - label: "Check integrations local stacks and basic subscription" - # trigger: "integrations" - # build: - # env: - # SERVERLESS: "false" - # FORCE_CHECK_ALL: "true" - # PUBLISH_COVERAGE_REPORTS: "false" - # ELASTIC_SUBSCRIPTION: "basic" - # depends_on: - # - step: "check" - # allow_failure: false - # if: | - # build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" + - label: "Check integrations local stacks and basic subscription" + trigger: "integrations" + build: + env: + SERVERLESS: "false" + FORCE_CHECK_ALL: "true" + PUBLISH_COVERAGE_REPORTS: "false" + ELASTIC_SUBSCRIPTION: "basic" + depends_on: + - step: "check" + allow_failure: false + if: | + build.env('TEST_PACKAGES_BASIC_SUBSCRIPTION') == "true" - label: "Check integrations local stacks - Stack Version v9.1" trigger: "integrations" @@ -97,7 +94,6 @@ steps: FORCE_CHECK_ALL: "true" STACK_VERSION: 9.1.0-SNAPSHOT PUBLISH_COVERAGE_REPORTS: "false" - BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -110,7 +106,6 @@ steps: build: env: SERVERLESS_PROJECT: observability - BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false @@ -123,7 +118,6 @@ steps: build: env: SERVERLESS_PROJECT: security - BUILDKITE_REFSPEC: "refs/pull/13484/merge" depends_on: - step: "check" allow_failure: false diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a931dfc0cf6..3294a98ac1b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -104,7 +104,6 @@ steps: command: ".buildkite/scripts/report_issues.sh" env: CI_MAX_TESTS_REPORTED: 30 - DRY_RUN: "true" agents: image: "${LINUX_AGENT_IMAGE}" cpu: "8" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 62d343af7fb..1384cfb02bc 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -760,7 +760,7 @@ teardown_test_package() { } list_all_directories() { - find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort |grep -E '^(nginx|apache|elastic_package_registry|bluecoat|snort)$' + find . -maxdepth 1 -mindepth 1 -type d | xargs -I {} basename {} | sort } check_package() { diff --git a/catalog-info.yaml b/catalog-info.yaml index f3ccea38d8c..fa0734b0985 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -196,12 +196,12 @@ spec: message: "Run the daily jobs for 9.x" env: TEST_PACKAGES_9_BRANCH: "true" - # main_daily_basic_subscription: - # branch: "main" - # cronline: "30 4 * * *" - # message: "Run the daily jobs for basic subscription" - # env: - # TEST_PACKAGES_BASIC_SUBSCRIPTION: "true" + main_daily_basic_subscription: + branch: "main" + cronline: "30 4 * * *" + message: "Run the daily jobs for basic subscription" + env: + TEST_PACKAGES_BASIC_SUBSCRIPTION: "true" provider_settings: trigger_mode: none # don't trigger jobs from github activity