[7.17](backport #29047) CI: enable AWS cloud testing on a PR basis if changes in certain files

Jenkinsfile

@@ -277,13 +277,13 @@ def generateStages(Map args = [:]) {
 def cloud(Map args = [:]) {
   withGithubNotify(context: args.context) {
     withNode(labels: args.label, forceWorkspace: true){
-      startCloudTestEnv(name: args.directory, dirs: args.dirs)
+      startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS)
     }
-    withCloudTestEnv() {
+    withCloudTestEnv(args) {
       try {
         target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id)
       } finally {
-        terraformCleanup(name: args.directory, dir: args.directory)
+        terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS)
       }
     }
   }
@@ -851,14 +851,15 @@ def tarAndUploadArtifacts(Map args = [:]) {
 * This method executes a closure with credentials for cloud test
 * environments.
 */
-def withCloudTestEnv(Closure body) {
+def withCloudTestEnv(Map args = [:], Closure body) {
   def maskedVars = []
   def testTags = "${env.TEST_TAGS}"
 
   // Allow AWS credentials when the build was configured to do so with:
   //   - the cloudtests build parameters
   //   - the aws github label
-  if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws')) {
+  //   - forced with the cloud argument aws github label
+  if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws') || args.get('withAWS', false)) {
     testTags = "${testTags},aws"
     def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
     if (!aws.containsKey('access_key')) {
@@ -872,6 +873,7 @@ def withCloudTestEnv(Closure body) {
       [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
       [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
     ])
+    log(level: 'INFO', text: 'withCloudTestEnv: it has been configured to run in AWS.')
   }
 
   withEnv([
@@ -897,7 +899,7 @@ def startCloudTestEnv(Map args = [:]) {
   String name = normalise(args.name)
   def dirs = args.get('dirs',[])
   stage("${name}-prepare-cloud-env"){
-    withCloudTestEnv() {
+    withCloudTestEnv(args) {
       withBeatsEnv(archive: false, withModule: false) {
         try {
           dirs?.each { folder ->
@@ -940,7 +942,7 @@ def terraformCleanup(Map args = [:]) {
   String name = normalise(args.name)
   String directory = args.dir
   stage("${name}-tear-down-cloud-env"){
-    withCloudTestEnv() {
+    withCloudTestEnv(args) {
       withBeatsEnv(archive: false, withModule: false) {
         unstash("terraform-${name}")
         retryWithSleep(retries: 2, seconds: 5, backoff: true) {
@@ -1065,6 +1067,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
   public run(Map args = [:]){
     steps.stageStatusCache(args){
       def withModule = args.content.get('withModule', false)
+      def withAWS = args.content.get('withAWS', false)
       //
       // What's the retry policy for fighting the flakiness:
       //   1) Lint/Packaging/Cloud/k8sTest stages don't retry, since their failures are normally legitim
@@ -1123,7 +1126,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
         steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
       }
       if(args?.content?.containsKey('cloud')) {
-        steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
+        steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id, withAWS: withAWS)
       }
     }
   }

x-pack/filebeat/Jenkinsfile.yml

@@ -113,6 +113,20 @@ stages:
         labels:
           - "aws"
       stage: extended
+    cloudAWS:
+        cloud: "mage build test"
+        withAWS: true          ## Enable the tests to run in AWS
+        withModule: true       ## run the ITs only if the changeset affects a specific module.
+        dirs:                  ## run the cloud tests for the given modules.
+          - "x-pack/filebeat/input/awss3/_meta/terraform"
+        when:                  ## Override the top-level when.
+          changeset:           ## when PR contains any of those entries in the changeset
+            - "^x-pack/filebeat/input/awss3/.*"
+            - "^x-pack/filebeat/module/aws/.*"
+            - "^x-pack/filebeat/input/awscloudwatch/.*"
+            - "^x-pack/filebeat/Jenkinsfile.yml"
+            - "^x-pack/libbeat/common/aws/.*"
+        stage: extended
     #windows-7-32:  See https://github.com/elastic/beats/issues/22315
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.

x-pack/filebeat/input/awss3/input_integration_test.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"gopkg.in/yaml.v2"
 
 	awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws"
 
@@ -29,7 +30,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/sync/errgroup"
-	"gopkg.in/yaml.v2"
 
 	"github.com/elastic/beats/v7/filebeat/beater"
 	v2 "github.com/elastic/beats/v7/filebeat/input/v2"
@@ -373,11 +373,6 @@ func drainSQS(t *testing.T, region string, queueURL string) {
 	t.Logf("Drained %d SQS messages.", deletedCount)
 }
 
-func TestGetBucketNameFromARN(t *testing.T) {
-	bucketName := getBucketNameFromARN("arn:aws:s3:::my_corporate_bucket")
-	assert.Equal(t, "my_corporate_bucket", bucketName)
-}
-
 func TestGetRegionForBucketARN(t *testing.T) {
 	logp.TestingSetup()
 
@@ -444,13 +439,6 @@ func TestPaginatorListPrefix(t *testing.T) {
 	assert.Equal(t, expected, objects)
 }
 
-func TestGetProviderFromDomain(t *testing.T) {
-	assert.Equal(t, "aws", getProviderFromDomain("", ""))
-	assert.Equal(t, "aws", getProviderFromDomain("c2s.ic.gov", ""))
-	assert.Equal(t, "abc", getProviderFromDomain("abc.com", "abc"))
-	assert.Equal(t, "xyz", getProviderFromDomain("oraclecloud.com", "xyz"))
-}
-
 func TestInputRunSNS(t *testing.T) {
 	logp.TestingSetup()
 

x-pack/metricbeat/Jenkinsfile.yml

@@ -45,6 +45,18 @@ stages:
             labels:
                 - "aws"
         stage: extended
+    cloudAWS:
+        cloud: "mage build test"
+        withAWS: true          ## Enable the tests to run in AWS
+        withModule: true       ## run the ITs only if the changeset affects a specific module.
+        dirs:                  ## run the cloud tests for the given modules.
+          - "x-pack/metricbeat/module/aws"
+        when:                  ## Override the top-level when.
+          changeset:         ## when PR contains any of those entries in the changeset
+            - "^x-pack/metricbeat/module/aws/.*"
+            - "^x-pack/metricbeat/Jenkinsfile.yml"
+            - "^x-pack/libbeat/common/aws/.*"
+        stage: extended
     macos:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.