From c5d4f61fc8f8fadb5b22392bbbe1dbb7ceb90491 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 14 May 2020 16:50:33 +0100 Subject: [PATCH 1/9] wip ci: getRegionFromPattern step --- .../GetRegionFromPatternStepTests.groovy | 205 ++++++++++++++++++ vars/getRegionFromPattern.groovy | 65 ++++++ vars/getRegionFromPattern.txt | 20 ++ 3 files changed, 290 insertions(+) create mode 100644 src/test/groovy/GetRegionFromPatternStepTests.groovy create mode 100644 vars/getRegionFromPattern.groovy create mode 100644 vars/getRegionFromPattern.txt diff --git a/src/test/groovy/GetRegionFromPatternStepTests.groovy b/src/test/groovy/GetRegionFromPatternStepTests.groovy new file mode 100644 index 000000000..c32b5ba69 --- /dev/null +++ b/src/test/groovy/GetRegionFromPatternStepTests.groovy @@ -0,0 +1,205 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Before +import org.junit.Test +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertTrue + +class GetRegionFromPatternStepTests extends ApmBasePipelineTest { + String scriptName = 'vars/getRegionFromPattern.groovy' + + @Override + @Before + void setUp() throws Exception { + super.setUp() + env.CHANGE_TARGET = 'foo' + env.GIT_BASE_COMMIT = 'bar' + } + + @Test + void test_without_pattern_parameter() throws Exception { + def script = loadScript(scriptName) + try { + script.call() + } catch(e){ + //NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('error', 'getRegionFromPattern: Missing pattern argument.')) + assertJobStatusFailure() + } + + @Test + void test_without_env_variables() throws Exception { + def script = loadScript(scriptName) + def result = true + env.remove('CHANGE_TARGET') + result = script.call(pattern: 'foo') + printCallStack() + assertFalse(result) + assertTrue(assertMethodCallContainsPattern('echo', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertJobStatusSuccess() + } + + @Test + void test_simple_match() throws Exception { + def script = loadScript(scriptName) + def changeset = 'file.txt' + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + def module = script.call(patterns: '^file.txt') + printCallStack() + // TODO assertTrue(result) + assertTrue(assertMethodCallContainsPattern('log', "getRegionFromPattern: found with regex [^file.txt]")) + assertJobStatusSuccess() + } + + @Test + void test_simple_match_with_previous_commit_env_variable() throws Exception { + env.GIT_PREVIOUS_COMMIT = "foo-1" + env.remove('CHANGE_TARGET') + def script = loadScript(scriptName) + def changeset = 'file.txt' + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + // TODO + def module = script.call(pattern: '^file.txt') + printCallStack() + //assertTrue(result) + assertJobStatusSuccess() + } + + @Test + void test_complex_match() throws Exception { + def script = loadScript(scriptName) + def changeset = 'foo/anotherfolder/file.txt' + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + helper.registerAllowedMethod('sh', [Map.class], { m -> + if (m.script.contains('git diff')) { + return 'foo/anotherfolder/file.txt' + } else { + return 0 + } + }) + def module = script.call(pattern: '^foo/.*/file.txt') + printCallStack() + //assertTrue(result) + assertJobStatusSuccess() + } + + @Test + void test_simple_unmatch() throws Exception { + def script = loadScript(scriptName) + def changeset = 'foo/anotherfolder/file.txt' + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + def module = script.call(pattern: '^unknown.txt') + printCallStack() + assertTrue(module.equals('')) + assertTrue(assertMethodCallContainsPattern('log', "getRegionFromPattern: not found with regex [^unknown.txt]")) + assertJobStatusSuccess() + } + + @Test + void test_windows() throws Exception { + def script = loadScript(scriptName) + helper.registerAllowedMethod('isUnix', [], { false }) + try { + script.call() + } catch(e){ + //NOOP + } + printCallStack() + assertTrue(assertMethodCallContainsPattern('error', 'getRegionFromPattern: windows is not supported yet.')) + assertJobStatusFailure() + } + + @Test + void test_without_change_request_env_variable() throws Exception { + env.GIT_PREVIOUS_COMMIT = "foo-1" + env.remove('CHANGE_TARGET') + def script = loadScript(scriptName) + def changeset = 'foo/anotherfolder/file.txt' + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + helper.registerAllowedMethod('sh', [Map.class], { m -> + assertFalse(m.script.contains('origin/')) + }) + def module = script.call(pattern: 'foo') + printCallStack() + assertJobStatusSuccess() + // TODO + } + + @Test + void test_with_empty_change_target_env_variable() throws Exception { + env.CHANGE_TARGET = " " + def script = loadScript(scriptName) + def changeset = 'foo/anotherfolder/file.txt' + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + helper.registerAllowedMethod('sh', [Map.class], { m -> + assertFalse(m.script.contains('origin/')) + }) + def module = script.call(pattern: 'foo') + printCallStack() + assertJobStatusSuccess() + // TODO + } + + @Test + void test_with_from_parameter() throws Exception { + def script = loadScript(scriptName) + def changeset = ''' foo/bar/file.txt + '''.stripMargin().stripIndent() + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + def module = script.call(pattern: '^foo/.*/file.txt', from: 'something') + printCallStack() + // TODO assertTrue(result) + assertTrue(assertMethodCallContainsPattern('sh', 'something...bar')) + assertJobStatusSuccess() + } + + @Test + void test_with_from_and_to_parameters() throws Exception { + def script = loadScript(scriptName) + def changeset = ''' foo/bar/file.txt + '''.stripMargin().stripIndent() + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + def module = script.call(pattern: '^foo/.*/file.txt', from: 'something', to: 'else') + printCallStack() + // TODO assertTrue(result) + assertTrue(assertMethodCallContainsPattern('sh', 'something...else')) + assertJobStatusSuccess() + } + + @Test + void test_with_empty_values_for_from_and_to_parameters() throws Exception { + def script = loadScript(scriptName) + def module = script.call(pattern: '^foo/.*/file.txt', from: '', to: '') + printCallStack() + //TODO assertFalse(module) + assertTrue(assertMethodCallContainsPattern('echo', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertJobStatusSuccess() + } + + @Test + void test_eith_empty_value_for_to_parameter() throws Exception { + def script = loadScript(scriptName) + def module = script.call(patterns: [ '^foo/.*/file.txt' ], to: '') + printCallStack() + //TODO assertFalse(result) + assertTrue(assertMethodCallContainsPattern('echo', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertJobStatusSuccess() + } +} diff --git a/vars/getRegionFromPattern.groovy b/vars/getRegionFromPattern.groovy new file mode 100644 index 000000000..64aa1968f --- /dev/null +++ b/vars/getRegionFromPattern.groovy @@ -0,0 +1,65 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.cloudbees.groovy.cps.NonCPS + +/** + Given the regex pattern, the CHANGE_TARGET, GIT_SHA env variables then it + evaluates the change list: + + - When exact match then all the files should match those patterns then it + returns the region otherwise and empty string. + + // def module = getRegionFromPattern(pattern: "beat/module/([^\/]+)/") + whenTrue(module.trim()) { + // ... + } + + NOTE: This particular implementation requires to checkout with the step gitCheckout + +*/ +def call(Map params = [:]) { + if(!isUnix()){ + error('getRegionFromPattern: windows is not supported yet.') + } + def pattern = params.containsKey('pattern') ? params.pattern : error('getRegionFromPattern: Missing pattern argument.') + def from = params.get('from', env.CHANGE_TARGET?.trim() ? "origin/${env.CHANGE_TARGET}" : env.GIT_PREVIOUS_COMMIT) + def to = params.get('to', env.GIT_BASE_COMMIT) + + def gitDiffFile = 'git-diff.txt' + def match = false + if (from?.trim() && to?.trim()) { + def changes = sh(script: "git diff --name-only ${from}...${to} > ${gitDiffFile}", returnStdout: true) + match = getRegion(gitDiffFile, pattern) + log(level: 'INFO', text: "getRegionFromPattern: ${match.trim() ?: 'not found'} with regex ${pattern}") + } else { + echo 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes. Or the from/to arguments are required.' + } + return match +} + +def getRegion(gitDiffFile, patterns) { + def fileContent = readFile(gitDiffFile) + def match = true + fileContent.split('\n').each { String line -> + log(level: 'DEBUG', text: "changeset element: '${line}'") + if (!patterns.every { line ==~ it }) { + match = false + } + } + return match +} diff --git a/vars/getRegionFromPattern.txt b/vars/getRegionFromPattern.txt new file mode 100644 index 000000000..0cf504401 --- /dev/null +++ b/vars/getRegionFromPattern.txt @@ -0,0 +1,20 @@ +Given the regex pattern, the CHANGE_TARGET, GIT_SHA env variables then it +evaluates the change list: + +- When exact match then all the files should match those patterns then it + returns the region otherwise and empty string. + + NOTE: This particular implementation requires to checkout with the step gitCheckout + +``` + def module = getRegionFromPattern(pattern: "beat/module/([^\/]+)/") + whenTrue(module.trim()) { + // ... + } +``` + +* pattern: the regex pattern with the group to look for. Mandatory +* from: to override the diff from sha. Optional. If MPB, and PR then origin/${env.CHANGE_TARGET} otherwise env.GIT_PREVIOUS_COMMIT +* to: to override the commit to. Optional. Default: env.GIT_BASE_COMMIT + +**NOTE**: This particular implementation requires to checkout with the step `gitCheckout` From cc5224035c9cd1d7d97c3b882252c83d81875768 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 14 May 2020 17:10:35 +0100 Subject: [PATCH 2/9] ci: minor changes --- .../GetRegionFromPatternStepTests.groovy | 52 +++++++++++-------- vars/getRegionFromPattern.groovy | 13 ++--- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/src/test/groovy/GetRegionFromPatternStepTests.groovy b/src/test/groovy/GetRegionFromPatternStepTests.groovy index c32b5ba69..f0e6d01ea 100644 --- a/src/test/groovy/GetRegionFromPatternStepTests.groovy +++ b/src/test/groovy/GetRegionFromPatternStepTests.groovy @@ -18,6 +18,7 @@ import org.junit.Before import org.junit.Test import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertEquals import static org.junit.Assert.assertTrue class GetRegionFromPatternStepTests extends ApmBasePipelineTest { @@ -49,10 +50,10 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { def script = loadScript(scriptName) def result = true env.remove('CHANGE_TARGET') - result = script.call(pattern: 'foo') + def module = script.call(pattern: 'foo') printCallStack() - assertFalse(result) - assertTrue(assertMethodCallContainsPattern('echo', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertEquals('', module) + assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } @@ -61,10 +62,11 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { def script = loadScript(scriptName) def changeset = 'file.txt' helper.registerAllowedMethod('readFile', [String.class], { return changeset }) - def module = script.call(patterns: '^file.txt') + def module = script.call(pattern: '^file.txt') printCallStack() - // TODO assertTrue(result) - assertTrue(assertMethodCallContainsPattern('log', "getRegionFromPattern: found with regex [^file.txt]")) + // TODO + assertEquals('', module) + assertTrue(assertMethodCallContainsPattern('log', "getRegionFromPattern: not found with regex ^file.txt")) assertJobStatusSuccess() } @@ -75,10 +77,10 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { def script = loadScript(scriptName) def changeset = 'file.txt' helper.registerAllowedMethod('readFile', [String.class], { return changeset }) - // TODO def module = script.call(pattern: '^file.txt') printCallStack() - //assertTrue(result) + // TODO + assertEquals('', module) assertJobStatusSuccess() } @@ -96,7 +98,8 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { }) def module = script.call(pattern: '^foo/.*/file.txt') printCallStack() - //assertTrue(result) + // TODO + assertEquals('', module) assertJobStatusSuccess() } @@ -107,8 +110,9 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { helper.registerAllowedMethod('readFile', [String.class], { return changeset }) def module = script.call(pattern: '^unknown.txt') printCallStack() - assertTrue(module.equals('')) - assertTrue(assertMethodCallContainsPattern('log', "getRegionFromPattern: not found with regex [^unknown.txt]")) + // TODO + assertEquals('', module) + assertTrue(assertMethodCallContainsPattern('log', "getRegionFromPattern: not found with regex ^unknown.txt")) assertJobStatusSuccess() } @@ -138,8 +142,9 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { }) def module = script.call(pattern: 'foo') printCallStack() - assertJobStatusSuccess() // TODO + assertEquals('', module) + assertJobStatusSuccess() } @Test @@ -153,8 +158,9 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { }) def module = script.call(pattern: 'foo') printCallStack() - assertJobStatusSuccess() // TODO + assertEquals('', module) + assertJobStatusSuccess() } @Test @@ -165,7 +171,8 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { helper.registerAllowedMethod('readFile', [String.class], { return changeset }) def module = script.call(pattern: '^foo/.*/file.txt', from: 'something') printCallStack() - // TODO assertTrue(result) + // TODO + assertEquals('', module) assertTrue(assertMethodCallContainsPattern('sh', 'something...bar')) assertJobStatusSuccess() } @@ -178,7 +185,8 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { helper.registerAllowedMethod('readFile', [String.class], { return changeset }) def module = script.call(pattern: '^foo/.*/file.txt', from: 'something', to: 'else') printCallStack() - // TODO assertTrue(result) + // TODO + assertEquals('', module) assertTrue(assertMethodCallContainsPattern('sh', 'something...else')) assertJobStatusSuccess() } @@ -188,18 +196,20 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { def script = loadScript(scriptName) def module = script.call(pattern: '^foo/.*/file.txt', from: '', to: '') printCallStack() - //TODO assertFalse(module) - assertTrue(assertMethodCallContainsPattern('echo', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + // TODO + assertEquals('', module) + assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } @Test - void test_eith_empty_value_for_to_parameter() throws Exception { + void test_with_empty_value_for_to_parameter() throws Exception { def script = loadScript(scriptName) - def module = script.call(patterns: [ '^foo/.*/file.txt' ], to: '') + def module = script.call(pattern: '^foo/.*/file.txt', to: '') printCallStack() - //TODO assertFalse(result) - assertTrue(assertMethodCallContainsPattern('echo', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + // TODO + assertEquals('', module) + assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } } diff --git a/vars/getRegionFromPattern.groovy b/vars/getRegionFromPattern.groovy index 64aa1968f..88783f3e6 100644 --- a/vars/getRegionFromPattern.groovy +++ b/vars/getRegionFromPattern.groovy @@ -41,18 +41,19 @@ def call(Map params = [:]) { def to = params.get('to', env.GIT_BASE_COMMIT) def gitDiffFile = 'git-diff.txt' - def match = false + def group = '' if (from?.trim() && to?.trim()) { def changes = sh(script: "git diff --name-only ${from}...${to} > ${gitDiffFile}", returnStdout: true) - match = getRegion(gitDiffFile, pattern) - log(level: 'INFO', text: "getRegionFromPattern: ${match.trim() ?: 'not found'} with regex ${pattern}") + group = getGroup(gitDiffFile, pattern) + log(level: 'INFO', text: "getRegionFromPattern: ${group.trim() ?: 'not found'} with regex ${pattern}") } else { - echo 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes. Or the from/to arguments are required.' + log(level: 'INFO', text: 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes. Or the from/to arguments are required.') } - return match + return group } -def getRegion(gitDiffFile, patterns) { +def getGroup(gitDiffFile, patterns) { + return '' def fileContent = readFile(gitDiffFile) def match = true fileContent.split('\n').each { String line -> From d0a1bf68b20bc408d2cb7cefddfb54ff97569969 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 15 May 2020 14:11:22 +0100 Subject: [PATCH 3/9] Implemented --- .../GetRegionFromPatternStepTests.groovy | 97 ++++++++++--------- vars/getRegionFromPattern.groovy | 17 ++-- 2 files changed, 63 insertions(+), 51 deletions(-) diff --git a/src/test/groovy/GetRegionFromPatternStepTests.groovy b/src/test/groovy/GetRegionFromPatternStepTests.groovy index f0e6d01ea..b38d0eaaa 100644 --- a/src/test/groovy/GetRegionFromPatternStepTests.groovy +++ b/src/test/groovy/GetRegionFromPatternStepTests.groovy @@ -60,13 +60,10 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { @Test void test_simple_match() throws Exception { def script = loadScript(scriptName) - def changeset = 'file.txt' + def changeset = 'foo/bar/file.txt' helper.registerAllowedMethod('readFile', [String.class], { return changeset }) - def module = script.call(pattern: '^file.txt') - printCallStack() - // TODO - assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', "getRegionFromPattern: not found with regex ^file.txt")) + def module = script.call(pattern: '([^\\/]+)\\/.*') + assertEquals('foo', module) assertJobStatusSuccess() } @@ -75,31 +72,47 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { env.GIT_PREVIOUS_COMMIT = "foo-1" env.remove('CHANGE_TARGET') def script = loadScript(scriptName) - def changeset = 'file.txt' + def changeset = 'foo/bar/file.txt' helper.registerAllowedMethod('readFile', [String.class], { return changeset }) - def module = script.call(pattern: '^file.txt') + def module = script.call(pattern: '([^\\/]+)\\/.*') printCallStack() - // TODO - assertEquals('', module) + assertEquals('foo', module) assertJobStatusSuccess() } @Test - void test_complex_match() throws Exception { + void test_multiple_match() throws Exception { def script = loadScript(scriptName) - def changeset = 'foo/anotherfolder/file.txt' + def changeset = '''foo/bar/file.txt +foo/bar/subfolder'''.stripMargin().stripIndent() helper.registerAllowedMethod('readFile', [String.class], { return changeset }) - helper.registerAllowedMethod('sh', [Map.class], { m -> - if (m.script.contains('git diff')) { - return 'foo/anotherfolder/file.txt' - } else { - return 0 - } - }) - def module = script.call(pattern: '^foo/.*/file.txt') - printCallStack() - // TODO + def module = script.call(pattern: '([^\\/]+)\\/.*') + assertEquals('foo', module) + assertJobStatusSuccess() + } + + @Test + void test_multiple_match_2() throws Exception { + def script = loadScript(scriptName) + def changeset = '''filebeat/README.md +filebeat/Dockerfile +filebeat/docs/faq.asciidoc +filebeat/autodiscover/builder/hints/config.go'''.stripMargin().stripIndent() + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + def module = script.call(pattern: '([^\\/]+)\\/.*') + assertEquals('filebeat', module) + assertJobStatusSuccess() + } + + @Test + void test_multiple_without_match() throws Exception { + def script = loadScript(scriptName) + def changeset = '''foo/bar/file.txt +bar/foo/subfolder'''.stripMargin().stripIndent() + helper.registerAllowedMethod('readFile', [String.class], { return changeset }) + def module = script.call(pattern: '([^\\/]+)\\/.*') assertEquals('', module) + assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: not found')) assertJobStatusSuccess() } @@ -110,9 +123,8 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { helper.registerAllowedMethod('readFile', [String.class], { return changeset }) def module = script.call(pattern: '^unknown.txt') printCallStack() - // TODO assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', "getRegionFromPattern: not found with regex ^unknown.txt")) + assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: not found with regex ^unknown.txt')) assertJobStatusSuccess() } @@ -135,15 +147,14 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { env.GIT_PREVIOUS_COMMIT = "foo-1" env.remove('CHANGE_TARGET') def script = loadScript(scriptName) - def changeset = 'foo/anotherfolder/file.txt' + def changeset = 'foo/bar/file.txt' helper.registerAllowedMethod('readFile', [String.class], { return changeset }) helper.registerAllowedMethod('sh', [Map.class], { m -> - assertFalse(m.script.contains('origin/')) - }) - def module = script.call(pattern: 'foo') + assertFalse(m.script.contains('origin/')) + }) + def module = script.call(pattern: '([^\\/]+)\\/.*') printCallStack() - // TODO - assertEquals('', module) + assertEquals('foo', module) assertJobStatusSuccess() } @@ -154,11 +165,10 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { def changeset = 'foo/anotherfolder/file.txt' helper.registerAllowedMethod('readFile', [String.class], { return changeset }) helper.registerAllowedMethod('sh', [Map.class], { m -> - assertFalse(m.script.contains('origin/')) - }) + assertFalse(m.script.contains('origin/')) + }) def module = script.call(pattern: 'foo') printCallStack() - // TODO assertEquals('', module) assertJobStatusSuccess() } @@ -166,13 +176,11 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { @Test void test_with_from_parameter() throws Exception { def script = loadScript(scriptName) - def changeset = ''' foo/bar/file.txt - '''.stripMargin().stripIndent() + def changeset = 'foo/anotherfolder/file.txt' helper.registerAllowedMethod('readFile', [String.class], { return changeset }) - def module = script.call(pattern: '^foo/.*/file.txt', from: 'something') + def module = script.call(pattern: '([^\\/]+)\\/.*', from: 'something') printCallStack() - // TODO - assertEquals('', module) + assertEquals('foo', module) assertTrue(assertMethodCallContainsPattern('sh', 'something...bar')) assertJobStatusSuccess() } @@ -180,13 +188,14 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { @Test void test_with_from_and_to_parameters() throws Exception { def script = loadScript(scriptName) - def changeset = ''' foo/bar/file.txt - '''.stripMargin().stripIndent() + def changeset = 'foo/bar/file.txt' helper.registerAllowedMethod('readFile', [String.class], { return changeset }) - def module = script.call(pattern: '^foo/.*/file.txt', from: 'something', to: 'else') + helper.registerAllowedMethod('sh', [Map.class], { m -> + assertFalse(m.script.contains('origin/')) + }) + def module = script.call(pattern: '([^\\/]+)\\/.*', from: 'something', to: 'else') printCallStack() - // TODO - assertEquals('', module) + assertEquals('foo', module) assertTrue(assertMethodCallContainsPattern('sh', 'something...else')) assertJobStatusSuccess() } @@ -196,7 +205,6 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { def script = loadScript(scriptName) def module = script.call(pattern: '^foo/.*/file.txt', from: '', to: '') printCallStack() - // TODO assertEquals('', module) assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() @@ -207,7 +215,6 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { def script = loadScript(scriptName) def module = script.call(pattern: '^foo/.*/file.txt', to: '') printCallStack() - // TODO assertEquals('', module) assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() diff --git a/vars/getRegionFromPattern.groovy b/vars/getRegionFromPattern.groovy index 88783f3e6..4378ee216 100644 --- a/vars/getRegionFromPattern.groovy +++ b/vars/getRegionFromPattern.groovy @@ -52,15 +52,20 @@ def call(Map params = [:]) { return group } -def getGroup(gitDiffFile, patterns) { - return '' +def getGroup(gitDiffFile, pattern) { def fileContent = readFile(gitDiffFile) - def match = true + def modules = [:] + // /beat\/module\/([^\/]+)\/.*/ fileContent.split('\n').each { String line -> log(level: 'DEBUG', text: "changeset element: '${line}'") - if (!patterns.every { line ==~ it }) { - match = false + matches = line =~ pattern + def auxModule = matches.collect { it[1] }[0] ?: '' + if (auxModule.trim()) { + modules[auxModule] = auxModule } } - return match + if (modules.size() == 1) { + return modules.values().toArray()[0] + } + return '' } From feba9cc317eaeb4be48419c01bb12a11d295352a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 15 May 2020 14:16:38 +0100 Subject: [PATCH 4/9] rename --- ...s.groovy => GetBeatsModuleStepTests.groovy} | 18 +++++++++--------- ...romPattern.groovy => getBeatsModule.groovy} | 13 ++++++------- ...egionFromPattern.txt => getBeatsModule.txt} | 4 ++-- 3 files changed, 17 insertions(+), 18 deletions(-) rename src/test/groovy/{GetRegionFromPatternStepTests.groovy => GetBeatsModuleStepTests.groovy} (86%) rename vars/{getRegionFromPattern.groovy => getBeatsModule.groovy} (79%) rename vars/{getRegionFromPattern.txt => getBeatsModule.txt} (86%) diff --git a/src/test/groovy/GetRegionFromPatternStepTests.groovy b/src/test/groovy/GetBeatsModuleStepTests.groovy similarity index 86% rename from src/test/groovy/GetRegionFromPatternStepTests.groovy rename to src/test/groovy/GetBeatsModuleStepTests.groovy index b38d0eaaa..df8639b85 100644 --- a/src/test/groovy/GetRegionFromPatternStepTests.groovy +++ b/src/test/groovy/GetBeatsModuleStepTests.groovy @@ -21,8 +21,8 @@ import static org.junit.Assert.assertFalse import static org.junit.Assert.assertEquals import static org.junit.Assert.assertTrue -class GetRegionFromPatternStepTests extends ApmBasePipelineTest { - String scriptName = 'vars/getRegionFromPattern.groovy' +class GetBeatsModuleStepTests extends ApmBasePipelineTest { + String scriptName = 'vars/getBeatsModule.groovy' @Override @Before @@ -41,7 +41,7 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { //NOOP } printCallStack() - assertTrue(assertMethodCallContainsPattern('error', 'getRegionFromPattern: Missing pattern argument.')) + assertTrue(assertMethodCallContainsPattern('error', 'getBeatsModule: Missing pattern argument.')) assertJobStatusFailure() } @@ -53,7 +53,7 @@ class GetRegionFromPatternStepTests extends ApmBasePipelineTest { def module = script.call(pattern: 'foo') printCallStack() assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } @@ -112,7 +112,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() helper.registerAllowedMethod('readFile', [String.class], { return changeset }) def module = script.call(pattern: '([^\\/]+)\\/.*') assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: not found')) + assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: not found')) assertJobStatusSuccess() } @@ -124,7 +124,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() def module = script.call(pattern: '^unknown.txt') printCallStack() assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: not found with regex ^unknown.txt')) + assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: not found with regex ^unknown.txt')) assertJobStatusSuccess() } @@ -138,7 +138,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() //NOOP } printCallStack() - assertTrue(assertMethodCallContainsPattern('error', 'getRegionFromPattern: windows is not supported yet.')) + assertTrue(assertMethodCallContainsPattern('error', 'getBeatsModule: windows is not supported yet.')) assertJobStatusFailure() } @@ -206,7 +206,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() def module = script.call(pattern: '^foo/.*/file.txt', from: '', to: '') printCallStack() assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } @@ -216,7 +216,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() def module = script.call(pattern: '^foo/.*/file.txt', to: '') printCallStack() assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } } diff --git a/vars/getRegionFromPattern.groovy b/vars/getBeatsModule.groovy similarity index 79% rename from vars/getRegionFromPattern.groovy rename to vars/getBeatsModule.groovy index 4378ee216..c3b4e0426 100644 --- a/vars/getRegionFromPattern.groovy +++ b/vars/getBeatsModule.groovy @@ -19,12 +19,12 @@ import com.cloudbees.groovy.cps.NonCPS /** Given the regex pattern, the CHANGE_TARGET, GIT_SHA env variables then it - evaluates the change list: + evaluates the change list and returns the module name. - When exact match then all the files should match those patterns then it returns the region otherwise and empty string. - // def module = getRegionFromPattern(pattern: "beat/module/([^\/]+)/") + // def module = getBeatsModule(pattern: '([^\\/]+)\\/.*') whenTrue(module.trim()) { // ... } @@ -34,9 +34,9 @@ import com.cloudbees.groovy.cps.NonCPS */ def call(Map params = [:]) { if(!isUnix()){ - error('getRegionFromPattern: windows is not supported yet.') + error('getBeatsModule: windows is not supported yet.') } - def pattern = params.containsKey('pattern') ? params.pattern : error('getRegionFromPattern: Missing pattern argument.') + def pattern = params.containsKey('pattern') ? params.pattern : error('getBeatsModule: Missing pattern argument.') def from = params.get('from', env.CHANGE_TARGET?.trim() ? "origin/${env.CHANGE_TARGET}" : env.GIT_PREVIOUS_COMMIT) def to = params.get('to', env.GIT_BASE_COMMIT) @@ -45,9 +45,9 @@ def call(Map params = [:]) { if (from?.trim() && to?.trim()) { def changes = sh(script: "git diff --name-only ${from}...${to} > ${gitDiffFile}", returnStdout: true) group = getGroup(gitDiffFile, pattern) - log(level: 'INFO', text: "getRegionFromPattern: ${group.trim() ?: 'not found'} with regex ${pattern}") + log(level: 'INFO', text: "getBeatsModule: ${group.trim() ?: 'not found'} with regex ${pattern}") } else { - log(level: 'INFO', text: 'getRegionFromPattern: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes. Or the from/to arguments are required.') + log(level: 'INFO', text: 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes. Or the from/to arguments are required.') } return group } @@ -55,7 +55,6 @@ def call(Map params = [:]) { def getGroup(gitDiffFile, pattern) { def fileContent = readFile(gitDiffFile) def modules = [:] - // /beat\/module\/([^\/]+)\/.*/ fileContent.split('\n').each { String line -> log(level: 'DEBUG', text: "changeset element: '${line}'") matches = line =~ pattern diff --git a/vars/getRegionFromPattern.txt b/vars/getBeatsModule.txt similarity index 86% rename from vars/getRegionFromPattern.txt rename to vars/getBeatsModule.txt index 0cf504401..548e516be 100644 --- a/vars/getRegionFromPattern.txt +++ b/vars/getBeatsModule.txt @@ -1,5 +1,5 @@ Given the regex pattern, the CHANGE_TARGET, GIT_SHA env variables then it -evaluates the change list: +evaluates the change list and returns the module name. - When exact match then all the files should match those patterns then it returns the region otherwise and empty string. @@ -7,7 +7,7 @@ evaluates the change list: NOTE: This particular implementation requires to checkout with the step gitCheckout ``` - def module = getRegionFromPattern(pattern: "beat/module/([^\/]+)/") + def module = getBeatsModule(pattern: '([^\\/]+)\\/.*') whenTrue(module.trim()) { // ... } From 427739cba9de0f9af4bb3a082005ae984f710697 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 15 May 2020 14:56:14 +0100 Subject: [PATCH 5/9] add exclude parameter --- .../groovy/GetBeatsModuleStepTests.groovy | 38 +++++++++++++++++++ vars/getBeatsModule.groovy | 24 +++++++++--- vars/getBeatsModule.txt | 4 ++ 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/test/groovy/GetBeatsModuleStepTests.groovy b/src/test/groovy/GetBeatsModuleStepTests.groovy index df8639b85..f867aa118 100644 --- a/src/test/groovy/GetBeatsModuleStepTests.groovy +++ b/src/test/groovy/GetBeatsModuleStepTests.groovy @@ -24,6 +24,18 @@ import static org.junit.Assert.assertTrue class GetBeatsModuleStepTests extends ApmBasePipelineTest { String scriptName = 'vars/getBeatsModule.groovy' + def realData = '''CHANGELOG.next.asciidoc +metricbeat/docs/modules/zookeeper.asciidoc +metricbeat/docs/modules/zookeeper/connection.asciidoc +metricbeat/docs/modules_list.asciidoc +metricbeat/module/zookeeper/_meta/docs.asciidoc +metricbeat/module/zookeeper/connection/_meta/docs.asciidoc +metricbeat/module/zookeeper/connection/_meta/fields.yml +metricbeat/module/zookeeper/connection/connection.go +metricbeat/module/zookeeper/fields.go +metricbeat/module/zookeeper/mntr/_meta/docs.asciidoc +metricbeat/module/zookeeper/server/_meta/docs.asciidoc'''.stripMargin().stripIndent() + @Override @Before void setUp() throws Exception { @@ -219,4 +231,30 @@ bar/foo/subfolder'''.stripMargin().stripIndent() assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } + + @Test + void test_multiple_match_with_real_data_with_exclude() throws Exception { + def script = loadScript(scriptName) + helper.registerAllowedMethod('readFile', [String.class], { return realData }) + def module = script.call(pattern: '.*\\/module\\/([^\\/]+)\\/.*', exclude: '(.*\\/docs\\/.*|.*\\.asciidoc)' ) + assertEquals('zookeeper', module) + assertJobStatusSuccess() + } + + @Test + void test_multiple_match_with_real_data_without_exclude() throws Exception { + def script = loadScript(scriptName) + helper.registerAllowedMethod('readFile', [String.class], { return realData }) + def module = script.call(pattern: '.*\\/module\\/([^\\/]+)\\/.*') + assertEquals('', module) + assertJobStatusSuccess() + } + + @Test + void test_is_excluded() throws Exception { + def script = loadScript(scriptName) + assertFalse(script.isExcluded('', '')) + assertTrue(script.isExcluded('metricbeat/docs/modules/zookeeper.asciidoc', '(.*\\/docs\\/.*|.*\\.asciidoc)')) + assertFalse(script.isExcluded('metricbeat/zookeeper.asciido', '(.*\\/docs\\/.*|.*\\.asciidoc)')) + } } diff --git a/vars/getBeatsModule.groovy b/vars/getBeatsModule.groovy index c3b4e0426..6d3d339ba 100644 --- a/vars/getBeatsModule.groovy +++ b/vars/getBeatsModule.groovy @@ -24,11 +24,15 @@ import com.cloudbees.groovy.cps.NonCPS - When exact match then all the files should match those patterns then it returns the region otherwise and empty string. - // def module = getBeatsModule(pattern: '([^\\/]+)\\/.*') + def module = getBeatsModule(pattern: '([^\\/]+)\\/.*') whenTrue(module.trim()) { // ... } + // Exclude the asciidoc files from the search. + def module = getBeatsModule(pattern: '([^\\/]+)\\/.*', exclude: '.*\\.asciidoc') + + NOTE: This particular implementation requires to checkout with the step gitCheckout */ @@ -37,6 +41,7 @@ def call(Map params = [:]) { error('getBeatsModule: windows is not supported yet.') } def pattern = params.containsKey('pattern') ? params.pattern : error('getBeatsModule: Missing pattern argument.') + def exclude = params.get('exclude', '') def from = params.get('from', env.CHANGE_TARGET?.trim() ? "origin/${env.CHANGE_TARGET}" : env.GIT_PREVIOUS_COMMIT) def to = params.get('to', env.GIT_BASE_COMMIT) @@ -44,7 +49,7 @@ def call(Map params = [:]) { def group = '' if (from?.trim() && to?.trim()) { def changes = sh(script: "git diff --name-only ${from}...${to} > ${gitDiffFile}", returnStdout: true) - group = getGroup(gitDiffFile, pattern) + group = getGroup(gitDiffFile, pattern, exclude) log(level: 'INFO', text: "getBeatsModule: ${group.trim() ?: 'not found'} with regex ${pattern}") } else { log(level: 'INFO', text: 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes. Or the from/to arguments are required.') @@ -52,14 +57,14 @@ def call(Map params = [:]) { return group } -def getGroup(gitDiffFile, pattern) { +def getGroup(gitDiffFile, pattern, exclude) { def fileContent = readFile(gitDiffFile) def modules = [:] fileContent.split('\n').each { String line -> log(level: 'DEBUG', text: "changeset element: '${line}'") - matches = line =~ pattern - def auxModule = matches.collect { it[1] }[0] ?: '' - if (auxModule.trim()) { + if (!isExcluded(line, exclude)) { + matches = line =~ pattern + def auxModule = matches.collect { it[1] }[0] ?: '' modules[auxModule] = auxModule } } @@ -68,3 +73,10 @@ def getGroup(gitDiffFile, pattern) { } return '' } + +def isExcluded(line, exclude) { + if (exclude.trim()) { + return (line ==~ exclude) + } + return false +} diff --git a/vars/getBeatsModule.txt b/vars/getBeatsModule.txt index 548e516be..693d38596 100644 --- a/vars/getBeatsModule.txt +++ b/vars/getBeatsModule.txt @@ -11,9 +11,13 @@ evaluates the change list and returns the module name. whenTrue(module.trim()) { // ... } + + // Exclude the asciidoc files from the search. + def module = getBeatsModule(pattern: '([^\\/]+)\\/.*', exclude: '.*\\.asciidoc') ``` * pattern: the regex pattern with the group to look for. Mandatory +* exclude: the regex pattern with the files to be excluded from the search. Optional * from: to override the diff from sha. Optional. If MPB, and PR then origin/${env.CHANGE_TARGET} otherwise env.GIT_PREVIOUS_COMMIT * to: to override the commit to. Optional. Default: env.GIT_BASE_COMMIT From 8972e1b9810136777303781d1a4526ec05d19a84 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 15 May 2020 18:12:05 +0100 Subject: [PATCH 6/9] fix: serialisation issue --- vars/getBeatsModule.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vars/getBeatsModule.groovy b/vars/getBeatsModule.groovy index 6d3d339ba..4a34040d2 100644 --- a/vars/getBeatsModule.groovy +++ b/vars/getBeatsModule.groovy @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -import com.cloudbees.groovy.cps.NonCPS - /** Given the regex pattern, the CHANGE_TARGET, GIT_SHA env variables then it evaluates the change list and returns the module name. @@ -63,7 +61,7 @@ def getGroup(gitDiffFile, pattern, exclude) { fileContent.split('\n').each { String line -> log(level: 'DEBUG', text: "changeset element: '${line}'") if (!isExcluded(line, exclude)) { - matches = line =~ pattern + def matches = line =~ pattern def auxModule = matches.collect { it[1] }[0] ?: '' modules[auxModule] = auxModule } From c2cca9eb1b81cc509b68b60b8f0e81ea83149b3d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 15 May 2020 18:12:37 +0100 Subject: [PATCH 7/9] test: ITs --- local/configs/jenkins.yaml | 1 + src/test/resources/jobs/getBeatsModule.dsl | 69 ++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 src/test/resources/jobs/getBeatsModule.dsl diff --git a/local/configs/jenkins.yaml b/local/configs/jenkins.yaml index 93be3ccf8..fa3d4c61e 100644 --- a/local/configs/jenkins.yaml +++ b/local/configs/jenkins.yaml @@ -109,6 +109,7 @@ jobs: - file: "/var/pipeline-library/src/test/resources/jobs/cancelPreviousRunningBuilds.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/dockerLogin.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/downstream.dsl" + - file: "/var/pipeline-library/src/test/resources/jobs/getBeatsModule.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/abort.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/connectionRefused.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/error.dsl" diff --git a/src/test/resources/jobs/getBeatsModule.dsl b/src/test/resources/jobs/getBeatsModule.dsl new file mode 100644 index 000000000..f502eda1f --- /dev/null +++ b/src/test/resources/jobs/getBeatsModule.dsl @@ -0,0 +1,69 @@ +NAME = 'it/getBeatsModule' +DSL = '''pipeline { + agent any + environment { + PIPELINE_LOG_LEVEL = 'DEBUG' + } + stages { + stage('checkout') { + steps { + gitCheckout(credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', + repo: 'https://github.com/elastic/apm-pipeline-library.git', + branch: 'master', + basedir: 'sub-folder', + shallow: false) + } + } + stage('match') { + steps { + dir('sub-folder') { + script { + def module = getBeatsModule(pattern: '^\\\\.ci\\\\/([^\\\\/]+)\\\\/.*', + from: '347185bd7e2b402ba8f6befa5ef4428ad417fbbc', + to: '4d9fc25d258622c767ec4d38df38520647cc7dda') + whenFalse(module.equals('jobs')){ + error("Expected module name 'jobs'") + } + } + } + } + } + stage('no_match') { + steps { + dir('sub-folder') { + script { + def module = getBeatsModule(pattern: '^\\\\ci\\\\/([^\\\\/]+)\\\\/.*', + from: '347185bd7e2b402ba8f6befa5ef4428ad417fbbc', + to: '4d9fc25d258622c767ec4d38df38520647cc7dda') + whenFalse(module.equals('')){ + error("Expected module name ''") + } + } + } + } + } + stage('match_with_exclude') { + steps { + dir('sub-folder') { + script { + def module = getBeatsModule(pattern: '^test-infra\\\\/([^\\\\/]+)\\\\/.*', + from: 'b0de59d0ec1e2ae52103a238a2e9cb5b0d7fd9b8', + to: '641fd600836abafa51def05260d63fab6eed4707', + exclude: '^resources.*') + whenFalse(module.equals('apm-ci')){ + error("Expected module name 'apm-ci'") + } + } + } + } + } + } +}''' + +pipelineJob(NAME) { + definition { + cps { + script(DSL.stripIndent()) + } + } +} From 2476123fd1a852bd71b9e714f0347bf6b39ef77b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 18 May 2020 07:56:39 +0100 Subject: [PATCH 8/9] rename --- local/configs/jenkins.yaml | 2 +- ...ovy => GetGitMatchingGroupStepTests.groovy} | 18 +++++++++--------- ...BeatsModule.dsl => getGitMatchingGroup.dsl} | 0 ...odule.groovy => getGitMatchingGroup.groovy} | 12 ++++++------ ...BeatsModule.txt => getGitMatchingGroup.txt} | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) rename src/test/groovy/{GetBeatsModuleStepTests.groovy => GetGitMatchingGroupStepTests.groovy} (89%) rename src/test/resources/jobs/{getBeatsModule.dsl => getGitMatchingGroup.dsl} (100%) rename vars/{getBeatsModule.groovy => getGitMatchingGroup.groovy} (80%) rename vars/{getBeatsModule.txt => getGitMatchingGroup.txt} (85%) diff --git a/local/configs/jenkins.yaml b/local/configs/jenkins.yaml index fa3d4c61e..dec920309 100644 --- a/local/configs/jenkins.yaml +++ b/local/configs/jenkins.yaml @@ -109,12 +109,12 @@ jobs: - file: "/var/pipeline-library/src/test/resources/jobs/cancelPreviousRunningBuilds.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/dockerLogin.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/downstream.dsl" - - file: "/var/pipeline-library/src/test/resources/jobs/getBeatsModule.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/abort.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/connectionRefused.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/error.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/success.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/getBuildInfoJsonFiles/unstable.dsl" + - file: "/var/pipeline-library/src/test/resources/jobs/getGitMatchingGroup.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/git.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/gitBaseCommit.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/gitCheckout.dsl" diff --git a/src/test/groovy/GetBeatsModuleStepTests.groovy b/src/test/groovy/GetGitMatchingGroupStepTests.groovy similarity index 89% rename from src/test/groovy/GetBeatsModuleStepTests.groovy rename to src/test/groovy/GetGitMatchingGroupStepTests.groovy index f867aa118..1093471ee 100644 --- a/src/test/groovy/GetBeatsModuleStepTests.groovy +++ b/src/test/groovy/GetGitMatchingGroupStepTests.groovy @@ -21,8 +21,8 @@ import static org.junit.Assert.assertFalse import static org.junit.Assert.assertEquals import static org.junit.Assert.assertTrue -class GetBeatsModuleStepTests extends ApmBasePipelineTest { - String scriptName = 'vars/getBeatsModule.groovy' +class GetGitMatchingGroupStepTests extends ApmBasePipelineTest { + String scriptName = 'vars/getGitMatchingGroup.groovy' def realData = '''CHANGELOG.next.asciidoc metricbeat/docs/modules/zookeeper.asciidoc @@ -53,7 +53,7 @@ metricbeat/module/zookeeper/server/_meta/docs.asciidoc'''.stripMargin().stripInd //NOOP } printCallStack() - assertTrue(assertMethodCallContainsPattern('error', 'getBeatsModule: Missing pattern argument.')) + assertTrue(assertMethodCallContainsPattern('error', 'Missing pattern argument.')) assertJobStatusFailure() } @@ -65,7 +65,7 @@ metricbeat/module/zookeeper/server/_meta/docs.asciidoc'''.stripMargin().stripInd def module = script.call(pattern: 'foo') printCallStack() assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertTrue(assertMethodCallContainsPattern('log', 'CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } @@ -124,7 +124,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() helper.registerAllowedMethod('readFile', [String.class], { return changeset }) def module = script.call(pattern: '([^\\/]+)\\/.*') assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: not found')) + assertTrue(assertMethodCallContainsPattern('log', 'not found')) assertJobStatusSuccess() } @@ -136,7 +136,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() def module = script.call(pattern: '^unknown.txt') printCallStack() assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: not found with regex ^unknown.txt')) + assertTrue(assertMethodCallContainsPattern('log', 'not found with regex ^unknown.txt')) assertJobStatusSuccess() } @@ -150,7 +150,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() //NOOP } printCallStack() - assertTrue(assertMethodCallContainsPattern('error', 'getBeatsModule: windows is not supported yet.')) + assertTrue(assertMethodCallContainsPattern('error', 'windows is not supported yet.')) assertJobStatusFailure() } @@ -218,7 +218,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() def module = script.call(pattern: '^foo/.*/file.txt', from: '', to: '') printCallStack() assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertTrue(assertMethodCallContainsPattern('log', 'CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } @@ -228,7 +228,7 @@ bar/foo/subfolder'''.stripMargin().stripIndent() def module = script.call(pattern: '^foo/.*/file.txt', to: '') printCallStack() assertEquals('', module) - assertTrue(assertMethodCallContainsPattern('log', 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) + assertTrue(assertMethodCallContainsPattern('log', 'CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes.')) assertJobStatusSuccess() } diff --git a/src/test/resources/jobs/getBeatsModule.dsl b/src/test/resources/jobs/getGitMatchingGroup.dsl similarity index 100% rename from src/test/resources/jobs/getBeatsModule.dsl rename to src/test/resources/jobs/getGitMatchingGroup.dsl diff --git a/vars/getBeatsModule.groovy b/vars/getGitMatchingGroup.groovy similarity index 80% rename from vars/getBeatsModule.groovy rename to vars/getGitMatchingGroup.groovy index 4a34040d2..2243928a5 100644 --- a/vars/getBeatsModule.groovy +++ b/vars/getGitMatchingGroup.groovy @@ -22,13 +22,13 @@ - When exact match then all the files should match those patterns then it returns the region otherwise and empty string. - def module = getBeatsModule(pattern: '([^\\/]+)\\/.*') + def module = getGitMatchingGroup(pattern: '([^\\/]+)\\/.*') whenTrue(module.trim()) { // ... } // Exclude the asciidoc files from the search. - def module = getBeatsModule(pattern: '([^\\/]+)\\/.*', exclude: '.*\\.asciidoc') + def module = getGitMatchingGroup(pattern: '([^\\/]+)\\/.*', exclude: '.*\\.asciidoc') NOTE: This particular implementation requires to checkout with the step gitCheckout @@ -36,9 +36,9 @@ */ def call(Map params = [:]) { if(!isUnix()){ - error('getBeatsModule: windows is not supported yet.') + error('getGitMatchingGroup: windows is not supported yet.') } - def pattern = params.containsKey('pattern') ? params.pattern : error('getBeatsModule: Missing pattern argument.') + def pattern = params.containsKey('pattern') ? params.pattern : error('getGitMatchingGroup: Missing pattern argument.') def exclude = params.get('exclude', '') def from = params.get('from', env.CHANGE_TARGET?.trim() ? "origin/${env.CHANGE_TARGET}" : env.GIT_PREVIOUS_COMMIT) def to = params.get('to', env.GIT_BASE_COMMIT) @@ -48,9 +48,9 @@ def call(Map params = [:]) { if (from?.trim() && to?.trim()) { def changes = sh(script: "git diff --name-only ${from}...${to} > ${gitDiffFile}", returnStdout: true) group = getGroup(gitDiffFile, pattern, exclude) - log(level: 'INFO', text: "getBeatsModule: ${group.trim() ?: 'not found'} with regex ${pattern}") + log(level: 'INFO', text: "getGitMatchingGroup: ${group.trim() ?: 'not found'} with regex ${pattern}") } else { - log(level: 'INFO', text: 'getBeatsModule: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes. Or the from/to arguments are required.') + log(level: 'INFO', text: 'getGitMatchingGroup: CHANGE_TARGET or GIT_PREVIOUS_COMMIT and GIT_BASE_COMMIT env variables are required to evaluate the changes. Or the from/to arguments are required.') } return group } diff --git a/vars/getBeatsModule.txt b/vars/getGitMatchingGroup.txt similarity index 85% rename from vars/getBeatsModule.txt rename to vars/getGitMatchingGroup.txt index 693d38596..61dd813d6 100644 --- a/vars/getBeatsModule.txt +++ b/vars/getGitMatchingGroup.txt @@ -7,13 +7,13 @@ evaluates the change list and returns the module name. NOTE: This particular implementation requires to checkout with the step gitCheckout ``` - def module = getBeatsModule(pattern: '([^\\/]+)\\/.*') + def module = getGitMatchingGroup(pattern: '([^\\/]+)\\/.*') whenTrue(module.trim()) { // ... } // Exclude the asciidoc files from the search. - def module = getBeatsModule(pattern: '([^\\/]+)\\/.*', exclude: '.*\\.asciidoc') + def module = getGitMatchingGroup(pattern: '([^\\/]+)\\/.*', exclude: '.*\\.asciidoc') ``` * pattern: the regex pattern with the group to look for. Mandatory From a4824dcbf39a60baf26515ab9d410c6eca8298cd Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 18 May 2020 12:41:12 +0200 Subject: [PATCH 9/9] Apply suggestions from code review --- src/test/resources/jobs/getGitMatchingGroup.dsl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/resources/jobs/getGitMatchingGroup.dsl b/src/test/resources/jobs/getGitMatchingGroup.dsl index f502eda1f..520293008 100644 --- a/src/test/resources/jobs/getGitMatchingGroup.dsl +++ b/src/test/resources/jobs/getGitMatchingGroup.dsl @@ -1,4 +1,4 @@ -NAME = 'it/getBeatsModule' +NAME = 'it/getGitMatchingGroup' DSL = '''pipeline { agent any environment { @@ -18,7 +18,7 @@ DSL = '''pipeline { steps { dir('sub-folder') { script { - def module = getBeatsModule(pattern: '^\\\\.ci\\\\/([^\\\\/]+)\\\\/.*', + def module = getGitMatchingGroup(pattern: '^\\\\.ci\\\\/([^\\\\/]+)\\\\/.*', from: '347185bd7e2b402ba8f6befa5ef4428ad417fbbc', to: '4d9fc25d258622c767ec4d38df38520647cc7dda') whenFalse(module.equals('jobs')){ @@ -32,7 +32,7 @@ DSL = '''pipeline { steps { dir('sub-folder') { script { - def module = getBeatsModule(pattern: '^\\\\ci\\\\/([^\\\\/]+)\\\\/.*', + def module = getGitMatchingGroup(pattern: '^\\\\ci\\\\/([^\\\\/]+)\\\\/.*', from: '347185bd7e2b402ba8f6befa5ef4428ad417fbbc', to: '4d9fc25d258622c767ec4d38df38520647cc7dda') whenFalse(module.equals('')){ @@ -46,7 +46,7 @@ DSL = '''pipeline { steps { dir('sub-folder') { script { - def module = getBeatsModule(pattern: '^test-infra\\\\/([^\\\\/]+)\\\\/.*', + def module = getGitMatchingGroup(pattern: '^test-infra\\\\/([^\\\\/]+)\\\\/.*', from: 'b0de59d0ec1e2ae52103a238a2e9cb5b0d7fd9b8', to: '641fd600836abafa51def05260d63fab6eed4707', exclude: '^resources.*')