Skip to content

Commit d9d13f3

Browse files
rjernstrjernst
authored
Use project dependency instead of substitutions for distributions (#37730)
Currently integration tests which use either bwc snapshot versions or the current version of elasticsearch depend on project substitutions to link to the build of those artifacts. Likewise, vagrant tests use dependency substitutions to get to bwc snapshots of rpm and debs. This commit changes those to depend on the relevant project/configuration and removes the dependency substitutions for distributions we do not publish.
1 parent 04c6414 commit d9d13f3

File tree

5 files changed

+89
-52
lines changed

5 files changed

+89
-52
lines changed

‎build.gradle‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -221,14 +221,6 @@ allprojects {
221221
"org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
222222
"org.elasticsearch.test:framework:${version}": ':test:framework',
223223
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
224-
"downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
225-
"downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
226-
"downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
227-
"downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
228-
"downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
229-
"downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
230-
"downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
231-
"downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
232224
"org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
233225
"org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
234226
// for transport client
@@ -240,19 +232,6 @@ allprojects {
240232
"org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
241233
"org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
242234
]
243-
// substitute unreleased versions with projects that check out and build locally
244-
bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
245-
Version unreleased = unreleasedVersion.version
246-
String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
247-
ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
248-
ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
249-
ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
250-
if (unreleased.onOrAfter('6.3.0')) {
251-
ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
252-
ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
253-
ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
254-
}
255-
}
256235

257236
/*
258237
* Gradle only resolve project substitutions during dependency resolution but

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
2323
import org.elasticsearch.gradle.BuildPlugin
2424
import org.elasticsearch.gradle.LoggedExec
2525
import org.elasticsearch.gradle.Version
26+
import org.elasticsearch.gradle.VersionCollection
2627
import org.elasticsearch.gradle.VersionProperties
2728
import org.elasticsearch.gradle.plugin.PluginBuildPlugin
2829
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
@@ -171,6 +172,12 @@ class ClusterFormationTasks {
171172

172173
/** Adds a dependency on the given distribution */
173174
static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
175+
if (distro.equals("integ-test-zip")) {
176+
// short circuit integ test so it doesn't complicate the rest of the distribution setup below
177+
project.dependencies.add(configuration.name,
178+
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip")
179+
return
180+
}
174181
// TEMP HACK
175182
// The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
176183
if (distro.equals('oss-zip')) {
@@ -180,22 +187,31 @@ class ClusterFormationTasks {
180187
distro = 'default'
181188
}
182189
// END TEMP HACK
183-
if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
190+
if (['oss', 'default'].contains(distro) == false) {
184191
throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
185192
}
186193
Version version = Version.fromString(elasticsearchVersion)
187-
if (version.before('6.3.0') && distro.startsWith('oss-')) {
188-
distro = distro.substring('oss-'.length())
189-
}
190-
String group = "downloads.zip"
191-
if (distro.equals("integ-test-zip")) {
192-
group = "org.elasticsearch.distribution.integ-test-zip"
193-
}
194+
String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
194195
String artifactName = 'elasticsearch'
195196
if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
196197
artifactName += '-oss'
197198
}
198-
project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
199+
String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
200+
Object dependency
201+
boolean internalBuild = project.hasProperty('bwcVersions')
202+
VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
203+
if (project.hasProperty('bwcVersions')) {
204+
// NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
205+
unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
206+
}
207+
if (unreleasedInfo != null) {
208+
dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
209+
} else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
210+
dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
211+
} else {
212+
dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
213+
}
214+
project.dependencies.add(configuration.name, dependency)
199215
}
200216

201217
/** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
44
import org.elasticsearch.gradle.FileContentsTask
55
import org.elasticsearch.gradle.LoggedExec
66
import org.elasticsearch.gradle.Version
7+
import org.elasticsearch.gradle.VersionCollection
78
import org.gradle.api.*
89
import org.gradle.api.artifacts.dsl.RepositoryHandler
910
import org.gradle.api.execution.TaskExecutionAdapter
@@ -184,22 +185,38 @@ class VagrantTestPlugin implements Plugin<Project> {
184185
upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
185186
}
186187

188+
List<Object> dependencies = new ArrayList<>()
187189
DISTRIBUTIONS.each {
188190
// Adds a dependency for the current version
189-
project.dependencies.add(PACKAGING_CONFIGURATION,
190-
project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
191+
dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
191192
}
192193

193-
UPGRADE_FROM_ARCHIVES.each {
194-
// The version of elasticsearch that we upgrade *from*
195-
project.dependencies.add(PACKAGING_CONFIGURATION,
196-
"downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
197-
if (upgradeFromVersion.onOrAfter('6.3.0')) {
198-
project.dependencies.add(PACKAGING_CONFIGURATION,
199-
"downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
194+
// The version of elasticsearch that we upgrade *from*
195+
VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion)
196+
if (unreleasedInfo != null) {
197+
// handle snapshots pointing to bwc build
198+
UPGRADE_FROM_ARCHIVES.each {
199+
dependencies.add(project.dependencies.project(
200+
path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it))
201+
if (upgradeFromVersion.onOrAfter('6.3.0')) {
202+
dependencies.add(project.dependencies.project(
203+
path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}"))
204+
}
205+
}
206+
} else {
207+
UPGRADE_FROM_ARCHIVES.each {
208+
// The version of elasticsearch that we upgrade *from*
209+
dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
210+
if (upgradeFromVersion.onOrAfter('6.3.0')) {
211+
dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
212+
}
200213
}
201214
}
202215

216+
for (Object dependency : dependencies) {
217+
project.dependencies.add(PACKAGING_CONFIGURATION, dependency)
218+
}
219+
203220
project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
204221
}
205222

‎buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020

2121
import java.util.ArrayList;
2222
import java.util.Collection;
23+
import java.util.Collections;
24+
import java.util.HashMap;
2325
import java.util.HashSet;
2426
import java.util.List;
2527
import java.util.Map;
@@ -86,6 +88,7 @@ public class VersionCollection {
8688

8789
private final Version currentVersion;
8890
private final Map<Integer, List<Version>> groupByMajor;
91+
private final Map<Version, UnreleasedVersionInfo> unreleased;
8992

9093
public class UnreleasedVersionInfo {
9194
public final Version version;
@@ -129,6 +132,16 @@ protected VersionCollection(List<String> versionLines, Version currentVersionPro
129132
assertCurrentVersionMatchesParsed(currentVersionProperty);
130133

131134
assertNoOlderThanTwoMajors();
135+
136+
Map<Version, UnreleasedVersionInfo> unreleased = new HashMap<>();
137+
for (Version unreleasedVersion : getUnreleased()) {
138+
if (unreleasedVersion.equals(currentVersion)) {
139+
continue;
140+
}
141+
unreleased.put(unreleasedVersion,
142+
new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion)));
143+
}
144+
this.unreleased = Collections.unmodifiableMap(unreleased);
132145
}
133146

134147
private void assertNoOlderThanTwoMajors() {
@@ -150,6 +163,13 @@ private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) {
150163
}
151164
}
152165

166+
/**
167+
* Returns info about the unreleased version, or {@code null} if the version is released.
168+
*/
169+
public UnreleasedVersionInfo unreleasedInfo(Version version) {
170+
return unreleased.get(version);
171+
}
172+
153173
public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
154174
getUnreleased().stream()
155175
.filter(version -> version.equals(currentVersion) == false)

‎distribution/bwc/build.gradle‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -120,18 +120,19 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
120120
}
121121
}
122122

123-
List<File> artifactFiles = []
123+
Map<String, File> artifactFiles = [:]
124124
List<String> projectDirs = []
125-
for (String project : ['zip', 'deb', 'rpm']) {
125+
List<String> projects = ['zip', 'deb', 'rpm']
126+
for (String projectName : projects) {
126127
String baseDir = "distribution"
127128
if (bwcVersion.onOrAfter('6.3.0')) {
128-
baseDir += project == 'zip' ? '/archives' : '/packages'
129+
baseDir += projectName == 'zip' ? '/archives' : '/packages'
129130
// add oss variant first
130-
projectDirs.add("${baseDir}/oss-${project}")
131-
artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${project}"))
131+
projectDirs.add("${baseDir}/oss-${projectName}")
132+
artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}"))
132133
}
133-
projectDirs.add("${baseDir}/${project}")
134-
artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${project}"))
134+
projectDirs.add("${baseDir}/${projectName}")
135+
artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}"))
135136
}
136137

137138
task buildBwcVersion(type: Exec) {
@@ -187,7 +188,7 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
187188
standardOutput = new IndentingOutputStream(System.out, bwcVersion)
188189
errorOutput = new IndentingOutputStream(System.err, bwcVersion)
189190
doLast {
190-
List missing = artifactFiles.grep { file ->
191+
List missing = artifactFiles.values().grep { file ->
191192
false == file.exists()
192193
}
193194
if (false == missing.empty) {
@@ -197,12 +198,16 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
197198
}
198199
}
199200

200-
artifacts {
201-
for (File artifactFile : artifactFiles) {
202-
String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
201+
for (e in artifactFiles) {
202+
String projectName = e.key
203+
File artifactFile = e.value
204+
String artifactFileName = artifactFile.name
205+
String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
203206
String suffix = artifactFile.toString()[-3..-1]
204-
'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion
205-
}
207+
configurations.create(projectName)
208+
artifacts {
209+
it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion])
210+
}
206211
}
207212
// make sure no dependencies were added to assemble; we want it to be a no-op
208213
assemble.dependsOn = []

0 commit comments

Comments
 (0)