Enable Dockerfile from artifacts.elastic.co (#38585)

This commit enables the copyDockerfile task to render a Dockerfile that
sources the Elasticsearch binary from artifacts.elastic.co. This is
needed for reproducibility and transparency for the official Docker
images in the Docker library.

Author: jasontedor

distribution/docker/build.gradle

@@ -19,17 +19,38 @@ dependencies {
 }
 
 ext.expansions = { oss ->
+  final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz"
+  final String ingestGeoip = "ingest-geoip-${VersionProperties.elasticsearch}.zip"
+  final String ingestUserAgent = "ingest-user-agent-${VersionProperties.elasticsearch}.zip"
   return [
-    'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz",
-    'jdkUrl' : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
-    'jdkVersion' : '11.0.1',
-    'license': oss ? 'Apache-2.0' : 'Elastic License',
-    'ingest-geoip' : "ingest-geoip-${VersionProperties.elasticsearch}.zip",
-    'ingest-user-agent' : "ingest-user-agent-${VersionProperties.elasticsearch}.zip",
-    'version' : VersionProperties.elasticsearch
+    'elasticsearch'       : elasticsearch,
+    'jdkUrl'              : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
+    'jdkVersion'          : '11.0.1',
+    'license'             : oss ? 'Apache-2.0' : 'Elastic License',
+    'source_elasticsearch': local() ? "COPY $elasticsearch $ingestGeoip $ingestUserAgent /opt/" : "RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch} && cd -",
+    'ingest-geoip-plugin' : local() ? "file:///opt/$ingestGeoip" : "ingest-geoip",
+    'ingest-user-agent-plugin' : local() ? "file:///opt/$ingestUserAgent" : "ingest-user-agent",
+    'version'             : VersionProperties.elasticsearch
   ]
 }
 
+/*
+ * We need to be able to render a Dockerfile that references the official artifacts on https://artifacts.elastic.co. For this, we use a
+ * substitution in the Dockerfile template where we can either replace source_elasticsearch with a COPY from the Docker build context, or
+ * a RUN curl command to retrieve the artifact from https://artifacts.elastic.co. The system property build.docker.source, which can be
+ * either "local" (default) or "remote" controls which version of the Dockerfile is produced.
+ */
+private static boolean local() {
+  final String buildDockerSource = System.getProperty("build.docker.source")
+  if (buildDockerSource == null || "local".equals(buildDockerSource)) {
+    return true
+  } else if ("remote".equals(buildDockerSource)) {
+    return false
+  } else {
+    throw new IllegalArgumentException("expected build.docker.source to be [local] or [remote] but was [" + buildDockerSource + "]")
+  }
+}
+
 private static String files(final boolean oss) {
   return "build/${ oss ? 'oss-' : ''}docker"
 }
@@ -50,19 +71,21 @@ void addCopyDockerContextTask(final boolean oss) {
       from 'src/docker/config'
     }
 
-    if (oss) {
-      from configurations.ossDockerSource
-    } else {
-      from configurations.dockerSource
-    }
+    if (local()) {
+      if (oss) {
+        from configurations.ossDockerSource
+      } else {
+        from configurations.dockerSource
+      }
 
-    from configurations.dockerPlugins
+      from configurations.dockerPlugins
+    }
   }
 }
 
 void addCopyDockerfileTask(final boolean oss) {
   task(taskName("copy", oss, "Dockerfile"), type: Copy) {
-    mustRunAfter(taskName("copy", oss, "DockerContext"))
+    dependsOn taskName("copy", oss, "DockerContext")
     into files(oss)
 
     from('src/docker/Dockerfile') {
@@ -73,7 +96,6 @@ void addCopyDockerfileTask(final boolean oss) {
 
 void addBuildDockerImage(final boolean oss) {
   final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
-    dependsOn taskName("copy", oss, "DockerContext")
     dependsOn taskName("copy", oss, "Dockerfile")
     List<String> tags
     if (oss) {

distribution/docker/src/docker/Dockerfile

@@ -30,15 +30,15 @@ RUN groupadd -g 1000 elasticsearch && \
 
 WORKDIR /usr/share/elasticsearch
 
-COPY ${elasticsearch} ${ingest-geoip} ${ingest-user-agent} /opt/
+${source_elasticsearch}
+
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
-RUN elasticsearch-plugin install --batch file:///opt/${ingest-geoip}
-RUN elasticsearch-plugin install --batch file:///opt/${ingest-user-agent}
+RUN elasticsearch-plugin install --batch ${ingest-geoip-plugin}
+RUN elasticsearch-plugin install --batch ${ingest-user-agent-plugin}
 RUN mkdir -p config data logs
 RUN chmod 0775 config data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
 
-
 ################################################################################
 # Build stage 1 (the actual elasticsearch image):
 # Copy elasticsearch from stage 0