Limit _FILE env var support to specific vars (#52525)

Closes #52503. Implement a list of `_FILE` env vars that will be used to
populate env vars with file content, instead of processing all `_FILE`
vars in the environment.

Author: pugnascotia

distribution/src/bin/elasticsearch-env-from-file

@@ -7,11 +7,15 @@ set -e -o pipefail
 # point to it. This can be used to provide secrets to a container, without
 # the values being specified explicitly when running the container.
 #
+# Note that only supported environment variables are processed, in order
+# to avoid unexpected failures when an environment sets a "*_FILE" variable
+# that doesn't contain a filename.
+#
 # This script is intended to be sourced, not executed, and modifies the
 # environment.
 
-for VAR_NAME_FILE in $(env | cut -f1 -d= | grep '_FILE$'); do
-  if [[ -n "$VAR_NAME_FILE" ]]; then
+for VAR_NAME_FILE in ELASTIC_PASSWORD_FILE KEYSTORE_PASSWORD_FILE ; do
+  if [[ -n "${!VAR_NAME_FILE}" ]]; then
     VAR_NAME="${VAR_NAME_FILE%_FILE}"
 
     if env | grep "^${VAR_NAME}="; then

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -56,7 +56,6 @@
 import static org.elasticsearch.packaging.util.FileUtils.append;
 import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
 import static org.elasticsearch.packaging.util.FileUtils.rm;
-import static org.elasticsearch.packaging.util.ServerUtils.makeRequest;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyString;
@@ -211,38 +210,10 @@ public void test071BindMountCustomPathWithDifferentUID() throws Exception {
         });
     }
 
-    /**
-     * Check that environment variables can be populated by setting variables with the suffix "_FILE",
-     * which point to files that hold the required values.
-     */
-    public void test080SetEnvironmentVariablesUsingFiles() throws Exception {
-        final String optionsFilename = "esJavaOpts.txt";
-
-        // ES_JAVA_OPTS_FILE
-        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
-
-        Map<String, String> envVars = Map.of("ES_JAVA_OPTS_FILE", "/run/secrets/" + optionsFilename);
-
-        // File permissions need to be secured in order for the ES wrapper to accept
-        // them for populating env var values
-        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p600);
-
-        final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
-
-        // Restart the container
-        runContainer(distribution(), volumes, envVars);
-
-        waitForElasticsearch(installation);
-
-        final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
-
-        assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
-    }
-
     /**
      * Check that the elastic user's password can be configured via a file and the ELASTIC_PASSWORD_FILE environment variable.
      */
-    public void test081ConfigurePasswordThroughEnvironmentVariableFile() throws Exception {
+    public void test080ConfigurePasswordThroughEnvironmentVariableFile() throws Exception {
         // Test relies on configuring security
         assumeTrue(distribution.isDefault());
 
@@ -289,7 +260,7 @@ public void test081ConfigurePasswordThroughEnvironmentVariableFile() throws Exce
      * Check that when verifying the file permissions of _FILE environment variables, symlinks
      * are followed.
      */
-    public void test082SymlinksAreFollowedWithEnvironmentVariableFiles() throws Exception {
+    public void test081SymlinksAreFollowedWithEnvironmentVariableFiles() throws Exception {
         // Test relies on configuring security
         assumeTrue(distribution.isDefault());
         // Test relies on symlinks
@@ -329,47 +300,42 @@ public void test082SymlinksAreFollowedWithEnvironmentVariableFiles() throws Exce
     /**
      * Check that environment variables cannot be used with _FILE environment variables.
      */
-    public void test083CannotUseEnvVarsAndFiles() throws Exception {
-        final String optionsFilename = "esJavaOpts.txt";
+    public void test082CannotUseEnvVarsAndFiles() throws Exception {
+        final String passwordFilename = "password.txt";
 
-        // ES_JAVA_OPTS_FILE
-        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
+        Files.writeString(tempDir.resolve(passwordFilename), "other_hunter2\n");
 
-        Map<String, String> envVars = Map.of(
-            "ES_JAVA_OPTS",
-            "-XX:+UseCompressedOops",
-            "ES_JAVA_OPTS_FILE",
-            "/run/secrets/" + optionsFilename
-        );
+        Map<String, String> envVars = new HashMap<>();
+        envVars.put("ELASTIC_PASSWORD", "hunter2");
+        envVars.put("ELASTIC_PASSWORD_FILE", "/run/secrets/" + passwordFilename);
 
         // File permissions need to be secured in order for the ES wrapper to accept
         // them for populating env var values
-        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p600);
+        Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p600);
 
         final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
 
         final Result dockerLogs = runContainerExpectingFailure(distribution, volumes, envVars);
 
         assertThat(
             dockerLogs.stderr,
-            containsString("ERROR: Both ES_JAVA_OPTS_FILE and ES_JAVA_OPTS are set. These are mutually exclusive.")
+            containsString("ERROR: Both ELASTIC_PASSWORD_FILE and ELASTIC_PASSWORD are set. These are mutually exclusive.")
         );
     }
 
     /**
      * Check that when populating environment variables by setting variables with the suffix "_FILE",
      * the files' permissions are checked.
      */
-    public void test084EnvironmentVariablesUsingFilesHaveCorrectPermissions() throws Exception {
-        final String optionsFilename = "esJavaOpts.txt";
+    public void test083EnvironmentVariablesUsingFilesHaveCorrectPermissions() throws Exception {
+        final String passwordFilename = "password.txt";
 
-        // ES_JAVA_OPTS_FILE
-        Files.writeString(tempDir.resolve(optionsFilename), "-XX:-UseCompressedOops\n");
+        Files.writeString(tempDir.resolve(passwordFilename), "hunter2\n");
 
-        Map<String, String> envVars = Map.of("ES_JAVA_OPTS_FILE", "/run/secrets/" + optionsFilename);
+        Map<String, String> envVars = Map.of("ELASTIC_PASSWORD_FILE", "/run/secrets/" + passwordFilename);
 
         // Set invalid file permissions
-        Files.setPosixFilePermissions(tempDir.resolve(optionsFilename), p660);
+        Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p660);
 
         final Map<Path, Path> volumes = Map.of(tempDir, Path.of("/run/secrets"));
 
@@ -378,15 +344,17 @@ public void test084EnvironmentVariablesUsingFilesHaveCorrectPermissions() throws
 
         assertThat(
             dockerLogs.stderr,
-            containsString("ERROR: File /run/secrets/" + optionsFilename + " from ES_JAVA_OPTS_FILE must have file permissions 400 or 600")
+            containsString(
+                "ERROR: File /run/secrets/" + passwordFilename + " from ELASTIC_PASSWORD_FILE must have file permissions 400 or 600"
+            )
         );
     }
 
     /**
      * Check that when verifying the file permissions of _FILE environment variables, symlinks
      * are followed, and that invalid target permissions are detected.
      */
-    public void test085SymlinkToFileWithInvalidPermissionsIsRejected() throws Exception {
+    public void test084SymlinkToFileWithInvalidPermissionsIsRejected() throws Exception {
         // Test relies on configuring security
         assumeTrue(distribution.isDefault());
         // Test relies on symlinks
@@ -435,7 +403,7 @@ public void test085SymlinkToFileWithInvalidPermissionsIsRejected() throws Except
      * Check that environment variables are translated to -E options even for commands invoked under
      * `docker exec`, where the Docker image's entrypoint is not executed.
      */
-    public void test086EnvironmentVariablesAreRespectedUnderDockerExec() {
+    public void test085EnvironmentVariablesAreRespectedUnderDockerExec() {
         // This test relies on a CLI tool attempting to connect to Elasticsearch, and the
         // tool in question is only in the default distribution.
         assumeTrue(distribution.isDefault());

qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -47,6 +47,7 @@
 import static org.elasticsearch.packaging.util.Docker.waitForPathToExist;
 import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File;
 import static org.elasticsearch.packaging.util.FileMatcher.file;
+import static org.elasticsearch.packaging.util.FileMatcher.p600;
 import static org.elasticsearch.packaging.util.FileMatcher.p660;
 import static org.elasticsearch.packaging.util.FileUtils.getTempDir;
 import static org.elasticsearch.packaging.util.FileUtils.rm;
@@ -305,11 +306,47 @@ public void test60DockerEnvironmentVariablePassword() throws Exception {
         ServerUtils.runElasticsearchTests();
     }
 
+    /**
+     * Check that we can mount a password-protected keystore to a docker image
+     * and provide a password via a file, pointed at from an environment variable.
+     */
+    public void test61DockerEnvironmentVariablePasswordFromFile() throws Exception {
+        assumeTrue(distribution().isDocker());
+
+        Path tempDir = null;
+        try {
+            tempDir = Files.createTempDirectory(getTempDir(), DockerTests.class.getSimpleName());
+
+            String password = "password";
+            String passwordFilename = "password.txt";
+            Files.writeString(tempDir.resolve(passwordFilename), password + "\n");
+            Files.setPosixFilePermissions(tempDir.resolve(passwordFilename), p600);
+
+            Path dockerKeystore = installation.config("elasticsearch.keystore");
+
+            Path localKeystoreFile = getKeystoreFileFromDockerContainer(password, dockerKeystore);
+
+            // restart ES with password and mounted keystore
+            Map<Path, Path> volumes = Map.of(localKeystoreFile, dockerKeystore, tempDir, Path.of("/run/secrets"));
+            Map<String, String> envVars = Map.of("KEYSTORE_PASSWORD_FILE", "/run/secrets/" + passwordFilename);
+
+            runContainer(distribution(), volumes, envVars);
+
+            waitForElasticsearch(installation);
+            ServerUtils.runElasticsearchTests();
+        }
+        finally {
+            if (tempDir != null) {
+                rm(tempDir);
+            }
+        }
+    }
+
     /**
      * Check that if we provide the wrong password for a mounted and password-protected
      * keystore, Elasticsearch doesn't start.
      */
-    public void test61DockerEnvironmentVariableBadPassword() throws Exception {
+    public void test62DockerEnvironmentVariableBadPassword() throws Exception {
         assumeTrue(distribution().isDocker());
         String password = "password";
         Path dockerKeystore = installation.config("elasticsearch.keystore");