Remove bootstrap.system_call_filter setting (#72848)

This commit removes the bootstrap.system_call_filter setting, as
starting in Elasticsearch 8.0.0 we are going to require that system call
filters be installed and that this is not user configurable. Note that
while we force bootstrap to attempt to install system call filters, we
only enforce that they are installed via a bootstrap check in production
environments. We can consider changing this behavior, but leave that for
future consideration and thus a potential follow-up change.

Author: jasontedor

docs/reference/migration/migrate_8_0/settings.asciidoc

@@ -226,16 +226,17 @@ Discontinue use of the removed settings. Specifying these settings in
 ====
 
 [[system-call-filter-setting]]
-.System call filter setting deprecated
+.System call filter setting removed
 [%collapsible]
 ====
 *Details* +
 Elasticsearch uses system call filters to remove its ability to fork another
 process. This is useful to mitigate remote code exploits. These system call
-filters are enabled by default, and controlled via the setting
+filters are enabled by default, and were previously controlled via the setting
 `bootstrap.system_call_filter`. Starting in Elasticsearch 8.0, system call
-filters will be required. As such, the setting `bootstrap.system_call_filter` is
-deprecated and will be removed in Elasticsearch 8.0.
+filters will be required. As such, the setting `bootstrap.system_call_filter`
+was deprecated in Elasticsearch 7.13.0, and is removed as of Elasticsearch
+8.0.0.
 
 *Impact* +
 Discontinue use of the removed setting. Specifying this setting in Elasticsearch

server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -89,17 +89,28 @@ public void run() {
         });
     }
 
-    /** initialize native resources */
-    public static void initializeNatives(Path tmpFile, boolean mlockAll, boolean systemCallFilter, boolean ctrlHandler) {
+    /**
+     * Initialize native resources.
+     *
+     * @param tmpFile          the temp directory
+     * @param mlockAll         whether or not to lock memory
+     * @param systemCallFilter whether or not to install system call filters
+     * @param ctrlHandler      whether or not to install the ctrl-c handler (applies to Windows only)
+     */
+    static void initializeNatives(final Path tmpFile, final boolean mlockAll, final boolean systemCallFilter, final boolean ctrlHandler) {
         final Logger logger = LogManager.getLogger(Bootstrap.class);
 
         // check if the user is running as root, and bail
         if (Natives.definitelyRunningAsRoot()) {
             throw new RuntimeException("can not run elasticsearch as root");
         }
 
-        // enable system call filter
         if (systemCallFilter) {
+            /*
+             * Try to install system call filters; if they fail to install; a bootstrap check will fail startup in production mode.
+             *
+             * TODO: should we fail hard here if system call filters fail to install, or remain lenient in non-production environments?
+             */
             Natives.tryInstallSystemCallFilter(tmpFile);
         }
 
@@ -165,7 +176,7 @@ private void setup(boolean addShutdownHook, Environment environment) throws Boot
         initializeNatives(
                 environment.tmpFile(),
                 BootstrapSettings.MEMORY_LOCK_SETTING.get(settings),
-                BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings),
+                true, // always install system call filters, not user-configurable since 8.0.0
                 BootstrapSettings.CTRLHANDLER_SETTING.get(settings));
 
         // initialize probes before the security manager is installed

server/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java

@@ -523,15 +523,14 @@ String getUseSerialGC() {
     }
 
     /**
-     * Bootstrap check that if system call filters are enabled, then system call filters must have installed successfully.
+     * Bootstrap check that system call filters must have installed successfully.
      */
     static class SystemCallFilterCheck implements BootstrapCheck {
 
         @Override
         public BootstrapCheckResult check(BootstrapContext context) {
-            if (BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(context.settings()) && isSystemCallFilterInstalled() == false) {
-                final String message =  "system call filters failed to install; " +
-                        "check the logs and fix your configuration or disable system call filters at your own risk";
+            if (isSystemCallFilterInstalled() == false) {
+                final String message = "system call filters failed to install; check the logs and fix your configuration";
                 return BootstrapCheckResult.failure(message);
             } else {
                 return BootstrapCheckResult.success();
@@ -590,10 +589,10 @@ String onError() {
         String message(BootstrapContext context) {
             return String.format(
                 Locale.ROOT,
-                "OnError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
+                "OnError [%s] requires forking but is prevented by system call filters;" +
                     " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
-                onError(),
-                BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.getKey());
+                onError()
+            );
         }
 
     }
@@ -614,10 +613,10 @@ String onOutOfMemoryError() {
         String message(BootstrapContext context) {
             return String.format(
                 Locale.ROOT,
-                "OnOutOfMemoryError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
+                "OnOutOfMemoryError [%s] requires forking but is prevented by system call filters;" +
                     " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
-                onOutOfMemoryError(),
-                BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.getKey());
+                onOutOfMemoryError()
+            );
         }
 
     }

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -430,7 +430,6 @@ public void apply(Settings value, Settings current, Settings previous) {
             PluginsService.MANDATORY_SETTING,
             BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING,
             BootstrapSettings.MEMORY_LOCK_SETTING,
-            BootstrapSettings.SYSTEM_CALL_FILTER_SETTING,
             BootstrapSettings.CTRLHANDLER_SETTING,
             KeyStoreWrapper.SEED_SETTING,
             IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING,

server/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java

@@ -417,37 +417,31 @@ String getUseSerialGC() {
 
     public void testSystemCallFilterCheck() throws NodeValidationException {
         final AtomicBoolean isSystemCallFilterInstalled = new AtomicBoolean();
-        BootstrapContext context = randomBoolean() ? createTestContext(Settings.builder().put("bootstrap.system_call_filter", true)
-            .build(), null) : emptyContext;
+        final BootstrapContext context;
+        if (randomBoolean()) {
+            context = createTestContext(Settings.builder().put("bootstrap.system_call_filter", true).build(), null);
+        } else {
+            context = emptyContext;
+        }
 
         final BootstrapChecks.SystemCallFilterCheck systemCallFilterEnabledCheck = new BootstrapChecks.SystemCallFilterCheck() {
+
             @Override
             boolean isSystemCallFilterInstalled() {
                 return isSystemCallFilterInstalled.get();
             }
+
         };
 
         final NodeValidationException e = expectThrows(
             NodeValidationException.class,
             () -> BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck)));
         assertThat(
             e.getMessage(),
-            containsString("system call filters failed to install; " +
-                "check the logs and fix your configuration or disable system call filters at your own risk"));
+            containsString("system call filters failed to install; check the logs and fix your configuration"));
 
         isSystemCallFilterInstalled.set(true);
         BootstrapChecks.check(context, true, Collections.singletonList(systemCallFilterEnabledCheck));
-        BootstrapContext context_1 = createTestContext(Settings.builder().put("bootstrap.system_call_filter", false).build(), null);
-        final BootstrapChecks.SystemCallFilterCheck systemCallFilterNotEnabledCheck = new BootstrapChecks.SystemCallFilterCheck() {
-            @Override
-            boolean isSystemCallFilterInstalled() {
-                return isSystemCallFilterInstalled.get();
-            }
-        };
-        isSystemCallFilterInstalled.set(false);
-        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck));
-        isSystemCallFilterInstalled.set(true);
-        BootstrapChecks.check(context_1, true, Collections.singletonList(systemCallFilterNotEnabledCheck));
     }
 
     public void testMightForkCheck() throws NodeValidationException {
@@ -482,6 +476,7 @@ public void testOnErrorCheck() throws NodeValidationException {
         final AtomicBoolean isSystemCallFilterInstalled = new AtomicBoolean();
         final AtomicReference<String> onError = new AtomicReference<>();
         final BootstrapChecks.MightForkCheck check = new BootstrapChecks.OnErrorCheck() {
+
             @Override
             boolean isSystemCallFilterInstalled() {
                 return isSystemCallFilterInstalled.get();
@@ -491,6 +486,7 @@ boolean isSystemCallFilterInstalled() {
             String onError() {
                 return onError.get();
             }
+
         };
 
         final String command = randomAlphaOfLength(16);
@@ -502,14 +498,15 @@ String onError() {
             e -> assertThat(
                 e.getMessage(),
                 containsString(
-                    "OnError [" + command + "] requires forking but is prevented by system call filters " +
-                        "([bootstrap.system_call_filter=true]); upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
+                    "OnError [" + command + "] requires forking but is prevented by system call filters;" +
+                        " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
     }
 
     public void testOnOutOfMemoryErrorCheck() throws NodeValidationException {
         final AtomicBoolean isSystemCallFilterInstalled = new AtomicBoolean();
         final AtomicReference<String> onOutOfMemoryError = new AtomicReference<>();
         final BootstrapChecks.MightForkCheck check = new BootstrapChecks.OnOutOfMemoryErrorCheck() {
+
             @Override
             boolean isSystemCallFilterInstalled() {
                 return isSystemCallFilterInstalled.get();
@@ -519,6 +516,7 @@ boolean isSystemCallFilterInstalled() {
             String onOutOfMemoryError() {
                 return onOutOfMemoryError.get();
             }
+
         };
 
         final String command = randomAlphaOfLength(16);
@@ -531,7 +529,7 @@ String onOutOfMemoryError() {
                 e.getMessage(),
                 containsString(
                     "OnOutOfMemoryError [" + command + "]"
-                        + " requires forking but is prevented by system call filters ([bootstrap.system_call_filter=true]);"
+                        + " requires forking but is prevented by system call filters;"
                         + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
     }
 

server/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java

@@ -16,7 +16,6 @@ public class BootstrapSettingsTests extends ESTestCase {
     public void testDefaultSettings() {
         assertTrue(BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(Settings.EMPTY));
         assertFalse(BootstrapSettings.MEMORY_LOCK_SETTING.get(Settings.EMPTY));
-        assertTrue(BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(Settings.EMPTY));
         assertTrue(BootstrapSettings.CTRLHANDLER_SETTING.get(Settings.EMPTY));
     }
 

test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

@@ -73,6 +73,7 @@ public class BootstrapForTesting {
         // just like bootstrap, initialize natives, then SM
         final boolean memoryLock =
                 BootstrapSettings.MEMORY_LOCK_SETTING.get(Settings.EMPTY); // use the default bootstrap.memory_lock setting
+        // some tests need the ability to disable system call filters (so they can fork other processes as part of test execution)
         final boolean systemCallFilter = Booleans.parseBoolean(System.getProperty("tests.system_call_filter", "true"));
         Bootstrap.initializeNatives(javaTmpDir, memoryLock, systemCallFilter, true);
 

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -32,8 +32,7 @@ private DeprecationChecks() {
     static List<Function<ClusterState, DeprecationIssue>> CLUSTER_SETTINGS_CHECKS =
         Collections.emptyList();
 
-    static List<BiFunction<Settings, PluginsAndModules, DeprecationIssue>> NODE_SETTINGS_CHECKS =
-        List.of(NodeDeprecationChecks::checkBootstrapSystemCallFilterSetting);
+    static List<BiFunction<Settings, PluginsAndModules, DeprecationIssue>> NODE_SETTINGS_CHECKS = List.of();
 
     static List<Function<IndexMetadata, DeprecationIssue>> INDEX_SETTINGS_CHECKS =
             List.of(IndexDeprecationChecks::oldIndicesCheck, IndexDeprecationChecks::translogRetentionSettingCheck);

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java

@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.deprecation;
 
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
-import org.elasticsearch.bootstrap.BootstrapSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@@ -18,14 +17,6 @@
 
 public class NodeDeprecationChecks {
 
-    static DeprecationIssue checkBootstrapSystemCallFilterSetting(final Settings settings, final PluginsAndModules pluginsAndModules) {
-        return checkRemovedSetting(
-            settings,
-            BootstrapSettings.SYSTEM_CALL_FILTER_SETTING,
-            "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/breaking-changes-7.13.html#deprecate-system-call-filter-setting"
-        );
-    }
-
     private static DeprecationIssue checkDeprecatedSetting(
         final Settings settings,
         final PluginsAndModules pluginsAndModules,

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java

@@ -7,38 +7,17 @@
 
 package org.elasticsearch.xpack.deprecation;
 
-import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
-import org.elasticsearch.bootstrap.BootstrapSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
-import java.util.List;
-
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
 public class NodeDeprecationChecksTests extends ESTestCase {
 
-    public void testCheckBootstrapSystemCallFilterSetting() {
-        final boolean boostrapSystemCallFilter = randomBoolean();
-        final Settings settings =
-            Settings.builder().put(BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.getKey(), boostrapSystemCallFilter).build();
-        final PluginsAndModules pluginsAndModules = new PluginsAndModules(List.of(), List.of());
-        final List<DeprecationIssue> issues =
-            DeprecationChecks.filterChecks(DeprecationChecks.NODE_SETTINGS_CHECKS, c -> c.apply(settings, pluginsAndModules));
-        final DeprecationIssue expected = new DeprecationIssue(
-            DeprecationIssue.Level.CRITICAL,
-            "setting [bootstrap.system_call_filter] is deprecated and will be removed in the next major version",
-            "https://www.elastic.co/guide/en/elasticsearch/reference/7.13/breaking-changes-7.13.html#deprecate-system-call-filter-setting",
-            "the setting [bootstrap.system_call_filter] is currently set to [" + boostrapSystemCallFilter + "], remove this setting");
-        assertThat(issues, hasItem(expected));
-        assertSettingDeprecationsAndWarnings(new Setting<?>[]{BootstrapSettings.SYSTEM_CALL_FILTER_SETTING});
-    }
-
     public void testRemovedSettingNotSet() {
         final Settings settings = Settings.EMPTY;
         final Setting<?> removedSetting = Setting.simpleString("node.removed_setting");