Merge branch 'master' into translog-generation

* master:
  Format RemovePluginCommand to 100-column limit
  Add Javadocs for RemovePluginCommand#execute
  Avoid overflow when computing total FS stats
  Mark EvilJNANativesTests as awaiting fixes
  Format EvilJNANativesTests to 100-column limit
  Upgrade from JNA 4.2.2 to JNA 4.4.0
  Task Manager should be able to support non-transport tasks (#23619)

Author: jasontedor

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -1578,7 +1578,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginSecurity.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]ProgressInputStream.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]RemovePluginCommand.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]RepositoryPlugin.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]ScriptPlugin.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]SearchPlugin.java" checks="LineLength" />
@@ -3945,7 +3944,6 @@
   <suppress files="plugins[/\\]store-smb[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]smbsimplefs[/\\]SmbSimpleFsDirectoryService.java" checks="LineLength" />
   <suppress files="qa[/\\]backwards-5.0[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]backwards[/\\]IndexingIT.java" checks="LineLength" />
   <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]ESPolicyUnitTests.java" checks="LineLength" />
-  <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]EvilJNANativesTests.java" checks="LineLength" />
   <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]EvilSecurityTests.java" checks="LineLength" />
   <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]SystemCallFilterTests.java" checks="LineLength" />
   <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cli[/\\]EvilCommandTests.java" checks="LineLength" />

buildSrc/version.properties

@@ -10,7 +10,7 @@ snakeyaml         = 1.15
 # When updating log4j, please update also docs/java-api/index.asciidoc
 log4j             = 2.7
 slf4j             = 1.6.2
-jna               = 4.2.2
+jna               = 4.4.0
 
 # test dependencies
 randomizedrunner  = 2.5.0

core/licenses/jna-4.2.2.jar.sha1

@@ -0,0 +1 @@
+cb208278274bf12ebdb56c61bd7407e6f774d65a

core/licenses/jna-4.4.0.jar.sha1

@@ -138,8 +138,8 @@ private double addDouble(double current, double other) {
 
         public void add(Path path) {
             total = FsProbe.adjustForHugeFilesystems(addLong(total, path.total));
-            free = addLong(free, path.free);
-            available = addLong(available, path.available);
+            free = FsProbe.adjustForHugeFilesystems(addLong(free, path.free));
+            available = FsProbe.adjustForHugeFilesystems(addLong(available, path.available));
             if (path.spins != null && path.spins.booleanValue()) {
                 // Spinning is contagious!
                 spins = Boolean.TRUE;

core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java

@@ -19,12 +19,14 @@
 
 package org.elasticsearch.plugins;
 
+import java.io.IOException;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
@@ -39,67 +41,93 @@
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
 
 /**
- * A command for the plugin cli to remove a plugin from elasticsearch.
+ * A command for the plugin CLI to remove a plugin from Elasticsearch.
  */
 class RemovePluginCommand extends EnvironmentAwareCommand {
 
     private final OptionSpec<String> arguments;
 
     RemovePluginCommand() {
-        super("Removes a plugin from elasticsearch");
+        super("removes a plugin from Elasticsearch");
         this.arguments = parser.nonOptions("plugin name");
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception {
-        String arg = arguments.value(options);
-        execute(terminal, arg, env);
+    protected void execute(final Terminal terminal, final OptionSet options, final Environment env)
+            throws Exception {
+        final String pluginName = arguments.value(options);
+        execute(terminal, pluginName, env);
     }
 
-    // pkg private for testing
-    void execute(Terminal terminal, String pluginName, Environment env) throws Exception {
+    /**
+     * Remove the plugin specified by {@code pluginName}.
+     *
+     * @param terminal   the terminal to use for input/output
+     * @param pluginName the name of the plugin to remove
+     * @param env        the environment for the local node
+     * @throws IOException   if any I/O exception occurs while performing a file operation
+     * @throws UserException if plugin name is null
+     * @throws UserException if plugin directory does not exist
+     * @throws UserException if the plugin bin directory is not a directory
+     */
+    void execute(final Terminal terminal, final String pluginName, final Environment env)
+            throws IOException, UserException {
         if (pluginName == null) {
             throw new UserException(ExitCodes.USAGE, "plugin name is required");
         }
 
-        terminal.println("-> Removing " + Strings.coalesceToEmpty(pluginName) + "...");
+        terminal.println("-> removing [" + Strings.coalesceToEmpty(pluginName) + "]...");
 
         final Path pluginDir = env.pluginsFile().resolve(pluginName);
         if (Files.exists(pluginDir) == false) {
-            throw new UserException(
-                    ExitCodes.CONFIG,
-                    "plugin " + pluginName + " not found; run 'elasticsearch-plugin list' to get list of installed plugins");
+            final String message = String.format(
+                    Locale.ROOT,
+                    "plugin [%s] not found; "
+                            + "run 'elasticsearch-plugin list' to get list of installed plugins",
+                    pluginName);
+            throw new UserException(ExitCodes.CONFIG, message);
         }
 
         final List<Path> pluginPaths = new ArrayList<>();
 
         final Path pluginBinDir = env.binFile().resolve(pluginName);
         if (Files.exists(pluginBinDir)) {
             if (Files.isDirectory(pluginBinDir) == false) {
-                throw new UserException(ExitCodes.IO_ERROR, "Bin dir for " + pluginName + " is not a directory");
+                throw new UserException(
+                        ExitCodes.IO_ERROR, "bin dir for " + pluginName + " is not a directory");
             }
             pluginPaths.add(pluginBinDir);
-            terminal.println(VERBOSE, "Removing: " + pluginBinDir);
+            terminal.println(VERBOSE, "removing [" + pluginBinDir + "]");
         }
 
-        terminal.println(VERBOSE, "Removing: " + pluginDir);
+        terminal.println(VERBOSE, "removing [" + pluginDir + "]");
         final Path tmpPluginDir = env.pluginsFile().resolve(".removing-" + pluginName);
         try {
             Files.move(pluginDir, tmpPluginDir, StandardCopyOption.ATOMIC_MOVE);
         } catch (final AtomicMoveNotSupportedException e) {
-            // this can happen on a union filesystem when a plugin is not installed on the top layer; we fall back to a non-atomic move
+            /*
+             * On a union file system if the plugin that we are removing is not installed on the
+             * top layer then atomic move will not be supported. In this case, we fall back to a
+             * non-atomic move.
+             */
             Files.move(pluginDir, tmpPluginDir);
         }
         pluginPaths.add(tmpPluginDir);
 
         IOUtils.rm(pluginPaths.toArray(new Path[pluginPaths.size()]));
 
-        // we preserve the config files in case the user is upgrading the plugin, but we print
-        // a message so the user knows in case they want to remove manually
+        /*
+         * We preserve the config files in case the user is upgrading the plugin, but we print a
+         * message so the user knows in case they want to remove manually.
+         */
         final Path pluginConfigDir = env.configFile().resolve(pluginName);
         if (Files.exists(pluginConfigDir)) {
-            terminal.println(
-                    "-> Preserving plugin config files [" + pluginConfigDir + "] in case of upgrade, delete manually if not needed");
+            final String message = String.format(
+                    Locale.ROOT,
+                    "-> preserving plugin config files [%s] in case of upgrade; "
+                            + "delete manually if not needed",
+                    pluginConfigDir);
+            terminal.println(message);
         }
     }
 

core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tasks;
+
+/**
+ * An interface for a request that can be used to register a task manager task
+ */
+public interface TaskAwareRequest {
+    /**
+     * Set a reference to task that caused this task to be run.
+     */
+    default void setParentTask(String parentTaskNode, long parentTaskId) {
+        setParentTask(new TaskId(parentTaskNode, parentTaskId));
+    }
+
+    /**
+     * Set a reference to task that created this request.
+     */
+    void setParentTask(TaskId taskId);
+
+    /**
+     * Get a reference to the task that created this request. Implementers should default to
+     * {@link TaskId#EMPTY_TASK_ID}, meaning "there is no parent".
+     */
+    TaskId getParentTask();
+
+    /**
+     * Returns the task object that should be used to keep track of the processing of the request.
+     *
+     * A request can override this method and return null to avoid being tracked by the task
+     * manager.
+     */
+    default Task createTask(long id, String type, String action, TaskId parentTaskId) {
+        return new Task(id, type, action, getDescription(), parentTaskId);
+    }
+
+    /**
+     * Returns optional description of the request to be displayed by the task manager
+     */
+    default String getDescription() {
+        return "";
+    }
+}

core/src/main/java/org/elasticsearch/tasks/TaskAwareRequest.java

@@ -35,18 +35,14 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
-import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Consumer;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
@@ -83,7 +79,7 @@ public void setTaskResultsService(TaskResultsService taskResultsService) {
      * <p>
      * Returns the task manager tracked task or null if the task doesn't support the task manager
      */
-    public Task register(String type, String action, TransportRequest request) {
+    public Task register(String type, String action, TaskAwareRequest request) {
         Task task = request.createTask(taskIdGenerator.incrementAndGet(), type, action, request.getParentTask());
         if (task == null) {
             return null;

core/src/main/java/org/elasticsearch/tasks/TaskManager.java

@@ -21,12 +21,12 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskAwareRequest;
 import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
 
-public abstract class TransportRequest extends TransportMessage {
+public abstract class TransportRequest extends TransportMessage implements TaskAwareRequest {
     public static class Empty extends TransportRequest {
         public static final Empty INSTANCE = new Empty();
     }
@@ -39,43 +39,22 @@ public static class Empty extends TransportRequest {
     public TransportRequest() {
     }
 
-    /**
-     * Set a reference to task that caused this task to be run.
-     */
-    public void setParentTask(String parentTaskNode, long parentTaskId) {
-        setParentTask(new TaskId(parentTaskNode, parentTaskId));
-    }
-
     /**
      * Set a reference to task that created this request.
      */
+    @Override
     public void setParentTask(TaskId taskId) {
         this.parentTaskId = taskId;
     }
 
     /**
      * Get a reference to the task that created this request. Defaults to {@link TaskId#EMPTY_TASK_ID}, meaning "there is no parent".
      */
+    @Override
     public TaskId getParentTask() {
         return parentTaskId;
     }
 
-    /**
-     * Returns the task object that should be used to keep track of the processing of the request.
-     *
-     * A request can override this method and return null to avoid being tracked by the task manager.
-     */
-    public Task createTask(long id, String type, String action, TaskId parentTaskId) {
-        return new Task(id, type, action, getDescription(), parentTaskId);
-    }
-
-    /**
-     * Returns optional description of the request to be displayed by the task manager
-     */
-    public String getDescription() {
-        return "";
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);