Merge pull request #597 from vlm/small-file-limit

Fix "Too many open files" if too small of a system limit is encountered

Author: vlm

include/llbuild/Basic/PlatformUtility.h

@@ -53,6 +53,10 @@ std::string makeTmpDir();
 // Return a string containing all valid path separators on the current platform
 std::string getPathSeparators();
 
+/// Gets the max open file limit for the current process.
+/// Returns: 0 on failure, otherwise the max number of open files.
+llbuild_rlim_t getOpenFileLimit();
+
 /// Sets the max open file limit to min(max(soft_limit, limit), hard_limit),
 /// where soft_limit and hard_limit are gathered from the system.
 ///

lib/Basic/LaneBasedExecutionQueue.cpp

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llbuild/Basic/ExecutionQueue.h"
+#include "llbuild/Basic/PlatformUtility.h"
 
 #include "llbuild/Basic/Tracing.h"
 
@@ -195,22 +196,16 @@ class LaneBasedExecutionQueue : public ExecutionQueue {
 
 public:
   LaneBasedExecutionQueue(ExecutionQueueDelegate& delegate,
-                          unsigned numLanes, SchedulerAlgorithm alg,
+                          unsigned numLanesSuggestion, SchedulerAlgorithm alg,
                           const char* const* environment)
-  : ExecutionQueue(delegate), buildID(std::random_device()()), numLanes(numLanes),
+  : ExecutionQueue(delegate), buildID(std::random_device()()),
         readyJobs(Scheduler::make(alg)), environment(environment)
   {
-    // Configure the background task maximum. We currently support an
-    // environmental override for experimentation pursposes, but otherwise limit
-    // to a modest multiple of the core count, since we currently burn one thread
-    // per background task.
-    char *p = getenv("LLBUILD_BACKGROUND_TASK_MAX");
-    if (p && !StringRef(p).getAsInteger(10, backgroundTaskMax)) {
-      // Parsed.
-    } else {
-      backgroundTaskMax = std::min(1024U, numLanes * 64U);
-    }
-            
+
+    auto taskLimits = estimateTaskLimits(numLanesSuggestion);
+    numLanes = taskLimits.first;
+    backgroundTaskMax = taskLimits.second;
+
     for (unsigned i = 0; i != numLanes; ++i) {
       lanes.push_back(std::unique_ptr<std::thread>(
                           new std::thread(
@@ -243,6 +238,52 @@ class LaneBasedExecutionQueue : public ExecutionQueue {
     }
   }
 
+  /// Returns the number of allowed foreground and background tasks.
+  static auto estimateTaskLimits(unsigned numLanes) -> std::pair<unsigned, unsigned> {
+    llbuild_rlim_t curOpenFileLimit = llbuild::basic::sys::getOpenFileLimit();
+    const unsigned reservedFileCount = (STDERR_FILENO+1) + 2 /* Database */
+                                       + 1 /* Logging */
+                                       + 2 /* Additional fds during spawn */
+                                       + 2 /* Fudge factor */;
+    if (curOpenFileLimit < reservedFileCount) {
+      assert(curOpenFileLimit < reservedFileCount);
+      // Certainly can't afford background tasks.
+      // Maybe even can't afford building altogether, but let's risk it.
+      return std::make_pair(1, 0);
+    }
+
+    unsigned allowedFilesForTasks = static_cast<unsigned>(std::min(curOpenFileLimit, static_cast<llbuild_rlim_t>(INT_MAX))) - reservedFileCount;
+    unsigned filesPerTask = 2;  // A task has output [and control] file descriptors.
+    unsigned maxConcurrentTasks = allowedFilesForTasks / filesPerTask;
+
+    if (numLanes > maxConcurrentTasks) {
+      // Can't afford background tasks, and maybe won't even support
+      // the full extent of requested concurrency.
+      numLanes = std::max(1u, maxConcurrentTasks);
+      return std::make_pair(numLanes, 0);
+    }
+
+    // Number of tasks that can be run, according to open file limits.
+    unsigned extraTasksMax = maxConcurrentTasks - numLanes;
+
+    // Configure the background task maximum. We currently support an
+    // environmental override for experimentation purposes, but otherwise
+    // limit to a modest multiple of the core count, since we currently burn
+    // one thread per background task.
+    unsigned backgroundTaskMax = 0;
+    char *p = getenv("LLBUILD_BACKGROUND_TASK_MAX");
+    if (p && !StringRef(p).getAsInteger(10, backgroundTaskMax)) {
+      // Parsed.
+    } else {
+      backgroundTaskMax = std::min(1024U, numLanes * 64U);
+    }
+
+    // The number of background can't exceed available concurrency.
+    backgroundTaskMax = std::min(backgroundTaskMax, extraTasksMax);
+
+    return std::make_pair(numLanes, backgroundTaskMax);
+  }
+
   virtual void addJob(QueueJob job) override {
     uint64_t readyJobsCount;
     {
@@ -328,22 +369,15 @@ class LaneBasedExecutionQueue : public ExecutionQueue {
     handle.id = context.jobID;
 
     ProcessReleaseFn releaseFn = [this](std::function<void()>&& processWait) {
-      bool releaseAllowed = false;
-      // This check is not guaranteed to prevent more than backgroundTaskMax
-      // tasks from releasing. We could race between the check and increment and
-      // thus have a few extra. However, for our purposes, this should be fine.
-      // The cap is primarly intended to prevent runaway explosions of tasks.
-      if (backgroundTaskCount < backgroundTaskMax) {
-        backgroundTaskCount++;
-        releaseAllowed = true;
-      }
-      if (releaseAllowed) {
+      auto previousTaskCount = backgroundTaskCount.fetch_add(1);
+      if (previousTaskCount < backgroundTaskMax) {
         // Launch the process wait on a detached thread
         std::thread([this, processWait=std::move(processWait)]() mutable {
           processWait();
           backgroundTaskCount--;
         }).detach();
       } else {
+        backgroundTaskCount--;
         // not allowed to release, call wait directly
         processWait();
       }

lib/Basic/PlatformUtility.cpp

@@ -212,6 +212,22 @@ int sys::write(int fileHandle, void *destinationBuffer,
 #endif
 }
 
+// Get the current process' open file limit. Returns -1 on failure.
+llbuild_rlim_t sys::getOpenFileLimit() {
+#if defined(_WIN32)
+  int value = _getmaxstdio();
+  return std::min(0, value);
+#else
+  struct rlimit rl;
+  int ret = getrlimit(RLIMIT_NOFILE, &rl);
+  if (ret != 0) {
+    return 0;
+  }
+
+  return rl.rlim_cur;
+#endif
+}
+
 // Raise the open file limit, returns 0 on success, -1 on failure
 int sys::raiseOpenFileLimit(llbuild_rlim_t limit) {
 #if defined(_WIN32)

tests/Ninja/Build/Inputs/close-control-fd

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+# Close the control channel and run a given command.
+# Example usage:
+#
+#   > exec ./run-releasing-control-fd sleep 10
+#
+# The `exec` prefix is necessary to prevent the file descriptor being held
+# by the outer shell while running the `run-relesing-control-fd`.
+
+from __future__ import print_function
+import subprocess
+import sys
+import os
+
+def usage():
+    print("Expected: [--require-control-fd] <command> [args...]")
+    sys.exit(1)
+
+def main(argv):
+    require_control_fd = False
+
+    # Parse command line options
+    if len(argv) <= 1:
+        usage()
+    elif argv[1] == "--require-control-fd":
+        if len(argv) == 2:
+            usage()
+        argv=argv[1:]
+        require_control_fd = True
+    elif argv[1][0:1] == "-":
+        usage()
+
+    exec_args = argv[1:]
+
+    # Close the control descriptor, if given.
+    control_fd = os.environ.get("LLBUILD_CONTROL_FD")
+    if control_fd is None and require_control_fd:
+        print("%s: missing LLBUILD_CONTROL_FD" % exec_args[0], file=sys.stderr)
+        sys.exit(1)
+    elif control_fd is not None:
+        del os.environ["LLBUILD_CONTROL_FD"]
+        os.close(int(control_fd))
+
+    use_shell = len(exec_args) == 1 and ' ' in exec_args[0]
+    subprocess.check_call(exec_args, shell=use_shell)
+
+if __name__ == '__main__':
+    main(sys.argv)

tests/Ninja/Build/Inputs/run-releasing-control-fd

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+# Example usage:
+#   > ./ulimited -n 50 echo foo
+#   Prior RLIMIT_NOFILE: soft=256, hard=RLIM_INFINITY
+#   Reset RLIMIT_NOFILE: soft=50, hard=RLIM_INFINITY
+#   foo
+#   >
+
+
+from __future__ import print_function
+import errno
+import subprocess
+import resource
+import sys
+import os
+
+
+def main(argv):
+    if len(argv) < 4 or argv[1] != "-n":
+        print("Expected: -n <n> <command-exe> [args...]")
+        sys.exit(1)
+
+    try:
+        new_limit = int(argv[2])
+    except ValueError as e:
+        print("-n: %s" % e)
+        sys.exit(os.EX_USAGE)
+
+    execute_with_limit(new_limit, argv[3:])
+
+    print("%s: Cannot execute", argv[3])
+    exit(os.EX_OSERR)
+
+
+def execute_with_limit(new_limit, arguments):
+    max_fd_soft, max_fd_hard = get_and_show_limit("Prior", "RLIMIT_NOFILE")
+    resource.setrlimit(resource.RLIMIT_NOFILE, [new_limit, max_fd_hard])
+    _ = get_and_show_limit("Reset", "RLIMIT_NOFILE")
+    os.execvp(arguments[0], arguments)
+
+
+def stringify_limit(limit):
+    if limit == resource.RLIM_INFINITY:
+        return "RLIM_INFINITY"
+    return limit
+
+
+def get_and_show_limit(prefix, name):
+    max_fd_soft, max_fd_hard = resource.getrlimit(getattr(resource, name))
+    max_fd_soft_s = stringify_limit(max_fd_soft)
+    max_fd_hard_s = stringify_limit(max_fd_hard)
+    print("%s %s: {soft: %s, hard: %s}"
+          % (prefix, name, max_fd_soft_s, max_fd_hard_s), file=sys.stderr)
+    return (max_fd_soft, max_fd_hard)
+
+
+if __name__ == '__main__':
+    main(sys.argv)

tests/Ninja/Build/Inputs/ulimited

@@ -3,7 +3,7 @@
 # RUN: rm -rf %t.build
 # RUN: mkdir -p %t.build
 # RUN: cp %s %t.build/build.ninja
-# RUN: cp %S/Inputs/close-control-fd %t.build
+# RUN: cp %S/Inputs/run-releasing-control-fd %t.build
 # RUN: cp %S/Inputs/wait-for-file %t.build
 # RUN: %{llbuild} ninja build --jobs 3 --no-db --chdir %t.build &> %t.out
 # RUN: %{FileCheck} < %t.out %s
@@ -22,7 +22,7 @@ rule CUSTOM
   command = ${COMMAND}
 
 build first: CUSTOM
-  command = touch executing && . ./close-control-fd && ./wait-for-file stop && rm -f executing
+  command = touch executing && exec ./run-releasing-control-fd "./wait-for-file stop && rm -f executing"
 
 build second: CUSTOM
   command = test ! -f executing

tests/Ninja/Build/console-pool-no-control-fd.ninja

@@ -0,0 +1,63 @@
+# Check operation under max file descriptor constraints
+
+# RUN: rm -rf %t.build
+# RUN: mkdir -p %t.build
+# RUN: cp %s %t.build/build.ninja
+# RUN: cp %S/Inputs/run-releasing-control-fd %t.build
+# RUN: cp %S/Inputs/ulimited %t.build
+# RUN: %t.build/ulimited -n 30 %{llbuild} ninja build --jobs 02 --chdir %t.build &> %t.out
+# RUN: %t.build/ulimited -n 40 %{llbuild} ninja build --jobs 04 --chdir %t.build &> %t.out
+# RUN: %t.build/ulimited -n 60 %{llbuild} ninja build --jobs 10 --chdir %t.build &> %t.out
+# RUN: %t.build/ulimited -n 80 %{llbuild} ninja build --jobs 10 --chdir %t.build &> %t.out
+#
+rule WAIT
+  command = exec ./run-releasing-control-fd "sleep 0.1"
+
+build 00: WAIT
+build 01: WAIT
+build 02: WAIT
+build 03: WAIT
+build 04: WAIT
+build 05: WAIT
+build 06: WAIT
+build 07: WAIT
+build 08: WAIT
+build 09: WAIT
+build 10: WAIT
+build 11: WAIT
+build 12: WAIT
+build 13: WAIT
+build 14: WAIT
+build 15: WAIT
+build 16: WAIT
+build 17: WAIT
+build 18: WAIT
+build 19: WAIT
+build 20: WAIT
+build 21: WAIT
+build 22: WAIT
+build 23: WAIT
+build 24: WAIT
+build 25: WAIT
+build 26: WAIT
+build 27: WAIT
+build 28: WAIT
+build 29: WAIT
+build 30: WAIT
+build 31: WAIT
+build 32: WAIT
+build 33: WAIT
+build 34: WAIT
+build 35: WAIT
+build 36: WAIT
+build 37: WAIT
+build 38: WAIT
+build 39: WAIT
+
+build output: phony $
+  00 01 02 03 04 05 06 07 08 09 $
+  10 11 12 13 14 15 16 17 18 19 $
+  20 21 22 23 24 25 26 27 28 29 $
+  30 31 32 33 34 35 36 37 38 39
+
+default output