diff --git a/include/llbuild/Basic/PlatformUtility.h b/include/llbuild/Basic/PlatformUtility.h index 49afc308..c077eb74 100644 --- a/include/llbuild/Basic/PlatformUtility.h +++ b/include/llbuild/Basic/PlatformUtility.h @@ -53,6 +53,10 @@ std::string makeTmpDir(); // Return a string containing all valid path separators on the current platform std::string getPathSeparators(); +/// Gets the max open file limit for the current process. +/// Returns: 0 on failure, otherwise the max number of open files. +llbuild_rlim_t getOpenFileLimit(); + /// Sets the max open file limit to min(max(soft_limit, limit), hard_limit), /// where soft_limit and hard_limit are gathered from the system. /// diff --git a/lib/Basic/LaneBasedExecutionQueue.cpp b/lib/Basic/LaneBasedExecutionQueue.cpp index 8affe95a..8ec5ae96 100644 --- a/lib/Basic/LaneBasedExecutionQueue.cpp +++ b/lib/Basic/LaneBasedExecutionQueue.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "llbuild/Basic/ExecutionQueue.h" +#include "llbuild/Basic/PlatformUtility.h" #include "llbuild/Basic/Tracing.h" @@ -195,22 +196,16 @@ class LaneBasedExecutionQueue : public ExecutionQueue { public: LaneBasedExecutionQueue(ExecutionQueueDelegate& delegate, - unsigned numLanes, SchedulerAlgorithm alg, + unsigned numLanesSuggestion, SchedulerAlgorithm alg, const char* const* environment) - : ExecutionQueue(delegate), buildID(std::random_device()()), numLanes(numLanes), + : ExecutionQueue(delegate), buildID(std::random_device()()), readyJobs(Scheduler::make(alg)), environment(environment) { - // Configure the background task maximum. We currently support an - // environmental override for experimentation pursposes, but otherwise limit - // to a modest multiple of the core count, since we currently burn one thread - // per background task. - char *p = getenv("LLBUILD_BACKGROUND_TASK_MAX"); - if (p && !StringRef(p).getAsInteger(10, backgroundTaskMax)) { - // Parsed. - } else { - backgroundTaskMax = std::min(1024U, numLanes * 64U); - } - + + auto taskLimits = estimateTaskLimits(numLanesSuggestion); + numLanes = taskLimits.first; + backgroundTaskMax = taskLimits.second; + for (unsigned i = 0; i != numLanes; ++i) { lanes.push_back(std::unique_ptr( new std::thread( @@ -243,6 +238,52 @@ class LaneBasedExecutionQueue : public ExecutionQueue { } } + /// Returns the number of allowed foreground and background tasks. + static auto estimateTaskLimits(unsigned numLanes) -> std::pair { + llbuild_rlim_t curOpenFileLimit = llbuild::basic::sys::getOpenFileLimit(); + const unsigned reservedFileCount = (STDERR_FILENO+1) + 2 /* Database */ + + 1 /* Logging */ + + 2 /* Additional fds during spawn */ + + 2 /* Fudge factor */; + if (curOpenFileLimit < reservedFileCount) { + assert(curOpenFileLimit < reservedFileCount); + // Certainly can't afford background tasks. + // Maybe even can't afford building altogether, but let's risk it. + return std::make_pair(1, 0); + } + + unsigned allowedFilesForTasks = static_cast(std::min(curOpenFileLimit, static_cast(INT_MAX))) - reservedFileCount; + unsigned filesPerTask = 2; // A task has output [and control] file descriptors. + unsigned maxConcurrentTasks = allowedFilesForTasks / filesPerTask; + + if (numLanes > maxConcurrentTasks) { + // Can't afford background tasks, and maybe won't even support + // the full extent of requested concurrency. + numLanes = std::max(1u, maxConcurrentTasks); + return std::make_pair(numLanes, 0); + } + + // Number of tasks that can be run, according to open file limits. + unsigned extraTasksMax = maxConcurrentTasks - numLanes; + + // Configure the background task maximum. We currently support an + // environmental override for experimentation purposes, but otherwise + // limit to a modest multiple of the core count, since we currently burn + // one thread per background task. + unsigned backgroundTaskMax = 0; + char *p = getenv("LLBUILD_BACKGROUND_TASK_MAX"); + if (p && !StringRef(p).getAsInteger(10, backgroundTaskMax)) { + // Parsed. + } else { + backgroundTaskMax = std::min(1024U, numLanes * 64U); + } + + // The number of background can't exceed available concurrency. + backgroundTaskMax = std::min(backgroundTaskMax, extraTasksMax); + + return std::make_pair(numLanes, backgroundTaskMax); + } + virtual void addJob(QueueJob job) override { uint64_t readyJobsCount; { @@ -328,22 +369,15 @@ class LaneBasedExecutionQueue : public ExecutionQueue { handle.id = context.jobID; ProcessReleaseFn releaseFn = [this](std::function&& processWait) { - bool releaseAllowed = false; - // This check is not guaranteed to prevent more than backgroundTaskMax - // tasks from releasing. We could race between the check and increment and - // thus have a few extra. However, for our purposes, this should be fine. - // The cap is primarly intended to prevent runaway explosions of tasks. - if (backgroundTaskCount < backgroundTaskMax) { - backgroundTaskCount++; - releaseAllowed = true; - } - if (releaseAllowed) { + auto previousTaskCount = backgroundTaskCount.fetch_add(1); + if (previousTaskCount < backgroundTaskMax) { // Launch the process wait on a detached thread std::thread([this, processWait=std::move(processWait)]() mutable { processWait(); backgroundTaskCount--; }).detach(); } else { + backgroundTaskCount--; // not allowed to release, call wait directly processWait(); } diff --git a/lib/Basic/PlatformUtility.cpp b/lib/Basic/PlatformUtility.cpp index a851c86b..03d29149 100644 --- a/lib/Basic/PlatformUtility.cpp +++ b/lib/Basic/PlatformUtility.cpp @@ -212,6 +212,22 @@ int sys::write(int fileHandle, void *destinationBuffer, #endif } +// Get the current process' open file limit. Returns -1 on failure. +llbuild_rlim_t sys::getOpenFileLimit() { +#if defined(_WIN32) + int value = _getmaxstdio(); + return std::min(0, value); +#else + struct rlimit rl; + int ret = getrlimit(RLIMIT_NOFILE, &rl); + if (ret != 0) { + return 0; + } + + return rl.rlim_cur; +#endif +} + // Raise the open file limit, returns 0 on success, -1 on failure int sys::raiseOpenFileLimit(llbuild_rlim_t limit) { #if defined(_WIN32) diff --git a/tests/Ninja/Build/Inputs/close-control-fd b/tests/Ninja/Build/Inputs/close-control-fd deleted file mode 100755 index e73ed99d..00000000 --- a/tests/Ninja/Build/Inputs/close-control-fd +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -# Close the control channel to unblock llbuild -# Usage: `source ./close-control-fd ; other shell commands` - -if [ "${LLBUILD_CONTROL_FD}" -a "${LLBUILD_TASK_ID}" ]; then - printf "llbuild.1\n${LLBUILD_TASK_ID}\n" >&${LLBUILD_CONTROL_FD} - eval "exec ${LLBUILD_CONTROL_FD}>&-" -fi - diff --git a/tests/Ninja/Build/Inputs/run-releasing-control-fd b/tests/Ninja/Build/Inputs/run-releasing-control-fd new file mode 100755 index 00000000..093cf863 --- /dev/null +++ b/tests/Ninja/Build/Inputs/run-releasing-control-fd @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Close the control channel and run a given command. +# Example usage: +# +# > exec ./run-releasing-control-fd sleep 10 +# +# The `exec` prefix is necessary to prevent the file descriptor being held +# by the outer shell while running the `run-relesing-control-fd`. + +from __future__ import print_function +import subprocess +import sys +import os + +def usage(): + print("Expected: [--require-control-fd] [args...]") + sys.exit(1) + +def main(argv): + require_control_fd = False + + # Parse command line options + if len(argv) <= 1: + usage() + elif argv[1] == "--require-control-fd": + if len(argv) == 2: + usage() + argv=argv[1:] + require_control_fd = True + elif argv[1][0:1] == "-": + usage() + + exec_args = argv[1:] + + # Close the control descriptor, if given. + control_fd = os.environ.get("LLBUILD_CONTROL_FD") + if control_fd is None and require_control_fd: + print("%s: missing LLBUILD_CONTROL_FD" % exec_args[0], file=sys.stderr) + sys.exit(1) + elif control_fd is not None: + del os.environ["LLBUILD_CONTROL_FD"] + os.close(int(control_fd)) + + use_shell = len(exec_args) == 1 and ' ' in exec_args[0] + subprocess.check_call(exec_args, shell=use_shell) + +if __name__ == '__main__': + main(sys.argv) diff --git a/tests/Ninja/Build/Inputs/ulimited b/tests/Ninja/Build/Inputs/ulimited new file mode 100755 index 00000000..b54f5ace --- /dev/null +++ b/tests/Ninja/Build/Inputs/ulimited @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +# Example usage: +# > ./ulimited -n 50 echo foo +# Prior RLIMIT_NOFILE: soft=256, hard=RLIM_INFINITY +# Reset RLIMIT_NOFILE: soft=50, hard=RLIM_INFINITY +# foo +# > + + +from __future__ import print_function +import errno +import subprocess +import resource +import sys +import os + + +def main(argv): + if len(argv) < 4 or argv[1] != "-n": + print("Expected: -n [args...]") + sys.exit(1) + + try: + new_limit = int(argv[2]) + except ValueError as e: + print("-n: %s" % e) + sys.exit(os.EX_USAGE) + + execute_with_limit(new_limit, argv[3:]) + + print("%s: Cannot execute", argv[3]) + exit(os.EX_OSERR) + + +def execute_with_limit(new_limit, arguments): + max_fd_soft, max_fd_hard = get_and_show_limit("Prior", "RLIMIT_NOFILE") + resource.setrlimit(resource.RLIMIT_NOFILE, [new_limit, max_fd_hard]) + _ = get_and_show_limit("Reset", "RLIMIT_NOFILE") + os.execvp(arguments[0], arguments) + + +def stringify_limit(limit): + if limit == resource.RLIM_INFINITY: + return "RLIM_INFINITY" + return limit + + +def get_and_show_limit(prefix, name): + max_fd_soft, max_fd_hard = resource.getrlimit(getattr(resource, name)) + max_fd_soft_s = stringify_limit(max_fd_soft) + max_fd_hard_s = stringify_limit(max_fd_hard) + print("%s %s: {soft: %s, hard: %s}" + % (prefix, name, max_fd_soft_s, max_fd_hard_s), file=sys.stderr) + return (max_fd_soft, max_fd_hard) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/tests/Ninja/Build/console-pool-no-control-fd.ninja b/tests/Ninja/Build/console-pool-no-control-fd.ninja index 7d311859..4a7d26e1 100644 --- a/tests/Ninja/Build/console-pool-no-control-fd.ninja +++ b/tests/Ninja/Build/console-pool-no-control-fd.ninja @@ -3,7 +3,7 @@ # RUN: rm -rf %t.build # RUN: mkdir -p %t.build # RUN: cp %s %t.build/build.ninja -# RUN: cp %S/Inputs/close-control-fd %t.build +# RUN: cp %S/Inputs/run-releasing-control-fd %t.build # RUN: cp %S/Inputs/wait-for-file %t.build # RUN: %{llbuild} ninja build --jobs 3 --no-db --chdir %t.build &> %t.out # RUN: %{FileCheck} < %t.out %s @@ -22,7 +22,7 @@ rule CUSTOM command = ${COMMAND} build first: CUSTOM - command = touch executing && . ./close-control-fd && ./wait-for-file stop && rm -f executing + command = touch executing && exec ./run-releasing-control-fd "./wait-for-file stop && rm -f executing" build second: CUSTOM command = test ! -f executing diff --git a/tests/Ninja/Build/file-limit.ninja b/tests/Ninja/Build/file-limit.ninja new file mode 100644 index 00000000..94ec02a4 --- /dev/null +++ b/tests/Ninja/Build/file-limit.ninja @@ -0,0 +1,63 @@ +# Check operation under max file descriptor constraints + +# RUN: rm -rf %t.build +# RUN: mkdir -p %t.build +# RUN: cp %s %t.build/build.ninja +# RUN: cp %S/Inputs/run-releasing-control-fd %t.build +# RUN: cp %S/Inputs/ulimited %t.build +# RUN: %t.build/ulimited -n 30 %{llbuild} ninja build --jobs 02 --chdir %t.build &> %t.out +# RUN: %t.build/ulimited -n 40 %{llbuild} ninja build --jobs 04 --chdir %t.build &> %t.out +# RUN: %t.build/ulimited -n 60 %{llbuild} ninja build --jobs 10 --chdir %t.build &> %t.out +# RUN: %t.build/ulimited -n 80 %{llbuild} ninja build --jobs 10 --chdir %t.build &> %t.out +# +rule WAIT + command = exec ./run-releasing-control-fd "sleep 0.1" + +build 00: WAIT +build 01: WAIT +build 02: WAIT +build 03: WAIT +build 04: WAIT +build 05: WAIT +build 06: WAIT +build 07: WAIT +build 08: WAIT +build 09: WAIT +build 10: WAIT +build 11: WAIT +build 12: WAIT +build 13: WAIT +build 14: WAIT +build 15: WAIT +build 16: WAIT +build 17: WAIT +build 18: WAIT +build 19: WAIT +build 20: WAIT +build 21: WAIT +build 22: WAIT +build 23: WAIT +build 24: WAIT +build 25: WAIT +build 26: WAIT +build 27: WAIT +build 28: WAIT +build 29: WAIT +build 30: WAIT +build 31: WAIT +build 32: WAIT +build 33: WAIT +build 34: WAIT +build 35: WAIT +build 36: WAIT +build 37: WAIT +build 38: WAIT +build 39: WAIT + +build output: phony $ + 00 01 02 03 04 05 06 07 08 09 $ + 10 11 12 13 14 15 16 17 18 19 $ + 20 21 22 23 24 25 26 27 28 29 $ + 30 31 32 33 34 35 36 37 38 39 + +default output