diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6d40313d509..88a734efc84 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,5 +1,6 @@ #include "nix/store/derivations.hh" #include "nix/store/globals.hh" +#include "nix/store/local-gc.hh" #include "nix/store/local-store.hh" #include "nix/store/path.hh" #include "nix/util/finally.hh" @@ -12,11 +13,6 @@ #include "store-config-private.hh" -#if !defined(__linux__) -// For shelling out to lsof -# include "nix/util/processes.hh" -#endif - #include #include #include @@ -310,151 +306,18 @@ Roots LocalStore::findRoots(bool censor) return roots; } -/** - * Key is a mere string because cannot has path with macOS's libc++ - */ -typedef boost::unordered_flat_map< - std::string, - boost::unordered_flat_set>, - StringViewHash, - std::equal_to<>> - UncheckedRoots; - -static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) -{ - std::filesystem::path buf; - try { - buf = std::filesystem::read_symlink(file); - } catch (std::filesystem::filesystem_error & e) { - if (e.code() == std::errc::no_such_file_or_directory || e.code() == std::errc::permission_denied - || e.code() == std::errc::no_such_process) - return; - throw; - } - if (buf.is_absolute()) - roots[buf.string()].emplace(file.string()); -} - -static std::string quoteRegexChars(const std::string & raw) -{ - static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])"); - return boost::regex_replace(raw, specialRegex, R"(\\$&)"); -} - -#ifdef __linux__ -static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots) -{ - try { - roots[readFile(path)].emplace(path.string()); - } catch (SystemError & e) { - if (!e.is(std::errc::no_such_file_or_directory) && !e.is(std::errc::permission_denied)) - throw; - } -} -#endif - void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { - UncheckedRoots unchecked; - - auto procDir = AutoCloseDir{opendir("/proc")}; - if (procDir) { - struct dirent * ent; - static const auto digitsRegex = boost::regex(R"(^\d+$)"); - static const auto mapRegex = boost::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); - auto storePathRegex = boost::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); - while (errno = 0, ent = readdir(procDir.get())) { - checkInterrupt(); - if (boost::regex_match(ent->d_name, digitsRegex)) { - try { - readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked); - readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - - auto fdStr = fmt("/proc/%s/fd", ent->d_name); - auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); - if (!fdDir) { - if (errno == ENOENT || errno == EACCES) - continue; - throw SysError("opening %1%", fdStr); - } - struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') - readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); - } - if (errno) { - if (errno == ESRCH) - continue; - throw SysError("iterating /proc/%1%/fd", ent->d_name); - } - fdDir.reset(); - - std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString>(readFile(mapFile.string()), "\n"); - for (const auto & line : mapLines) { - auto match = boost::smatch{}; - if (boost::regex_match(line, match, mapRegex)) - unchecked[match[1]].emplace(mapFile.string()); - } + auto unchecked = findRuntimeRootsUnchecked(*config); - auto envFile = fmt("/proc/%s/environ", ent->d_name); - auto envString = readFile(envFile); - auto env_end = boost::sregex_iterator{}; - for (auto i = boost::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; - i != env_end; - ++i) - unchecked[i->str()].emplace(envFile); - } catch (SystemError & e) { - if (errno == ENOENT || errno == EACCES || errno == ESRCH) - continue; - throw; - } - } - } - if (errno) - throw SysError("iterating /proc"); - } - -#if !defined(__linux__) - // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. - // See: https://github.com/NixOS/nix/issues/3011 - // Because of this we disable lsof when running the tests. - if (getEnv("_NIX_TEST_NO_LSOF") != "1") { - try { - boost::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString>(runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n"); - for (const auto & line : lsofLines) { - boost::smatch match; - if (boost::regex_match(line, match, lsofRegex)) - unchecked[match[1].str()].emplace("{lsof}"); - } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ - } - } -#endif - -#ifdef __linux__ - readFileRoots("/proc/sys/kernel/modprobe", unchecked); - readFileRoots("/proc/sys/kernel/fbsplash", unchecked); - readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); -#endif - - for (auto & [target, links] : unchecked) { - if (!isInStore(target)) + for (auto & [path, links] : unchecked) { + if (!isValidPath(path)) continue; - try { - auto path = toStorePath(target).first; - if (!isValidPath(path)) - continue; - debug("got additional root '%1%'", printStorePath(path)); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); - } catch (BadStorePath &) { - } + debug("got additional root '%1%'", printStorePath(path)); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); } } diff --git a/src/libstore/include/nix/store/local-gc.hh b/src/libstore/include/nix/store/local-gc.hh new file mode 100644 index 00000000000..d609626e2f5 --- /dev/null +++ b/src/libstore/include/nix/store/local-gc.hh @@ -0,0 +1,20 @@ +#include "nix/store/gc-store.hh" +#include +#include + +namespace nix { + +/** + * Finds a list of "runtime roots", i.e. store paths currently open, + * mapped, or in the environment of a process and should not be deleted. + * + * This function does not attempt to check the nix database and find if paths are + * valid. It may return paths in the store that look like nix paths, + * but are not known to the nix daemon or may not even exist. + * + * @param config Configuration for the store, needed to find the store dir + * @return a map from store paths to processes that are using them + */ +Roots findRuntimeRootsUnchecked(const StoreDirConfig & config); + +} // namespace nix diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index c7026818ea7..724d68ece19 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -50,6 +50,7 @@ headers = [ config_pub_h ] + files( 'length-prefixed-protocol-helper.hh', 'local-binary-cache-store.hh', 'local-fs-store.hh', + 'local-gc.hh', 'local-overlay-store.hh', 'local-store.hh', 'log-store.hh', diff --git a/src/libstore/local-gc.cc b/src/libstore/local-gc.cc new file mode 100644 index 00000000000..9e3bbfb5ccc --- /dev/null +++ b/src/libstore/local-gc.cc @@ -0,0 +1,165 @@ +#include "nix/store/gc-store.hh" +#include "nix/store/store-dir-config.hh" +#include "nix/util/file-system.hh" +#include "nix/util/signals.hh" +#include "nix/util/types.hh" +#include "nix/store/local-gc.hh" +#include +#include + +#if !defined(__linux__) +// For shelling out to lsof +# include "store-config-private.hh" +# include "nix/util/environment-variables.hh" +# include "nix/util/processes.hh" +#endif + +namespace nix { + +/** + * Key is a mere string because cannot has path with macOS's libc++ + */ +typedef boost::unordered_flat_map< + std::string, + boost::unordered_flat_set>, + StringViewHash, + std::equal_to<>> + UncheckedRoots; + +static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) +{ + std::filesystem::path buf; + try { + buf = std::filesystem::read_symlink(file); + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::no_such_file_or_directory || e.code() == std::errc::permission_denied + || e.code() == std::errc::no_such_process) + return; + throw; + } + if (buf.is_absolute()) + roots[buf.string()].emplace(file.string()); +} + +static std::string quoteRegexChars(const std::string & raw) +{ + static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])"); + return boost::regex_replace(raw, specialRegex, R"(\\$&)"); +} + +#ifdef __linux__ +static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots) +{ + try { + roots[readFile(path)].emplace(path.string()); + } catch (SystemError & e) { + if (!e.is(std::errc::no_such_file_or_directory) && !e.is(std::errc::permission_denied)) + throw; + } +} +#endif + +Roots findRuntimeRootsUnchecked(const StoreDirConfig & config) +{ + UncheckedRoots unchecked; + + auto procDir = AutoCloseDir{opendir("/proc")}; + if (procDir) { + struct dirent * ent; + static const auto digitsRegex = boost::regex(R"(^\d+$)"); + static const auto mapRegex = boost::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); + auto storePathRegex = boost::regex(quoteRegexChars(config.storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); + while (errno = 0, ent = readdir(procDir.get())) { + checkInterrupt(); + if (boost::regex_match(ent->d_name, digitsRegex)) { + try { + readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) + continue; + throw SysError("opening %1%", fdStr); + } + struct dirent * fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + if (errno) { + if (errno == ESRCH) + continue; + throw SysError("iterating /proc/%1%/fd", ent->d_name); + } + fdDir.reset(); + + std::filesystem::path mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString>(readFile(mapFile.string()), "\n"); + for (const auto & line : mapLines) { + auto match = boost::smatch{}; + if (boost::regex_match(line, match, mapRegex)) + unchecked[match[1]].emplace(mapFile.string()); + } + + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile); + auto env_end = boost::sregex_iterator{}; + for (auto i = boost::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; + i != env_end; + ++i) + unchecked[i->str()].emplace(envFile); + } catch (SystemError & e) { + if (errno == ENOENT || errno == EACCES || errno == ESRCH) + continue; + throw; + } + } + } + if (errno) + throw SysError("iterating /proc"); + } + +#if !defined(__linux__) + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") != "1") { + try { + boost::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = + tokenizeString>(runProgram(LSOF, true, {"-n", "-w", "-F", "n"}), "\n"); + for (const auto & line : lsofLines) { + boost::smatch match; + if (boost::regex_match(line, match, lsofRegex)) + unchecked[match[1].str()].emplace("{lsof}"); + } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ + } + } +#endif + +#ifdef __linux__ + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); +#endif + + Roots roots; + + for (auto & [target, links] : unchecked) { + if (!config.isInStore(target)) + continue; + try { + auto path = config.toStorePath(target).first; + roots[path].insert(links.begin(), links.end()); + } catch (BadStorePath &) { + } + } + + return roots; +} + +} // namespace nix diff --git a/src/libstore/meson.build b/src/libstore/meson.build index a01aa903ae2..3bd4796b73e 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -315,6 +315,7 @@ sources = files( 'legacy-ssh-store.cc', 'local-binary-cache-store.cc', 'local-fs-store.cc', + 'local-gc.cc', 'local-overlay-store.cc', 'local-store.cc', 'log-store.cc',