diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 3f00ff6444c..ae8dab8d015 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -408,8 +408,12 @@ void Worker::waitForInput() is a build timeout, then wait for input until the first deadline for any child. */ auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. + + auto localStore = dynamic_cast(&store); + if (localStore && localStore->config->getGCSettings().minFree.get() != 0) + // If we have a local store (and thus are capable of automatically collecting garbage) and configured to do so, + // periodically wake up to see if we need to run the garbage collector. (See the `autoGC` call site above in + // this file, also gated on having a local store. when we wake up, we intended to reach that call site.) nearest = before + std::chrono::seconds(10); for (auto & i : children) { if (!i.respectTimeouts) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 669977f5230..ad97d6fae9a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -465,9 +465,11 @@ struct GCLimitReached void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { + const auto & gcSettings = config->getGCSettings(); + bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; - bool gcKeepOutputs = settings.gcKeepOutputs; - bool gcKeepDerivations = settings.gcKeepDerivations; + bool keepOutputs = gcSettings.keepOutputs; + bool keepDerivations = gcSettings.keepDerivations; boost::unordered_flat_set> roots, dead, alive; @@ -491,8 +493,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) (the garbage collector will recurse into deleting the outputs or derivers, respectively). So disable them. */ if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - gcKeepOutputs = false; - gcKeepDerivations = false; + keepOutputs = false; + keepDerivations = false; } if (shouldDelete) @@ -728,8 +730,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) *path, closure, /* flipDirection */ false, - gcKeepOutputs, - gcKeepDerivations); + keepOutputs, + keepDerivations); for (auto & p : closure) alive.insert(p); } catch (InvalidPath &) { @@ -770,7 +772,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* If keep-derivations is set and this is a derivation, then visit the derivation outputs. */ - if (gcKeepDerivations && path->isDerivation()) { + if (keepDerivations && path->isDerivation()) { for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) if (maybeOutPath && isValidPath(*maybeOutPath) && queryPathInfo(*maybeOutPath)->deriver == *path) @@ -778,7 +780,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } /* If keep-outputs is set, then visit the derivers. */ - if (gcKeepOutputs) { + if (keepOutputs) { auto derivers = queryValidDerivers(*path); for (auto & i : derivers) enqueue(i); @@ -920,6 +922,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { #if HAVE_STATVFS + const auto & gcSettings = config->getGCSettings(); + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { @@ -946,14 +950,14 @@ void LocalStore::autoGC(bool sync) auto now = std::chrono::steady_clock::now(); - if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) + if (now < state->lastGCCheck + std::chrono::seconds(gcSettings.minFreeCheckInterval)) return; auto avail = getAvail(); state->lastGCCheck = now; - if (avail >= settings.minFree || avail >= settings.maxFree) + if (avail >= gcSettings.minFree || avail >= gcSettings.maxFree) return; if (avail > state->availAfterGC * 0.97) @@ -964,7 +968,7 @@ void LocalStore::autoGC(bool sync) std::promise promise; future = state->gcFuture = promise.get_future().share(); - std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { + std::thread([promise{std::move(promise)}, this, avail, getAvail, &gcSettings]() mutable { try { /* Wake up any threads waiting for the auto-GC to finish. */ @@ -976,7 +980,7 @@ void LocalStore::autoGC(bool sync) }); GCOptions options; - options.maxFreed = settings.maxFree - avail; + options.maxFreed = gcSettings.maxFree - avail; printInfo("running auto-GC to free %d bytes", options.maxFreed); diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 454b7f3e5a2..9e8eb0dffab 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -62,7 +62,88 @@ const uint32_t maxIdsPerBuild = #endif ; -class Settings : public Config +struct GCSettings : public virtual Config +{ + Setting reservedSize{ + this, + 8 * 1024 * 1024, + "gc-reserved-space", + "Amount of reserved disk space for the garbage collector.", + }; + + Setting keepOutputs{ + this, + false, + "keep-outputs", + R"( + If `true`, the garbage collector keeps the outputs of + non-garbage derivations. If `false` (default), outputs are + deleted unless they are GC roots themselves (or reachable from other + roots). + + In general, outputs must be registered as roots separately. However, + even if the output of a derivation is registered as a root, the + collector still deletes store paths that are used only at build + time (e.g., the C compiler, or source tarballs downloaded from the + network). To prevent it from doing so, set this option to `true`. + )", + {"gc-keep-outputs"}, + }; + + Setting keepDerivations{ + this, + true, + "keep-derivations", + R"( + If `true` (default), the garbage collector keeps the derivations + from which non-garbage store paths were built. If `false`, they are + deleted unless explicitly registered as a root (or reachable from + other roots). + + Keeping derivation around is useful for querying and traceability + (e.g., it allows you to ask with what dependencies or options a + store path was built), so by default this option is on. Turn it off + to save a bit of disk space (or a lot if `keep-outputs` is also + turned on). + )", + {"gc-keep-derivations"}, + }; + + Setting minFree{ + this, + 0, + "min-free", + R"( + When free disk space in `/nix/store` drops below `min-free` during a + build, Nix performs a garbage-collection until `max-free` bytes are + available or there is no more garbage. A value of `0` (the default) + disables this feature. + )", + }; + + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + Setting maxFree{ + this, + std::numeric_limits::max(), + "max-free", + R"( + When a garbage collection is triggered by the `min-free` option, it + stops as soon as `max-free` bytes are available. The default is + infinity (i.e. delete all garbage). + )", + }; + + Setting minFreeCheckInterval{ + this, + 5, + "min-free-check-interval", + "Number of seconds between checking free disk space.", + }; +}; + +class Settings : public virtual Config, private GCSettings { StringSet getDefaultSystemFeatures(); @@ -77,6 +158,19 @@ public: Settings(); + /** + * Get the GC settings. + */ + GCSettings & getGCSettings() + { + return *this; + } + + const GCSettings & getGCSettings() const + { + return *this; + } + static unsigned int getDefaultCores(); /** @@ -427,9 +521,6 @@ public: This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. )"}; - Setting reservedSize{ - this, 8 * 1024 * 1024, "gc-reserved-space", "Amount of reserved disk space for the garbage collector."}; - Setting fsyncMetadata{ this, true, @@ -601,42 +692,6 @@ public: Setting pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; - Setting gcKeepOutputs{ - this, - false, - "keep-outputs", - R"( - If `true`, the garbage collector keeps the outputs of - non-garbage derivations. If `false` (default), outputs are - deleted unless they are GC roots themselves (or reachable from other - roots). - - In general, outputs must be registered as roots separately. However, - even if the output of a derivation is registered as a root, the - collector still deletes store paths that are used only at build - time (e.g., the C compiler, or source tarballs downloaded from the - network). To prevent it from doing so, set this option to `true`. - )", - {"gc-keep-outputs"}}; - - Setting gcKeepDerivations{ - this, - true, - "keep-derivations", - R"( - If `true` (default), the garbage collector keeps the derivations - from which non-garbage store paths were built. If `false`, they are - deleted unless explicitly registered as a root (or reachable from - other roots). - - Keeping derivation around is useful for querying and traceability - (e.g., it allows you to ask with what dependencies or options a - store path was built), so by default this option is on. Turn it off - to save a bit of disk space (or a lot if `keep-outputs` is also - turned on). - )", - {"gc-keep-derivations"}}; - Setting autoOptimiseStore{ this, false, @@ -1252,32 +1307,6 @@ public: first. If it is not available there, it tries the original URI. )"}; - Setting minFree{ - this, - 0, - "min-free", - R"( - When free disk space in `/nix/store` drops below `min-free` during a - build, Nix performs a garbage-collection until `max-free` bytes are - available or there is no more garbage. A value of `0` (the default) - disables this feature. - )"}; - - Setting maxFree{// n.b. this is deliberately int64 max rather than uint64 max because - // this goes through the Nix language JSON parser and thus needs to be - // representable in Nix language integers. - this, - std::numeric_limits::max(), - "max-free", - R"( - When a garbage collection is triggered by the `min-free` option, it - stops as soon as `max-free` bytes are available. The default is - infinity (i.e. delete all garbage). - )"}; - - Setting minFreeCheckInterval{ - this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - Setting narBufferSize{ this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 5460c6e786a..320e9ff9035 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -31,6 +31,8 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; +struct GCSettings; + struct LocalBuildStoreConfig : virtual LocalFSStoreConfig { @@ -85,6 +87,11 @@ private: bool getDefaultRequireSigs(); public: + /** + * For now, this just grabs the global GC settings, but by having this method we get ready for these being per-store + * settings instead. + */ + const GCSettings & getGCSettings() const; Setting requireSigs{ this, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 17f0216b581..1e255068ff6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -140,6 +140,7 @@ LocalStore::LocalStore(ref config) createDirs(tempRootsDir); createDirs(dbDir); Path gcRootsDir = config->stateDir + "/gcroots"; + const auto & gcSettings = settings.getGCSettings(); if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); replaceSymlink(profilesDir, gcRootsDir + "/profiles"); @@ -200,7 +201,7 @@ LocalStore::LocalStore(ref config) before doing a garbage collection. */ try { struct stat st; - if (stat(reservedPath.c_str(), &st) == -1 || st.st_size != settings.reservedSize) { + if (stat(reservedPath.c_str(), &st) == -1 || st.st_size != gcSettings.reservedSize) { AutoCloseFD fd = toDescriptor(open( reservedPath.c_str(), O_WRONLY | O_CREAT @@ -211,16 +212,16 @@ LocalStore::LocalStore(ref config) 0600)); int res = -1; #if HAVE_POSIX_FALLOCATE - res = posix_fallocate(fd.get(), 0, settings.reservedSize); + res = posix_fallocate(fd.get(), 0, gcSettings.reservedSize); #endif if (res == -1) { - writeFull(fd.get(), std::string(settings.reservedSize, 'X')); + writeFull(fd.get(), std::string(gcSettings.reservedSize, 'X')); [[gnu::unused]] auto res2 = #ifdef _WIN32 SetEndOfFile(fd.get()) #else - ftruncate(fd.get(), settings.reservedSize) + ftruncate(fd.get(), gcSettings.reservedSize) #endif ; } @@ -438,6 +439,11 @@ LocalStore::~LocalStore() } } +const GCSettings & LocalStoreConfig::getGCSettings() const +{ + return settings.getGCSettings(); +} + StoreReference LocalStoreConfig::getReference() const { auto params = getQueryParams(); diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 4e57723634a..b3926d9e6a5 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -503,7 +503,8 @@ static void opQuery(Strings opFlags, Strings opArgs) args.insert(p); StorePathSet referrers; - store->computeFSClosure(args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); + auto & gcSettings = settings.getGCSettings(); + store->computeFSClosure(args, referrers, true, gcSettings.keepOutputs, gcSettings.keepDerivations); auto & gcStore = require(*store); Roots roots = gcStore.findRoots(false);