diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 35ae73e5d1f..bf94c24e286 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -19,7 +19,6 @@ in prefix, inlineHTML ? true, }: -settingsInfo: let @@ -27,11 +26,25 @@ let prefix: setting: { description, - documentDefault, - defaultValue, - aliases, - value, + experimentalFeature, + + # Whether we document the default, because it is machine agostic, + # or don't because because it is machine-specific. + documentDefault ? true, + + # The default value is JSON for new-style config, rather than then + # a string or boolean, for old-style config. + isJson ? false, + + defaultValue ? null, + + subSettings ? null, + + aliases ? [ ], + + # The current value for this setting. Purposefully unused. + value ? null, }: let result = squash '' @@ -50,7 +63,7 @@ let ${description} - **Default:** ${showDefault documentDefault defaultValue} + ${showDefaultOrSubSettings} ${showAliases aliases} ''; @@ -72,9 +85,24 @@ let > ``` ''; + showDefaultOrSubSettings = + if !isAttrs subSettings then + # No subsettings, instead single setting. Show the default value. + '' + **Default:** ${showDefault} + '' + else + # Indent the nested sub-settings, and append the outer setting name onto the prefix + indent " " '' + **Nullable sub-settings**: ${if subSettings.nullable then "true" else "false"} + ${builtins.trace prefix (showSettings "${prefix}-${setting}" subSettings.map)} + ''; + showDefault = - documentDefault: defaultValue: if documentDefault then + if isJson then + "`${builtins.toJSON defaultValue}`" + else # a StringMap value type is specified as a string, but # this shows the value type. The empty stringmap is `null` in # JSON, but that converts to `{ }` here. @@ -95,5 +123,7 @@ let in result; + showSettings = + prefix: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)); in -concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) +showSettings prefix diff --git a/meson.build b/meson.build index c072a482163..6042a438a12 100644 --- a/meson.build +++ b/meson.build @@ -26,8 +26,8 @@ subproject('nix') # Docs if get_option('doc-gen') - subproject('internal-api-docs') - subproject('external-api-docs') + #subproject('internal-api-docs') + #subproject('external-api-docs') if meson.can_run_host_binaries() subproject('nix-manual') endif diff --git a/src/libstore-tests/data/store-reference/auto.json b/src/libstore-tests/data/store-reference/auto.json new file mode 100644 index 00000000000..da510f64949 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto.json @@ -0,0 +1,3 @@ +{ + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/auto_param.json b/src/libstore-tests/data/store-reference/auto_param.json new file mode 100644 index 00000000000..41826c83b56 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto_param.json @@ -0,0 +1,4 @@ +{ + "root": "/foo/bar/baz", + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/local_1.json b/src/libstore-tests/data/store-reference/local_1.json new file mode 100644 index 00000000000..3b44e382b7f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_1.json @@ -0,0 +1,5 @@ +{ + "authority": "", + "root": "/foo/bar/baz", + "scheme": "local" +} diff --git a/src/libstore-tests/data/store-reference/local_2.json b/src/libstore-tests/data/store-reference/local_2.json new file mode 100644 index 00000000000..346b0cf225f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_2.json @@ -0,0 +1,5 @@ +{ + "authority": "/foo/bar/baz", + "scheme": "local", + "trusted": true +} diff --git a/src/libstore-tests/data/store-reference/ssh.json b/src/libstore-tests/data/store-reference/ssh.json new file mode 100644 index 00000000000..e5b0b367ffd --- /dev/null +++ b/src/libstore-tests/data/store-reference/ssh.json @@ -0,0 +1,4 @@ +{ + "authority": "localhost", + "scheme": "ssh" +} diff --git a/src/libstore-tests/data/store-reference/unix.json b/src/libstore-tests/data/store-reference/unix.json new file mode 100644 index 00000000000..8943e959dd6 --- /dev/null +++ b/src/libstore-tests/data/store-reference/unix.json @@ -0,0 +1,6 @@ +{ + "authority": "", + "max-connections": 7, + "scheme": "unix", + "trusted": true +} diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc index 3dd8137a329..cf2eb61608e 100644 --- a/src/libstore-tests/dummy-store.cc +++ b/src/libstore-tests/dummy-store.cc @@ -6,13 +6,25 @@ namespace nix { +TEST(DummyStore, constructConfig) +{ + DummyStoreConfig config{"dummy", "", {}}; + + EXPECT_EQ(config.storeDir, settings.nixStore); +} + +TEST(DummyStore, constructConfigNoAuthority) +{ + EXPECT_THROW(DummyStoreConfig("dummy", "not-allowed", {}), UsageError); +} + TEST(DummyStore, realisation_read) { initLibStore(/*loadConfig=*/false); auto store = [] { auto cfg = make_ref(StoreReference::Params{}); - cfg->readOnly = false; + cfg->readOnly = {false}; return cfg->openDummyStore(); }(); diff --git a/src/libstore-tests/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc index d60ecc424c5..bf518660f9b 100644 --- a/src/libstore-tests/legacy-ssh-store.cc +++ b/src/libstore-tests/legacy-ssh-store.cc @@ -12,13 +12,15 @@ TEST(LegacySSHStore, constructConfig) StoreConfig::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }); EXPECT_EQ( - config.remoteProgram.get(), + config.remoteProgram, (Strings{ "foo", "bar", diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index 175e5d0f44e..59418a528dd 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -17,14 +17,14 @@ TEST(LocalOverlayStore, constructConfig_rootQueryParam) }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir, std::optional{"/foo/bar"}); } TEST(LocalOverlayStore, constructConfig_rootPath) { LocalOverlayStoreConfig config{"local-overlay", "/foo/bar", {}}; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir, std::optional{"/foo/bar"}); } } // namespace nix diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index d008888974b..e0a2549e5e0 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -2,12 +2,6 @@ #include "nix/store/local-store.hh" -// Needed for template specialisations. This is not good! When we -// overhaul how store configs work, this should be fixed. -#include "nix/util/args.hh" -#include "nix/util/config-impl.hh" -#include "nix/util/abstract-setting-to-json.hh" - namespace nix { TEST(LocalStore, constructConfig_rootQueryParam) @@ -23,14 +17,14 @@ TEST(LocalStore, constructConfig_rootQueryParam) }, }; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir, std::optional{"/foo/bar"}); } TEST(LocalStore, constructConfig_rootPath) { LocalStoreConfig config{"local", "/foo/bar", {}}; - EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); + EXPECT_EQ(config.rootDir, std::optional{"/foo/bar"}); } TEST(LocalStore, constructConfig_to_string) diff --git a/src/libstore-tests/s3-binary-cache-store.cc b/src/libstore-tests/s3-binary-cache-store.cc index 59090a589f0..0444a231fe7 100644 --- a/src/libstore-tests/s3-binary-cache-store.cc +++ b/src/libstore-tests/s3-binary-cache-store.cc @@ -32,7 +32,7 @@ TEST(S3BinaryCacheStore, constructConfigWithRegion) .authority = ParsedURL::Authority{.host = "my-bucket"}, .query = (StringMap) {{"region", "eu-west-1"}}, })); - EXPECT_EQ(config.region.get(), "eu-west-1"); + EXPECT_EQ(config.region, "eu-west-1"); } TEST(S3BinaryCacheStore, defaultSettings) @@ -47,10 +47,10 @@ TEST(S3BinaryCacheStore, defaultSettings) })); // Check default values - EXPECT_EQ(config.region.get(), "us-east-1"); - EXPECT_EQ(config.profile.get(), "default"); - EXPECT_EQ(config.scheme.get(), "https"); - EXPECT_EQ(config.endpoint.get(), ""); + EXPECT_EQ(config.region, "us-east-1"); + EXPECT_EQ(config.profile, "default"); + EXPECT_EQ(config.scheme, "https"); + EXPECT_EQ(config.endpoint, ""); } /** @@ -58,7 +58,7 @@ TEST(S3BinaryCacheStore, defaultSettings) */ TEST(S3BinaryCacheStore, s3StoreConfigPreservesParameters) { - StringMap params; + StoreReference::Params params; params["region"] = "eu-west-1"; params["endpoint"] = "custom.s3.com"; @@ -93,7 +93,7 @@ TEST(S3BinaryCacheStore, s3SchemeRegistration) */ TEST(S3BinaryCacheStore, parameterFiltering) { - StringMap params; + StoreReference::Params params; params["region"] = "eu-west-1"; params["endpoint"] = "minio.local"; params["want-mass-query"] = "true"; // Non-S3 store parameter @@ -111,8 +111,8 @@ TEST(S3BinaryCacheStore, parameterFiltering) })); // But the non-S3 params should still be set on the config - EXPECT_EQ(config.wantMassQuery.get(), true); - EXPECT_EQ(config.priority.get(), 10); + EXPECT_EQ(config.wantMassQuery, true); + EXPECT_EQ(config.priority, 10); // And all params (S3 and non-S3) should be returned by getReference() auto ref = config.getReference(); @@ -128,16 +128,16 @@ TEST(S3BinaryCacheStore, parameterFiltering) TEST(S3BinaryCacheStore, storageClassDefault) { S3BinaryCacheStoreConfig config{"s3", "test-bucket", {}}; - EXPECT_EQ(config.storageClass.get(), std::nullopt); + EXPECT_EQ(config.storageClass, std::nullopt); } TEST(S3BinaryCacheStore, storageClassConfiguration) { - StringMap params; + StoreReference::Params params; params["storage-class"] = "GLACIER"; - S3BinaryCacheStoreConfig config("s3", "test-bucket", params); - EXPECT_EQ(config.storageClass.get(), std::optional("GLACIER")); + S3BinaryCacheStoreConfig config{"s3", "test-bucket", params}; + EXPECT_EQ(config.storageClass, std::optional("GLACIER")); } } // namespace nix diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index a156da52b71..1fb23467092 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -1,8 +1,6 @@ #include #include "nix/store/ssh-store.hh" -#include "nix/util/config-impl.hh" -#include "nix/util/abstract-setting-to-json.hh" namespace nix { @@ -14,44 +12,98 @@ TEST(SSHStore, constructConfig) StoreConfig::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }, }; EXPECT_EQ( - config.remoteProgram.get(), + config.remoteProgram, (Strings{ "foo", "bar", })); EXPECT_EQ(config.getReference().render(/*withParams=*/true), "ssh-ng://me@localhost:2222?remote-program=foo%20bar"); - config.resetOverridden(); + config.authority.port = std::nullopt; EXPECT_EQ(config.getReference().render(/*withParams=*/true), "ssh-ng://me@localhost:2222"); } TEST(MountedSSHStore, constructConfig) { - MountedSSHStoreConfig config{ - "mounted-ssh", + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + + SSHStoreConfig config{ + "ssh-ng", + "localhost", + StoreConfig::Params{ + { + "remote-program", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{}, + }, + }, + mockXpSettings, + }; + + EXPECT_EQ( + config.remoteProgram, + (Strings{ + "foo", + "bar", + })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/nix/store"); +} + +TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir) +{ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + + SSHStoreConfig config{ + "ssh-ng", "localhost", StoreConfig::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{ + {"real", "/foo/bar"}, + }, }, }, + mockXpSettings, }; EXPECT_EQ( - config.remoteProgram.get(), + config.remoteProgram, (Strings{ "foo", "bar", })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/foo/bar"); } } // namespace nix diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index 272d6732a85..c2368ff5018 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -17,14 +17,14 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest std::filesystem::path goldenMaster(PathView testStem) const override { - return unitTestData / (testStem + ".txt"); + return unitTestData / testStem; } }; #define URI_TEST_READ(STEM, OBJ) \ TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \ { \ - readTest(#STEM, ([&](const auto & encoded) { \ + readTest(#STEM ".txt", ([&](const auto & encoded) { \ StoreReference expected = OBJ; \ auto got = StoreReference::parse(encoded); \ ASSERT_EQ(got, expected); \ @@ -35,7 +35,7 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \ { \ writeTest( \ - #STEM, \ + #STEM ".txt", \ [&]() -> StoreReference { return OBJ; }, \ [](const auto & file) { return StoreReference::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \ @@ -45,14 +45,43 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest URI_TEST_READ(STEM, OBJ) \ URI_TEST_WRITE(STEM, OBJ) -URI_TEST( +#define JSON_TEST_READ(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM ".json", ([&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + StoreReference expected = OBJ; \ + StoreReference got = encoded; \ + ASSERT_EQ(got, expected); \ + })); \ + } + +#define JSON_TEST_WRITE(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM ".json", \ + [&]() -> StoreReference { return OBJ; }, \ + [](const auto & file) -> StoreReference { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, json(got).dump(2) + "\n"); }); \ + } + +#define JSON_TEST(STEM, OBJ) \ + JSON_TEST_READ(STEM, OBJ) \ + JSON_TEST_WRITE(STEM, OBJ) + +#define BOTH_FORMATS_TEST(STEM, OBJ) \ + URI_TEST(STEM, OBJ) \ + JSON_TEST(STEM, OBJ) + +BOTH_FORMATS_TEST( auto, (StoreReference{ .variant = StoreReference::Auto{}, .params = {}, })) -URI_TEST( +BOTH_FORMATS_TEST( auto_param, (StoreReference{ .variant = StoreReference::Auto{}, @@ -81,7 +110,7 @@ static StoreReference localExample_2{ }, .params = { - {"trusted", "true"}, + {"trusted", true}, }, }; @@ -96,9 +125,9 @@ static StoreReference localExample_3{ }, }; -URI_TEST(local_1, localExample_1) +BOTH_FORMATS_TEST(local_1, localExample_1) -URI_TEST(local_2, localExample_2) +BOTH_FORMATS_TEST(local_2, localExample_2) /* Test path with encoded spaces */ URI_TEST(local_3, localExample_3) @@ -124,16 +153,16 @@ static StoreReference unixExample{ }, .params = { - {"max-connections", "7"}, - {"trusted", "true"}, + {"max-connections", 7}, + {"trusted", true}, }, }; -URI_TEST(unix, unixExample) +BOTH_FORMATS_TEST(unix, unixExample) URI_TEST_READ(unix_shorthand, unixExample) -URI_TEST( +BOTH_FORMATS_TEST( ssh, (StoreReference{ .variant = diff --git a/src/libstore-tests/write-derivation.cc b/src/libstore-tests/write-derivation.cc index c320f92faf3..9eab14bc74d 100644 --- a/src/libstore-tests/write-derivation.cc +++ b/src/libstore-tests/write-derivation.cc @@ -16,7 +16,7 @@ class WriteDerivationTest : public LibStoreTest : LibStoreTest(config_->openDummyStore()) , config(std::move(config_)) { - config->readOnly = false; + config->readOnly = {false}; } WriteDerivationTest() @@ -45,7 +45,7 @@ TEST_F(WriteDerivationTest, addToStoreFromDumpCalledOnce) auto drv = makeSimpleDrv(); auto path1 = writeDerivation(*store, drv, NoRepair); - config->readOnly = true; + config->readOnly = {true}; auto path2 = writeDerivation(*store, drv, NoRepair); EXPECT_EQ(path1, path2); EXPECT_THAT( diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index caae7247948..86476590137 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -13,6 +13,7 @@ #include "nix/util/callback.hh" #include "nix/util/signals.hh" #include "nix/util/archive.hh" +#include "nix/store/config-parse-impl.hh" #include #include @@ -24,23 +25,130 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(Config & config) - : config{config} +constexpr static const BinaryCacheStoreConfigT binaryCacheStoreConfigDescriptions = { + .compression{ + { + .name = "compression", + .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", + }, + { + .makeDefault = []() -> std::string { return "xz"; }, + }, + }, + .writeNARListing{ + { + .name = "write-nar-listing", + .description = "Whether to write a JSON file that lists the files in each NAR.", + }, + { + .makeDefault = [] { return false; }, + }, + }, + .writeDebugInfo{ + { + .name = "index-debug-info", + .description = R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )", + }, + { + .makeDefault = [] { return false; }, + }, + }, + .secretKeyFile{ + { + .name = "secret-key", + .description = "Path to the secret key used to sign the binary cache.", + }, + { + .makeDefault = []() -> Path { return ""; }, + }, + }, + .secretKeyFiles{ + { + .name = "secret-keys", + .description = "List of paths to the secret keys used to sign the binary cache.", + }, + { + .makeDefault = []() -> std::vector { return {}; }, + }, + }, + .localNarCache{ + { + .name = "local-nar-cache", + .description = + "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", + }, + { + .makeDefault = []() -> Path { return ""; }, + }, + }, + .parallelCompression{ + { + .name = "parallel-compression", + .description = + "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", + }, + { + .makeDefault = [] { return false; }, + }, + }, + .compressionLevel{ + { + .name = "compression-level", + .description = R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )", + }, + { + .makeDefault = [] { return -1; }, + }, + }, +}; + +#define BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(compression), X(writeNARListing), X(writeDebugInfo), X(secretKeyFile), X(secretKeyFiles), X(localNarCache), \ + X(parallelCompression), X(compressionLevel), + +MAKE_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +BinaryCacheStore::Config::BinaryCacheStoreConfig(const Store::Config & storeConfig, const StoreConfig::Params & params) + : BinaryCacheStoreConfigT{binaryCacheStoreConfigApplyParse(params)} + , storeConfig{storeConfig} +{ +} + +config::SettingDescriptionMap BinaryCacheStoreConfig::descriptions() +{ + constexpr auto & descriptions = binaryCacheStoreConfigDescriptions; + return {BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}; +} + +BinaryCacheStore::BinaryCacheStore(const Config & config) + : Store{config.storeConfig} + , config{config} { if (config.secretKeyFile != "") signers.push_back(std::make_unique(SecretKey{readFile(config.secretKeyFile)})); - if (config.secretKeyFiles != "") { - std::stringstream ss(config.secretKeyFiles); - Path keyPath; - while (std::getline(ss, keyPath, ',')) { - signers.push_back(std::make_unique(SecretKey{readFile(keyPath)})); - } + for (auto & keyPath : config.secretKeyFiles) { + signers.push_back(std::make_unique(SecretKey{readFile(keyPath)})); } StringSink sink; sink << narVersionMagic1; narMagic = sink.s; + + // Want to call this but cannot, because virtual function lookup is + // disabled in a constructor. It is thus left to instances to call + // it instead. + + // init(); } void BinaryCacheStore::init() @@ -59,13 +167,13 @@ void BinaryCacheStore::init() if (value != storeDir) throw Error( "binary cache '%s' is for Nix stores with prefix '%s', not '%s'", - config.getHumanReadableURI(), + config.storeConfig.getHumanReadableURI(), value, storeDir); } else if (name == "WantMassQuery") { - config.wantMassQuery.setDefault(value == "1"); + resolvedSubstConfig.wantMassQuery = config.storeConfig.wantMassQuery.value_or(value == "1"); } else if (name == "Priority") { - config.priority.setDefault(std::stoi(value)); + resolvedSubstConfig.priority = config.storeConfig.priority.value_or(std::stoi(value)); } } } @@ -132,7 +240,7 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) if (diskCache) diskCache->upsertNarInfo( - config.getReference().render(/*FIXME withParams=*/false), + config.storeConfig.getReference().render(/*FIXME withParams=*/false), std::string(narInfo->path.hashPart()), std::shared_ptr(narInfo)); } @@ -439,7 +547,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) void BinaryCacheStore::queryPathInfoUncached( const StorePath & storePath, Callback> callback) noexcept { - auto uri = config.getReference().render(/*FIXME withParams=*/false); + auto uri = config.storeConfig.getReference().render(/*FIXME withParams=*/false); auto storePathS = printStorePath(storePath); auto act = std::make_shared( *logger, @@ -550,7 +658,7 @@ void BinaryCacheStore::queryRealisationUncached( void BinaryCacheStore::registerDrvOutput(const Realisation & info) { if (diskCache) - diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); + diskCache->upsertRealisation(config.storeConfig.getReference().render(/*FIXME withParams=*/false), info); upsertFile(makeRealisationPath(info.id), static_cast(info).dump(), "application/json"); } @@ -587,7 +695,7 @@ std::optional BinaryCacheStore::getBuildLogExact(const StorePath & { auto logPath = "log/" + std::string(baseNameOf(printStorePath(path))); - debug("fetching build log from binary cache '%s/%s'", config.getHumanReadableURI(), logPath); + debug("fetching build log from binary cache '%s/%s'", config.storeConfig.getHumanReadableURI(), logPath); return getFile(logPath); } diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index b4fc997a554..08052867be2 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -626,7 +626,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild() .initialOutputs = initialOutputs, .buildMode = buildMode, .defaultPathsInChroot = std::move(defaultPathsInChroot), - .systemFeatures = worker.store.config.systemFeatures.get(), + .systemFeatures = worker.store.config.systemFeatures, .desugaredEnv = std::move(desugaredEnv), }; diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 12f187b4c9e..3271eb7e29b 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -2,17 +2,74 @@ #include "nix/store/common-ssh-store-config.hh" #include "nix/store/ssh.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { -CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) +constexpr static const CommonSSHStoreConfigT commonSSHStoreConfigDescriptions = { + .sshKey{ + { + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + { + .makeDefault = []() -> Path { return ""; }, + }, + }, + .sshPublicHostKey{ + { + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + { + .makeDefault = []() -> Path { return ""; }, + }, + }, + .compress{ + { + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + { + .makeDefault = [] { return false; }, + }, + }, + .remoteStore{ + { + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )", + }, + { + .makeDefault = []() -> Path { return ""; }, + }, + }, +}; + +#define COMMON_SSH_STORE_CONFIG_FIELDS(X) X(sshKey), X(sshPublicHostKey), X(compress), X(remoteStore), + +MAKE_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap CommonSSHStoreConfig::descriptions() +{ + constexpr auto & descriptions = commonSSHStoreConfigDescriptions; + return {COMMON_SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}; +} + +CommonSSHStoreConfig::CommonSSHStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) : CommonSSHStoreConfig(scheme, ParsedURL::Authority::parse(authority), params) { } CommonSSHStoreConfig::CommonSSHStoreConfig( - std::string_view scheme, const ParsedURL::Authority & authority, const Params & params) - : StoreConfig(params) + std::string_view scheme, const ParsedURL::Authority & authority, const StoreConfig::Params & params) + : CommonSSHStoreConfigT{commonSSHStoreConfigApplyParse(params)} , authority(authority) { } @@ -21,8 +78,8 @@ SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD { return { authority, - sshKey.get(), - sshPublicHostKey.get(), + sshKey, + sshPublicHostKey, useMaster, compress, logFD, diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc new file mode 100644 index 00000000000..ef5795b935b --- /dev/null +++ b/src/libstore/config-parse.cc @@ -0,0 +1,71 @@ +#include + +#include "nix/store/config-parse.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/util.hh" + +namespace nix::config { + +}; + +namespace nlohmann { + +using namespace nix::config; + +SettingDescription adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return { + .description = getString(valueAt(obj, "description")), + .experimentalFeature = valueAt(obj, "experimentalFeature").get>(), + .info = [&]() -> decltype(SettingDescription::info) { + if (auto documentDefault = optionalValueAt(obj, "documentDefault")) { + return SettingDescription::Single{ + .defaultValue = *documentDefault ? (std::optional{valueAt(obj, "defaultValue")}) + : (std::optional{}), + }; + } else { + auto & subObj = getObject(valueAt(obj, "subSettings")); + return SettingDescription::Sub{ + .nullable = valueAt(subObj, "nullable"), + .map = valueAt(subObj, "map"), + }; + } + }(), + }; +} + +void adl_serializer::to_json(json & obj, const SettingDescription & sd) +{ + obj.emplace("description", sd.description); + // obj.emplace("aliases", sd.aliases); + obj.emplace("experimentalFeature", sd.experimentalFeature); + + std::visit( + overloaded{ + [&](const SettingDescription::Single & single) { + // Indicate the default value is JSON, rather than a legacy setting + // boolean or string. + // + // TODO remove if we no longer have the legacy setting system / the + // code handling doc rendering of the settings is decoupled. + obj.emplace("isJson", true); + + // Cannot just use `null` because the default value might itself be + // `null`. + obj.emplace("documentDefault", single.defaultValue.has_value()); + + if (single.defaultValue.has_value()) + obj.emplace("defaultValue", *single.defaultValue); + }, + [&](const SettingDescription::Sub & sub) { + json subJson; + subJson.emplace("nullable", sub.nullable); + subJson.emplace("map", sub.map); + obj.emplace("subSettings", std::move(subJson)); + }, + }, + sd.info); +} + +} // namespace nlohmann diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 2ead0c444c9..91a7bff7ebd 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -404,7 +404,7 @@ bool DerivationOptions::canBuildLocally(Store & localStore, const BasicDe return false; for (auto & feature : getRequiredSystemFeatures(drv)) - if (!localStore.config.systemFeatures.get().count(feature)) + if (!localStore.config.systemFeatures.count(feature)) return false; return true; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c45a13cc30a..b881f91b606 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,14 +1,57 @@ +#include "nix/store/dummy-store.hh" #include "nix/store/store-registration.hh" #include "nix/util/archive.hh" #include "nix/util/callback.hh" #include "nix/util/memory-source-accessor.hh" #include "nix/store/dummy-store-impl.hh" #include "nix/store/realisation.hh" +#include "nix/store/config-parse-impl.hh" #include namespace nix { +constexpr static const DummyStoreConfigT dummyStoreConfigDescriptions = { + .readOnly{ + { + .name = "read-only", + .description = R"( + Make any sort of write fail instead of succeeding. + No additional memory will be used, because no information needs to be stored. + )", + }, + { + .makeDefault = [] { return true; }, + }, + }, +}; + +#define DUMMY_STORE_CONFIG_FIELDS(X) X(readOnly) + +MAKE_PARSE(DummyStoreConfig, dummyStoreConfig, DUMMY_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(DummyStoreConfig, dummyStoreConfig, DUMMY_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap DummyStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + { + constexpr auto & descriptions = dummyStoreConfigDescriptions; + ret.merge(decltype(ret){DUMMY_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +DummyStoreConfig::DummyStoreConfig(const Params & params) + : Store::Config(params) + , DummyStoreConfigT{dummyStoreConfigApplyParse(params)} +{ + // Disable caching since this a temporary in-memory store. + pathInfoCacheSize = 0; +} + std::string DummyStoreConfig::doc() { return @@ -16,6 +59,17 @@ std::string DummyStoreConfig::doc() ; } +StoreReference DummyStoreConfig::getReference() const +{ + return { + .variant = + StoreReference::Specified{ + .scheme = *uriSchemes().begin(), + }, + .params = getQueryParams(), + }; +} + namespace { class WholeStoreViewAccessor : public SourceAccessor diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 4846d445fe1..ee2219d6985 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -109,7 +109,7 @@ void LocalStore::addTempRoot(const StorePath & path) auto fdRootsSocket(_fdRootsSocket.lock()); if (!*fdRootsSocket) { - auto socketPath = config->stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir + gcSocketPath; debug("connecting to '%s'", socketPath); *fdRootsSocket = createUnixDomainSocket(); try { @@ -248,7 +248,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R else { target = absPath(target, dirOf(path)); if (!pathExists(target)) { - if (isInDir(path, std::filesystem::path{config->stateDir.get()} / gcRootsDir / "auto")) { + if (isInDir(path, std::filesystem::path{config->stateDir} / gcRootsDir / "auto")) { printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } @@ -508,7 +508,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) readFile(*p); /* Start the server for receiving new roots. */ - auto socketPath = config->stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir + gcSocketPath; createDirs(dirOf(socketPath)); auto fdServer = createUnixDomainSocket(socketPath, 0666); @@ -821,7 +821,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(config->realStoreDir.get().c_str())); + AutoCloseDir dir(opendir(config->realStoreDir.c_str())); if (!dir) throw SysError("opening directory '%1%'", config->realStoreDir); @@ -924,7 +924,7 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(config->realStoreDir.get().c_str(), &st)) + if (statvfs(config->realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", config->realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ef6ae92a44d..e0031668b71 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -3,10 +3,64 @@ #include "nix/store/globals.hh" #include "nix/store/nar-info-disk-cache.hh" #include "nix/util/callback.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" namespace nix { +constexpr static const HttpBinaryCacheStoreConfigT + httpBinaryCacheStoreConfigDescriptions = { + .narinfoCompression{ + { + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + { + .makeDefault = [] { return std::string{}; }, + }, + }, + .lsCompression{ + { + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + { + .makeDefault = [] { return std::string{}; }, + }, + }, + .logCompression{ + { + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + { + .makeDefault = [] { return std::string{}; }, + }, + }, +}; + +#define HTTP_BINARY_CACHE_STORE_CONFIG_FIELDS(X) X(narinfoCompression), X(lsCompression), X(logCompression) + +MAKE_PARSE(HttpBinaryCacheStoreConfig, httpBinaryCacheStoreConfig, HTTP_BINARY_CACHE_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(HttpBinaryCacheStoreConfig, httpBinaryCacheStoreConfig, HTTP_BINARY_CACHE_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap HttpBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + { + constexpr auto & descriptions = httpBinaryCacheStoreConfigDescriptions; + ret.merge(decltype(ret){HTTP_BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + MakeError(UploadToHTTP, Error); StringSet HttpBinaryCacheStoreConfig::uriSchemes() @@ -19,9 +73,10 @@ StringSet HttpBinaryCacheStoreConfig::uriSchemes() } HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( - std::string_view scheme, std::string_view _cacheUri, const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + std::string_view scheme, std::string_view _cacheUri, const StoreConfig::Params & params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} + , HttpBinaryCacheStoreConfigT{httpBinaryCacheStoreConfigApplyParse(params)} , cacheUri(parseURL( std::string{scheme} + "://" + (!_cacheUri.empty() ? _cacheUri @@ -29,6 +84,7 @@ HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( { while (!cacheUri.path.empty() && cacheUri.path.back() == "") cacheUri.path.pop_back(); + assert(cacheUri.query.empty()); } StoreReference HttpBinaryCacheStoreConfig::getReference() const @@ -50,8 +106,8 @@ std::string HttpBinaryCacheStoreConfig::doc() ; } -HttpBinaryCacheStore::HttpBinaryCacheStore(ref config) - : Store{*config} // TODO it will actually mutate the configuration +HttpBinaryCacheStore::HttpBinaryCacheStore(ref config) + : Store{*config} , BinaryCacheStore{*config} , config{config} { @@ -66,25 +122,25 @@ void HttpBinaryCacheStore::init() auto cacheKey = config->getReference().render(/*withParams=*/false); if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) { - config->wantMassQuery.setDefault(cacheInfo->wantMassQuery); - config->priority.setDefault(cacheInfo->priority); + resolvedSubstConfig.wantMassQuery = config->storeConfig.wantMassQuery.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority = config->storeConfig.priority.value_or(cacheInfo->priority); } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string()); } - diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority); + diskCache->createCache(cacheKey, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } std::optional HttpBinaryCacheStore::getCompressionMethod(const std::string & path) { - if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty()) + if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.empty()) return config->narinfoCompression; - else if (hasSuffix(path, ".ls") && !config->lsCompression.get().empty()) + else if (hasSuffix(path, ".ls") && !config->lsCompression.empty()) return config->lsCompression; - else if (hasPrefix(path, "log/") && !config->logCompression.get().empty()) + else if (hasPrefix(path, "log/") && !config->logCompression.empty()) return config->logCompression; else return std::nullopt; @@ -268,9 +324,7 @@ std::optional HttpBinaryCacheStore::isTrustedClient() ref HttpBinaryCacheStore::Config::openStore() const { - return make_ref( - ref{// FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this())}); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index e64dc3eae58..06b36482358 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -14,51 +14,26 @@ namespace nix { struct NarInfo; class RemoteFSAccessor; -struct BinaryCacheStoreConfig : virtual StoreConfig +template class F> +struct BinaryCacheStoreConfigT { - using StoreConfig::StoreConfig; - - const Setting compression{ - this, "xz", "compression", "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - - const Setting writeNARListing{ - this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; - - const Setting writeDebugInfo{ - this, - false, - "index-debug-info", - R"( - Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to - fetch debug info on demand - )"}; - - const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; - - const Setting secretKeyFiles{ - this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."}; - - const Setting localNarCache{ - this, - "", - "local-nar-cache", - "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; - - const Setting parallelCompression{ - this, - false, - "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - - const Setting compressionLevel{ - this, - -1, - "compression-level", - R"( - The *preset level* to be used when compressing NARs. - The meaning and accepted values depend on the compression method selected. - `-1` specifies that the default compression level should be used. - )"}; + F::type compression; + F::type writeNARListing; + F::type writeDebugInfo; + F::type secretKeyFile; + F>::type secretKeyFiles; + F::type localNarCache; + F::type parallelCompression; + F::type compressionLevel; +}; + +struct BinaryCacheStoreConfig : BinaryCacheStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + const Store::Config & storeConfig; + + BinaryCacheStoreConfig(const Store::Config &, const StoreConfig::Params &); }; /** @@ -69,11 +44,7 @@ struct BinaryCacheStore : virtual Store, virtual LogStore { using Config = BinaryCacheStoreConfig; - /** - * Intentionally mutable because some things we update due to the - * cache's own (remote side) settings. - */ - Config & config; + const Config & config; private: std::vector> signers; @@ -87,7 +58,7 @@ protected: constexpr const static std::string cacheInfoFile = "nix-cache-info"; - BinaryCacheStore(Config &); + BinaryCacheStore(const Config &); /** * Compute the path to the given realisation @@ -141,7 +112,11 @@ public: public: - virtual void init() override; + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init(); private: diff --git a/src/libstore/include/nix/store/common-ssh-store-config.hh b/src/libstore/include/nix/store/common-ssh-store-config.hh index bbd81835d4f..971d8ca2c82 100644 --- a/src/libstore/include/nix/store/common-ssh-store-config.hh +++ b/src/libstore/include/nix/store/common-ssh-store-config.hh @@ -8,30 +8,26 @@ namespace nix { class SSHMaster; -struct CommonSSHStoreConfig : virtual StoreConfig +template class F> +struct CommonSSHStoreConfigT { - using StoreConfig::StoreConfig; - - CommonSSHStoreConfig(std::string_view scheme, const ParsedURL::Authority & authority, const Params & params); - CommonSSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); - - const Setting sshKey{ - this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; - - const Setting sshPublicHostKey{ - this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; + F::type sshKey; + F::type sshPublicHostKey; + F::type compress; + F::type remoteStore; +}; - const Setting compress{this, false, "compress", "Whether to enable SSH compression."}; +struct CommonSSHStoreConfig : CommonSSHStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting remoteStore{ - this, - "", - "remote-store", - R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )"}; + /** + * @param scheme Note this isn't stored by this mix-in class, but + * just used for better error messages. + */ + CommonSSHStoreConfig( + std::string_view scheme, const ParsedURL::Authority & authority, const StoreConfig::Params & params); + CommonSSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); /** * Authority representing the SSH host to connect to. diff --git a/src/libstore/include/nix/store/config-parse-impl.hh b/src/libstore/include/nix/store/config-parse-impl.hh new file mode 100644 index 00000000000..13a8558ede5 --- /dev/null +++ b/src/libstore/include/nix/store/config-parse-impl.hh @@ -0,0 +1,94 @@ +#pragma once +///@file + +#include + +#include "nix/store/config-parse.hh" +#include "nix/util/util.hh" +#include "nix/util/json-utils.hh" +#include "nix/util/configuration.hh" + +namespace nix::config { + +template +std::optional +SettingInfo::parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const +{ + const nlohmann::json * p = get(map, name); + if (p && experimentalFeature) + xpSettings.require(*experimentalFeature); + return p ? (std::optional{p->get()}) : std::nullopt; +} + +template +std::pair SettingInfo::describe(const T & def) const +{ + return { + std::string{name}, + SettingDescription{ + .description = stripIndentation(description), + .experimentalFeature = experimentalFeature, + .info = + SettingDescription::Single{ + .defaultValue = + documentDefault ? (std::optional{nlohmann::json(def)}) : (std::optional{}), + }, + }, + }; +} + +/** + * Look up the setting's name in a map, falling back on the default if + * it does not exist. + */ +#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params, xpSettings) + +#define DESCRIBE_ROW(FIELD) \ + { \ + descriptions.FIELD.describe(descriptions.FIELD.makeDefault()), \ + } + +#define APPLY_ROW(FIELD) .FIELD = parsed.FIELD ? *parsed.FIELD : descriptions.FIELD.makeDefault() + +#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##Parse( \ + const StoreReference::Params & params, \ + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) \ + { \ + constexpr auto & descriptions = LOWER##Descriptions; \ + return {FIELDS(CONFIG_ROW)}; \ + } + +#define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##ApplyParse(const StoreReference::Params & params) \ + { \ + auto parsed = LOWER##Parse(params); \ + constexpr auto & descriptions = LOWER##Descriptions; \ + return {FIELDS(APPLY_ROW)}; \ + } + +/** + * Version for separate defaults + */ +#define DESCRIBE_ROW_SEP_DEFAULTS(FIELD) \ + { \ + descriptions.FIELD.describe(defaults.FIELD), \ + } + +/** + * Version for separate defaults + */ +#define APPLY_ROW_SEP_DEFAULTS(FIELD) .FIELD = parsed.FIELD.value_or(std::move(defaults.FIELD)) + +/** + * Version for separate defaults + */ +#define MAKE_APPLY_PARSE_SEP_DEFAULTS(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##ApplyParse(const StoreReference::Params & params) \ + { \ + auto defaults = LOWER##Defaults(); \ + auto parsed = LOWER##Parse(params); \ + return {FIELDS(APPLY_ROW_SEP_DEFAULTS)}; \ + } + +} // namespace nix::config diff --git a/src/libstore/include/nix/store/config-parse.hh b/src/libstore/include/nix/store/config-parse.hh new file mode 100644 index 00000000000..55d102aba7c --- /dev/null +++ b/src/libstore/include/nix/store/config-parse.hh @@ -0,0 +1,173 @@ +#pragma once +///@file + +#include + +#include "nix/util/config-abstract.hh" +#include "nix/util/json-impls.hh" +#include "nix/util/experimental-features.hh" + +namespace nix { + +struct ExperimentalFeatureSettings; + +}; + +namespace nix::config { + +struct SettingDescription; + +/** + * Typed version used as source of truth, and for operations like + * defaulting configurations. + * + * It is important that this type support `constexpr` values to avoid + * running into issues with static initialization order. + */ +template +struct SettingInfo +{ + /** + * The "self reference" is a trick For higher order stuff to work without extra wrappers + */ + using type = SettingInfo; + + /** + * Name of the setting, used when parsing configuration maps + */ + std::string_view name; + + /** + * Description of the setting. It is used just for documentation. + */ + std::string_view description; + +#if 0 + /** + * Other names of the setting also used when parsing configuration + * maps. This is useful for back-compat, etc. + */ + std::set aliases; +#endif + + /** + * `ExperimentalFeature` that must be enabled if the setting is + * allowed to be used + */ + std::optional experimentalFeature; + + /** + * Whether to document the default value. (Some defaults are + * system-specific and should not be documented.) + */ + bool documentDefault = true; + + /** + * Describe the setting as a key-value pair (name -> other info). + * The default value will be rendered to JSON if it is to be + * documented. + */ + std::pair describe(const T & def) const; + + std::optional + parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const; +}; + +template +struct MakeDefault +{ + T (*makeDefault)(); +}; + +/** + * For the common case where the defaults are completely independent + * from one another. + * + * @note occasionally, when this is not the case, the defaulting logic + * can be written more manually instead. This is needed e.g. for + * `LocaFSStore` in libnixstore. + */ +template +struct SettingInfoWithDefault : SettingInfo, MakeDefault +{ + /** + * The "self reference" is a trick For higher order stuff to work without extra wrappers + */ + using type = SettingInfoWithDefault; +}; + +struct SettingDescription; + +/** + * Map of setting names to descriptions of those settings. + */ +using SettingDescriptionMap = std::map; + +/** + * Untyped version used for rendering docs. This is not the source of + * truth, it is generated from the typed one. + * + * @note No `name` field because this is intended to be used as the value type + * of a map + */ +struct SettingDescription +{ + /** + * @see SettingInfo::description + */ + std::string description; + +#if 0 + /** + * @see SettingInfo::aliases + */ + StringSet aliases; +#endif + + /** + * @see SettingInfo::experimentalFeature + */ + std::optional experimentalFeature; + + /** + * A single leaf setting, to be optionally specified by arbitrary + * value (of some type) or left default. + */ + struct Single + { + /** + * Optional, for the `SettingInfo::documentDefault = false` case. + */ + std::optional defaultValue; + }; + + /** + * A nested settings object + */ + struct Sub + { + /** + * If `false`, this is just pure namespaceing. If `true`, we + * have a distinction between `null` and `{}`, meaning + * enabling/disabling the entire settings group. + */ + bool nullable = true; + + SettingDescriptionMap map; + }; + + /** + * Variant for `info` below + */ + using Info = std::variant; + + /** + * More information about this setting, depending on whether its the + * single leaf setting or subsettings case + */ + Info info; +}; + +} // namespace nix::config + +JSON_IMPL(config::SettingDescription) diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh index d371c4e51f9..728c8e0846f 100644 --- a/src/libstore/include/nix/store/dummy-store.hh +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -9,14 +9,19 @@ namespace nix { struct DummyStore; -struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig +template class F> +struct DummyStoreConfigT { - DummyStoreConfig(const Params & params) - : StoreConfig(params) - { - // Disable caching since this a temporary in-memory store. - pathInfoCacheSize = 0; - } + F::type readOnly; +}; + +struct DummyStoreConfig : public std::enable_shared_from_this, + Store::Config, + DummyStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + DummyStoreConfig(const Params & params); DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) : DummyStoreConfig(params) @@ -25,15 +30,6 @@ struct DummyStoreConfig : public std::enable_shared_from_this, throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); } - Setting readOnly{ - this, - true, - "read-only", - R"( - Make any sort of write fail instead of succeeding. - No additional memory will be used, because no information needs to be stored. - )"}; - static const std::string name() { return "Dummy Store"; @@ -53,16 +49,7 @@ struct DummyStoreConfig : public std::enable_shared_from_this, ref openStore() const override; - StoreReference getReference() const override - { - return { - .variant = - StoreReference::Specified{ - .scheme = *uriSchemes().begin(), - }, - .params = getQueryParams(), - }; - } + StoreReference getReference() const override; }; } // namespace nix diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index ea3d77b7987..26c1a491dae 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -10,32 +10,25 @@ namespace nix { +template class F> +struct HttpBinaryCacheStoreConfigT +{ + F::type narinfoCompression; + F::type lsCompression; + F::type logCompression; +}; + struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, - virtual Store::Config, - BinaryCacheStoreConfig + Store::Config, + BinaryCacheStoreConfig, + HttpBinaryCacheStoreConfigT { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + static config::SettingDescriptionMap descriptions(); - HttpBinaryCacheStoreConfig( - std::string_view scheme, std::string_view cacheUri, const Store::Config::Params & params); + HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view cacheUri, const StoreConfig::Params & params); ParsedURL cacheUri; - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - static const std::string name() { return "HTTP Binary Cache Store"; @@ -64,9 +57,9 @@ public: using Config = HttpBinaryCacheStoreConfig; - ref config; + ref config; - HttpBinaryCacheStore(ref config); + HttpBinaryCacheStore(ref config); void init() override; diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index 994918f90f0..668a7fb96ff 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -10,25 +10,27 @@ namespace nix { -struct LegacySSHStoreConfig : std::enable_shared_from_this, virtual CommonSSHStoreConfig +template class F> +struct LegacySSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + F::type remoteProgram; + F::type maxConnections; +}; - LegacySSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); +struct LegacySSHStoreConfig : std::enable_shared_from_this, + Store::Config, + CommonSSHStoreConfig, + LegacySSHStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); -#ifndef _WIN32 - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; -#else + /** + * Hack for getting remote build log output. Intentionally not a + * documented user-visible setting. + */ Descriptor logFD = INVALID_DESCRIPTOR; -#endif - - const Setting remoteProgram{ - this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; + LegacySSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); /** * Hack for hydra diff --git a/src/libstore/include/nix/store/local-binary-cache-store.hh b/src/libstore/include/nix/store/local-binary-cache-store.hh index 2846a9225c7..87e6ed375da 100644 --- a/src/libstore/include/nix/store/local-binary-cache-store.hh +++ b/src/libstore/include/nix/store/local-binary-cache-store.hh @@ -3,16 +3,16 @@ namespace nix { struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this, - virtual Store::Config, + Store::Config, BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + static config::SettingDescriptionMap descriptions(); /** * @param binaryCacheDir `file://` is a short-hand for `file:///` * for now. */ - LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const StoreConfig::Params & params); Path binaryCacheDir; diff --git a/src/libstore/include/nix/store/local-fs-store.hh b/src/libstore/include/nix/store/local-fs-store.hh index 2fb9e140206..97c5bba8420 100644 --- a/src/libstore/include/nix/store/local-fs-store.hh +++ b/src/libstore/include/nix/store/local-fs-store.hh @@ -7,21 +7,22 @@ namespace nix { -struct LocalFSStoreConfig : virtual StoreConfig +template class F> +struct LocalFSStoreConfigT { -private: - static OptionalPathSetting makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) - { - return { - &self, - std::move(defaultValue), - "root", - "Directory prefixed to all other paths.", - }; - } + F>::type rootDir; + F::type stateDir; + F::type logDir; + F::type realStoreDir; +}; + +struct LocalFSStoreConfig : LocalFSStoreConfigT +{ + const Store::Config & storeConfig; -public: - using StoreConfig::StoreConfig; + static config::SettingDescriptionMap descriptions(); + + LocalFSStoreConfig(const Store::Config & storeConfig, const StoreConfig::Params &); /** * Used to override the `root` settings. Can't be done via modifying @@ -30,40 +31,7 @@ public: * * @todo Make this less error-prone with new store settings system. */ - LocalFSStoreConfig(PathView path, const Params & params); - - OptionalPathSetting rootDir = makeRootDirSetting(*this, std::nullopt); - -private: - - /** - * An indirection so that we don't need to refer to global settings - * in headers. - */ - static Path getDefaultStateDir(); - - /** - * An indirection so that we don't need to refer to global settings - * in headers. - */ - static Path getDefaultLogDir(); - -public: - - PathSetting stateDir{ - this, - rootDir.get() ? *rootDir.get() + "/nix/var/nix" : getDefaultStateDir(), - "state", - "Directory where Nix stores state."}; - - PathSetting logDir{ - this, - rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : getDefaultLogDir(), - "log", - "directory where Nix stores log files."}; - - PathSetting realStoreDir{ - this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; + LocalFSStoreConfig(const Store::Config & storeConfig, PathView path, const StoreConfig::Params & params); }; struct LocalFSStore : virtual Store, virtual GcStore, virtual LogStore diff --git a/src/libstore/include/nix/store/local-overlay-store.hh b/src/libstore/include/nix/store/local-overlay-store.hh index 1d69d341708..b8cc7ee68d9 100644 --- a/src/libstore/include/nix/store/local-overlay-store.hh +++ b/src/libstore/include/nix/store/local-overlay-store.hh @@ -2,72 +2,23 @@ namespace nix { +template class F> +struct LocalOverlayStoreConfigT +{ + F>::type lowerStoreConfig; + F::type upperLayer; + F::type checkMount; + F::type remountHook; +}; + /** * Configuration for `LocalOverlayStore`. */ -struct LocalOverlayStoreConfig : virtual LocalStoreConfig +struct LocalOverlayStoreConfig : LocalStoreConfig, LocalOverlayStoreConfigT { - LocalOverlayStoreConfig(const StringMap & params) - : LocalOverlayStoreConfig("local-overlay", "", params) - { - } - - LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) - { - } + static config::SettingDescriptionMap descriptions(); - const Setting lowerStoreUri{ - (StoreConfig *) this, - "", - "lower-store", - R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )"}; - - const PathSetting upperLayer{ - (StoreConfig *) this, - "", - "upper-layer", - R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )"}; - - Setting checkMount{ - (StoreConfig *) this, - true, - "check-mount", - R"( - Check that the overlay filesystem is correctly mounted. - - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )"}; - - const PathSetting remountHook{ - (StoreConfig *) this, - "", - "remount-hook", - R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )"}; + LocalOverlayStoreConfig(std::string_view scheme, PathView path, const StoreConfig::Params & params); static const std::string name() { diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index 212229e42f9..bb6a41f3407 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -31,82 +31,35 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; -struct LocalBuildStoreConfig : virtual LocalFSStoreConfig +template class F> +struct LocalStoreConfigT { - -private: + F::type requireSigs; + F::type readOnly; /** Input for computing the build directory. See `getBuildDir()`. */ - Setting> buildDir{ - this, - std::nullopt, - "build-dir", - R"( - The directory on the host, in which derivations' temporary build directories are created. - - If not set, Nix will use the `builds` subdirectory of its configured state directory. - - Note that builds are often performed by the Nix daemon, so its `build-dir` applies. - - Nix will create this directory automatically with suitable permissions if it does not exist. - Otherwise its permissions must allow all users to traverse the directory (i.e. it must have `o+x` set, in unix parlance) for non-sandboxed builds to work correctly. - - This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files. - - If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](@docroot@/command-ref/conf-file.md#conf-sandbox-build-dir). - - > **Warning** - > - > `build-dir` must not be set to a world-writable directory. - > Placing temporary build directories in a world-writable place allows other users to access or modify build data that is currently in use. - > This alone is merely an impurity, but combined with another factor this has allowed malicious derivations to escape the build sandbox. - )"}; -public: - Path getBuildDir() const; + F>::type buildDir; }; struct LocalStoreConfig : std::enable_shared_from_this, - virtual LocalFSStoreConfig, - virtual LocalBuildStoreConfig + Store::Config, + LocalFSStore::Config, + LocalStoreConfigT { - using LocalFSStoreConfig::LocalFSStoreConfig; + static config::SettingDescriptionMap descriptions(); - LocalStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + LocalStoreConfig(const StoreConfig::Params & params) + : LocalStoreConfig{"local", "", params} + { + } -private: + LocalStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); /** - * An indirection so that we don't need to refer to global settings - * in headers. + * For `RestrictedStore` */ - bool getDefaultRequireSigs(); - -public: - - Setting requireSigs{ - this, - getDefaultRequireSigs(), - "require-sigs", - "Whether store paths copied into this store should have a trusted signature."}; - - Setting readOnly{ - this, - false, - "read-only", - R"( - Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - - Normally Nix attempts to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. - - Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. - - > **Warning** - > Do not use this unless the filesystem is read-only. - > - > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. - )"}; + LocalStoreConfig(const LocalStoreConfig &); static const std::string name() { @@ -123,6 +76,8 @@ public: ref openStore() const override; StoreReference getReference() const override; + + Path getBuildDir() const; }; class LocalStore : public virtual IndirectRootStore, public virtual GcStore diff --git a/src/libstore/include/nix/store/machines.hh b/src/libstore/include/nix/store/machines.hh index 1f7bb669ab5..cea054dc142 100644 --- a/src/libstore/include/nix/store/machines.hh +++ b/src/libstore/include/nix/store/machines.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "nix/util/ref.hh" #include "nix/store/store-reference.hh" diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index c17d6a9cb5a..2ce14876e22 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -29,6 +29,8 @@ headers = [ config_pub_h ] + files( 'common-protocol-impl.hh', 'common-protocol.hh', 'common-ssh-store-config.hh', + 'config-parse-impl.hh', + 'config-parse.hh', 'content-address.hh', 'daemon.hh', 'derivation-options.hh', diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index b152e054b9d..e599d6046d2 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -18,18 +18,20 @@ template class Pool; class RemoteFSAccessor; -struct RemoteStoreConfig : virtual StoreConfig +template class F> +struct RemoteStoreConfigT { - using StoreConfig::StoreConfig; + F::type maxConnections; + F::type maxConnectionAge; +}; + +struct RemoteStoreConfig : RemoteStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting maxConnections{ - this, 1, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; + const Store::Config & storeConfig; - const Setting maxConnectionAge{ - this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; + RemoteStoreConfig(const Store::Config &, const StoreConfig::Params &); }; /** diff --git a/src/libstore/include/nix/store/s3-binary-cache-store.hh b/src/libstore/include/nix/store/s3-binary-cache-store.hh index 5896293f1c4..dddf3770c4e 100644 --- a/src/libstore/include/nix/store/s3-binary-cache-store.hh +++ b/src/libstore/include/nix/store/s3-binary-cache-store.hh @@ -6,118 +6,27 @@ namespace nix { -struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig +template class F> +struct S3BinaryCacheStoreConfigT { - using HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig; - - S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); - - const Setting profile{ - this, - "default", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix uses the `default` profile. - )"}; - - const Setting region{ - this, - "us-east-1", - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us-east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "https", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The S3 endpoint to use. When empty (default), uses AWS S3 with - region-specific endpoints (e.g., s3.us-east-1.amazonaws.com). - For S3-compatible services such as MinIO, set this to your service's endpoint. - - > **Note** - > - > Custom endpoints must support HTTPS and use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting multipartUpload{ - this, - false, - "multipart-upload", - R"( - Whether to use multipart uploads for large files. When enabled, - files exceeding the multipart threshold will be uploaded in - multiple parts, which is required for files larger than 5 GiB and - can improve performance and reliability for large uploads. - )"}; - - const Setting multipartChunkSize{ - this, - 5 * 1024 * 1024, - "multipart-chunk-size", - R"( - The size (in bytes) of each part in multipart uploads. Must be - at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the - number of requests but use more memory. Default is 5 MiB. - )", - {"buffer-size"}}; - - const Setting multipartThreshold{ - this, - 100 * 1024 * 1024, - "multipart-threshold", - R"( - The minimum file size (in bytes) for using multipart uploads. - Files smaller than this threshold will use regular PUT requests. - Default is 100 MiB. Only takes effect when multipart-upload is enabled. - )"}; - - const Setting> storageClass{ - this, - std::nullopt, - "storage-class", - R"( - The S3 storage class to use for uploaded objects. When not set (default), - uses the bucket's default storage class. Valid values include: - - STANDARD (default, frequently accessed data) - - REDUCED_REDUNDANCY (less frequently accessed data) - - STANDARD_IA (infrequent access) - - ONEZONE_IA (infrequent access, single AZ) - - INTELLIGENT_TIERING (automatic cost optimization) - - GLACIER (archival with retrieval times in minutes to hours) - - DEEP_ARCHIVE (long-term archival with 12-hour retrieval) - - GLACIER_IR (instant retrieval archival) + F::type profile; + F::type region; + F::type scheme; + F::type endpoint; + F::type multipartUpload; + F::type multipartChunkSize; + F::type multipartThreshold; + F>::type storageClass; +}; - See AWS S3 documentation for detailed storage class descriptions and pricing: - https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html - )"}; +struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, + HttpBinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - /** - * Set of settings that are part of the S3 URI itself. - * These are needed for region specification and other S3-specific settings. - */ - const std::set s3UriSettings = {&profile, ®ion, &scheme, &endpoint}; + S3BinaryCacheStoreConfig( + std::string_view uriScheme, std::string_view bucketName, const StoreConfig::Params & params); static const std::string name() { diff --git a/src/libstore/include/nix/store/ssh-store.hh b/src/libstore/include/nix/store/ssh-store.hh index 9584a1a862c..ab42eeeb35e 100644 --- a/src/libstore/include/nix/store/ssh-store.hh +++ b/src/libstore/include/nix/store/ssh-store.hh @@ -8,17 +8,27 @@ namespace nix { +template class F> +struct SSHStoreConfigT +{ + F::type remoteProgram; +}; + struct SSHStoreConfig : std::enable_shared_from_this, - virtual RemoteStoreConfig, - virtual CommonSSHStoreConfig + Store::Config, + RemoteStore::Config, + CommonSSHStoreConfig, + SSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + static config::SettingDescriptionMap descriptions(); - SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + std::optional mounted; - const Setting remoteProgram{ - this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreConfig::Params & params, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); static const std::string name() { @@ -37,29 +47,4 @@ struct SSHStoreConfig : std::enable_shared_from_this, StoreReference getReference() const override; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig -{ - MountedSSHStoreConfig(StringMap params); - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); - - static const std::string name() - { - return "Experimental SSH Store with filesystem mounted"; - } - - static StringSet uriSchemes() - { - return {"mounted-ssh-ng"}; - } - - static std::string doc(); - - static std::optional experimentalFeature() - { - return ExperimentalFeature::MountedSSHStore; - } - - ref openStore() const override; -}; - } // namespace nix diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 4c0b156faaa..e1c00a32d3f 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -8,7 +8,6 @@ #include "nix/util/serialise.hh" #include "nix/util/lru-cache.hh" #include "nix/util/sync.hh" -#include "nix/util/configuration.hh" #include "nix/store/path-info.hh" #include "nix/util/repair-flag.hh" #include "nix/store/store-dir-config.hh" @@ -71,46 +70,45 @@ struct MissingPaths uint64_t narSize{0}; }; -/** - * Need to make this a separate class so I can get the right - * initialization order in the constructor for `StoreConfig`. +/* + * Underlying store configuration type. + * + * Don't worry to much about the `F` parameter, it just some abstract + * nonsense for the "higher-kinded data" pattern. It is used in each + * settings record in order to ensure don't forgot to parse or document + * settings field. */ -struct StoreConfigBase : Config +template class F> +struct StoreConfigT { - using Config::Config; - -private: - - /** - * An indirection so that we don't need to refer to global settings - * in headers. - */ - static Path getDefaultNixStoreDir(); - -public: + F::type _storeDir; + F::type pathInfoCacheSize; + F::type isTrusted; + F::type systemFeatures; +}; - const PathSetting storeDir_{ - this, - getDefaultNixStoreDir(), - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; +template class F> +struct SubstituterConfigT +{ + F::type priority; + F::type wantMassQuery; }; +/** + * @note In other cases we don't expose this function directly, but in + * this case we must because of `Store::resolvedSubstConfig` below. As + * the docs of that field describe, this is a case where the + * configuration is intentionally stateful. + */ +SubstituterConfigT substituterConfigDefaults(); + /** * About the class hierarchy of the store types: * * Each store type `Foo` consists of two classes: * - * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration - * for the store - * - * It should only contain members of type `Setting` (or subclasses - * of it) and inherit the constructors of `StoreConfig` - * (`using StoreConfig::StoreConfig`). + * 1. A class `FooConfig : StoreConfigT regStore; * ``` * - * @note The order of `StoreConfigBase` and then `StorerConfig` is - * very important. This ensures that `StoreConfigBase::storeDir_` + * @note The order of `StoreConfigT` and then `StoreDirConfig` is + * very important. This ensures that `StoreConfigT<..>::storeDir_` * is initialized before we have our one chance (because references are - * immutable) to initialize `StoreConfig::storeDir`. + * immutable) to initialize `StoreDirConfig::storeDir`. + * + * @note `std::optional` rather than `config::PlainValue` is applied to + * `SubstitutorConfigT` because these are overrides. Caches themselves (not our + * config) can update default settings, but aren't allowed to update settings + * specified by the client (i.e. us). */ -struct StoreConfig : public StoreConfigBase, public StoreDirConfig +struct StoreConfig : StoreConfigT, StoreDirConfig, SubstituterConfigT { using Params = StoreReference::Params; - StoreConfig(const Params & params); + static config::SettingDescriptionMap descriptions(); - StoreConfig() = delete; + StoreConfig(const StoreConfig::Params &); virtual ~StoreConfig() {} @@ -155,14 +158,17 @@ struct StoreConfig : public StoreConfigBase, public StoreDirConfig /** * Get overridden store reference query parameters. */ - StringMap getQueryParams() const + StoreReference::Params getQueryParams() const { +#if 0 // FIXME render back to query parms auto queryParams = std::map{}; getSettings(queryParams, /*overriddenOnly=*/true); StringMap res; for (const auto & [name, info] : queryParams) res.insert({name, info.value}); return res; +#endif + return {}; } /** @@ -174,50 +180,6 @@ struct StoreConfig : public StoreConfigBase, public StoreDirConfig return std::nullopt; } - Setting pathInfoCacheSize{ - this, 65536, "path-info-cache-size", "Size of the in-memory store path metadata cache."}; - - Setting isTrusted{ - this, - false, - "trusted", - R"( - Whether paths from this store can be used as substitutes - even if they are not signed by a key listed in the - [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) - setting. - )"}; - - Setting priority{ - this, - 0, - "priority", - R"( - Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - A lower value means a higher priority. - )"}; - - Setting wantMassQuery{ - this, - false, - "want-mass-query", - R"( - Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - )"}; - - Setting systemFeatures{ - this, - getDefaultSystemFeatures(), - "system-features", - R"( - Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. - - Example: `"kvm"` - )", - {}, - // Don't document the machine-specific default value - false}; - /** * Open a store of the type corresponding to this configuration * type. @@ -275,6 +237,14 @@ public: return config; } + /** + * Resolved substituter configuration. This is intentionally mutable + * as store clients may do IO to ask the underlying store for their + * default setting values if the client config did not statically + * override them. + */ + SubstituterConfigT resolvedSubstConfig = substituterConfigDefaults(); + protected: struct PathInfoCacheValue @@ -317,11 +287,6 @@ protected: Store(const Store::Config & config); public: - /** - * Perform any necessary effectful operation to make the store up and - * running - */ - virtual void init() {}; virtual ~Store() {} @@ -1011,3 +976,5 @@ struct json_avoids_null : std::true_type } // namespace nix JSON_IMPL(nix::TrustedFlag) +// Parses a Store URL, uses global state not pure so think about this +JSON_IMPL(ref) diff --git a/src/libstore/include/nix/store/store-dir-config.hh b/src/libstore/include/nix/store/store-dir-config.hh index 34e928182ad..1a52aa7c3c9 100644 --- a/src/libstore/include/nix/store/store-dir-config.hh +++ b/src/libstore/include/nix/store/store-dir-config.hh @@ -3,7 +3,8 @@ #include "nix/store/path.hh" #include "nix/util/hash.hh" #include "nix/store/content-address.hh" -#include "nix/util/configuration.hh" +#include "nix/store/store-reference.hh" +#include "nix/store/config-parse.hh" #include #include @@ -31,8 +32,6 @@ struct StoreDirConfig { const Path & storeDir; - // pure methods - StorePath parseStorePath(std::string_view path) const; std::optional maybeParseStorePath(std::string_view path) const; diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index dc34500d9cb..a2d69c3b774 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -2,8 +2,10 @@ ///@file #include +#include #include "nix/util/types.hh" +#include "nix/util/json-impls.hh" namespace nix { @@ -41,7 +43,17 @@ namespace nix { */ struct StoreReference { - using Params = StringMap; + /** + * Would do + * + * ``` + * using Params = nlohmann::json::object_t; + * ``` + * + * but cannot because `` doesn't have that. + * + */ + using Params = std::map>; /** * Special store reference `""` or `"auto"` @@ -92,7 +104,7 @@ struct StoreReference Params params; - bool operator==(const StoreReference & rhs) const = default; + bool operator==(const StoreReference & rhs) const; /** * Render the whole store reference as a URI, optionally including parameters. @@ -121,3 +133,5 @@ static inline std::ostream & operator<<(std::ostream & os, const StoreReference std::pair splitUriAndParams(const std::string & uri); } // namespace nix + +JSON_IMPL(StoreReference) diff --git a/src/libstore/include/nix/store/store-registration.hh b/src/libstore/include/nix/store/store-registration.hh index 8b0f344ba38..e48d5fd40af 100644 --- a/src/libstore/include/nix/store/store-registration.hh +++ b/src/libstore/include/nix/store/store-registration.hh @@ -29,6 +29,19 @@ struct StoreFactory */ StringSet uriSchemes; + /** + * @note This is a functional pointer for now because this situation: + * + * - We register store types with global initializers + * + * - The default values for some settings maybe depend on the settings globals. + * + * And because the ordering of global initialization is arbitrary, + * this is not allowed. For now, we can simply defer actually + * creating these maps until we need to later. + */ + config::SettingDescriptionMap (*configDescriptions)(); + /** * An experimental feature this type store is gated, if it is to be * experimental. @@ -40,21 +53,21 @@ struct StoreFactory * whatever comes after `://` and before `?`. */ std::function( - std::string_view scheme, std::string_view authorityPath, const Store::Config::Params & params)> + std::string_view scheme, std::string_view authorityPath, const StoreConfig::Params & params)> parseConfig; - - /** - * Just for dumping the defaults. Kind of awkward this exists, - * because it means we cannot require fields to be manually - * specified so easily. - */ - std::function()> getConfig; }; struct Implementations { +private: + + /** + * The name of this type of store, and a factory for it. + */ using Map = std::map; +public: + static Map & registered(); template @@ -63,11 +76,11 @@ struct Implementations StoreFactory factory{ .doc = TConfig::doc(), .uriSchemes = TConfig::uriSchemes(), + .configDescriptions = TConfig::descriptions, .experimentalFeature = TConfig::experimentalFeature(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { return make_ref(scheme, uri, params); }), - .getConfig = ([]() -> ref { return make_ref(Store::Config::Params{}); }), }; auto [it, didInsert] = registered().insert({TConfig::name(), std::move(factory)}); if (!didInsert) { diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 764e8768a32..d0070e44188 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -8,20 +8,21 @@ namespace nix { struct UDSRemoteStoreConfig : std::enable_shared_from_this, - virtual LocalFSStoreConfig, - virtual RemoteStoreConfig + Store::Config, + LocalFSStore::Config, + RemoteStore::Config { - // TODO(fzakaria): Delete this constructor once moved over to the factory pattern - // outlined in https://github.com/NixOS/nix/issues/10766 - using LocalFSStoreConfig::LocalFSStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + static config::SettingDescriptionMap descriptions(); + + UDSRemoteStoreConfig(const StoreConfig::Params & params) + : UDSRemoteStoreConfig{"unix", "", params} + { + } /** * @param authority is the socket path. */ - UDSRemoteStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); - - UDSRemoteStoreConfig(const Params & params); + UDSRemoteStoreConfig(std::string_view scheme, std::string_view authority, const StoreConfig::Params & params); static const std::string name() { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 3b466c9bb8b..82c6264f54c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,15 +12,63 @@ #include "nix/store/ssh.hh" #include "nix/store/derivations.hh" #include "nix/util/callback.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" #include "nix/store/globals.hh" namespace nix { -LegacySSHStoreConfig::LegacySSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, ParsedURL::Authority::parse(authority), params) +constexpr static const LegacySSHStoreConfigT legacySSHStoreConfigDescriptions = { + .remoteProgram{ + { + .name = "remote-program", + .description = "Path to the `nix-store` executable on the remote machine.", + }, + { + .makeDefault = []() -> Strings { return {"nix-store"}; }, + }, + }, + .maxConnections{ + { + .name = "max-connections", + .description = "Maximum number of concurrent SSH connections.", + }, + { + .makeDefault = [] { return 1; }, + }, + }, +}; + +#define LEGACY_SSH_STORE_CONFIG_FIELDS(X) X(remoteProgram), X(maxConnections) + +MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LegacySSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = legacySSHStoreConfigDescriptions; + ret.merge(decltype(ret){LEGACY_SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +LegacySSHStore::Config::LegacySSHStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : Store::Config{params} + , CommonSSHStoreConfig{scheme, ParsedURL::Authority::parse(authority), params} + , LegacySSHStoreConfigT{legacySSHStoreConfigApplyParse(params)} { +#ifndef _WIN32 + if (auto * p = get(params, "log-fd")) { + logFD = p->get(); + } +#endif } std::string LegacySSHStoreConfig::doc() @@ -54,12 +102,12 @@ LegacySSHStore::LegacySSHStore(ref config) ref LegacySSHStore::openConnection() { auto conn = make_ref(); - Strings command = config->remoteProgram.get(); + Strings command = config->remoteProgram; command.push_back("--serve"); command.push_back("--write"); - if (config->remoteStore.get() != "") { + if (config->remoteStore != "") { command.push_back("--store"); - command.push_back(config->remoteStore.get()); + command.push_back(config->remoteStore); } conn->sshConn = master.startCommand(std::move(command), std::list{config->extraSshArgs}); if (config->connPipeSize) { diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 63730a01bd7..36fc8540a6c 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -8,10 +8,18 @@ namespace nix { +config::SettingDescriptionMap LocalBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params) : Store::Config{params} - , BinaryCacheStoreConfig{params} + , BinaryCacheStoreConfig{*this, params} , binaryCacheDir(binaryCacheDir) { } @@ -38,9 +46,9 @@ struct LocalBinaryCacheStore : virtual BinaryCacheStore { using Config = LocalBinaryCacheStoreConfig; - ref config; + ref config; - LocalBinaryCacheStore(ref config) + LocalBinaryCacheStore(ref config) : Store{*config} , BinaryCacheStore{*config} , config{config} @@ -122,9 +130,7 @@ StringSet LocalBinaryCacheStoreConfig::uriSchemes() ref LocalBinaryCacheStoreConfig::openStore() const { - auto store = make_ref( - ref{// FIXME we shouldn't actually need a mutable config - std::const_pointer_cast(shared_from_this())}); + auto store = make_ref(ref{shared_from_this()}); store->init(); return store; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 1a38cac3b7f..75fac862ec4 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,3 +1,4 @@ +#include "nix/util/json-utils.hh" #include "nix/util/archive.hh" #include "nix/util/posix-source-accessor.hh" #include "nix/store/store-api.hh" @@ -5,32 +6,81 @@ #include "nix/store/globals.hh" #include "nix/util/compression.hh" #include "nix/store/derivations.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { -Path LocalFSStoreConfig::getDefaultStateDir() +constexpr static const LocalFSStoreConfigT localFSStoreConfigDescriptions = { + .rootDir{ + .name = "root", + .description = "Directory prefixed to all other paths.", + }, + .stateDir{ + .name = "state", + .description = "Directory where Nix stores state.", + }, + .logDir{ + .name = "log", + .description = "directory where Nix stores log files.", + }, + .realStoreDir{ + .name = "real", + .description = "Physical path of the Nix store.", + }, +}; + +#define LOCAL_FS_STORE_CONFIG_FIELDS(X) X(rootDir), X(stateDir), X(logDir), X(realStoreDir), + +MAKE_PARSE(LocalFSStoreConfig, localFSStoreConfig, LOCAL_FS_STORE_CONFIG_FIELDS) + +/** + * @param rootDir Fallback if not in `params` + */ +static LocalFSStoreConfigT +localFSStoreConfigDefaults(const Path & storeDir, const std::optional & rootDir) +{ + return { + .rootDir = std::nullopt, + .stateDir = rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir, + .logDir = rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir, + .realStoreDir = rootDir ? *rootDir + "/nix/store" : storeDir, + }; +} + +static LocalFSStoreConfigT +localFSStoreConfigApplyParse(const Path & storeDir, LocalFSStoreConfigT parsed) +{ + auto defaults = localFSStoreConfigDefaults(storeDir, parsed.rootDir.value_or(std::nullopt)); + return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW_SEP_DEFAULTS)}; +} + +config::SettingDescriptionMap LocalFSStoreConfig::descriptions() +{ + constexpr auto & descriptions = localFSStoreConfigDescriptions; + auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt); + return {LOCAL_FS_STORE_CONFIG_FIELDS(DESCRIBE_ROW_SEP_DEFAULTS)}; +} + +LocalFSStore::Config::LocalFSStoreConfig(const Store::Config & storeConfig, const StoreConfig::Params & params) + : LocalFSStoreConfigT{localFSStoreConfigApplyParse( + storeConfig.storeDir, localFSStoreConfigParse(params))} + , storeConfig{storeConfig} { - return settings.nixStateDir; } -Path LocalFSStoreConfig::getDefaultLogDir() +static LocalFSStoreConfigT +applyAuthority(LocalFSStoreConfigT parsed, PathView rootDir) { - return settings.nixLogDir; + if (!rootDir.empty()) + parsed.rootDir = std::optional{Path{rootDir}}; + return parsed; } -LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) - : StoreConfig(params) - /* Default `?root` from `rootDir` if non set - * NOTE: We would like to just do rootDir.set(...), which would take care of - * all normalization and error checking for us. Unfortunately we cannot do - * that because of the complicated initialization order of other fields with - * the virtual class hierarchy of nix store configs, and the design of the - * settings system. As such, we have no choice but to redefine the field and - * manually repeat the same normalization logic. - */ - , rootDir{makeRootDirSetting( - *this, - !rootDir.empty() && params.count("root") == 0 ? std::optional{canonPath(rootDir)} : std::nullopt)} +LocalFSStore::Config::LocalFSStoreConfig( + const Store::Config & storeConfig, PathView rootDir, const StoreConfig::Params & params) + : LocalFSStoreConfigT{localFSStoreConfigApplyParse( + storeConfig.storeDir, applyAuthority(localFSStoreConfigParse(params), rootDir))} + , storeConfig{storeConfig} { } @@ -46,7 +96,7 @@ struct LocalStoreAccessor : PosixSourceAccessor bool requireValidPath; LocalStoreAccessor(ref store, bool requireValidPath) - : PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}) + : PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir}) , store(store) , requireValidPath(requireValidPath) { @@ -97,7 +147,7 @@ ref LocalFSStore::getFSAccessor(bool requireValidPath) std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & path, bool requireValidPath) { - auto absPath = std::filesystem::path{config.realStoreDir.get()} / path.to_string(); + auto absPath = std::filesystem::path{config.realStoreDir} / path.to_string(); if (requireValidPath) { /* Only return non-null if the store object is a fully-valid member of the store. */ @@ -120,9 +170,8 @@ std::optional LocalFSStore::getBuildLogExact(const StorePath & path for (int j = 0; j < 2; j++) { - Path logPath = - j == 0 ? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) - : fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName); + Path logPath = j == 0 ? fmt("%s/%s/%s/%s", config.logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) + : fmt("%s/%s/%s", config.logDir, drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index c8aa1d1a2b6..ce132f239eb 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,11 +5,108 @@ #include "nix/store/realisation.hh" #include "nix/util/processes.hh" #include "nix/util/url.hh" -#include "nix/store/store-open.hh" +#include "nix/store/store-api.hh" #include "nix/store/store-registration.hh" +#include "nix/store/config-parse-impl.hh" namespace nix { +static LocalOverlayStoreConfigT localOverlayStoreConfigDescriptions = { + .lowerStoreConfig{ + { + .name = "lower-store", + .description = R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )", + // It's not actually machine-specific, but we don't yet have a + // `to_json` for `StoreConfig`. + .documentDefault = false, + }, + { + .makeDefault = []() -> ref { + return make_ref(StoreConfig::Params{}); + }, + }, + }, + .upperLayer{ + { + .name = "upper-layer", + .description = R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )", + }, + { + .makeDefault = [] { return Path{}; }, + }, + }, + .checkMount{ + { + .name = "check-mount", + .description = R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )", + }, + { + .makeDefault = [] { return true; }, + }, + }, + .remountHook{ + { + .name = "remount-hook", + .description = R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )", + }, + { + .makeDefault = [] { return Path{}; }, + }, + }, +}; + +#define LOCAL_OVERLAY_STORE_CONFIG_FIELDS(X) X(lowerStoreConfig), X(upperLayer), X(checkMount), X(remountHook), + +MAKE_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LocalOverlayStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(LocalStoreConfig::descriptions()); + { + constexpr auto & descriptions = localOverlayStoreConfigDescriptions; + ret.merge(decltype(ret){LOCAL_OVERLAY_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +LocalOverlayStore::Config::LocalOverlayStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : LocalStore::Config(scheme, authority, params) + , LocalOverlayStoreConfigT{localOverlayStoreConfigApplyParse(params)} +{ +} + std::string LocalOverlayStoreConfig::doc() { return @@ -43,13 +140,13 @@ LocalOverlayStore::LocalOverlayStore(ref config) , LocalFSStore{*config} , LocalStore{static_cast>(config)} , config{config} - , lowerStore(openStore(percentDecode(config->lowerStoreUri.get())).dynamic_pointer_cast()) + , lowerStore(config->lowerStoreConfig->openStore().dynamic_pointer_cast()) { - if (config->checkMount.get()) { + if (config->checkMount) { std::smatch match; std::string mountInfo; auto mounts = readFile(std::filesystem::path{"/proc/self/mounts"}); - auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get() + R"( .*(\n|$))"); + auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. // Loop until the last match, which will be the current state of the mount point. @@ -62,12 +159,12 @@ LocalOverlayStore::LocalOverlayStore(ref config) return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); }; - auto expectedLowerDir = lowerStore->config.realStoreDir.get(); + auto expectedLowerDir = lowerStore->config.realStoreDir; if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer)) { debug("expected lowerdir: %s", expectedLowerDir); debug("expected upperdir: %s", config->upperLayer); debug("actual mount: %s", mountInfo); - throw Error("overlay filesystem '%s' mounted incorrectly", config->realStoreDir.get()); + throw Error("overlay filesystem '%s' mounted incorrectly", config->realStoreDir); } } } @@ -206,7 +303,7 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { - auto mergedDir = config->realStoreDir.get() + "/"; + auto mergedDir = config->realStoreDir + "/"; if (path.substr(0, mergedDir.length()) != mergedDir) { warn("local-overlay: unexpected gc path '%s' ", path); return; @@ -260,7 +357,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists(config->realStoreDir.get() + "/" + storePath.to_string()); + return pathExists(config->realStoreDir + "/" + storePath.to_string()); }; bool errors = false; @@ -280,8 +377,8 @@ void LocalOverlayStore::remountIfNecessary() if (!_remountRequired) return; - if (config->remountHook.get().empty()) { - warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir.get()); + if (config->remountHook.empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir); } else { runProgram(config->remountHook, false, {config->realStoreDir}); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a849576f6ee..1588b26b7e8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -18,6 +18,7 @@ #include "nix/store/keys.hh" #include "nix/util/url.hh" #include "nix/util/users.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-open.hh" #include "nix/store/store-registration.hh" @@ -56,17 +57,101 @@ #include #include "nix/util/strings.hh" +#include "nix/util/json-utils.hh" #include "store-config-private.hh" namespace nix { -LocalStoreConfig::LocalStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(authority, params) +constexpr static const LocalStoreConfigT localStoreConfigDescriptions = { + .requireSigs{ + { + .name = "require-sigs", + .description = "Whether store paths copied into this store should have a trusted signature.", + }, + { + .makeDefault = [] { return settings.requireSigs.get(); }, + }, + }, + .readOnly{ + { + .name = "read-only", + .description = R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix attempts to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )", + }, + { + .makeDefault = [] { return false; }, + }, + }, + .buildDir{ + { + .name = "build-dir", + .description = R"( + The directory on the host, in which derivations' temporary build directories are created. + + If not set, Nix will use the `builds` subdirectory of its configured state directory. + + Note that builds are often performed by the Nix daemon, so its `build-dir` applies. + + Nix will create this directory automatically with suitable permissions if it does not exist. + Otherwise its permissions must allow all users to traverse the directory (i.e. it must have `o+x` set, in unix parlance) for non-sandboxed builds to work correctly. + + This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files. + + If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](@docroot@/command-ref/conf-file.md#conf-sandbox-build-dir). + + > **Warning** + > + > `build-dir` must not be set to a world-writable directory. + > Placing temporary build directories in a world-writable place allows other users to access or modify build data that is currently in use. + > This alone is merely an impurity, but combined with another factor this has allowed malicious derivations to escape the build sandbox. + )", + }, + { + .makeDefault = []() -> std::optional { return std::nullopt; }, + }, + }, +}; + +#define LOCAL_STORE_CONFIG_FIELDS(X) X(requireSigs), X(readOnly), X(buildDir), + +MAKE_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LocalStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + { + constexpr auto & descriptions = localStoreConfigDescriptions; + ret.merge(decltype(ret){LOCAL_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +LocalStore::Config::LocalStoreConfig( + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : Store::Config(params) + , LocalFSStore::Config(*this, authority, params) + , LocalStoreConfigT{localStoreConfigApplyParse(params)} { } +LocalStoreConfig::LocalStoreConfig(const LocalStoreConfig &) = default; + std::string LocalStoreConfig::doc() { return @@ -74,11 +159,11 @@ std::string LocalStoreConfig::doc() ; } -Path LocalBuildStoreConfig::getBuildDir() const +Path LocalStoreConfig::getBuildDir() const { return settings.buildDir.get().has_value() ? *settings.buildDir.get() - : buildDir.get().has_value() ? *buildDir.get() - : stateDir.get() + "/builds"; + : buildDir.has_value() ? *buildDir + : stateDir + "/builds"; } ref LocalStore::Config::openStore() const @@ -86,11 +171,6 @@ ref LocalStore::Config::openStore() const return make_ref(ref{shared_from_this()}); } -bool LocalStoreConfig::getDefaultRequireSigs() -{ - return settings.requireSigs; -} - struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ @@ -130,7 +210,7 @@ LocalStore::LocalStore(ref config) state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ - createDirs(config->realStoreDir.get()); + createDirs(config->realStoreDir); if (config->readOnly) { experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); } else { @@ -169,13 +249,13 @@ LocalStore::LocalStore(ref config) "warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); else if (!config->readOnly) { struct stat st; - if (stat(config->realStoreDir.get().c_str(), &st)) + if (stat(config->realStoreDir.c_str(), &st)) throw SysError("getting attributes of path '%1%'", config->realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) + if (chown(config->realStoreDir.c_str(), 0, gr->gr_gid) == -1) throw SysError("changing ownership of path '%1%'", config->realStoreDir); - if (chmod(config->realStoreDir.get().c_str(), perm) == -1) + if (chmod(config->realStoreDir.c_str(), perm) == -1) throw SysError("changing permissions on path '%1%'", config->realStoreDir); } } @@ -184,7 +264,7 @@ LocalStore::LocalStore(ref config) /* Ensure that the store and its parents are not symlinks. */ if (!settings.allowSymlinkedStore) { - std::filesystem::path path = config->realStoreDir.get(); + std::filesystem::path path = config->realStoreDir; std::filesystem::path root = path.root_path(); while (path != root) { if (std::filesystem::is_symlink(path)) @@ -606,11 +686,11 @@ void LocalStore::makeStoreWritable() return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) + if (statvfs(config->realStoreDir.c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + if (mount(0, config->realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError("remounting %1% writable", config->realStoreDir); } #endif @@ -921,7 +1001,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) break; if (sub->storeDir != storeDir) continue; - if (!sub->config.wantMassQuery) + if (!sub->resolvedSubstConfig.wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); @@ -1473,7 +1553,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair database and the filesystem) in the loop below, in order to catch invalid states. */ - for (auto & i : DirectoryIterator{config->realStoreDir.get()}) { + for (auto & i : DirectoryIterator{config->realStoreDir}) { checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index d614676668b..79496a2cbcd 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -64,8 +64,8 @@ StoreReference Machine::completeStoreReference() const auto * generic = std::get_if(&storeUri.variant); if (generic && generic->scheme == "ssh") { - storeUri.params["max-connections"] = "1"; - storeUri.params["log-fd"] = "4"; + storeUri.params["max-connections"] = 1; + storeUri.params["log-fd"] = 4; } if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { @@ -77,15 +77,12 @@ StoreReference Machine::completeStoreReference() const { auto & fs = storeUri.params["system-features"]; - auto append = [&](auto feats) { - for (auto & f : feats) { - if (fs.size() > 0) - fs += ' '; - fs += f; - } - }; - append(supportedFeatures); - append(mandatoryFeatures); + if (!fs.is_array()) + fs = nlohmann::json::array(); + auto features = supportedFeatures; + features.insert(supportedFeatures.begin(), supportedFeatures.end()); + for (auto & feat : features) + fs += feat; } return storeUri; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index e3425deb5e8..d990897568f 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -275,6 +275,7 @@ sources = files( 'builtins/unpack-channel.cc', 'common-protocol.cc', 'common-ssh-store-config.cc', + 'config-parse.cc', 'content-address.cc', 'daemon.cc', 'derivation-options.cc', diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md deleted file mode 100644 index 1ebfe3081dc..00000000000 --- a/src/libstore/mounted-ssh-store.md +++ /dev/null @@ -1,18 +0,0 @@ -R"( - -**Store URL format**: `mounted-ssh-ng://[username@]hostname` - -Experimental store type that allows full access to a Nix store on a remote machine, -and additionally requires that store be mounted in the local file system. - -The mounting of that store is not managed by Nix, and must by managed manually. -It could be accomplished with SSHFS or NFS, for example. - -The local file system is used to optimize certain operations. -For example, rather than serializing Nix archives and sending over the Nix channel, -we can directly access the file system data via the mount-point. - -The local file system is also used to make certain operations possible that wouldn't otherwise be. -For example, persistent GC roots can be created if they reside on the same file system as the remote store: -the remote side will create the symlinks necessary to avoid race conditions. -)" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index dca093e04c7..5090e97c5c9 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -226,7 +226,7 @@ void LocalStore::optimisePath_( the store itself (we don't want or need to mess with its permissions). */ const Path dirOfPath(dirOf(path)); - bool mustToggle = dirOfPath != config->realStoreDir.get(); + bool mustToggle = dirOfPath != config->realStoreDir; if (mustToggle) makeWritable(dirOfPath); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 949a51f187a..45ff09e9106 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -18,18 +18,58 @@ #include "nix/util/callback.hh" #include "nix/store/filetransfer.hh" #include "nix/util/signals.hh" +#include "nix/store/config-parse-impl.hh" #include namespace nix { +constexpr static const RemoteStoreConfigT remoteStoreConfigDescriptions = { + .maxConnections{ + { + .name = "max-connections", + .description = "Maximum number of concurrent connections to the Nix daemon.", + }, + { + .makeDefault = [] { return 1; }, + }, + }, + .maxConnectionAge{ + { + .name = "max-connection-age", + .description = "Maximum age of a connection before it is closed.", + }, + { + .makeDefault = [] { return std::numeric_limits::max(); }, + }, + }, +}; + +#define REMOTE_STORE_CONFIG_FIELDS(X) X(maxConnections), X(maxConnectionAge), + +MAKE_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap RemoteStoreConfig::descriptions() +{ + constexpr auto & descriptions = remoteStoreConfigDescriptions; + return {REMOTE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}; +} + +RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreConfig::Params & params) + : RemoteStoreConfigT{remoteStoreConfigApplyParse(params)} + , storeConfig{storeConfig} +{ +} + /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) - : Store{config} + : Store{config.storeConfig} , config{config} , connections( make_ref>( - std::max(1, config.maxConnections.get()), + std::max(1, config.maxConnections), [this]() { auto conn = openConnectionWrapper(); try { @@ -53,7 +93,8 @@ RemoteStore::RemoteStore(const Config & config) ref RemoteStore::openConnectionWrapper() { if (failed) - throw Error("opening a connection to remote store '%s' previously failed", config.getHumanReadableURI()); + throw Error( + "opening a connection to remote store '%s' previously failed", config.storeConfig.getHumanReadableURI()); try { return openConnection(); } catch (...) { @@ -97,7 +138,8 @@ void RemoteStore::initConnection(Connection & conn) if (ex) std::rethrow_exception(ex); } catch (Error & e) { - throw Error("cannot open connection to remote store '%s': %s", config.getHumanReadableURI(), e.what()); + throw Error( + "cannot open connection to remote store '%s': %s", config.storeConfig.getHumanReadableURI(), e.what()); } setOptions(conn); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 5dfc7f11de5..60610704b68 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,5 +1,6 @@ #include "nix/store/s3-binary-cache-store.hh" #include "nix/store/http-binary-cache-store.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" #include "nix/util/error.hh" #include "nix/util/logging.hh" @@ -23,7 +24,7 @@ static constexpr uint64_t AWS_MAX_PART_COUNT = 10000; class S3BinaryCacheStore : public virtual HttpBinaryCacheStore { public: - S3BinaryCacheStore(ref config) + S3BinaryCacheStore(ref config) : Store{*config} , BinaryCacheStore{*config} , HttpBinaryCacheStore{config} @@ -35,7 +36,7 @@ class S3BinaryCacheStore : public virtual HttpBinaryCacheStore const std::string & path, RestartableSource & source, const std::string & mimeType, uint64_t sizeHint) override; private: - ref s3Config; + ref s3Config; /** * Uploads a file to S3 using a regular (non-multipart) upload. @@ -135,8 +136,8 @@ void S3BinaryCacheStore::upsertFile( { auto doUpload = [&](RestartableSource & src, uint64_t size, std::optional headers) { Headers uploadHeaders = headers.value_or(Headers()); - if (auto storageClass = s3Config->storageClass.get()) { - uploadHeaders.emplace_back("x-amz-storage-class", *storageClass); + if (s3Config->storageClass) { + uploadHeaders.emplace_back("x-amz-storage-class", *s3Config->storageClass); } if (s3Config->multipartUpload && size > s3Config->multipartThreshold) { uploadMultipart(path, src, size, mimeType, std::move(uploadHeaders)); @@ -220,7 +221,7 @@ S3BinaryCacheStore::MultipartSink::MultipartSink( warn( "adjusting S3 multipart chunk size from %s to %s " "to stay within %d part limit for %s file", - renderSize(store.s3Config->multipartChunkSize.get()), + renderSize(store.s3Config->multipartChunkSize), renderSize(minChunkSize), AWS_MAX_PART_COUNT, renderSize(sizeHint)); @@ -400,18 +401,170 @@ StringSet S3BinaryCacheStoreConfig::uriSchemes() return {"s3"}; } +// We don't want clang-format to move the brance to the next line causing +// everything to be indented even more. + +// clang-format off +constexpr static const S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDescriptions = { + // clang-format on + .profile{ + { + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix uses the `default` profile. + )", + }, + { + .makeDefault = [] { return std::string{}; }, + }, + }, + .region{ + { + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + { + .makeDefault = [] { return std::string{"us-east-1"}; }, + }, + }, + .scheme{ + { + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + { + .makeDefault = [] { return std::string{}; }, + }, + }, + .endpoint{ + { + .name = "endpoint", + .description = R"( + The S3 endpoint to use. When empty (default), uses AWS S3 with + region-specific endpoints (e.g., s3.us-east-1.amazonaws.com). + For S3-compatible services such as MinIO, set this to your service's endpoint. + + > **Note** + > + > This endpoint must support HTTPS and uses path-based + > addressing instead of virtual host based addressing. + )", + }, + { + .makeDefault = [] { return std::string{}; }, + }, + }, + .multipartUpload{ + { + .name = "multipart-upload", + .description = R"( + Whether to use multipart uploads for large files. When enabled, + files exceeding the multipart threshold will be uploaded in + multiple parts, which is required for files larger than 5 GiB and + can improve performance and reliability for large uploads. + )", + }, + { + .makeDefault = [] { return false; }, + }, + }, + .multipartChunkSize{ + { + .name = "multipart-chunk-size", + .description = R"( + The size (in bytes) of each part in multipart uploads. Must be + at least 5 MiB (AWS S3 requirement). Larger chunk sizes reduce the + number of requests but use more memory. Default is 5 MiB. + )", + }, + { + .makeDefault = [] { return (uint64_t) (5 * 1024 * 1024); }, + }, + }, + .multipartThreshold{ + { + .name = "multipart-threshold", + .description = R"( + The minimum file size (in bytes) for using multipart uploads. + Files smaller than this threshold will use regular PUT requests. + Default is 100 MiB. Only takes effect when multipart-upload is enabled. + )", + }, + { + .makeDefault = [] { return (uint64_t) (100 * 1024 * 1024); }, + }, + }, + .storageClass{ + { + .name = "storage-class", + .description = R"( + The S3 storage class to use for uploaded objects. When not set (default), + uses the bucket's default storage class. Valid values include: + - STANDARD (default, frequently accessed data) + - REDUCED_REDUNDANCY (less frequently accessed data) + - STANDARD_IA (infrequent access) + - ONEZONE_IA (infrequent access, single AZ) + - INTELLIGENT_TIERING (automatic cost optimization) + - GLACIER (archival with retrieval times in minutes to hours) + - DEEP_ARCHIVE (long-term archival with 12-hour retrieval) + - GLACIER_IR (instant retrieval archival) + + See AWS S3 documentation for detailed storage class descriptions and pricing: + https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html + )", + }, + { + .makeDefault = [] { return std::optional{}; }, + }, + }, +}; + +#define S3_BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(profile), X(region), X(scheme), X(endpoint), X(multipartUpload), X(multipartChunkSize), X(multipartThreshold), \ + X(storageClass) + +MAKE_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap S3BinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + { + constexpr auto & descriptions = s3BinaryCacheStoreConfigDescriptions; + ret.merge(decltype(ret){S3_BINARY_CACHE_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + return ret; +} + +static const std::set s3UriParams = {"profile", "region", "scheme", "endpoint"}; + S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( - std::string_view scheme, std::string_view _cacheUri, const Params & params) - : StoreConfig(params) - , HttpBinaryCacheStoreConfig(scheme, _cacheUri, params) + std::string_view scheme, std::string_view authority, const StoreConfig::Params & params) + : HttpBinaryCacheStoreConfig{scheme, authority, params} + , S3BinaryCacheStoreConfigT{s3BinaryCacheStoreConfigApplyParse(params)} { assert(cacheUri.query.empty()); assert(cacheUri.scheme == "s3"); for (const auto & [key, value] : params) { - auto s3Params = - std::views::transform(s3UriSettings, [](const AbstractSetting * setting) { return setting->name; }); - if (std::ranges::contains(s3Params, key)) { + if (s3UriParams.contains(key)) { cacheUri.query[key] = value; } } @@ -420,22 +573,22 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( throw UsageError( "multipart-chunk-size must be at least %s, got %s", renderSize(AWS_MIN_PART_SIZE), - renderSize(multipartChunkSize.get())); + renderSize(multipartChunkSize)); } if (multipartChunkSize > AWS_MAX_PART_SIZE) { throw UsageError( "multipart-chunk-size must be at most %s, got %s", renderSize(AWS_MAX_PART_SIZE), - renderSize(multipartChunkSize.get())); + renderSize(multipartChunkSize)); } if (multipartUpload && multipartThreshold < multipartChunkSize) { warn( "multipart-threshold (%s) is less than multipart-chunk-size (%s), " "which may result in single-part multipart uploads", - renderSize(multipartThreshold.get()), - renderSize(multipartChunkSize.get())); + renderSize(multipartThreshold), + renderSize(multipartChunkSize)); } } @@ -444,9 +597,9 @@ std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const auto reference = getReference(); reference.params = [&]() { Params relevantParams; - for (auto & setting : s3UriSettings) - if (setting->overridden) - relevantParams.insert({setting->name, reference.params.at(setting->name)}); + for (const auto & param : s3UriParams) + if (auto it = reference.params.find(param); it != reference.params.end()) + relevantParams.insert(*it); return relevantParams; }(); return reference.render(); @@ -463,9 +616,8 @@ std::string S3BinaryCacheStoreConfig::doc() ref S3BinaryCacheStoreConfig::openStore() const { - auto sharedThis = std::const_pointer_cast( - std::static_pointer_cast(shared_from_this())); - return make_ref(ref{sharedThis}); + return make_ref( + ref{std::enable_shared_from_this::shared_from_this()}); } static RegisterStoreImplementation registerS3BinaryCacheStore; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index ce973e7343a..1753895c8dd 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,3 +1,4 @@ +#include "nix/util/json-utils.hh" #include "nix/store/ssh-store.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/remote-store-connection.hh" @@ -7,14 +8,88 @@ #include "nix/store/worker-protocol-impl.hh" #include "nix/util/pool.hh" #include "nix/store/ssh.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/store/store-registration.hh" namespace nix { -SSHStoreConfig::SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) +constexpr static const SSHStoreConfigT sshStoreConfigDescriptions = { + .remoteProgram{ + { + .name = "remote-program", + .description = "Path to the `nix-daemon` executable on the remote machine.", + }, + { + .makeDefault = []() -> Strings { return {"nix-daemon"}; }, + }, + }, +}; + +#define SSH_STORE_CONFIG_FIELDS(X) X(remoteProgram) + +MAKE_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap SSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = sshStoreConfigDescriptions; + ret.merge(decltype(ret){SSH_STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + ret.insert_or_assign( + "mounted", + config::SettingDescription{ + .description = stripIndentation(R"( + If this nested settings object is defined (`{..}` not `null`), additionally requires that store be mounted in the local file system. + + The mounting of that store is not managed by Nix, and must by managed manually. + It could be accomplished with SSHFS or NFS, for example. + + The local file system is used to optimize certain operations. + For example, rather than serializing Nix archives and sending over the Nix channel, + we can directly access the file system data via the mount-point. + + The local file system is also used to make certain operations possible that wouldn't otherwise be. + For example, persistent GC roots can be created if they reside on the same file system as the remote store: + the remote side will create the symlinks necessary to avoid race conditions. + )"), + .experimentalFeature = Xp::MountedSSHStore, + .info = config::SettingDescription::Sub{.nullable = true, .map = LocalFSStoreConfig::descriptions()}, + }); + return ret; +} + +static std::optional getMounted( + const Store::Config & storeConfig, + const StoreConfig::Params & params, + const ExperimentalFeatureSettings & xpSettings) +{ + auto mountedParamsOpt = optionalValueAt(params, "mounted"); + if (!mountedParamsOpt) + return {}; + auto * mountedParamsP = getNullable(*mountedParamsOpt); + xpSettings.require(Xp::MountedSSHStore); + if (!mountedParamsP) + return {}; + auto & mountedParams = getObject(*mountedParamsP); + return {{storeConfig, mountedParams}}; +} + +SSHStoreConfig::SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreConfig::Params & params, + const ExperimentalFeatureSettings & xpSettings) : Store::Config{params} - , RemoteStore::Config{params} + , RemoteStore::Config{*this, params} , CommonSSHStoreConfig{scheme, authority, params} + , SSHStoreConfigT{sshStoreConfigApplyParse(params)} + , mounted{getMounted(*this, params, xpSettings)} { } @@ -87,31 +162,6 @@ struct SSHStore : virtual RemoteStore }; }; -MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) -{ -} - -MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) - , LocalFSStoreConfig(params) -{ -} - -std::string MountedSSHStoreConfig::doc() -{ - return -#include "mounted-ssh-store.md" - ; -} - /** * The mounted ssh store assumes that filesystems on the remote host are * shared with the local host. This means that the remote nix store is @@ -128,13 +178,16 @@ std::string MountedSSHStoreConfig::doc() */ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { - using Config = MountedSSHStoreConfig; + using Config = SSHStore::Config; + + const LocalFSStore::Config & mountedConfig; - MountedSSHStore(ref config) + MountedSSHStore(ref config, const LocalFSStore::Config & mountedConfig) : Store{*config} , RemoteStore{*config} , SSHStore{config} - , LocalFSStore{*config} + , LocalFSStore{mountedConfig} + , mountedConfig{mountedConfig} { extraRemoteProgramArgs = { "--process-ops", @@ -187,24 +240,24 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore } }; -ref SSHStore::Config::openStore() const -{ - return make_ref(ref{shared_from_this()}); -} - ref MountedSSHStore::Config::openStore() const { - return make_ref(ref{std::dynamic_pointer_cast(shared_from_this())}); + ref config{shared_from_this()}; + + if (config->mounted) + return make_ref(config, *config->mounted); + else + return make_ref(config); } ref SSHStore::openConnection() { auto conn = make_ref(); - Strings command = config->remoteProgram.get(); + Strings command = config->remoteProgram; command.push_back("--stdio"); - if (config->remoteStore.get() != "") { + if (config->remoteStore != "") { command.push_back("--store"); - command.push_back(config->remoteStore.get()); + command.push_back(config->remoteStore); } command.insert(command.end(), extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); conn->sshConn = master.startCommand(std::move(command)); @@ -214,6 +267,5 @@ ref SSHStore::openConnection() } static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; } // namespace nix diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md index 26e0d6e39e0..63a455966ee 100644 --- a/src/libstore/ssh-store.md +++ b/src/libstore/ssh-store.md @@ -4,5 +4,4 @@ R"( Experimental store type that allows full access to a Nix store on a remote machine. - )" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c292e2e431d..7a0217273e2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -18,6 +18,7 @@ // `addMultipleToStore`. #include "nix/store/worker-protocol.hh" #include "nix/util/signals.hh" +#include "nix/store/config-parse-impl.hh" #include #include @@ -28,17 +29,6 @@ using json = nlohmann::json; namespace nix { -Path StoreConfigBase::getDefaultNixStoreDir() -{ - return settings.nixStore; -} - -StoreConfig::StoreConfig(const Params & params) - : StoreConfigBase(params) - , StoreDirConfig{storeDir_} -{ -} - bool StoreDirConfig::isInStore(PathView path) const { return isInDir(path, storeDir); @@ -84,6 +74,124 @@ StorePath Store::followLinksToStorePath(std::string_view path) const return toStorePath(followLinksToStore(path)).first; } +constexpr static const StoreConfigT storeConfigDescriptions = { + ._storeDir{ + { + .name = "store", + .description = R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )", + }, + { + .makeDefault = []() -> Path { return settings.nixStore; }, + }, + }, + .pathInfoCacheSize{ + { + .name = "path-info-cache-size", + .description = "Size of the in-memory store path metadata cache.", + }, + { + .makeDefault = [] { return 65536; }, + }, + }, + .isTrusted{ + { + .name = "trusted", + .description = R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )", + }, + { + .makeDefault = [] { return false; }, + }, + }, + .systemFeatures{ + { + .name = "system-features", + .description = R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + + Example: `"kvm"` + )", + // The default value is CPU- and OS-specific, and thus + // unsuitable to be rendered in the documentation. + .documentDefault = false, + }, + { + .makeDefault = StoreConfig::getDefaultSystemFeatures, + }, + }, +}; + +constexpr static const SubstituterConfigT substituterConfigDescriptions = { + .priority{ + { + .name = "priority", + .description = R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )", + }, + { + .makeDefault = [] { return 0; }, + }, + }, + .wantMassQuery{ + { + .name = "want-mass-query", + .description = R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )", + }, + { + .makeDefault = [] { return false; }, + }, + }, +}; + +#define STORE_CONFIG_FIELDS(X) X(_storeDir), X(pathInfoCacheSize), X(isTrusted), X(systemFeatures), + +#define SUBSTITUTER_CONFIG_FIELDS(X) X(priority), X(wantMassQuery), + +MAKE_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) +MAKE_PARSE(SubstituterConfig, substituterConfig, SUBSTITUTER_CONFIG_FIELDS) + +MAKE_APPLY_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) + +SubstituterConfigT substituterConfigDefaults() +{ +#define F(FIELD) .FIELD = substituterConfigDescriptions.FIELD.makeDefault() + return {SUBSTITUTER_CONFIG_FIELDS(F)}; +#undef F +} + +Store::Config::StoreConfig(const StoreConfig::Params & params) + : StoreConfigT{storeConfigApplyParse(params)} + , StoreDirConfig{_storeDir} + , SubstituterConfigT{substituterConfigParse(params)} +{ +} + +config::SettingDescriptionMap StoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + { + constexpr auto & descriptions = storeConfigDescriptions; + ret.merge(config::SettingDescriptionMap{STORE_CONFIG_FIELDS(DESCRIBE_ROW)}); + } + { + constexpr auto & descriptions = substituterConfigDescriptions; + ret.merge(config::SettingDescriptionMap{SUBSTITUTER_CONFIG_FIELDS(DESCRIBE_ROW)}); + }; + return ret; +} + StorePath Store::addToStore( std::string_view name, const SourcePath & path, diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 8c756ff5819..2951fd3d3ce 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -1,4 +1,5 @@ #include "nix/util/source-path.hh" +#include "nix/store/config-parse-impl.hh" #include "nix/util/util.hh" #include "nix/store/store-dir-config.hh" #include "nix/store/derivations.hh" diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 01e197be76d..af5bce594f7 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -1,14 +1,19 @@ +#include + #include "nix/util/error.hh" #include "nix/util/split.hh" #include "nix/util/url.hh" #include "nix/store/store-reference.hh" #include "nix/util/file-system.hh" #include "nix/util/util.hh" +#include "nix/util/json-utils.hh" #include namespace nix { +bool StoreReference::operator==(const StoreReference & rhs) const = default; + static bool isNonUriPath(const std::string & spec) { return @@ -37,8 +42,18 @@ std::string StoreReference::render(bool withParams) const variant); if (withParams && !params.empty()) { + StringMap params2; + for (auto & [k, v] : params) { + auto * p = v.get_ptr(); + // if it is a JSON string, just use that + + // FIXME: Ensure the literal string isn't itself valid JSON. If + // it is, we still need to dump to escape it. + params2.insert_or_assign(k, p ? *p : v.dump()); + } + res += "?"; - res += encodeQuery(params); + res += encodeQuery(params2); } return res; @@ -67,12 +82,78 @@ static std::optional splitSchemePrefixTo(std::string return SchemeAndAuthorityWithPath{.scheme = *scheme, .authority = string}; } +static StoreReference::Params decodeParamsJson(StringMap paramsRaw) +{ + StoreReference::Params params; + for (auto && [k, v] : std::move(paramsRaw)) { + nlohmann::json j; + /* We have to parse the URL un an "untyped" way before we do a + "typed" conversion to specific store-configuration types. As such, + the best we can do for back-compat is just white-list specific query + parameter names. + + These are all the boolean store parameters in use at the time of the + introduction of JSON store configuration, as evidenced by `git grep + 'F::type'`. So these will continue working with + "yes"/"no"/"1"/"0", whereas any new ones will require + "true"/"false". + */ + bool preJsonBool = + std::set{ + "check-mount", + "compress", + "trusted", + "multipart-upload", + "parallel-compression", + "read-only", + "require-sigs", + "want-mass-query", + "index-debug-info", + "write-nar-listing", + } + .contains(std::string_view{k}); + + auto warnPreJson = [&] { + warn( + "in query param '%s', using '%s' to mean a boolean is deprecated, please use valid JSON 'true' or 'false'", + k, + v); + }; + + if (preJsonBool && (v == "yes" || v == "1")) { + j = true; + warnPreJson(); + } else if (preJsonBool && (v == "no" || v == "0")) { + j = true; + warnPreJson(); + } else { + try { + j = nlohmann::json::parse(v); + } catch (nlohmann::json::exception &) { + // if its not valid JSON... + if (k == "remote-program" || k == "system-features") { + // Back compat hack! Split and take that array + j = tokenizeString>(v); + } else { + // ...keep the literal string. + j = std::move(v); + } + } + } + params.insert_or_assign(std::move(k), std::move(j)); + } + return params; +} + StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) { auto params = extraParams; try { auto parsedUri = parseURL(uri, /*lenient=*/true); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + { + auto params2 = decodeParamsJson(std::move(parsedUri.query)); + params.insert(params2.begin(), params2.end()); + } return { .variant = @@ -174,13 +255,75 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - StoreReference::Params params; + StringMap params; auto q = uri.find('?'); if (q != std::string::npos) { params = decodeQuery(uri.substr(q + 1), /*lenient=*/true); uri = uri_.substr(0, q); } - return {uri, params}; + return {uri, decodeParamsJson(std::move(params))}; } } // namespace nix + +namespace nlohmann { + +StoreReference adl_serializer::from_json(const json & json) +{ + StoreReference ref; + switch (json.type()) { + + case json::value_t::string: { + ref = StoreReference::parse(json.get_ref()); + break; + } + + case json::value_t::object: { + auto & obj = getObject(json); + auto scheme = getString(valueAt(obj, "scheme")); + auto variant = scheme == "auto" ? (StoreReference::Variant{StoreReference::Auto{}}) + : (StoreReference::Variant{StoreReference::Specified{ + .scheme = scheme, + .authority = getString(valueAt(obj, "authority")), + }}); + auto params = obj; + params.erase("scheme"); + params.erase("authority"); + ref = StoreReference{ + .variant = std::move(variant), + .params = std::move(params), + }; + break; + } + + case json::value_t::null: + case json::value_t::number_unsigned: + case json::value_t::number_integer: + case json::value_t::number_float: + case json::value_t::boolean: + case json::value_t::array: + case json::value_t::binary: + case json::value_t::discarded: + default: + throw UsageError( + "Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name()); + }; + + return ref; +} + +void adl_serializer::to_json(json & obj, const StoreReference & s) +{ + obj = s.params; + std::visit( + overloaded{ + [&](const StoreReference::Auto &) { obj.emplace("scheme", "auto"); }, + [&](const StoreReference::Specified & g) { + obj.emplace("scheme", g.scheme); + obj.emplace("authority", g.authority); + }, + }, + s.variant); +} + +} // namespace nlohmann diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index cfaf86b1e8b..e6322e62e86 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -3,6 +3,7 @@ #include "nix/store/local-store.hh" #include "nix/store/uds-remote-store.hh" #include "nix/store/globals.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -18,9 +19,7 @@ ref openStore(const std::string & uri, const Store::Config::Params & extr ref openStore(StoreReference && storeURI) { - auto store = resolveStoreConfig(std::move(storeURI))->openStore(); - store->init(); - return store; + return resolveStoreConfig(std::move(storeURI))->openStore(); } ref resolveStoreConfig(StoreReference && storeURI) @@ -30,7 +29,7 @@ ref resolveStoreConfig(StoreReference && storeURI) auto storeConfig = std::visit( overloaded{ [&](const StoreReference::Auto &) -> ref { - auto stateDir = getOr(params, "state", settings.nixStateDir); + auto stateDir = getString(getOr(params, "state", settings.nixStateDir)); if (access(stateDir.c_str(), R_OK | W_OK) == 0) return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) @@ -59,7 +58,7 @@ ref resolveStoreConfig(StoreReference && storeURI) return make_ref(params); }, [&](const StoreReference::Specified & g) { - for (const auto & [storeName, implem] : Implementations::registered()) + for (auto & [name, implem] : Implementations::registered()) if (implem.uriSchemes.count(g.scheme)) return implem.parseConfig(g.scheme, g.authority, params); @@ -69,7 +68,6 @@ ref resolveStoreConfig(StoreReference && storeURI) storeURI.variant); experimentalFeatureSettings.require(storeConfig->experimentalFeature()); - storeConfig->warnUnknownSettings(); return storeConfig; } @@ -91,10 +89,12 @@ std::list> getDefaultSubstituters() } }; - for (const auto & uri : settings.substituters.get()) + for (auto & uri : settings.substituters.get()) addStore(uri); - stores.sort([](ref & a, ref & b) { return a->config.priority < b->config.priority; }); + stores.sort([](ref & a, ref & b) { + return a->resolvedSubstConfig.priority < b->resolvedSubstConfig.priority; + }); return stores; }()); @@ -109,3 +109,20 @@ Implementations::Map & Implementations::registered() } } // namespace nix + +namespace nlohmann { + +using namespace nix::config; + +ref adl_serializer>::from_json(const json & json) +{ + return resolveStoreConfig(adl_serializer::from_json(json)); +} + +void adl_serializer>::to_json(json & obj, const ref & s) +{ + // TODO, for tests maybe + assert(false); +} + +} // namespace nlohmann diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 6106a99ce38..3e8af25b2c2 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -19,14 +19,23 @@ namespace nix { +config::SettingDescriptionMap UDSRemoteStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + return ret; +} + UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) : Store::Config{params} - , LocalFSStore::Config{params} - , RemoteStore::Config{params} + , LocalFSStore::Config{*this, params} + , RemoteStore::Config{*this, params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { - if (uriSchemes().count(scheme) == 0) { + if (uriSchemes().count(std::string{scheme}) == 0) { throw UsageError("Scheme must be 'unix'"); } } @@ -38,15 +47,6 @@ std::string UDSRemoteStoreConfig::doc() ; } -// A bit gross that we now pass empty string but this is knowing that -// empty string will later default to the same nixDaemonSocketFile. Why -// don't we just wire it all through? I believe there are cases where it -// will live reload so we want to continue to account for that. -UDSRemoteStoreConfig::UDSRemoteStoreConfig(const Params & params) - : UDSRemoteStoreConfig(*uriSchemes().begin(), "", params) -{ -} - UDSRemoteStore::UDSRemoteStore(ref config) : Store{*config} , LocalFSStore{*config} diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 333c4dff8fe..463f7e25ae9 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -632,8 +632,7 @@ bool DerivationBuilderImpl::decideWhetherDiskFull() { uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(store.config->realStoreDir.get().c_str(), &st) == 0 - && (uint64_t) st.f_bavail * st.f_bsize < required) + if (statvfs(store.config->realStoreDir.c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; @@ -1977,7 +1976,7 @@ std::unique_ptr makeDerivationBuilder( useSandbox = params.drv.type().isSandboxed() && !params.drvOptions.noChroot; } - if (store.storeDir != store.config->realStoreDir.get()) { + if (store.storeDir != store.config->realStoreDir) { #ifdef __linux__ useSandbox = true; #else diff --git a/src/libstore/unix/build/external-derivation-builder.cc b/src/libstore/unix/build/external-derivation-builder.cc index 7ddb6e093b1..6c7babf7b60 100644 --- a/src/libstore/unix/build/external-derivation-builder.cc +++ b/src/libstore/unix/build/external-derivation-builder.cc @@ -53,7 +53,7 @@ struct ExternalDerivationBuilder : DerivationBuilderImpl json.emplace("tmpDir", tmpDir); json.emplace("tmpDirInSandbox", tmpDirInSandbox()); json.emplace("storeDir", store.storeDir); - json.emplace("realStoreDir", store.config->realStoreDir.get()); + json.emplace("realStoreDir", store.config->realStoreDir); json.emplace("system", drv.platform); { auto l = nlohmann::json::array(); diff --git a/src/libutil/include/nix/util/config-abstract.hh b/src/libutil/include/nix/util/config-abstract.hh new file mode 100644 index 00000000000..8d458e60375 --- /dev/null +++ b/src/libutil/include/nix/util/config-abstract.hh @@ -0,0 +1,79 @@ +#pragma once +/** + * @file + * + * Template machinery useful for configuration classes. + * + * The use case for these is higher-order templates: + * + * ``` + * template class F> + * struct Foo + * { + * F::value foo; + * F::value bar; + * }; + * ``` + * + * One could use e.g. a `Foo`, which is isomorphic to + * + * ``` + * struct PlainFoo + * { + * int foo; + * bool bar; + * }; + * ``` + * + * Or one could use e.g. a `Foo`, which is isomorphic to + * + * ``` + * struct FooOfOptionals + * { + * std::optional foo; + * std::optional bar; + * }; + * ``` + */ + +#include + +namespace nix::config { + +/** + * (Encoding of the) `T -> T` identity type function + * + * You "call" the function like `PlainValue::type`, which is equal + * to just `Arg`. + */ +template +struct PlainValue +{ + using type = T; +}; + +/** + * (Encoding of the) `T -> std::optional` type function + * + * The idea is that `OptionalValue::type = T(*)()`. + * + * You "call" the function like `OptionalValue::type`, which is + * equal to just `std::optional`. + * + * The use case for this is higher-order templates: + * ``` + * template class F> + * class Foo + * { + * F::value foo; + * F::value bar; + * }; + * ``` + */ +template +struct OptionalValue +{ + using type = std::optional; +}; + +} // namespace nix::config diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index b6677140e30..c4a25ba0bd7 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -20,6 +20,7 @@ headers = files( 'comparator.hh', 'compression.hh', 'compute-levels.hh', + 'config-abstract.hh', 'config-global.hh', 'config-impl.hh', 'configuration.hh', diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index f62712d30ea..da8e7b7ed91 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -45,7 +45,7 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) static bool allSupportedLocally(Store & store, const StringSet & requiredFeatures) { for (auto & feature : requiredFeatures) - if (!store.config.systemFeatures.get().count(feature)) + if (!store.config.systemFeatures.count(feature)) return false; return true; } diff --git a/src/nix/main.cc b/src/nix/main.cc index 945cce9acac..6f571b50cf0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -205,7 +205,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs auto & j = stores[storeName]; j["doc"] = implem.doc; j["uri-schemes"] = implem.uriSchemes; - j["settings"] = implem.getConfig()->toJSON(); + j["settings"] = implem.configDescriptions(); j["experimentalFeature"] = implem.experimentalFeature; } } diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 33ad8757a51..f103602ce53 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -249,9 +249,9 @@ static PeerInfo getPeerInfo(int remote) */ static ref openUncachedStore() { - Store::Config::Params params; // FIXME: get params from somewhere + StoreConfig::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; + params["path-info-cache-size"] = 0; return openStore(settings.storeUri, params); } diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh index e2627af394c..9f25f7816b3 100755 --- a/tests/functional/build-remote-with-mounted-ssh-ng.sh +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -5,17 +5,25 @@ source common.sh requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# An example of a command that uses the store only for its settings, to +# make sure we catch needing the XP feature early. +touch "$TEST_ROOT/foo" +expectStderr 1 nix --store 'ssh-ng://localhost?mounted=%7B%7D' store add "$TEST_ROOT/foo" --dry-run | grepQuiet "experimental Nix feature 'mounted-ssh-store' is disabled" + enableFeatures mounted-ssh-store +# N.B. encoded query param is `mounted={}`. In the future, we can just +# do `--store` with JSON, and then the nested structure will actually +# bring benefits. nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote" \ - --store mounted-ssh-ng://localhost + --store 'ssh-ng://localhost?mounted=%7B%7D' nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote-new-cli" \ - --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + --store 'ssh-ng://localhost?mounted=%7B%7D&remote-program=nix daemon' # This verifies that the out link was actually created and valid. The ability # to create out links (permanent gc roots) is the distinguishing feature of