diff --git a/.gitignore b/.gitignore index 3f81f8ef5d8..4711fa7faf6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,15 +12,6 @@ perl/Makefile.config /svn-revision /libtool -/corepkgs/config.nix - -# /corepkgs/channels/ -/corepkgs/channels/unpack.sh - -# /corepkgs/nar/ -/corepkgs/nar/nar.sh -/corepkgs/nar/unnar.sh - # /doc/manual/ /doc/manual/*.1 /doc/manual/*.5 diff --git a/corepkgs/config.nix.in b/corepkgs/config.nix.in deleted file mode 100644 index cb994594420..00000000000 --- a/corepkgs/config.nix.in +++ /dev/null @@ -1,13 +0,0 @@ -# FIXME: remove this file? -let - fromEnv = var: def: - let val = builtins.getEnv var; in - if val != "" then val else def; -in rec { - nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; - nixPrefix = "@prefix@"; - nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@"; - nixLocalstateDir = "@localstatedir@"; - nixSysconfDir = "@sysconfdir@"; - nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@"; -} diff --git a/corepkgs/local.mk b/corepkgs/local.mk index 2c72d3a3199..0bc91cfab00 100644 --- a/corepkgs/local.mk +++ b/corepkgs/local.mk @@ -1,8 +1,4 @@ corepkgs_FILES = \ - unpack-channel.nix \ - derivation.nix \ fetchurl.nix -$(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) - -template-files += $(d)/config.nix +$(foreach file,$(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) diff --git a/doc/manual/generate-builtins.jq b/doc/manual/generate-builtins.jq deleted file mode 100644 index c3879947967..00000000000 --- a/doc/manual/generate-builtins.jq +++ /dev/null @@ -1,6 +0,0 @@ -. | to_entries | sort_by(.key) | map( - " - `builtins." + .key + "` " - + (.value.args | map("*" + . + "*") | join(" ")) - + " \n\n" - + (.value.doc | split("\n") | map(" " + . + "\n") | join("")) + "\n\n" -) | join("") diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix new file mode 100644 index 00000000000..416a7fdba1c --- /dev/null +++ b/doc/manual/generate-builtins.nix @@ -0,0 +1,14 @@ +with builtins; +with import ./utils.nix; + +builtins: + +concatStrings (map + (name: + let builtin = builtins.${name}; in + " - `builtins.${name}` " + concatStringsSep " " (map (s: "*${s}*") builtin.args) + + " \n\n" + + concatStrings (map (s: " ${s}\n") (splitLines builtin.doc)) + "\n\n" + ) + (attrNames builtins)) + diff --git a/doc/manual/generate-manpage.jq b/doc/manual/generate-manpage.jq deleted file mode 100644 index dd632f162a7..00000000000 --- a/doc/manual/generate-manpage.jq +++ /dev/null @@ -1,44 +0,0 @@ -def show_flags: - .flags - | map_values(select(.category != "config")) - | to_entries - | map( - " - `--" + .key + "`" - + (if .value.shortName then " / `" + .value.shortName + "`" else "" end) - + (if .value.labels then " " + (.value.labels | map("*" + . + "*") | join(" ")) else "" end) - + " \n" - + " " + .value.description + "\n\n") - | join("") - ; - -def show_synopsis: - "`" + .command + "` [*flags*...] " + (.args | map("*" + .label + "*" + (if has("arity") then "" else "..." end)) | join(" ")) + "\n\n" - ; - -def show_command: - . as $top | - .section + " Name\n\n" - + "`" + .command + "` - " + .def.description + "\n\n" - + .section + " Synopsis\n\n" - + ({"command": .command, "args": .def.args} | show_synopsis) - + (if .def | has("doc") - then .section + " Description\n\n" + .def.doc + "\n\n" - else "" - end) - + (if (.def.flags | length) > 0 then - .section + " Flags\n\n" - + (.def | show_flags) - else "" end) - + (if (.def.examples | length) > 0 then - .section + " Examples\n\n" - + (.def.examples | map(.description + "\n\n```console\n" + .command + "\n```\n" ) | join("\n")) - + "\n" - else "" end) - + (if .def.commands then .def.commands | to_entries | map( - "# Subcommand `" + ($top.command + " " + .key) + "`\n\n" - + ({"command": ($top.command + " " + .key), "section": "##", "def": .value} | show_command) - ) | join("") else "" end) - ; - -"Title: nix\n\n" -+ ({"command": "nix", "section": "#", "def": .} | show_command) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix new file mode 100644 index 00000000000..4709c0e2c99 --- /dev/null +++ b/doc/manual/generate-manpage.nix @@ -0,0 +1,56 @@ +with builtins; +with import ./utils.nix; + +let + + showCommand = + { command, section, def }: + "${section} Name\n\n" + + "`${command}` - ${def.description}\n\n" + + "${section} Synopsis\n\n" + + showSynopsis { inherit command; args = def.args; } + + (if def ? doc + then "${section} Description\n\n" + def.doc + "\n\n" + else "") + + (let s = showFlags def.flags; in + if s != "" + then "${section} Flags\n\n${s}" + else "") + + (if def.examples or [] != [] + then + "${section} Examples\n\n" + + concatStrings (map ({ description, command }: "${description}\n\n```console\n${command}\n```\n\n") def.examples) + else "") + + (if def.commands or [] != [] + then concatStrings ( + map (name: + "# Subcommand `${command} ${name}`\n\n" + + showCommand { command = command + " " + name; section = "##"; def = def.commands.${name}; }) + (attrNames def.commands)) + else ""); + + showFlags = flags: + concatStrings + (map (longName: + let flag = flags.${longName}; in + if flag.category or "" != "config" + then + " - `--${longName}`" + + (if flag ? shortName then " / `${flag.shortName}`" else "") + + (if flag ? labels then " " + (concatStringsSep " " (map (s: "*${s}*") flag.labels)) else "") + + " \n" + + " " + flag.description + "\n\n" + else "") + (attrNames flags)); + + showSynopsis = + { command, args }: + "`${command}` [*flags*...] ${concatStringsSep " " + (map (arg: "*${arg.label}*" + (if arg ? arity then "" else "...")) args)}\n\n"; + +in + +command: + +"Title: nix\n\n" ++ showCommand { command = "nix"; section = "#"; def = command; } diff --git a/doc/manual/generate-options.jq b/doc/manual/generate-options.jq deleted file mode 100644 index ccf62e8edde..00000000000 --- a/doc/manual/generate-options.jq +++ /dev/null @@ -1,16 +0,0 @@ -. | to_entries | sort_by(.key) | map( - " - `" + .key + "` \n\n" - + (.value.description | split("\n") | map(" " + . + "\n") | join("")) + "\n\n" - + " **Default:** " + ( - if .value.value == "" or .value.value == [] - then "*empty*" - elif (.value.value | type) == "array" - then "`" + (.value.value | join(" ")) + "`" - else "`" + (.value.value | tostring) + "`" - end) - + "\n\n" - + (if (.value.aliases | length) > 0 - then " **Deprecated alias:** " + (.value.aliases | map("`" + . + "`") | join(", ")) + "\n\n" - else "" - end) -) | join("") diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix new file mode 100644 index 00000000000..7afe279c373 --- /dev/null +++ b/doc/manual/generate-options.nix @@ -0,0 +1,21 @@ +with builtins; +with import ./utils.nix; + +options: + +concatStrings (map + (name: + let option = options.${name}; in + " - `${name}` \n\n" + + concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n" + + " **Default:** " + ( + if option.value == "" || option.value == [] + then "*empty*" + else if isBool option.value + then (if option.value then "`true`" else "`false`") + else "`" + toString option.value + "`") + "\n\n" + + (if option.aliases != [] + then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n" + else "") + ) + (attrNames options)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 297a7341499..3b8e7e2df70 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -15,6 +15,8 @@ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 dist-files += $(man-pages) +nix-eval = $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw --expr + $(d)/%.1: $(d)/src/command-ref/%.md $(trace-gen) lowdown -sT man $^ -o $@ @@ -24,25 +26,31 @@ $(d)/%.8: $(d)/src/command-ref/%.md $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md $(trace-gen) lowdown -sT man $^ -o $@ -$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.jq - jq -r -f doc/manual/generate-manpage.jq $< > $@ +$(d)/src/command-ref/nix.md: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix + $(trace-gen) $(nix-eval) 'import doc/manual/generate-manpage.nix (builtins.fromJSON (builtins.readFile $<))' > $@.tmp + @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.jq $(d)/src/command-ref/conf-file-prefix.md - cat doc/manual/src/command-ref/conf-file-prefix.md > $@ - jq -r -f doc/manual/generate-options.jq $< >> $@ +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix + @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp + $(trace-gen) $(nix-eval) 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix - $(trace-gen) $(bindir)/nix __dump-args > $@ + $(trace-gen) $(bindir)/nix __dump-args > $@.tmp + @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix - $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@ + $(trace-gen) env -i NIX_CONF_DIR=/dummy HOME=/dummy $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp + @mv $@.tmp $@ -$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.jq $(d)/src/expressions/builtins-prefix.md - cat doc/manual/src/expressions/builtins-prefix.md > $@ - jq -r -f doc/manual/generate-builtins.jq $< >> $@ +$(d)/src/expressions/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/expressions/builtins-prefix.md $(bindir)/nix + @cat doc/manual/src/expressions/builtins-prefix.md > $@.tmp + $(trace-gen) $(nix-eval) 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + @mv $@.tmp $@ $(d)/builtins.json: $(bindir)/nix - $(trace-gen) $(bindir)/nix __dump-builtins > $@ + $(trace-gen) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp + mv $@.tmp $@ # Generate the HTML manual. install: $(docdir)/manual/index.html diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 1cd1063cd24..1307c7c3776 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -51,6 +51,9 @@ Nix store is also printed. result to the Nix store. The resulting hash can be used with functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + - `--executable` + Set the executable bit on the downloaded file. + - `--name` *name* Override the name of the file in the Nix store. By default, this is `hash-basename`, where *basename* is the last component of *url*. diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix new file mode 100644 index 00000000000..50150bf3e46 --- /dev/null +++ b/doc/manual/utils.nix @@ -0,0 +1,7 @@ +with builtins; + +{ + splitLines = s: filter (x: !isList x) (split "\n" s); + + concatStrings = concatStringsSep ""; +} diff --git a/flake.lock b/flake.lock index f4368b1702c..822a73332c8 100644 --- a/flake.lock +++ b/flake.lock @@ -3,16 +3,15 @@ "lowdown-src": { "flake": false, "locked": { - "lastModified": 1598296217, - "narHash": "sha256-ha7lyNY1d8m+osmDpPc9f/bfZ3ZC1IVIXwfyklSWg8I=", - "owner": "edolstra", + "lastModified": 1598695561, + "narHash": "sha256-gyH/5j+h/nWw0W8AcR2WKvNBUsiQ7QuxqSJNXAwV+8E=", + "owner": "kristapsdz", "repo": "lowdown", - "rev": "c7a4e715af1e233080842db82d15b261cb74cb28", + "rev": "1705b4a26fbf065d9574dce47a94e8c7c79e052f", "type": "github" }, "original": { - "owner": "edolstra", - "ref": "no-structs-in-anonymous-unions", + "owner": "kristapsdz", "repo": "lowdown", "type": "github" } diff --git a/flake.nix b/flake.nix index b1707abcd39..4798fedd8aa 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "The purely functional package manager"; inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small"; - inputs.lowdown-src = { url = "github:edolstra/lowdown/no-structs-in-anonymous-unions"; flake = false; }; + inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; outputs = { self, nixpkgs, lowdown-src }: @@ -137,7 +137,7 @@ enableParallelBuilding = true; - makeFlags = "profiledir=$(out)/etc/profile.d"; + makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; doCheck = true; @@ -335,9 +335,6 @@ # syntax-check generated dot files, it still requires some # fonts. So provide those. FONTCONFIG_FILE = texFunctions.fontsConf; - - # To test building without precompiled headers. - makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ]; }; # System tests. diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 9f26296a9d7..c334639e2ec 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -4,6 +4,8 @@ EnvironmentVariables + NIX_SSL_CERT_FILE + /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt OBJC_DISABLE_INITIALIZE_FORK_SAFETY YES diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 500c99e4ac6..1fdb4b3a44e 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -1,4 +1,4 @@ -PRECOMPILE_HEADERS ?= 1 +PRECOMPILE_HEADERS ?= 0 print-var-help += \ echo " PRECOMPILE_HEADERS ($(PRECOMPILE_HEADERS)): Whether to use precompiled headers to speed up the build"; diff --git a/perl/configure.ac b/perl/configure.ac index c3769e1426c..255744afd1c 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -70,19 +70,6 @@ PKG_CHECK_MODULES([NIX], [nix-store]) NEED_PROG([NIX], [nix]) -# Get nix configure values -export NIX_REMOTE=daemon -nixbindir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixBinDir) -nixlibexecdir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixLibexecDir) -nixlocalstatedir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixLocalstateDir) -nixsysconfdir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixSysconfDir) -nixstoredir=$("$NIX" --experimental-features nix-command eval --raw -f '' nixStoreDir) -AC_SUBST(nixbindir) -AC_SUBST(nixlibexecdir) -AC_SUBST(nixlocalstatedir) -AC_SUBST(nixsysconfdir) -AC_SUBST(nixstoredir) - # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix test "$exec_prefix" = NONE && exec_prefix='${prefix}' diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index bc1749e601e..f7c6f2484c3 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -4,14 +4,8 @@ use MIME::Base64; $version = "@PACKAGE_VERSION@"; -$binDir = $ENV{"NIX_BIN_DIR"} || "@nixbindir@"; -$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@nixlibexecdir@"; -$stateDir = $ENV{"NIX_STATE_DIR"} || "@nixlocalstatedir@/nix"; -$logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix"; -$confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix"; -$storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@"; - -$useBindings = 1; +$binDir = Nix::Store::getBinDir; +$storeDir = Nix::Store::getStoreDir; %config = (); diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index d226264d4df..179f1dc9048 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -2,7 +2,6 @@ package Nix::Store; use strict; use warnings; -use Nix::Config; require Exporter; @@ -22,6 +21,7 @@ our @EXPORT = qw( addToStore makeFixedOutputPath derivationFromPath addTempRoot + getBinDir getStoreDir ); our $VERSION = '0.15'; @@ -34,62 +34,8 @@ sub backtick { return $res; } -if ($Nix::Config::useBindings) { - require XSLoader; - XSLoader::load('Nix::Store', $VERSION); -} else { - - # Provide slow fallbacks of some functions on platforms that don't - # support the Perl bindings. - - use File::Temp; - use Fcntl qw/F_SETFD/; - - *hashFile = sub { - my ($algo, $base32, $path) = @_; - my $res = backtick("$Nix::Config::binDir/nix-hash", "--flat", $path, "--type", $algo, $base32 ? "--base32" : ()); - chomp $res; - return $res; - }; - - *hashPath = sub { - my ($algo, $base32, $path) = @_; - my $res = backtick("$Nix::Config::binDir/nix-hash", $path, "--type", $algo, $base32 ? "--base32" : ()); - chomp $res; - return $res; - }; - - *hashString = sub { - my ($algo, $base32, $s) = @_; - my $fh = File::Temp->new(); - print $fh $s; - my $res = backtick("$Nix::Config::binDir/nix-hash", $fh->filename, "--type", $algo, $base32 ? "--base32" : ()); - chomp $res; - return $res; - }; - - *addToStore = sub { - my ($srcPath, $recursive, $algo) = @_; - die "not implemented" if $recursive || $algo ne "sha256"; - my $res = backtick("$Nix::Config::binDir/nix-store", "--add", $srcPath); - chomp $res; - return $res; - }; - - *isValidPath = sub { - my ($path) = @_; - my $res = backtick("$Nix::Config::binDir/nix-store", "--check-validity", "--print-invalid", $path); - chomp $res; - return $res ne $path; - }; - - *queryPathHash = sub { - my ($path) = @_; - my $res = backtick("$Nix::Config::binDir/nix-store", "--query", "--hash", $path); - chomp $res; - return $res; - }; -} +require XSLoader; +XSLoader::load('Nix::Store', $VERSION); 1; __END__ diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 8546f63078d..599921151d4 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -351,3 +351,13 @@ void addTempRoot(char * storePath) } catch (Error & e) { croak("%s", e.what()); } + + +SV * getBinDir() + PPCODE: + XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0))); + + +SV * getStoreDir() + PPCODE: + XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0))); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d678231c671..883fc27a7ea 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -356,6 +356,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sEpsilon(symbols.create("")) , repair(NoRepair) , store(store) + , regexCache(makeRegexCache()) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -370,7 +371,11 @@ EvalState::EvalState(const Strings & _searchPath, ref store) for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); } - addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + + try { + addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + } catch (Error &) { + } if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 80078d8a5a6..0e1f61baab3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,7 +6,6 @@ #include "symbol-table.hh" #include "config.hh" -#include #include #include #include @@ -65,6 +64,11 @@ typedef std::list SearchPath; void initGC(); +struct RegexCache; + +std::shared_ptr makeRegexCache(); + + class EvalState { public: @@ -120,7 +124,7 @@ private: std::unordered_map resolvedPaths; /* Cache used by prim_match(). */ - std::unordered_map regexCache; + std::shared_ptr regexCache; public: diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 6363446f6ca..d5c2ffe66c8 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -1,6 +1,7 @@ #include "flakeref.hh" #include "store-api.hh" #include "url.hh" +#include "url-parts.hh" #include "fetchers.hh" #include "registry.hh" diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a748469442e..78431f000ea 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -1,5 +1,6 @@ #include "lockfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 5e7cfda3e3d..9ec8b39c35e 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -6,7 +6,7 @@ namespace nix { class Store; -struct StorePath; +class StorePath; } namespace nix::flake { diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index d84b150e090..687a8ccdaaa 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -42,6 +42,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4a0dd554455..9cfe3f402ff 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1704,7 +1704,7 @@ static RegisterPrimOp primop_toFile({ ... cp ${configFile} $out/etc/foo.conf "; - ``` + ``` Note that `${configFile}` is an [antiquotation](language-values.md), so the result of the @@ -3085,17 +3085,25 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); -/* Match a regular expression against a string and return either - ‘null’ or a list containing substring matches. */ +struct RegexCache +{ + std::unordered_map cache; +}; + +std::shared_ptr makeRegexCache() +{ + return std::make_shared(); +} + void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto re = state.forceStringNoCtx(*args[0], pos); try { - auto regex = state.regexCache.find(re); - if (regex == state.regexCache.end()) - regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first; + auto regex = state.regexCache->cache.find(re); + if (regex == state.regexCache->cache.end()) + regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first; PathSet context; const std::string str = state.forceString(*args[1], context, pos); @@ -3565,9 +3573,10 @@ void EvalState::createBaseEnv() /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); - sDerivationNix = symbols.create(path); - evalFile(path, v); + sDerivationNix = symbols.create("//builtin/derivation.nix"); + eval(parse( + #include "primops/derivation.nix.gen.hh" + , foFile, sDerivationNix, "/", staticBaseEnv), v); addConstant("derivation", v); /* Now that we've added all primops, sort the `builtins' set, diff --git a/corepkgs/derivation.nix b/src/libexpr/primops/derivation.nix similarity index 100% rename from corepkgs/derivation.nix rename to src/libexpr/primops/derivation.nix diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index cef85cfef48..1a064ed5caf 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -3,8 +3,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "url.hh" - -#include +#include "url-parts.hh" namespace nix { diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index be71b786b96..89b1e6e7dda 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -23,7 +23,7 @@ struct InputScheme; struct Input { - friend class InputScheme; + friend struct InputScheme; std::shared_ptr scheme; // note: can be null Attrs attrs; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 5ca0f852143..ad7638d732f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1cc0c5e2e7c..1737658a7de 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -3,14 +3,15 @@ #include "fetchers.hh" #include "globals.hh" #include "store-api.hh" +#include "url-parts.hh" #include namespace nix::fetchers { -// A github or gitlab url -const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check -std::regex urlRegex(urlRegexS, std::regex::ECMAScript); +// A github or gitlab host +const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { @@ -50,9 +51,9 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains multiple branch/tag names", url.url); ref = value; } - else if (name == "url") { - if (!std::regex_match(value, urlRegex)) - throw BadURL("URL '%s' contains an invalid instance url", url.url); + else if (name == "host") { + if (!std::regex_match(value, hostRegex)) + throw BadURL("URL '%s' contains an invalid instance host", url.url); host_url = value; } // FIXME: barf on unsupported attributes @@ -67,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (ref) input.attrs.insert_or_assign("ref", *ref); - if (host_url) input.attrs.insert_or_assign("url", *host_url); + if (host_url) input.attrs.insert_or_assign("host", *host_url); return input; } @@ -77,7 +78,7 @@ struct GitArchiveInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != type()) return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") throw Error("unsupported input attribute '%s'", name); getStrAttr(attrs, "owner"); @@ -210,7 +211,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); + auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); @@ -236,7 +237,7 @@ struct GitLabInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); + auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b981d4d8ea6..74332ae3dda 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,4 +1,5 @@ #include "fetchers.hh" +#include "url-parts.hh" namespace nix::fetchers { diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 3e76ffc4dd7..d80c2ea7ad9 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" +#include "url-parts.hh" #include diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5433fe50d25..ebc0bd6a4e4 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -11,6 +11,7 @@ #include "nar-accessor.hh" #include "json.hh" #include "thread-pool.hh" +#include "callback.hh" #include #include @@ -22,7 +23,8 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(const Params & params) - : Store(params) + : BinaryCacheStoreConfig(params) + , Store(params) { if (secretKeyFile != "") secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 9bcdf5901b8..4b779cdd4ea 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -11,17 +11,21 @@ namespace nix { struct NarInfo; -class BinaryCacheStore : public Store +struct BinaryCacheStoreConfig : virtual StoreConfig { -public: - - const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; - const Setting writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; - const Setting parallelCompression{this, false, "parallel-compression", + using StoreConfig::StoreConfig; + + const Setting compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; + const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", "enable multi-threading compression, available for xz only currently"}; +}; + +class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig +{ private: @@ -58,7 +62,7 @@ public: public: - virtual void init(); + virtual void init() override; private: diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ce973862d4d..0b51d90eae3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -17,6 +17,7 @@ #include "daemon.hh" #include "worker-protocol.hh" #include "topo-sort.hh" +#include "callback.hh" #include #include @@ -2872,18 +2873,23 @@ void DerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } +struct RestrictedStoreConfig : LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + const std::string name() { return "Restricted Store"; } +}; /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore +struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig { ref next; DerivationGoal & goal; RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : Store(params), LocalFSStore(params), next(next), goal(goal) + : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) { } Path getRealStoreDir() override diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 7a5744bc159..49641c2acc0 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,18 +1,29 @@ #include "store-api.hh" +#include "callback.hh" namespace nix { -static std::string uriScheme = "dummy://"; +struct DummyStoreConfig : virtual StoreConfig { + using StoreConfig::StoreConfig; -struct DummyStore : public Store + const std::string name() override { return "Dummy Store"; } +}; + +struct DummyStore : public Store, public virtual DummyStoreConfig { - DummyStore(const Params & params) - : Store(params) + DummyStore(const std::string scheme, const std::string uri, const Params & params) + : DummyStore(params) { } + DummyStore(const Params & params) + : StoreConfig(params) + , Store(params) + { + } + string getUri() override { - return uriScheme; + return *uriSchemes().begin(); } void queryPathInfoUncached(const StorePath & path, @@ -21,6 +32,10 @@ struct DummyStore : public Store callback(nullptr); } + static std::set uriSchemes() { + return {"dummy"}; + } + std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } @@ -48,12 +63,6 @@ struct DummyStore : public Store { unsupported("buildDerivation"); } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr -{ - if (uri != uriScheme) return nullptr; - return std::make_shared(params); -}); +static RegisterStoreImplementation regStore; } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 17afba6081a..a601fd1647a 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -5,6 +5,7 @@ #include "s3.hh" #include "compression.hh" #include "finally.hh" +#include "callback.hh" #ifdef ENABLE_S3 #include diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e6cbc525d18..08b53c702f7 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -574,9 +574,12 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto /* If keep-derivations is set and this is a derivation, then don't delete the derivation if any of the outputs are alive. */ if (state.gcKeepDerivations && path.isDerivation()) { - for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i); + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && + isValidPath(*maybeOutPath) && + queryPathInfo(*maybeOutPath)->deriver == path + ) + incoming.insert(*maybeOutPath); } /* If keep-outputs is set, then don't delete this path if there diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4a5971c3f8f..0beb9b2b715 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "archive.hh" #include "args.hh" +#include "abstract-setting-to-json.hh" #include #include @@ -162,11 +163,6 @@ template<> std::string BaseSetting::to_string() const else abort(); } -template<> nlohmann::json BaseSetting::toJSON() -{ - return AbstractSetting::toJSON(); -} - template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1733239fb8d..86be7c006ce 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,12 +2,20 @@ #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "callback.hh" namespace nix { MakeError(UploadToHTTP, Error); -class HttpBinaryCacheStore : public BinaryCacheStore +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override { return "Http Binary Cache Store"; } +}; + +class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig { private: @@ -24,9 +32,12 @@ class HttpBinaryCacheStore : public BinaryCacheStore public: HttpBinaryCacheStore( - const Params & params, const Path & _cacheUri) - : BinaryCacheStore(params) - , cacheUri(_cacheUri) + const std::string & scheme, + const Path & _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStore(params) + , cacheUri(scheme + "://" + _cacheUri) { if (cacheUri.back() == '/') cacheUri.pop_back(); @@ -55,6 +66,13 @@ class HttpBinaryCacheStore : public BinaryCacheStore } } + static std::set uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) ret.insert("file"); + return ret; + } protected: void maybeDisable() @@ -162,18 +180,6 @@ class HttpBinaryCacheStore : public BinaryCacheStore }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr -{ - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - if (std::string(uri, 0, 7) != "http://" && - std::string(uri, 0, 8) != "https://" && - (!forceHttp || std::string(uri, 0, 7) != "file://")) - return 0; - auto store = std::make_shared(params, uri); - store->init(); - return store; -}); +static RegisterStoreImplementation regStore; } diff --git a/src/libstore/ipfs-binary-cache-store.cc b/src/libstore/ipfs-binary-cache-store.cc index 8d65ca242bc..626dfdd938e 100644 --- a/src/libstore/ipfs-binary-cache-store.cc +++ b/src/libstore/ipfs-binary-cache-store.cc @@ -7,13 +7,20 @@ #include "archive.hh" #include "compression.hh" #include "names.hh" +#include "callback.hh" namespace nix { +IPFSBinaryCacheStore::IPFSBinaryCacheStore(const Params & params) + : IPFSBinaryCacheStore("ipfs", "", params) +{ } + IPFSBinaryCacheStore::IPFSBinaryCacheStore( - const Params & params, const Path & _cacheUri) + const std::string & scheme, const std::string & uri, const Params & params) : Store(params) - , cacheUri(_cacheUri) + , IPFSBinaryCacheStoreConfig(params) + , cacheScheme(scheme) + , cacheUri(uri) { auto state(_state.lock()); @@ -27,22 +34,18 @@ IPFSBinaryCacheStore::IPFSBinaryCacheStore( if (cacheUri.back() == '/') cacheUri.pop_back(); - if (hasPrefix(cacheUri, "ipfs://")) { - if (cacheUri == "ipfs://") { - allowModify = true; - } else { - initialIpfsPath = "/ipfs/" + std::string(cacheUri, 7); - state->ipfsPath = initialIpfsPath; - allowModify = get(params, "allow-modify").value_or("") == "true"; - } - } else if (hasPrefix(cacheUri, "ipns://")) { - ipnsPath = "/ipns/" + std::string(cacheUri, 7); + if (cacheScheme == "ipfs") { + initialIpfsPath = "/ipfs/" + cacheUri; + state->ipfsPath = initialIpfsPath; + allowModify = get(params, "allow-modify").value_or("") == "true"; + } else if (cacheScheme == "ipns") { + ipnsPath = "/ipns/" + cacheUri; // TODO: we should try to determine if we are able to modify // this ipns allowModify = true; } else - throw Error("unknown IPNS URI '%s'", cacheUri); + throw Error("unknown IPNS URI '%s'", getUri()); std::string ipfsAPIHost(get(params, "host").value_or("127.0.0.1")); std::string ipfsAPIPort(get(params, "port").value_or("5001")); @@ -173,7 +176,7 @@ void IPFSBinaryCacheStore::sync() if (!ipnsPath) { warn("created new store at '%s'. The old store at %s is immutable, so we can't update it", - "ipfs://" + std::string(state->ipfsPath, 6), cacheUri); + "ipfs://" + std::string(state->ipfsPath, 6), getUri()); return; } @@ -276,7 +279,7 @@ void IPFSBinaryCacheStore::upsertFile(const std::string & path, const std::strin addLink(path, "/ipfs/" + addFile(data)); } catch (FileTransferError & e) { // TODO: may wrap the inner error in a better way. - throw UploadToIPFS("while uploading to IPFS binary cache at '%s': %s", cacheUri, e.msg()); + throw UploadToIPFS("while uploading to IPFS binary cache at '%s': %s", getUri(), e.msg()); } } @@ -413,7 +416,7 @@ void IPFSBinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSo if (!repair && isValidPath(info.path)) return; if (!allowModify) - throw Error("can't update '%s'", cacheUri); + throw Error("can't update '%s'", getUri()); /* Verify that all references are valid. This may do some .narinfo reads, but typically they'll already be cached. */ @@ -624,7 +627,7 @@ StorePath IPFSBinaryCacheStore::addTextToStore(const string & name, const string void IPFSBinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { if (!allowModify) - throw Error("can't update '%s'", cacheUri); + throw Error("can't update '%s'", getUri()); /* Note: this is inherently racy since there is no locking on binary caches. In particular, with S3 this unreliable, even @@ -651,15 +654,6 @@ void IPFSBinaryCacheStore::addTempRoot(const StorePath & path) getFileTransfer()->upload(request); } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr -{ - if (uri.substr(0, strlen("ipfs://")) != "ipfs://" && - uri.substr(0, strlen("ipns://")) != "ipns://") - return 0; - auto store = std::make_shared(params, uri); - return store; -}); +static RegisterStoreImplementation regStore; } diff --git a/src/libstore/ipfs-binary-cache-store.hh b/src/libstore/ipfs-binary-cache-store.hh index dcb7ad0ac25..1239d21cb13 100644 --- a/src/libstore/ipfs-binary-cache-store.hh +++ b/src/libstore/ipfs-binary-cache-store.hh @@ -6,10 +6,9 @@ namespace nix { MakeError(UploadToIPFS, Error); -class IPFSBinaryCacheStore : public Store +struct IPFSBinaryCacheStoreConfig : virtual StoreConfig { - -public: + using StoreConfig::StoreConfig; const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; @@ -20,13 +19,20 @@ public: const Setting _allowModify{this, false, "allow-modify", "allow Nix to update IPFS/IPNS address when appropriate"}; -private: + const std::string name() override { return "IPFS Store"; } +}; + +class IPFSBinaryCacheStore : public virtual Store, public virtual IPFSBinaryCacheStoreConfig +{ + +public: bool allowModify; std::unique_ptr secretKey; std::string narMagic; + std::string cacheScheme; std::string cacheUri; std::string daemonUri; @@ -45,13 +51,18 @@ private: public: - IPFSBinaryCacheStore(const Params & params, const Path & _cacheUri); + IPFSBinaryCacheStore(const Params & params); + + IPFSBinaryCacheStore(const std::string & scheme, const std::string & uri, const Params & params); std::string getUri() override { - return cacheUri; + return cacheScheme + "://" + cacheUri; } + static std::set uriSchemes() + { return {"ipfs", "ipns"}; } + private: std::string putIpfsDag(nlohmann::json data); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index dc03313f01c..5af75669afc 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -6,21 +6,28 @@ #include "worker-protocol.hh" #include "ssh.hh" #include "derivations.hh" +#include "callback.hh" namespace nix { -static std::string uriScheme = "ssh://"; - -struct LegacySSHStore : public Store +struct LegacySSHStoreConfig : virtual StoreConfig { - const Setting maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting sshKey{this, "", "ssh-key", "path to an SSH private key"}; - const Setting compress{this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; + using StoreConfig::StoreConfig; + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; + const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; + const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + + const std::string name() override { return "Legacy SSH Store"; } +}; +struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig +{ // Hack for getting remote build log output. - const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; struct Connection { @@ -37,8 +44,11 @@ struct LegacySSHStore : public Store SSHMaster master; - LegacySSHStore(const string & host, const Params & params) - : Store(params) + static std::set uriSchemes() { return {"ssh"}; } + + LegacySSHStore(const string & scheme, const string & host, const Params & params) + : StoreConfig(params) + , Store(params) , host(host) , connections(make_ref>( std::max(1, (int) maxConnections), @@ -84,7 +94,7 @@ struct LegacySSHStore : public Store string getUri() override { - return uriScheme + host; + return *uriSchemes().begin() + "://" + host; } void queryPathInfoUncached(const StorePath & path, @@ -325,12 +335,6 @@ struct LegacySSHStore : public Store } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation regStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 87d8334d74d..b5744448e47 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -4,7 +4,14 @@ namespace nix { -class LocalBinaryCacheStore : public BinaryCacheStore +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override { return "Local Binary Cache Store"; } +}; + +class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig { private: @@ -13,8 +20,11 @@ class LocalBinaryCacheStore : public BinaryCacheStore public: LocalBinaryCacheStore( - const Params & params, const Path & binaryCacheDir) - : BinaryCacheStore(params) + const std::string scheme, + const Path & binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { } @@ -26,6 +36,8 @@ class LocalBinaryCacheStore : public BinaryCacheStore return "file://" + binaryCacheDir; } + static std::set uriSchemes(); + protected: bool fileExists(const std::string & path) override; @@ -85,16 +97,14 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr +std::set LocalBinaryCacheStore::uriSchemes() { - if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || - std::string(uri, 0, 7) != "file://") - return 0; - auto store = std::make_shared(params, std::string(uri, 7)); - store->init(); - return store; -}); + if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1") + return {}; + else + return {"file"}; +} + +static RegisterStoreImplementation regStore; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0086bb13e38..94ff34cb8e7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,6 +6,7 @@ #include "derivations.hh" #include "nar-info.hh" #include "references.hh" +#include "callback.hh" #include #include @@ -42,7 +43,8 @@ namespace nix { LocalStore::LocalStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) , LocalFSStore(params) , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", "physical path to the Nix store"} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d5e6d68ef28..e7c9d1605be 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -30,8 +30,19 @@ struct OptimiseStats uint64_t blocksFreed = 0; }; +struct LocalStoreConfig : virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + + Setting requireSigs{(StoreConfig*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + + const std::string name() override { return "Local Store"; } +}; + -class LocalStore : public LocalFSStore +class LocalStore : public LocalFSStore, public virtual LocalStoreConfig { private: @@ -95,10 +106,6 @@ public: private: - Setting requireSigs{(Store*) this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - const PublicKeys & getPublicKeys(); public: diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index da398169621..ad4dccef995 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -5,7 +5,7 @@ #include "store-api.hh" #include "thread-pool.hh" #include "topo-sort.hh" - +#include "callback.hh" namespace nix { diff --git a/src/libstore/names.cc b/src/libstore/names.cc index d1c8a6101f8..41e28dc9931 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -1,10 +1,18 @@ #include "names.hh" #include "util.hh" +#include + namespace nix { +struct Regex +{ + std::regex regex; +}; + + DrvName::DrvName() { name = ""; @@ -30,11 +38,18 @@ DrvName::DrvName(std::string_view s) : hits(0) } +DrvName::~DrvName() +{ } + + bool DrvName::matches(DrvName & n) { if (name != "*") { - if (!regex) regex = std::unique_ptr(new std::regex(name, std::regex::extended)); - if (!std::regex_match(n.name, *regex)) return false; + if (!regex) { + regex = std::make_unique(); + regex->regex = std::regex(name, std::regex::extended); + } + if (!std::regex_match(n.name, regex->regex)) return false; } if (version != "" && version != n.version) return false; return true; @@ -99,7 +114,7 @@ DrvNames drvNamesFromArgs(const Strings & opArgs) { DrvNames result; for (auto & i : opArgs) - result.push_back(DrvName(i)); + result.emplace_back(i); return result; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 00e14b8c797..bc62aac9371 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -3,10 +3,11 @@ #include #include "types.hh" -#include namespace nix { +struct Regex; + struct DrvName { string fullName; @@ -16,10 +17,12 @@ struct DrvName DrvName(); DrvName(std::string_view s); + ~DrvName(); + bool matches(DrvName & n); private: - std::unique_ptr regex; + std::unique_ptr regex; }; typedef list DrvNames; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 61420e66159..0ca132e58ac 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -10,6 +10,7 @@ #include "pool.hh" #include "finally.hh" #include "logging.hh" +#include "callback.hh" #include #include @@ -94,9 +95,19 @@ void write(const Store & store, Sink & out, const std::optional & sto /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : Store(params) + , RemoteStoreConfig(params) , connections(make_ref>( std::max(1, (int) maxConnections), - [this]() { return openConnectionWrapper(); }, + [this]() { + auto conn = openConnectionWrapper(); + try { + initConnection(*conn); + } catch (...) { + failed = true; + throw; + } + return conn; + }, [this](const ref & r) { return r->to.good() @@ -123,19 +134,21 @@ ref RemoteStore::openConnectionWrapper() UDSRemoteStore::UDSRemoteStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) , LocalFSStore(params) , RemoteStore(params) { } -UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) - : Store(params) - , LocalFSStore(params) - , RemoteStore(params) - , path(socket_path) +UDSRemoteStore::UDSRemoteStore( + const std::string scheme, + std::string socket_path, + const Params & params) + : UDSRemoteStore(params) { + path.emplace(socket_path); } @@ -179,8 +192,6 @@ ref UDSRemoteStore::openConnection() conn->startTime = std::chrono::steady_clock::now(); - initConnection(*conn); - return conn; } @@ -991,14 +1002,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -static std::string uriScheme = "unix://"; - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation regStore; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 490840bd86f..566bc2943ab 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -16,18 +16,22 @@ struct FdSource; template class Pool; struct ConnectionHandle; - -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ -class RemoteStore : public virtual Store +struct RemoteStoreConfig : virtual StoreConfig { -public: + using StoreConfig::StoreConfig; - const Setting maxConnections{(Store*) this, 1, + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent connections to the Nix daemon"}; - const Setting maxConnectionAge{(Store*) this, std::numeric_limits::max(), + const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), "max-connection-age", "number of seconds to reuse a connection"}; +}; + +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +{ +public: virtual bool sameMachine() = 0; @@ -104,8 +108,6 @@ public: void flushBadConnections(); -protected: - struct Connection { AutoCloseFD fd; @@ -121,6 +123,8 @@ protected: ref openConnectionWrapper(); +protected: + virtual ref openConnection() = 0; void initConnection(Connection & conn); @@ -143,15 +147,35 @@ private: }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore +struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +{ + UDSRemoteStoreConfig(const Store::Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + { + } + + UDSRemoteStoreConfig() + : UDSRemoteStoreConfig(Store::Params({})) + { + } + + const std::string name() override { return "Local Daemon Store"; } +}; + +class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig { public: UDSRemoteStore(const Params & params); - UDSRemoteStore(std::string path, const Params & params); + UDSRemoteStore(const std::string scheme, std::string path, const Params & params); std::string getUri() override; + static std::set uriSchemes() + { return {"unix"}; } + bool sameMachine() override { return true; } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index a0a446bd399..d43f267e00f 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -172,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { - const Setting profile{this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting lsCompression{this, "", "ls-compression", "compression method for .ls files"}; - const Setting logCompression{this, "", "log-compression", "compression method for log/* files"}; + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + const Setting profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; + const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; + const Setting endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; + const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; + const Setting logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; const Setting multipartUpload{ - this, false, "multipart-upload", "whether to use multi-part uploads"}; + (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + const std::string name() override { return "S3 Binary Cache Store"; } +}; + +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig +{ std::string bucketName; Stats stats; @@ -193,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore S3Helper s3Helper; S3BinaryCacheStoreImpl( - const Params & params, const std::string & bucketName) - : S3BinaryCacheStore(params) + const std::string & scheme, + const std::string & bucketName, + const Params & params) + : StoreConfig(params) + , S3BinaryCacheStore(params) , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { @@ -426,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return paths; } + static std::set uriSchemes() { return {"s3"}; } + }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr -{ - if (std::string(uri, 0, 5) != "s3://") return 0; - auto store = std::make_shared(params, std::string(uri, 5)); - store->init(); - return store; -}); +static RegisterStoreImplementation regStore; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 6cb97c1f11a..6d6eca98d78 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -8,19 +8,25 @@ namespace nix { -static std::string uriScheme = "ssh-ng://"; +struct SSHStoreConfig : virtual RemoteStoreConfig +{ + using RemoteStoreConfig::RemoteStoreConfig; + + const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; + const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; + const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + + const std::string name() override { return "SSH Store"; } +}; -class SSHStore : public RemoteStore +class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig { public: - const Setting sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting compress{(Store*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"}; - - SSHStore(const std::string & host, const Params & params) - : Store(params) + SSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , Store(params) , RemoteStore(params) , host(host) , master( @@ -32,9 +38,11 @@ class SSHStore : public RemoteStore { } + static std::set uriSchemes() { return {"ssh-ng"}; } + std::string getUri() override { - return uriScheme + host; + return *uriSchemes().begin() + "://" + host; } bool sameMachine() override @@ -72,16 +80,9 @@ ref SSHStore::openConnection() + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); - initConnection(*conn); return conn; } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation regStore; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b3877487ccd..1bbc74db80d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -8,9 +8,7 @@ #include "json.hh" #include "url.hh" #include "archive.hh" - -#include - +#include "callback.hh" namespace nix { @@ -346,7 +344,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, Store::Store(const Params & params) - : Config(params) + : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { } @@ -1009,7 +1007,6 @@ Derivation Store::readDerivation(const StorePath & drvPath) } } - } @@ -1019,9 +1016,6 @@ Derivation Store::readDerivation(const StorePath & drvPath) namespace nix { - -RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair splitUriAndParams(const std::string & uri_) { @@ -1035,24 +1029,6 @@ std::pair splitUriAndParams(const std::string & uri_ return {uri, params}; } -ref openStore(const std::string & uri_, - const Store::Params & extraParams) -{ - auto [uri, uriParams] = splitUriAndParams(uri_); - auto params = extraParams; - params.insert(uriParams.begin(), uriParams.end()); - - for (auto fun : *RegisterStoreImplementation::implementations) { - auto store = fun(uri, params); - if (store) { - store->warnUnknownSettings(); - return ref(store); - } - } - - throw Error("don't know how to open Nix store '%s'", uri); -} - static bool isNonUriPath(const std::string & spec) { return // is not a URL @@ -1062,44 +1038,62 @@ static bool isNonUriPath(const std::string & spec) { && spec.find("/") != std::string::npos; } -StoreType getStoreType(const std::string & uri, const std::string & stateDir) +std::shared_ptr openFromNonUri(const std::string & uri, const Store::Params & params) { - if (uri == "daemon") { - return tDaemon; - } else if (uri == "local" || isNonUriPath(uri)) { - return tLocal; - } else if (uri == "" || uri == "auto") { + if (uri == "" || uri == "auto") { + auto stateDir = get(params, "state").value_or(settings.nixStateDir); if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return tLocal; + return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) - return tDaemon; + return std::make_shared(params); else - return tLocal; + return std::make_shared(params); + } else if (uri == "daemon") { + return std::make_shared(params); + } else if (uri == "local") { + return std::make_shared(params); + } else if (isNonUriPath(uri)) { + Store::Params params2 = params; + params2["root"] = absPath(uri); + return std::make_shared(params2); } else { - return tOther; + return nullptr; } } - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr +ref openStore(const std::string & uri_, + const Store::Params & extraParams) { - switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) { - case tDaemon: - return std::shared_ptr(std::make_shared(params)); - case tLocal: { - Store::Params params2 = params; - if (isNonUriPath(uri)) { - params2["root"] = absPath(uri); + auto params = extraParams; + try { + auto parsedUri = parseURL(uri_); + params.insert(parsedUri.query.begin(), parsedUri.query.end()); + + auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + + for (auto implem : *Implementations::registered) { + if (implem.uriSchemes.count(parsedUri.scheme)) { + auto store = implem.create(parsedUri.scheme, baseURI, params); + if (store) { + store->init(); + store->warnUnknownSettings(); + return ref(store); + } } - return std::shared_ptr(std::make_shared(params2)); } - default: - return nullptr; } -}); + catch (BadURL &) { + auto [uri, uriParams] = splitUriAndParams(uri_); + params.insert(uriParams.begin(), uriParams.end()); + if (auto store = openFromNonUri(uri, params)) { + store->warnUnknownSettings(); + return ref(store); + } + } + + throw Error("don't know how to open Nix store '%s'", uri_); +} std::list> getDefaultSubstituters() { @@ -1133,5 +1127,6 @@ std::list> getDefaultSubstituters() return stores; } +std::vector * Implementations::registered = 0; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0e456ac115e..4226cff42b9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -24,6 +24,31 @@ namespace nix { +/** + * About the class hierarchy of the store implementations: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `const Setting` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the + * implementation of the store. + * + * This class is expected to have a constructor `Foo(const Params & params)` + * that calls `StoreConfig(params)` (otherwise you're gonna encounter an + * `assertion failure` when trying to instantiate it). + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation regStore; + * ``` + */ MakeError(SubstError, Error); MakeError(BuildError, Error); // denotes a permanent build failure @@ -33,6 +58,7 @@ MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); MakeError(BadStorePath, Error); +MakeError(InvalidStoreURI, Error); class FSAccessor; class NarInfoDiskCache; @@ -144,12 +170,31 @@ struct BuildResult } }; - -class Store : public std::enable_shared_from_this, public Config +struct StoreConfig : public Config { -public: - - typedef std::map Params; + using Config::Config; + + /** + * When constructing a store implementation, we pass in a map `params` of + * parameters that's supposed to initialize the associated config. + * To do that, we must use the `StoreConfig(StringMap & params)` + * constructor, so we'd like to `delete` its default constructor to enforce + * it. + * + * However, actually deleting it means that all the subclasses of + * `StoreConfig` will have their default constructor deleted (because it's + * supposed to call the deleted default constructor of `StoreConfig`). But + * because we're always using virtual inheritance, the constructors of + * child classes will never implicitely call this one, so deleting it will + * be more painful than anything else. + * + * So we `assert(false)` here to ensure at runtime that the right + * constructor is always called without having to redefine a custom + * constructor for each `*Config` class. + */ + StoreConfig() { assert(false); } + + virtual const std::string name() = 0; const PathSetting storeDir_{this, false, settings.nixStore, "store", "path to the Nix store"}; @@ -167,6 +212,14 @@ public: "system-features", "Optional features that the system this store builds on implements (like \"kvm\")."}; +}; + +class Store : public std::enable_shared_from_this, public virtual StoreConfig +{ +public: + + typedef std::map Params; + protected: struct PathInfoCacheValue { @@ -200,6 +253,11 @@ protected: Store(const Params & params); public: + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init() {}; virtual ~Store() { } @@ -630,22 +688,25 @@ protected: }; - -class LocalFSStore : public virtual Store +struct LocalFSStoreConfig : virtual StoreConfig { -public: - - // FIXME: the (Store*) cast works around a bug in gcc that causes + using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. - const PathSetting rootDir{(Store*) this, true, "", + const PathSetting rootDir{(StoreConfig*) this, true, "", "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(Store*) this, false, + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state", "directory where Nix will store state"}; - const PathSetting logDir{(Store*) this, false, + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store state"}; +}; + +class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +{ +public: const static string drvsLogDir; @@ -734,39 +795,49 @@ ref openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -enum StoreType { - tDaemon, - tLocal, - tOther -}; - - -StoreType getStoreType(const std::string & uri = settings.storeUri.get(), - const std::string & stateDir = settings.nixStateDir); - /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options. */ std::list> getDefaultSubstituters(); +struct StoreFactory +{ + std::set uriSchemes; + std::function (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; + std::function ()> getConfig; +}; +struct Implementations +{ + static std::vector * registered; -/* Store implementation registration. */ -typedef std::function( - const std::string & uri, const Store::Params & params)> OpenStore; + template + static void add() + { + if (!registered) registered = new std::vector(); + StoreFactory factory{ + .uriSchemes = T::uriSchemes(), + .create = + ([](const std::string & scheme, const std::string & uri, const Store::Params & params) + -> std::shared_ptr + { return std::make_shared(scheme, uri, params); }), + .getConfig = + ([]() + -> std::shared_ptr + { return std::make_shared(StringMap({})); }) + }; + registered->push_back(factory); + } +}; +template struct RegisterStoreImplementation { - typedef std::vector Implementations; - static Implementations * implementations; - - RegisterStoreImplementation(OpenStore fun) + RegisterStoreImplementation() { - if (!implementations) implementations = new Implementations; - implementations->push_back(fun); + Implementations::add(); } }; - /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ string showPaths(const PathSet & paths); diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh new file mode 100644 index 00000000000..b3fbc84f76a --- /dev/null +++ b/src/libutil/abstract-setting-to-json.hh @@ -0,0 +1,15 @@ +#pragma once + +#include +#include "config.hh" + +namespace nix { +template +std::map BaseSetting::toJSONObject() +{ + auto obj = AbstractSetting::toJSONObject(); + obj.emplace("value", value); + obj.emplace("defaultValue", defaultValue); + return obj; +} +} diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh new file mode 100644 index 00000000000..ef31794bee3 --- /dev/null +++ b/src/libutil/callback.hh @@ -0,0 +1,46 @@ +#pragma once + +#include +#include + +namespace nix { + +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future to pass the value or + exception.) */ +template +class Callback +{ + std::function)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: + + Callback(std::function)> fun) : fun(fun) { } + + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } + + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_exception(exc); + fun(promise.get_future()); + } +}; + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 3cf720bce8d..5e6a211df3c 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,5 +1,6 @@ #include "config.hh" #include "args.hh" +#include "abstract-setting-to-json.hh" #include @@ -137,11 +138,7 @@ nlohmann::json Config::toJSON() auto res = nlohmann::json::object(); for (auto & s : _settings) if (!s.second.isAlias) { - auto obj = nlohmann::json::object(); - obj.emplace("description", s.second.setting->description); - obj.emplace("aliases", s.second.setting->aliases); - obj.emplace("value", s.second.setting->toJSON()); - res.emplace(s.first, obj); + res.emplace(s.first, s.second.setting->toJSON()); } return res; } @@ -168,17 +165,19 @@ void AbstractSetting::setDefault(const std::string & str) nlohmann::json AbstractSetting::toJSON() { - return to_string(); + return nlohmann::json(toJSONObject()); } -void AbstractSetting::convertToArg(Args & args, const std::string & category) +std::map AbstractSetting::toJSONObject() { + std::map obj; + obj.emplace("description", description); + obj.emplace("aliases", aliases); + return obj; } -template -nlohmann::json BaseSetting::toJSON() +void AbstractSetting::convertToArg(Args & args, const std::string & category) { - return value; } template @@ -259,11 +258,6 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> nlohmann::json BaseSetting::toJSON() -{ - return value; -} - template<> void BaseSetting::set(const std::string & str) { value = tokenizeString(str); @@ -274,11 +268,6 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> nlohmann::json BaseSetting::toJSON() -{ - return value; -} - template class BaseSetting; template class BaseSetting; template class BaseSetting; diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 2b42658067a..1f5f4e7b9df 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -206,7 +206,9 @@ protected: virtual std::string to_string() const = 0; - virtual nlohmann::json toJSON(); + nlohmann::json toJSON(); + + virtual std::map toJSONObject(); virtual void convertToArg(Args & args, const std::string & category); @@ -220,6 +222,7 @@ class BaseSetting : public AbstractSetting protected: T value; + const T defaultValue; public: @@ -229,6 +232,7 @@ public: const std::set & aliases = {}) : AbstractSetting(name, description, aliases) , value(def) + , defaultValue(def) { } operator const T &() const { return value; } @@ -251,7 +255,7 @@ public: void convertToArg(Args & args, const std::string & category) override; - nlohmann::json toJSON() override; + std::map toJSONObject() override; }; template diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index a39de041fc8..6e69bdce2fc 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include "ansicolor.hh" diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index c5abefe113a..c7777a21f60 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -161,7 +161,7 @@ namespace nix { Setting setting{&config, "", "name-of-the-setting", "description"}; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"description":"description\n","value":"value"}})#"); + ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#"); } TEST(Config, setSettingAlias) { diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh new file mode 100644 index 00000000000..64e06cfbc8c --- /dev/null +++ b/src/libutil/url-parts.hh @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +namespace nix { + +// URI stuff. +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z+.-]+)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +// This is because of the definition of a ref in refs.c in https://github.com/git/git +// See tests/fetchGitRefs.sh for the full definition +const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; +extern std::regex badGitRefRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; + +} diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 88c09eef9d7..c1bab866c15 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,4 +1,5 @@ #include "url.hh" +#include "url-parts.hh" #include "util.hh" namespace nix { diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 2ef88ef2a1e..6e77142e35a 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -2,8 +2,6 @@ #include "error.hh" -#include - namespace nix { struct ParsedURL @@ -29,40 +27,4 @@ std::map decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); -// URI stuff. -const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z+]+)"; -const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; -const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; -const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; -const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; -const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; -const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; -const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; -const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; -const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; -const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; -const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; -const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; - -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check -extern std::regex refRegex; - -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition -const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; -extern std::regex badGitRefRegex; - -// A Git revision (a SHA-1 commit hash). -const static std::string revRegexS = "[0-9a-fA-F]{40}"; -extern std::regex revRegex; - -// A ref or revision, or a ref followed by a revision. -const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; - -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 082e2637579..b8e20120354 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -12,13 +12,9 @@ #include #include -#include -#include #include #include #include -#include -#include #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -480,43 +476,8 @@ std::optional get(const T & map, const typename T::key_ } -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future to pass the value or - exception.) */ template -class Callback -{ - std::function)> fun; - std::atomic_flag done = ATOMIC_FLAG_INIT; - -public: - - Callback(std::function)> fun) : fun(fun) { } - - Callback(Callback && callback) : fun(std::move(callback.fun)) - { - auto prev = callback.done.test_and_set(); - if (prev) done.test_and_set(); - } - - void operator()(T && t) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise promise; - promise.set_value(std::move(t)); - fun(promise.get_future()); - } - - void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept - { - auto prev = done.test_and_set(); - assert(!prev); - std::promise promise; - promise.set_exception(exc); - fun(promise.get_future()); - } -}; +class Callback; /* Start a thread that handles various signals. Also block those signals diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 1ca263a68ea..be0ea5103f2 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -76,6 +76,13 @@ static void update(const StringSet & channelNames) auto store = openStore(); + auto [fd, unpackChannelPath] = createTempFile(); + writeFull(fd.get(), + #include "unpack-channel.nix.gen.hh" + ); + fd = -1; + AutoDelete del(unpackChannelPath, false); + // Download each channel. Strings exprs; for (const auto & channel : channels) { @@ -104,7 +111,7 @@ static void update(const StringSet & channelNames) bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); unpacked = true; } @@ -127,7 +134,7 @@ static void update(const StringSet & channelNames) // Unpack the channel tarballs into the Nix store and install them // into the channels profile. std::cerr << "unpacking channels...\n"; - Strings envArgs{ "--profile", profile, "--file", "", "--install", "--from-expression" }; + Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--from-expression" }; for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); diff --git a/corepkgs/unpack-channel.nix b/src/nix-channel/unpack-channel.nix similarity index 100% rename from corepkgs/unpack-channel.nix rename to src/nix-channel/unpack-channel.nix diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index cfa634a4413..6e652ccbf0a 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -1,5 +1,6 @@ #include "shared.hh" #include "local-store.hh" +#include "remote-store.hh" #include "util.hh" #include "serialise.hh" #include "archive.hh" @@ -285,44 +286,28 @@ static int _main(int argc, char * * argv) initPlugins(); if (stdio) { - if (getStoreType() == tDaemon) { - // Forward on this connection to the real daemon - auto socketPath = settings.nixDaemonSocketFile; - auto s = socket(PF_UNIX, SOCK_STREAM, 0); - if (s == -1) - throw SysError("creating Unix domain socket"); - - auto socketDir = dirOf(socketPath); - if (chdir(socketDir.c_str()) == -1) - throw SysError("changing to socket directory '%1%'", socketDir); - - auto socketName = std::string(baseNameOf(socketPath)); - auto addr = sockaddr_un{}; - addr.sun_family = AF_UNIX; - if (socketName.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket name %1% is too long", socketName); - strcpy(addr.sun_path, socketName.c_str()); - - if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to daemon at %1%", socketPath); - - auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; + if (auto store = openUncachedStore().dynamic_pointer_cast()) { + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; while (true) { fd_set fds; FD_ZERO(&fds); - FD_SET(s, &fds); + FD_SET(from, &fds); FD_SET(STDIN_FILENO, &fds); if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) throw SysError("waiting for data from client or server"); - if (FD_ISSET(s, &fds)) { - auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); if (res == -1) throw SysError("splicing data from daemon socket to stdout"); else if (res == 0) throw EndOfFile("unexpected EOF from daemon socket"); } if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); if (res == -1) throw SysError("splicing data from stdin to daemon socket"); else if (res == 0) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 8d786721069..ef2aee25df6 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -230,7 +230,7 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, { DrvNames selectors = drvNamesFromArgs(args); if (selectors.empty()) - selectors.push_back(DrvName("*")); + selectors.emplace_back("*"); DrvInfos elems; set done; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index e8eff445630..bdcac36c8dd 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -57,6 +57,7 @@ static int _main(int argc, char * * argv) bool fromExpr = false; string attrPath; bool unpack = false; + bool executable = false; string name; struct MyArgs : LegacyArgs, MixEvalArgs @@ -81,6 +82,8 @@ static int _main(int argc, char * * argv) } else if (*arg == "--unpack") unpack = true; + else if (*arg == "--executable") + executable = true; else if (*arg == "--name") name = getArg(*arg, arg, end); else if (*arg != "" && arg->at(0) == '-') @@ -175,7 +178,11 @@ static int _main(int argc, char * * argv) /* Download the file. */ { - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); if (!fd) throw SysError("creating temporary file '%s'", tmpFile); FdSink sink(fd.get()); @@ -201,7 +208,7 @@ static int _main(int argc, char * * argv) tmpFile = unpacked; } - const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); storePath = info.path; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 241c8699bb9..fc41da9e4e9 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -98,14 +98,14 @@ struct CmdBundle : InstallableCommand if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - auto attr1 = vRes->attrs->find(evalState->sDrvPath); + auto attr1 = vRes->attrs->get(evalState->sDrvPath); if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); PathSet context2; StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2)); - auto attr2 = vRes->attrs->find(evalState->sOutPath); + auto attr2 = vRes->attrs->get(evalState->sOutPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc new file mode 100644 index 00000000000..0cc2d93378f --- /dev/null +++ b/src/nix/describe-stores.cc @@ -0,0 +1,44 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +#include + +using namespace nix; + +struct CmdDescribeStores : Command, MixJSON +{ + std::string description() override + { + return "show registered store types and their available options"; + } + + Category category() override { return catUtility; } + + void run() override + { + auto res = nlohmann::json::object(); + for (auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + res[storeName] = storeConfig->toJSON(); + } + if (json) { + std::cout << res; + } else { + for (auto & [storeName, storeConfig] : res.items()) { + std::cout << "## " << storeName << std::endl << std::endl; + for (auto & [optionName, optionDesc] : storeConfig.items()) { + std::cout << "### " << optionName << std::endl << std::endl; + std::cout << optionDesc["description"].get() << std::endl; + std::cout << "default: " << optionDesc["defaultValue"] << std::endl <("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index a2ce9c8c1a6..f29fa71d2d4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -392,7 +392,7 @@ struct CmdDevelop : Common, MixEnvironment auto bashInstallable = std::make_shared( state, - std::move(installable->nixpkgsFlakeRef()), + installable->nixpkgsFlakeRef(), Strings{"bashInteractive"}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, lockFlags); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 4199dae0f51..0dc99d05e77 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -81,7 +81,7 @@ void printClosureDiff( auto beforeSize = totalSize(beforeVersions); auto afterSize = totalSize(afterVersions); auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize; - auto showDelta = abs(sizeDelta) >= 8 * 1024; + auto showDelta = std::abs(sizeDelta) >= 8 * 1024; std::set removed, unchanged; for (auto & [version, _] : beforeVersions) diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 82e92cdd0a9..683e9144602 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -49,9 +49,7 @@ struct CmdDoctor : StoreCommand { logger->log("Running checks against store uri: " + store->getUri()); - auto type = getStoreType(); - - if (type < tOther) { + if (store.dynamic_pointer_cast()) { success &= checkNixInPath(); success &= checkProfileRoots(store); } diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 41c75a4ed1a..c7c2f89811e 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -69,7 +69,7 @@ struct Installable virtual FlakeRef nixpkgsFlakeRef() const { - return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}})); + return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); } }; diff --git a/src/nix/local.mk b/src/nix/local.mk index e962006855d..ab4e9121b2e 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -29,3 +29,5 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh src/nix/develop.cc: src/nix/get-env.sh.gen.hh + +src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh diff --git a/src/nix/main.cc b/src/nix/main.cc index e9479f56407..1e9e07bc058 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -185,6 +185,7 @@ void mainWrapped(int argc, char * * argv) } if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { + evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); auto res = nlohmann::json::object(); auto builtins = state.baseEnv.values[0]->attrs; diff --git a/tests/content-addressed.sh b/tests/content-addressed.sh index ae9e3c59eb2..0ae2852d210 100644 --- a/tests/content-addressed.sh +++ b/tests/content-addressed.sh @@ -15,3 +15,6 @@ out1=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 1) out2=$(nix-build "${commonArgs[@]}" ./content-addressed.nix --arg seed 2) test $out1 == $out2 + +nix-instantiate --experimental-features ca-derivations ./content-addressed.nix -A rootCA --arg seed 5 +nix-collect-garbage --experimental-features ca-derivations --option keep-derivations true diff --git a/tests/describe-stores.sh b/tests/describe-stores.sh new file mode 100644 index 00000000000..3fea61483d5 --- /dev/null +++ b/tests/describe-stores.sh @@ -0,0 +1,8 @@ +source common.sh + +# Query an arbitrary value in `nix describe-stores --json`'s output just to +# check that it has the right structure +[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]] + +# Ensure that the output of `nix describe-stores` isn't empty +[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]] diff --git a/tests/local.mk b/tests/local.mk index bc40ecb5c28..413ef2610aa 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -15,6 +15,7 @@ nix_tests = \ linux-sandbox.sh \ build-dry.sh \ build-remote-input-addressed.sh \ + ssh-relay.sh \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ @@ -33,6 +34,7 @@ nix_tests = \ function-trace.sh \ recursive.sh \ ipfs.sh \ + describe-stores.sh \ flakes.sh \ content-addressed.sh # parallel.sh diff --git a/tests/recursive.sh b/tests/recursive.sh index cf10d55bfe4..80a178cc7f1 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -9,9 +9,8 @@ rm -f $TEST_ROOT/result export unreachable=$(nix add-to-store ./recursive.sh) -nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' +NIX_BIN_DIR=$(dirname $(type -p nix)) nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' with import ./config.nix; - with import ; mkDerivation { name = "recursive"; dummy = builtins.toFile "dummy" "bla bla"; @@ -24,9 +23,10 @@ nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/resu buildCommand = '\'\'' mkdir $out - PATH=${nixBinDir}:$PATH opts="--experimental-features nix-command" + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + # Check that we can query/build paths in our input closure. nix $opts path-info $dummy nix $opts build $dummy diff --git a/tests/ssh-relay.sh b/tests/ssh-relay.sh new file mode 100644 index 00000000000..dce50974b16 --- /dev/null +++ b/tests/ssh-relay.sh @@ -0,0 +1,16 @@ +source common.sh + +echo foo > $TEST_ROOT/hello.sh + +ssh_localhost=ssh://localhost +remote_store=?remote-store=$ssh_localhost + +store=$ssh_localhost + +store+=$remote_store +store+=$remote_store +store+=$remote_store + +out=$(nix add-to-store --store "$store" $TEST_ROOT/hello.sh) + +[ foo = $(< $out) ]