Add nix nario command

src/libmain/shared.cc

@@ -334,7 +334,7 @@ int handleExceptions(const std::string & programName, std::function<void()> fun)
             return e.status;
         } catch (UsageError & e) {
             logError(e.info());
-            printError("Try '%1% --help' for more information.", programName);
+            printError("\nTry '%1% --help' for more information.", programName);
             return 1;
         } catch (BaseError & e) {
             logError(e.info());

src/libstore/export-import.cc

@@ -4,92 +4,152 @@
 #include "nix/util/archive.hh"
 #include "nix/store/common-protocol.hh"
 #include "nix/store/common-protocol-impl.hh"
-
-#include <algorithm>
+#include "nix/store/worker-protocol.hh"
 
 namespace nix {
 
-static void exportPath(Store & store, const StorePath & path, Sink & sink)
-{
-    auto info = store.queryPathInfo(path);
-
-    HashSink hashSink(HashAlgorithm::SHA256);
-    TeeSink teeSink(sink, hashSink);
-
-    store.narFromPath(path, teeSink);
-
-    /* Refuse to export paths that have changed.  This prevents
-       filesystem corruption from spreading to other machines.
-       Don't complain if the stored hash is zero (unknown). */
-    Hash hash = hashSink.currentHash().hash;
-    if (hash != info->narHash && info->narHash != Hash(info->narHash.algo))
-        throw Error(
-            "hash of path '%s' has changed from '%s' to '%s'!",
-            store.printStorePath(path),
-            info->narHash.to_string(HashFormat::Nix32, true),
-            hash.to_string(HashFormat::Nix32, true));
-
-    teeSink << exportMagic << store.printStorePath(path);
-    CommonProto::write(store, CommonProto::WriteConn{.to = teeSink}, info->references);
-    teeSink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0;
-}
+static const uint32_t exportMagicV1 = 0x4558494e;
+static const uint64_t exportMagicV2 = 0x324f4952414e; // = 'NARIO2'
 
-void exportPaths(Store & store, const StorePathSet & paths, Sink & sink)
+void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version)
 {
     auto sorted = store.topoSortPaths(paths);
     std::reverse(sorted.begin(), sorted.end());
 
-    for (auto & path : sorted) {
-        sink << 1;
-        exportPath(store, path, sink);
+    auto dumpNar = [&](const ValidPathInfo & info) {
+        HashSink hashSink(HashAlgorithm::SHA256);
+        TeeSink teeSink(sink, hashSink);
+
+        store.narFromPath(info.path, teeSink);
+
+        /* Refuse to export paths that have changed.  This prevents
+           filesystem corruption from spreading to other machines.
+           Don't complain if the stored hash is zero (unknown). */
+        Hash hash = hashSink.currentHash().hash;
+        if (hash != info.narHash && info.narHash != Hash(info.narHash.algo))
+            throw Error(
+                "hash of path '%s' has changed from '%s' to '%s'!",
+                store.printStorePath(info.path),
+                info.narHash.to_string(HashFormat::Nix32, true),
+                hash.to_string(HashFormat::Nix32, true));
+    };
+
+    switch (version) {
+
+    case 1:
+        for (auto & path : sorted) {
+            sink << 1;
+            auto info = store.queryPathInfo(path);
+            dumpNar(*info);
+            sink << exportMagicV1 << store.printStorePath(path);
+            CommonProto::write(store, CommonProto::WriteConn{.to = sink}, info->references);
+            sink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0;
+        }
+        sink << 0;
+        break;
+
+    case 2:
+        sink << exportMagicV2;
+
+        for (auto & path : sorted) {
+            Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path)));
+            sink << 1;
+            auto info = store.queryPathInfo(path);
+            // FIXME: move to CommonProto?
+            WorkerProto::Serialise<ValidPathInfo>::write(
+                store, WorkerProto::WriteConn{.to = sink, .version = 16}, *info);
+            dumpNar(*info);
+        }
+
+        sink << 0;
+        break;
+
+    default:
+        throw Error("unsupported nario version %d", version);
     }
-
-    sink << 0;
 }
 
 StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs)
 {
     StorePaths res;
-    while (true) {
-        auto n = readNum<uint64_t>(source);
-        if (n == 0)
-            break;
-        if (n != 1)
-            throw Error("input doesn't look like something created by 'nix-store --export'");
-
-        /* Extract the NAR from the source. */
-        StringSink saved;
-        TeeSource tee{source, saved};
-        NullFileSystemObjectSink ether;
-        parseDump(ether, tee);
-
-        uint32_t magic = readInt(source);
-        if (magic != exportMagic)
-            throw Error("Nix archive cannot be imported; wrong format");
-
-        auto path = store.parseStorePath(readString(source));
-
-        // Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
-
-        auto references = CommonProto::Serialise<StorePathSet>::read(store, CommonProto::ReadConn{.from = source});
-        auto deriver = readString(source);
-        auto narHash = hashString(HashAlgorithm::SHA256, saved.s);
-
-        ValidPathInfo info{path, narHash};
-        if (deriver != "")
-            info.deriver = store.parseStorePath(deriver);
-        info.references = references;
-        info.narSize = saved.s.size();
-
-        // Ignore optional legacy signature.
-        if (readInt(source) == 1)
-            readString(source);
-
-        // Can't use underlying source, which would have been exhausted
-        auto source = StringSource(saved.s);
-        store.addToStore(info, source, NoRepair, checkSigs);
-
-        res.push_back(info.path);
+
+    auto version = readNum<uint64_t>(source);
+
+    /* Note: nario version 1 lacks an explicit header. The first
+       integer denotes whether a store path follows or not. So look
+       for 0 or 1. */
+    switch (version) {
+
+    case 0:
+        /* Empty version 1 nario, nothing to do. */
+        break;
+
+    case 1:
+        /* Non-empty version 1 nario. */
+        while (true) {
+            /* Extract the NAR from the source. */
+            StringSink saved;
+            TeeSource tee{source, saved};
+            NullFileSystemObjectSink ether;
+            parseDump(ether, tee);
+
+            uint32_t magic = readInt(source);
+            if (magic != exportMagicV1)
+                throw Error("nario cannot be imported; wrong format");
+
+            auto path = store.parseStorePath(readString(source));
+
+            auto references = CommonProto::Serialise<StorePathSet>::read(store, CommonProto::ReadConn{.from = source});
+            auto deriver = readString(source);
+            auto narHash = hashString(HashAlgorithm::SHA256, saved.s);
+
+            ValidPathInfo info{path, narHash};
+            if (deriver != "")
+                info.deriver = store.parseStorePath(deriver);
+            info.references = references;
+            info.narSize = saved.s.size();
+
+            // Ignore optional legacy signature.
+            if (readInt(source) == 1)
+                readString(source);
+
+            // Can't use underlying source, which would have been exhausted.
+            auto source2 = StringSource(saved.s);
+            store.addToStore(info, source2, NoRepair, checkSigs);
+
+            res.push_back(info.path);
+
+            auto n = readNum<uint64_t>(source);
+            if (n == 0)
+                break;
+            if (n != 1)
+                throw Error("input doesn't look like a nario");
+        }
+        break;
+
+    case exportMagicV2:
+        while (true) {
+            auto n = readNum<uint64_t>(source);
+            if (n == 0)
+                break;
+            if (n != 1)
+                throw Error("input doesn't look like a nario");
+
+            auto info = WorkerProto::Serialise<ValidPathInfo>::read(
+                store, WorkerProto::ReadConn{.from = source, .version = 16});
+
+            Activity act(
+                *logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path)));
+
+            store.addToStore(info, source, NoRepair, checkSigs);
+
+            res.push_back(info.path);
+        }
+
+        break;
+
+    default:
+        throw Error("input doesn't look like a nario");
     }
 
     return res;

src/libstore/include/nix/store/export-import.hh

@@ -4,16 +4,11 @@
 
 namespace nix {
 
-/**
- * Magic header of exportPath() output (obsolete).
- */
-const uint32_t exportMagic = 0x4558494e;
-
 /**
  * Export multiple paths in the format expected by `nix-store
  * --import`. The paths will be sorted topologically.
  */
-void exportPaths(Store & store, const StorePathSet & paths, Sink & sink);
+void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version);
 
 /**
  * Import a sequence of NAR dumps created by `exportPaths()` into the

src/libutil/args.cc

@@ -318,6 +318,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
         } catch (SystemError &) {
         }
     }
+
     for (auto pos = cmdline.begin(); pos != cmdline.end();) {
 
         auto arg = *pos;
@@ -354,6 +355,9 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang)
 
     processArgs(pendingArgs, true);
 
+    if (!completions)
+        checkArgs();
+
     initialFlagsProcessed();
 
     /* Now that we are done parsing, make sure that any experimental
@@ -384,7 +388,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
 
     auto & rootArgs = getRoot();
 
-    auto process = [&](const std::string & name, const Flag & flag) -> bool {
+    auto process = [&](const std::string & name, Flag & flag) -> bool {
         ++pos;
 
         if (auto & f = flag.experimentalFeature)
@@ -413,6 +417,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         }
         if (!anyCompleted)
             flag.handler.fun(std::move(args));
+        flag.timesUsed++;
         return true;
     };
 
@@ -504,6 +509,14 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
+void Args::checkArgs()
+{
+    for (auto & [name, flag] : longFlags) {
+        if (flag->required && flag->timesUsed == 0)
+            throw UsageError("required argument '--%s' is missing", name);
+    }
+}
+
 nlohmann::json Args::toJSON()
 {
     auto flags = nlohmann::json::object();
@@ -643,6 +656,13 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
+void MultiCommand::checkArgs()
+{
+    Args::checkArgs();
+    if (command)
+        command->second->checkArgs();
+}
+
 nlohmann::json MultiCommand::toJSON()
 {
     auto cmds = nlohmann::json::object();

src/libutil/include/nix/util/args.hh

@@ -202,8 +202,12 @@ public:
         Strings labels;
         Handler handler;
         CompleterClosure completer;
+        bool required = false;
 
         std::optional<ExperimentalFeature> experimentalFeature;
+
+        // FIXME: this should be private, but that breaks designated initializers.
+        size_t timesUsed = 0;
     };
 
 protected:
@@ -283,6 +287,8 @@ protected:
 
     StringSet hiddenCategories;
 
+    virtual void checkArgs();
+
     /**
      * Called after all command line flags before the first non-flag
      * argument (if any) have been processed.
@@ -428,6 +434,8 @@ public:
 protected:
     std::string commandName = "";
     bool aliasUsed = false;
+
+    void checkArgs() override;
 };
 
 Strings argvToStrings(int argc, char ** argv);

src/nix/meson.build

@@ -88,6 +88,7 @@ nix_sources = [ config_priv_h ] + files(
   'make-content-addressed.cc',
   'man-pages.cc',
   'nar.cc',
+  'nario.cc',
   'optimise-store.cc',
   'path-from-hash-part.cc',
   'path-info.cc',

src/nix/nario-export.md

@@ -0,0 +1,30 @@
+R""(
+
+# Examples
+
+* Export the closure of building `nixpkgs#hello`:
+
+  ```console
+  # nix nario export --format 1 -r nixpkgs#hello > dump
+  ```
+
+  It can be imported in another store:
+
+  ```console
+  # nix nario import < dump
+  ```
+
+# Description
+
+This command prints on standard output a serialization of the specified store paths in `nario` format. This serialization can be imported into another store using `nix nario import`.
+
+References of a path are not exported by default; use `-r` to export a complete closure.
+Paths are exported in topographically sorted order (i.e. if path `X` refers to `Y`, then `Y` appears before `X`).
+
+You must specify the desired `nario` version. Currently the following versions are supported:
+
+* `1`: This version is compatible with the legacy `nix-store --export` and `nix-store --import` commands. It should be avoided because it is not memory-efficient on import.
+
+* `2`: The latest version. Recommended.
+
+)""