From 437b9b9879558ce36788efb9db26f0baec87b570 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Nov 2025 17:25:07 -0500 Subject: [PATCH 1/4] Rename `MemorySourceAccessor::File::Directory::{contents -> entries}` This matches the "NAR Listing" JSON format, and also helps distinguish from regular file contents. Why we want to match that will become clear in the next comments, when we will in fact use (variations of) this data type for NAR listings. --- src/libstore-tests/references.cc | 4 ++-- src/libutil-tests/git.cc | 4 ++-- src/libutil/include/nix/util/memory-source-accessor.hh | 4 ++-- src/libutil/memory-source-accessor.cc | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libstore-tests/references.cc b/src/libstore-tests/references.cc index 9cecd573e55..f2c6fb51e5c 100644 --- a/src/libstore-tests/references.cc +++ b/src/libstore-tests/references.cc @@ -99,7 +99,7 @@ TEST(references, scanForReferencesDeep) // Create an in-memory file system with various reference patterns auto accessor = make_ref(); accessor->root = File::Directory{ - .contents{ + .entries{ { // file1.txt: contains hash1 "file1.txt", @@ -125,7 +125,7 @@ TEST(references, scanForReferencesDeep) // subdir: a subdirectory "subdir", File::Directory{ - .contents{ + .entries{ { // subdir/file4.txt: contains hash1 again "file4.txt", diff --git a/src/libutil-tests/git.cc b/src/libutil-tests/git.cc index 6180a4cfc7f..9d749b49260 100644 --- a/src/libutil-tests/git.cc +++ b/src/libutil-tests/git.cc @@ -230,7 +230,7 @@ TEST_F(GitTest, both_roundrip) auto files = make_ref(); files->root = File::Directory{ - .contents{ + .entries{ { "foo", File::Regular{ @@ -240,7 +240,7 @@ TEST_F(GitTest, both_roundrip) { "bar", File::Directory{ - .contents = + .entries = { { "baz", diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index eba282fe1c1..6aadc53a33d 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -35,7 +35,7 @@ struct MemorySourceAccessor : virtual SourceAccessor { using Name = std::string; - std::map> contents; + std::map> entries; bool operator==(const Directory &) const noexcept; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. @@ -95,7 +95,7 @@ inline bool MemorySourceAccessor::File::Directory::operator==( inline bool MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept { - return contents < other.contents; + return entries < other.entries; } inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default; diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index a9ffb77469c..6cc51a7fa95 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -29,13 +29,13 @@ MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, return nullptr; auto & curDir = *curDirP; - auto i = curDir.contents.find(name); - if (i == curDir.contents.end()) { + auto i = curDir.entries.find(name); + if (i == curDir.entries.end()) { if (!create) return nullptr; else { newF = true; - i = curDir.contents.insert( + i = curDir.entries.insert( i, { std::string{name}, @@ -106,7 +106,7 @@ MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const Canon throw Error("file '%s' does not exist", path); if (auto * d = std::get_if(&f->raw)) { DirEntries res; - for (auto & [name, file] : d->contents) + for (auto & [name, file] : d->entries) res.insert_or_assign(name, file.lstat().type); return res; } else From d17bfe3866dbd8760058b66908f7fc1bb01e87d7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Nov 2025 17:35:13 -0500 Subject: [PATCH 2/4] Move `nar-accessor.{cc,hh}` to libutil File-system-object-layer functionality doesn't depend on store-layer concets, and therefore doesn't need to live inside there. --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/include/nix/store/meson.build | 1 - src/libstore/meson.build | 1 - src/libstore/remote-fs-accessor.cc | 2 +- src/libutil/include/nix/util/meson.build | 1 + .../nix/store => libutil/include/nix/util}/nar-accessor.hh | 0 src/libutil/meson.build | 1 + src/{libstore => libutil}/nar-accessor.cc | 2 +- src/nix/cat.cc | 2 +- src/nix/ls.cc | 2 +- 10 files changed, 7 insertions(+), 7 deletions(-) rename src/{libstore/include/nix/store => libutil/include/nix/util}/nar-accessor.hh (100%) rename src/{libstore => libutil}/nar-accessor.cc (99%) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 274e47271fe..3cf2da70d27 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -8,7 +8,7 @@ #include "nix/util/sync.hh" #include "nix/store/remote-fs-accessor.hh" #include "nix/store/nar-info-disk-cache.hh" -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include "nix/util/thread-pool.hh" #include "nix/util/callback.hh" #include "nix/util/signals.hh" diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index 5d6626ff838..c17d6a9cb5a 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -55,7 +55,6 @@ headers = [ config_pub_h ] + files( 'machines.hh', 'make-content-addressed.hh', 'names.hh', - 'nar-accessor.hh', 'nar-info-disk-cache.hh', 'nar-info.hh', 'outputs-spec.hh', diff --git a/src/libstore/meson.build b/src/libstore/meson.build index d1b3666cc34..e3425deb5e8 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -300,7 +300,6 @@ sources = files( 'make-content-addressed.cc', 'misc.cc', 'names.cc', - 'nar-accessor.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', 'optimise-store.cc', diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index f7ca28ae2db..582599f0d5e 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -1,6 +1,6 @@ #include #include "nix/store/remote-fs-accessor.hh" -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include #include diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 9a606e15db9..b6677140e30 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -50,6 +50,7 @@ headers = files( 'memory-source-accessor.hh', 'mounted-source-accessor.hh', 'muxable-pipe.hh', + 'nar-accessor.hh', 'os-string.hh', 'pool.hh', 'pos-idx.hh', diff --git a/src/libstore/include/nix/store/nar-accessor.hh b/src/libutil/include/nix/util/nar-accessor.hh similarity index 100% rename from src/libstore/include/nix/store/nar-accessor.hh rename to src/libutil/include/nix/util/nar-accessor.hh diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8b7a5d97773..5290ff2d091 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -147,6 +147,7 @@ sources = [ config_priv_h ] + files( 'logging.cc', 'memory-source-accessor.cc', 'mounted-source-accessor.cc', + 'nar-accessor.cc', 'pos-table.cc', 'position.cc', 'posix-source-accessor.cc', diff --git a/src/libstore/nar-accessor.cc b/src/libutil/nar-accessor.cc similarity index 99% rename from src/libstore/nar-accessor.cc rename to src/libutil/nar-accessor.cc index 640b77540bb..6bcdf5f1384 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -1,4 +1,4 @@ -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include "nix/util/archive.hh" #include diff --git a/src/nix/cat.cc b/src/nix/cat.cc index bf58bb49236..114e8d38ec4 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include "nix/util/serialise.hh" #include "nix/util/source-accessor.hh" diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 9bf3c5996af..ca6e20be8da 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,6 @@ #include "nix/cmd/command.hh" #include "nix/store/store-api.hh" -#include "nix/store/nar-accessor.hh" +#include "nix/util/nar-accessor.hh" #include "nix/main/common-args.hh" #include From ac36d74b66387ba29733eedf526c1694dbea6e63 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Nov 2025 20:15:49 -0500 Subject: [PATCH 3/4] `listNar` should just take the source accessor by simple reference A shared pointer is not needed. --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/remote-fs-accessor.cc | 2 +- src/libutil/include/nix/util/nar-accessor.hh | 2 +- src/libutil/nar-accessor.cc | 8 ++++---- src/nix/cat.cc | 2 +- src/nix/ls.cc | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3cf2da70d27..49077524fe0 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), CanonPath::root, true)}, + {"root", listNar(*narAccessor, CanonPath::root, true)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 582599f0d5e..e2c8f65a79b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -39,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std: if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, CanonPath::root, true); + nlohmann::json j = listNar(*narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreExceptionExceptInterrupt(); diff --git a/src/libutil/include/nix/util/nar-accessor.hh b/src/libutil/include/nix/util/nar-accessor.hh index bfba5da73be..df7b0fcf2b9 100644 --- a/src/libutil/include/nix/util/nar-accessor.hh +++ b/src/libutil/include/nix/util/nar-accessor.hh @@ -38,6 +38,6 @@ ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBy * Write a JSON representation of the contents of a NAR (except file * contents). */ -nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); +nlohmann::json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse); } // namespace nix diff --git a/src/libutil/nar-accessor.cc b/src/libutil/nar-accessor.cc index 6bcdf5f1384..12db8ac7be5 100644 --- a/src/libutil/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -274,9 +274,9 @@ GetNarBytes seekableGetNarBytes(const Path & path) using nlohmann::json; -json listNar(ref accessor, const CanonPath & path, bool recurse) +json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse) { - auto st = accessor->lstat(path); + auto st = accessor.lstat(path); json obj = json::object(); @@ -295,7 +295,7 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) { obj["entries"] = json::object(); json & res2 = obj["entries"]; - for (const auto & [name, type] : accessor->readDirectory(path)) { + for (const auto & [name, type] : accessor.readDirectory(path)) { if (recurse) { res2[name] = listNar(accessor, path / name, true); } else @@ -305,7 +305,7 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) break; case SourceAccessor::Type::tSymlink: obj["type"] = "symlink"; - obj["target"] = accessor->readLink(path); + obj["target"] = accessor.readLink(path); break; case SourceAccessor::Type::tBlock: case SourceAccessor::Type::tChar: diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 114e8d38ec4..5c3d7cfd425 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -80,7 +80,7 @@ struct CmdCatNar : StoreCommand, MixCat throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); - auto listing = listNar(narAccessor, CanonPath::root, true); + auto listing = listNar(*narAccessor, CanonPath::root, true); cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index ca6e20be8da..ccd479c0af0 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -85,7 +85,7 @@ struct MixLs : virtual Args, MixJSON if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, path, recursive)); + logger->cout("%s", listNar(*accessor, path, recursive)); } else listText(accessor, std::move(path)); } @@ -150,7 +150,7 @@ struct CmdLsNar : Command, MixLs throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); - auto listing = listNar(narAccessor, CanonPath::root, true); + auto listing = listNar(*narAccessor, CanonPath::root, true); list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } }; From c4906741a122f8fa13adc279bfd053d6c1e85821 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Nov 2025 17:16:07 -0500 Subject: [PATCH 4/4] Deduplicate `listNar` and `MemorySourceAccessor::File` `listNar` did the not-so-pretty thing of going straight to JSON. Now it uses `MemorySourceAccessor::File`, or rather variations of it, to go to a C++ data type first, and only JSON second. To accomplish this we add some type parameters to the `File` data type. Actually, we need to do two rounds of this, because shallow NAR listings. There is `FileT` and `DirectoryT` accordingly. --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/remote-fs-accessor.cc | 2 +- .../nix/util/memory-source-accessor.hh | 135 +++++++++++------- src/libutil/include/nix/util/nar-accessor.hh | 53 ++++++- src/libutil/memory-source-accessor.cc | 15 +- src/libutil/nar-accessor.cc | 109 ++++++++++---- src/nix/cat.cc | 2 +- src/nix/ls.cc | 9 +- 8 files changed, 236 insertions(+), 91 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 49077524fe0..caae7247948 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -208,7 +208,7 @@ ref BinaryCacheStore::addToStoreCommon( if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(*narAccessor, CanonPath::root, true)}, + {"root", listNarDeep(*narAccessor, CanonPath::root)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index e2c8f65a79b..51bab995354 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -39,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std: if (cacheDir != "") { try { - nlohmann::json j = listNar(*narAccessor, CanonPath::root, true); + nlohmann::json j = listNarDeep(*narAccessor, CanonPath::root); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreExceptionExceptInterrupt(); diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index 6aadc53a33d..268f6b06f51 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -4,59 +4,111 @@ #include "nix/util/source-path.hh" #include "nix/util/fs-sink.hh" #include "nix/util/variant-wrapper.hh" +#include "nix/util/json-impls.hh" namespace nix { /** - * An source accessor for an in-memory file system. + * File System Object definitions + * + * @see https://nix.dev/manual/nix/latest/store/file-system-object.html */ -struct MemorySourceAccessor : virtual SourceAccessor +namespace fso { + +template +struct Regular +{ + bool executable = false; + RegularContents contents; + + auto operator<=>(const Regular &) const = default; +}; + +/** + * Child parameter because sometimes we want "shallow" directories without + * full file children. + */ +template +struct DirectoryT +{ + using Name = std::string; + + std::map> entries; + + inline bool operator==(const DirectoryT &) const noexcept; + inline std::strong_ordering operator<=>(const DirectoryT &) const noexcept; +}; + +struct Symlink +{ + std::string target; + + auto operator<=>(const Symlink &) const = default; +}; + +/** + * For when we know there is child, but don't know anything about it. + * + * This is not part of the core File System Object data model --- this + * represents not knowing, not an additional type of file. + */ +struct Opaque +{ + auto operator<=>(const Opaque &) const = default; +}; + +/** + * `File` nicely defining what a "file system object" + * is in Nix. + * + * With a different type arugment, it is also can be a "skeletal" + * version is that abstract syntax for a "NAR listing". + */ +template +struct VariantT { + bool operator==(const VariantT &) const noexcept; + std::strong_ordering operator<=>(const VariantT &) const noexcept; + + using Regular = nix::fso::Regular; + /** - * In addition to being part of the implementation of - * `MemorySourceAccessor`, this has a side benefit of nicely - * defining what a "file system object" is in Nix. + * In the default case, we do want full file children for our directory. */ - struct File - { - bool operator==(const File &) const noexcept; - std::strong_ordering operator<=>(const File &) const noexcept; + using Directory = nix::fso::DirectoryT>; - struct Regular - { - bool executable = false; - std::string contents; + using Symlink = nix::fso::Symlink; - bool operator==(const Regular &) const = default; - auto operator<=>(const Regular &) const = default; - }; + using Raw = std::variant; + Raw raw; - struct Directory - { - using Name = std::string; + MAKE_WRAPPER_CONSTRUCTOR(VariantT); - std::map> entries; + SourceAccessor::Stat lstat() const; +}; - bool operator==(const Directory &) const noexcept; - // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. - bool operator<(const Directory &) const noexcept; - }; +template +inline bool DirectoryT::operator==(const DirectoryT &) const noexcept = default; - struct Symlink - { - std::string target; +template +inline std::strong_ordering DirectoryT::operator<=>(const DirectoryT &) const noexcept = default; - bool operator==(const Symlink &) const = default; - auto operator<=>(const Symlink &) const = default; - }; +template +inline bool +VariantT::operator==(const VariantT &) const noexcept = default; - using Raw = std::variant; - Raw raw; +template +inline std::strong_ordering +VariantT::operator<=>(const VariantT &) const noexcept = default; - MAKE_WRAPPER_CONSTRUCTOR(File); +} // namespace fso - Stat lstat() const; - }; +/** + * An source accessor for an in-memory file system. + */ +struct MemorySourceAccessor : virtual SourceAccessor +{ + using File = fso::VariantT; std::optional root; @@ -89,19 +141,6 @@ struct MemorySourceAccessor : virtual SourceAccessor SourcePath addFile(CanonPath path, std::string && contents); }; -inline bool MemorySourceAccessor::File::Directory::operator==( - const MemorySourceAccessor::File::Directory &) const noexcept = default; - -inline bool -MemorySourceAccessor::File::Directory::operator<(const MemorySourceAccessor::File::Directory & other) const noexcept -{ - return entries < other.entries; -} - -inline bool MemorySourceAccessor::File::operator==(const MemorySourceAccessor::File &) const noexcept = default; -inline std::strong_ordering -MemorySourceAccessor::File::operator<=>(const MemorySourceAccessor::File &) const noexcept = default; - /** * Write to a `MemorySourceAccessor` at the given path */ diff --git a/src/libutil/include/nix/util/nar-accessor.hh b/src/libutil/include/nix/util/nar-accessor.hh index df7b0fcf2b9..9665af9bc98 100644 --- a/src/libutil/include/nix/util/nar-accessor.hh +++ b/src/libutil/include/nix/util/nar-accessor.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nix/util/source-accessor.hh" +#include "nix/util/memory-source-accessor.hh" #include @@ -34,10 +34,55 @@ GetNarBytes seekableGetNarBytes(const Path & path); ref makeLazyNarAccessor(const nlohmann::json & listing, GetNarBytes getNarBytes); +struct NarListingRegularFile +{ + /** + * @see `SourceAccessor::Stat::fileSize` + */ + std::optional fileSize; + + /** + * @see `SourceAccessor::Stat::narOffset` + * + * We only set to non-`std::nullopt` if it is also non-zero. + */ + std::optional narOffset; + + auto operator<=>(const NarListingRegularFile &) const = default; +}; + +/** + * Abstract syntax for a "NAR listing". + */ +using NarListing = fso::VariantT; + +/** + * Shallow NAR listing where directory children are not recursively expanded. + * Uses a variant that can hold Regular/Symlink fully, but Directory children + * are just unit types indicating presence without content. + */ +using ShallowNarListing = fso::VariantT; + +/** + * Return a deep structured representation of the contents of a NAR (except file + * contents), recursively listing all children. + */ +NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path); + +/** + * Return a shallow structured representation of the contents of a NAR (except file + * contents), only listing immediate children without recursing. + */ +ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path); + +/** + * Serialize a NarListing to JSON. + */ +void to_json(nlohmann::json & j, const NarListing & listing); + /** - * Write a JSON representation of the contents of a NAR (except file - * contents). + * Serialize a ShallowNarListing to JSON. */ -nlohmann::json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse); +void to_json(nlohmann::json & j, const ShallowNarListing & listing); } // namespace nix diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 6cc51a7fa95..6a9a0772b37 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -68,25 +68,26 @@ bool MemorySourceAccessor::pathExists(const CanonPath & path) return open(path, std::nullopt); } -MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +template<> +SourceAccessor::Stat MemorySourceAccessor::File::lstat() const { return std::visit( overloaded{ [](const Regular & r) { - return Stat{ - .type = tRegular, + return SourceAccessor::Stat{ + .type = SourceAccessor::tRegular, .fileSize = r.contents.size(), .isExecutable = r.executable, }; }, [](const Directory &) { - return Stat{ - .type = tDirectory, + return SourceAccessor::Stat{ + .type = SourceAccessor::tDirectory, }; }, [](const Symlink &) { - return Stat{ - .type = tSymlink, + return SourceAccessor::Stat{ + .type = SourceAccessor::tSymlink, }; }, }, diff --git a/src/libutil/nar-accessor.cc b/src/libutil/nar-accessor.cc index 12db8ac7be5..35ee4e5369e 100644 --- a/src/libutil/nar-accessor.cc +++ b/src/libutil/nar-accessor.cc @@ -272,41 +272,39 @@ GetNarBytes seekableGetNarBytes(const Path & path) }; } -using nlohmann::json; +template +using ListNarResult = std::conditional_t; -json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse) +template +static ListNarResult listNarImpl(SourceAccessor & accessor, const CanonPath & path) { auto st = accessor.lstat(path); - json obj = json::object(); - switch (st.type) { case SourceAccessor::Type::tRegular: - obj["type"] = "regular"; - if (st.fileSize) - obj["size"] = *st.fileSize; - if (st.isExecutable) - obj["executable"] = true; - if (st.narOffset && *st.narOffset) - obj["narOffset"] = *st.narOffset; - break; - case SourceAccessor::Type::tDirectory: - obj["type"] = "directory"; - { - obj["entries"] = json::object(); - json & res2 = obj["entries"]; - for (const auto & [name, type] : accessor.readDirectory(path)) { - if (recurse) { - res2[name] = listNar(accessor, path / name, true); - } else - res2[name] = json::object(); + return typename ListNarResult::Regular{ + .executable = st.isExecutable, + .contents = + NarListingRegularFile{ + .fileSize = st.fileSize, + .narOffset = st.narOffset && *st.narOffset ? st.narOffset : std::nullopt, + }, + }; + case SourceAccessor::Type::tDirectory: { + typename ListNarResult::Directory dir; + for (const auto & [name, type] : accessor.readDirectory(path)) { + if constexpr (deep) { + dir.entries.emplace(name, listNarImpl(accessor, path / name)); + } else { + dir.entries.emplace(name, fso::Opaque{}); } } - break; + return dir; + } case SourceAccessor::Type::tSymlink: - obj["type"] = "symlink"; - obj["target"] = accessor.readLink(path); - break; + return typename ListNarResult::Symlink{ + .target = accessor.readLink(path), + }; case SourceAccessor::Type::tBlock: case SourceAccessor::Type::tChar: case SourceAccessor::Type::tSocket: @@ -314,7 +312,64 @@ json listNar(SourceAccessor & accessor, const CanonPath & path, bool recurse) case SourceAccessor::Type::tUnknown: assert(false); // cannot happen for NARs } - return obj; +} + +NarListing listNarDeep(SourceAccessor & accessor, const CanonPath & path) +{ + return listNarImpl(accessor, path); +} + +ShallowNarListing listNarShallow(SourceAccessor & accessor, const CanonPath & path) +{ + return listNarImpl(accessor, path); +} + +template +static void to_json_impl(nlohmann::json & j, const Listing & listing) +{ + std::visit( + overloaded{ + [&](const typename Listing::Regular & r) { + j = nlohmann::json::object(); + j["type"] = "regular"; + if (r.contents.fileSize) + j["size"] = *r.contents.fileSize; + if (r.executable) + j["executable"] = true; + if (r.contents.narOffset) + j["narOffset"] = *r.contents.narOffset; + }, + [&](const typename Listing::Directory & d) { + j = nlohmann::json::object(); + j["type"] = "directory"; + j["entries"] = nlohmann::json::object(); + for (const auto & [name, child] : d.entries) { + if constexpr (std::is_same_v) { + to_json(j["entries"][name], child); + } else if constexpr (std::is_same_v) { + j["entries"][name] = nlohmann::json::object(); + } else { + static_assert(false); + } + } + }, + [&](const typename Listing::Symlink & s) { + j = nlohmann::json::object(); + j["type"] = "symlink"; + j["target"] = s.target; + }, + }, + listing.raw); +} + +void to_json(nlohmann::json & j, const NarListing & listing) +{ + to_json_impl(j, listing); +} + +void to_json(nlohmann::json & j, const ShallowNarListing & listing) +{ + to_json_impl(j, listing); } } // namespace nix diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 5c3d7cfd425..812dfdbcff8 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -80,7 +80,7 @@ struct CmdCatNar : StoreCommand, MixCat throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); - auto listing = listNar(*narAccessor, CanonPath::root, true); + nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root); cat(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index ccd479c0af0..fd4a98d7ae1 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -85,7 +85,12 @@ struct MixLs : virtual Args, MixJSON if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(*accessor, path, recursive)); + nlohmann::json j; + if (recursive) + j = listNarDeep(*accessor, path); + else + j = listNarShallow(*accessor, path); + logger->cout("%s", j.dump()); } else listText(accessor, std::move(path)); } @@ -150,7 +155,7 @@ struct CmdLsNar : Command, MixLs throw SysError("opening NAR file '%s'", narPath); auto source = FdSource{fd.get()}; auto narAccessor = makeNarAccessor(source); - auto listing = listNar(*narAccessor, CanonPath::root, true); + nlohmann::json listing = listNarDeep(*narAccessor, CanonPath::root); list(makeLazyNarAccessor(listing, seekableGetNarBytes(narPath)), CanonPath{path}); } };