diff --git a/doc/manual/rl-next/hash-convert-json.md b/doc/manual/rl-next/hash-convert-json.md new file mode 100644 index 00000000000..a3abcfe163b --- /dev/null +++ b/doc/manual/rl-next/hash-convert-json.md @@ -0,0 +1,25 @@ +--- +synopsis: "`nix hash convert` supports JSON format" +prs: [] +issues: [] +--- + +`nix hash convert` now supports a `json-base16` format for both input (`--from`) and output (`--to`). + +This format represents hashes as structured JSON objects: + +```json +{ + "format": "base16", + "algorithm": "sha256", + "hash": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +} +``` + +Currently, the `format` field must always `"base16"` (hexadecimal) for input, and will always be that for output. + +This is used in the new structured JSON outputs for store path info and derivations, and will be used whenever JSON formats needs to contain hashes going forward. + +JSON input is also auto-detected when `--from` is not specified. + +See [`nix hash convert`](@docroot@/command-ref/new-cli/nix3-hash-convert.md) for usage examples. diff --git a/doc/manual/rl-next/json-format-changes.md b/doc/manual/rl-next/json-format-changes.md index b5254d236d3..43998022458 100644 --- a/doc/manual/rl-next/json-format-changes.md +++ b/doc/manual/rl-next/json-format-changes.md @@ -59,7 +59,8 @@ The new structured format follows the [JSON guidelines](@docroot@/development/js - Old: `"narHash": "sha256:FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="` - New: `"narHash": {"algorithm": "sha256", "format": "base16", "hash": "15e3c5608946..."}` - Same structure applies to `downloadHash` in NAR info contexts - - The `format` field is always `"base16"` (hexadecimal) + + See `nix hash convert`'s new support for the JSON format for details on this format, and how to convert between it and other hash formats. Nix currently only produces, and doesn't consume this format. diff --git a/src/libcmd/include/nix/cmd/misc-store-flags.hh b/src/libcmd/include/nix/cmd/misc-store-flags.hh index 27e13907680..7020cfac8dc 100644 --- a/src/libcmd/include/nix/cmd/misc-store-flags.hh +++ b/src/libcmd/include/nix/cmd/misc-store-flags.hh @@ -1,5 +1,65 @@ #include "nix/util/args.hh" #include "nix/store/content-address.hh" +#include "nix/main/common-args.hh" + +namespace nix { + +/** + * @brief Tag type for JSON hash output format. + * + * JSON format outputs `{"algorithm": "", "hash": ""}`. + */ +struct OutputFormatJSON +{ + bool operator==(const OutputFormatJSON &) const = default; + auto operator<=>(const OutputFormatJSON &) const = default; +}; + +/** + * @brief Output hash format: either a HashFormat or JSON. + */ +struct OutputHashFormat +{ + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(OutputHashFormat); + + bool operator==(const OutputHashFormat &) const = default; + auto operator<=>(const OutputHashFormat &) const = default; + + /// Convenience constant for JSON format + static constexpr struct OutputFormatJSON JSON{}; + + /** + * Parse an output hash format from a string. + * + * Accepts all HashFormat names plus "json-base16". + */ + static OutputHashFormat parse(std::string_view s); + + /** + * The reverse of parse. + */ + std::string_view print() const; + + /** + * Parse a hash from a string representation, returning both the hash + * and the output format it was parsed from. + * + * Tries to parse as JSON first (returning OutputFormatJSON if successful), + * then falls back to Hash::parseAnyReturningFormat. + */ + static std::pair + parseAnyReturningFormat(std::string_view s, std::optional optAlgo); +}; + +/** + * Print a hash in the specified output format. + */ +void printHash(const Hash & h, const OutputHashFormat & format, MixPrintJSON & printer); + +} // namespace nix namespace nix::flag { @@ -11,8 +71,8 @@ static inline Args::Flag hashAlgo(HashAlgorithm * ha) } Args::Flag hashAlgoOpt(std::string && longName, std::optional * oha); -Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf); -Args::Flag hashFormatOpt(std::string && longName, std::optional * ohf); +Args::Flag hashFormatWithDefault(std::string && longName, OutputHashFormat * hf); +Args::Flag hashFormatOpt(std::string && longName, std::optional * ohf); static inline Args::Flag hashAlgoOpt(std::optional * oha) { diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index fd22118136b..1c2edd3df58 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -1,4 +1,61 @@ #include "nix/cmd/misc-store-flags.hh" +#include "nix/util/json-utils.hh" + +namespace nix { + +constexpr static std::string_view jsonFormatName = "json-base16"; + +OutputHashFormat OutputHashFormat::parse(std::string_view s) +{ + if (s == jsonFormatName) { + return OutputHashFormat::JSON; + } + return parseHashFormat(s); +} + +std::string_view OutputHashFormat::print() const +{ + return std::visit( + overloaded{ + [](const HashFormat & hf) -> std::string_view { return printHashFormat(hf); }, + [](const OutputFormatJSON &) -> std::string_view { return jsonFormatName; }, + }, + raw); +} + +std::pair +OutputHashFormat::parseAnyReturningFormat(std::string_view s, std::optional optAlgo) +{ + /* Try parsing as JSON first. If it is valid JSON, it must also be + in the right format. Otherwise, parse string formats. */ + std::optional jsonOpt; + try { + jsonOpt = nlohmann::json::parse(s); + } catch (nlohmann::json::parse_error &) { + } + + if (jsonOpt) { + auto hash = jsonOpt->get(); + if (optAlgo && hash.algo != *optAlgo) + throw BadHash("hash '%s' should have type '%s'", s, printHashAlgo(*optAlgo)); + return {hash, OutputHashFormat::JSON}; + } + + auto [hash, format] = Hash::parseAnyReturningFormat(s, optAlgo); + return {hash, format}; +} + +void printHash(const Hash & h, const OutputHashFormat & format, MixPrintJSON & printer) +{ + std::visit( + overloaded{ + [&](const HashFormat & hf) { logger->cout(h.to_string(hf, hf == HashFormat::SRI)); }, + [&](const OutputFormatJSON &) { printer.printJSON(nlohmann::json(h)); }, + }, + format.raw); +} + +} // namespace nix namespace nix::flag { @@ -9,27 +66,31 @@ static void hashFormatCompleter(AddCompletions & completions, size_t index, std: completions.add(format); } } + auto jsonName = OutputHashFormat(OutputHashFormat::JSON).print(); + if (hasPrefix(jsonName, prefix)) { + completions.add(std::string{jsonName}); + } } -Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf) +Args::Flag hashFormatWithDefault(std::string && longName, OutputHashFormat * hf) { assert(*hf == nix::HashFormat::SRI); return Args::Flag{ .longName = std::move(longName), - .description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.", + .description = "Hash format (`base16`, `nix32`, `base64`, `sri`, `json-base16`). Default: `sri`.", .labels = {"hash-format"}, - .handler = {[hf](std::string s) { *hf = parseHashFormat(s); }}, + .handler = {[hf](std::string s) { *hf = OutputHashFormat::parse(s); }}, .completer = hashFormatCompleter, }; } -Args::Flag hashFormatOpt(std::string && longName, std::optional * ohf) +Args::Flag hashFormatOpt(std::string && longName, std::optional * ohf) { return Args::Flag{ .longName = std::move(longName), - .description = "Hash format (`base16`, `nix32`, `base64`, `sri`).", + .description = "Hash format (`base16`, `nix32`, `base64`, `sri`, `json-base16`).", .labels = {"hash-format"}, - .handler = {[ohf](std::string s) { *ohf = std::optional{parseHashFormat(s)}; }}, + .handler = {[ohf](std::string s) { *ohf = std::optional{OutputHashFormat::parse(s)}; }}, .completer = hashFormatCompleter, }; } diff --git a/src/libutil/include/nix/util/variant-wrapper.hh b/src/libutil/include/nix/util/variant-wrapper.hh index 146ae07b635..14c9a37389c 100644 --- a/src/libutil/include/nix/util/variant-wrapper.hh +++ b/src/libutil/include/nix/util/variant-wrapper.hh @@ -22,10 +22,12 @@ * * The moral equivalent of `using Raw::Raw;` */ -#define MAKE_WRAPPER_CONSTRUCTOR(CLASS_NAME) \ - FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ - \ - CLASS_NAME(auto &&... arg) \ - : raw(std::forward(arg)...) \ - { \ +#define MAKE_WRAPPER_CONSTRUCTOR(CLASS_NAME) \ + FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ + \ + template \ + requires(!(sizeof...(Args) == 1 && (std::is_same_v, CLASS_NAME> && ...))) \ + CLASS_NAME(Args &&... arg) \ + : raw(std::forward(arg)...) \ + { \ } diff --git a/src/nix/hash-convert.md b/src/nix/hash-convert.md index dcebda74a3e..4ede877112b 100644 --- a/src/nix/hash-convert.md +++ b/src/nix/hash-convert.md @@ -33,6 +33,28 @@ R""( sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ``` +* Convert a hash to the `json-base16` format: + + ```console + $ nix hash convert --to json-base16 "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" + { + "algorithm":"sha256", + "format":"base16", + "hash":"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + } + ``` + +* Convert a hash from the `json-base16` format: + + ```console + $ nix hash convert --from json-base16 '{ + "format": "base16", + "algorithm": "sha256", + "hash": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + }' + sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= + ``` + # Description `nix hash convert` converts hashes from one encoding to another. diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2945c672c2c..2194696cd8f 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -17,10 +17,10 @@ using namespace nix; * * Deprecation Issue: https://github.com/NixOS/nix/issues/8876 */ -struct CmdHashBase : Command +struct CmdHashBase : Command, MixPrintJSON { FileIngestionMethod mode; - HashFormat hashFormat = HashFormat::SRI; + OutputHashFormat hashFormat = HashFormat::SRI; bool truncate = false; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; std::vector paths; @@ -37,25 +37,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&hashFormat, HashFormat::SRI}, + .handler = {&hashFormat, OutputHashFormat{HashFormat::SRI}}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&hashFormat, HashFormat::Base64}, + .handler = {&hashFormat, OutputHashFormat{HashFormat::Base64}}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&hashFormat, HashFormat::Nix32}, + .handler = {&hashFormat, OutputHashFormat{HashFormat::Nix32}}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&hashFormat, HashFormat::Base16}, + .handler = {&hashFormat, OutputHashFormat{HashFormat::Base16}}, }); addFlag(flag::hashAlgo("type", &hashAlgo)); @@ -129,7 +129,7 @@ struct CmdHashBase : Command if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI)); + printHash(h, hashFormat, *this); } } }; @@ -209,15 +209,14 @@ struct CmdToBase : Command /** * `nix hash convert` */ -struct CmdHashConvert : Command +struct CmdHashConvert : Command, MixPrintJSON { - std::optional from; - HashFormat to; + std::optional from; + OutputHashFormat to = HashFormat::SRI; std::optional algo; std::vector hashStrings; CmdHashConvert() - : to(HashFormat::SRI) { addFlag(flag::hashFormatOpt("from", &from)); addFlag(flag::hashFormatWithDefault("to", &to)); @@ -248,15 +247,15 @@ struct CmdHashConvert : Command void run() override { for (const auto & s : hashStrings) { - auto [h, parsedFormat] = Hash::parseAnyReturningFormat(s, algo); + auto [h, parsedFormat] = OutputHashFormat::parseAnyReturningFormat(s, algo); if (from && *from != parsedFormat) { throw BadHash( "input hash '%s' has format '%s', but '--from %s' was specified", s, - printHashFormat(parsedFormat), - printHashFormat(*from)); + parsedFormat.print(), + from->print()); } - logger->cout(h.to_string(to, to == HashFormat::SRI)); + printHash(h, to, *this); } } }; diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh index be76179ca87..76cc783a917 100755 --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -109,3 +109,66 @@ try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596f # Test SRI hashes that lack trailing '=' characters. These are incorrect but we need to support them for backward compatibility. [[ $(nix hash convert --from sri "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]] [[ $(nix hash convert --from sri "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ") = sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ== ]] + +# +# Test JSON format (json-base16) +# + +sha256_json='{ + "format": "base16", + "algorithm": "sha256", + "hash": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +}' + +sha512_json='{ + "format": "base16", + "algorithm": "sha512", + "hash": "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" +}' + +sha1_json='{ + "format": "base16", + "algorithm": "sha1", + "hash": "800d59cfcd3c05e900cb4e214be48f6b886a08df" +}' + +# Basic conversion to JSON format +nix hash convert --to json-base16 "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" \ + | jq -e --argjson expected "$sha256_json" '. == $expected' + +# JSON to SRI (default output) +[[ $(nix hash convert --from json-base16 "$sha256_json") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]] + +# JSON to JSON (round-trip) +for json in "$sha256_json" "$sha512_json" "$sha1_json"; do + nix hash convert --from json-base16 --to json-base16 "$json" \ + | jq -e --argjson expected "$json" '. == $expected' +done + +# JSON input for sha512 +[[ $(nix hash convert --from json-base16 "$sha512_json") = sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ== ]] + +# JSON input for sha1 +[[ $(nix hash convert --from json-base16 "$sha1_json") = sha1-gA1Zz808BekAy04hS+SPa4hqCN8= ]] + +# JSON auto-detection (without --from) +[[ $(nix hash convert "$sha256_json") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]] + +# JSON with --hash-algo validation (matching) +[[ $(nix hash convert --hash-algo sha256 "$sha256_json") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]] + +# JSON with --hash-algo validation (mismatched, should fail) +expectStderr 1 nix hash convert --hash-algo sha512 "$sha256_json" \ + | grepQuiet "should have type 'sha512'" + +# Asserting --from json-base16 with non-JSON input fails +expectStderr 1 nix hash convert --from json-base16 "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" \ + | grepQuiet "'sri', but '--from json-base16'" + +# Asserting --from with wrong format fails (JSON input with --from sri) +expectStderr 1 nix hash convert --from sri "$sha256_json" \ + | grepQuiet "'json-base16', but '--from sri'" + +# Asserting --from base16 with JSON input fails +expectStderr 1 nix hash convert --hash-algo sha256 --from base16 "$sha256_json" \ + | grepQuiet "'json-base16', but '--from base16'" diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix index 5363f8e72b4..ba7f3d3df39 100644 --- a/tests/nixos/fetchers-substitute.nix +++ b/tests/nixos/fetchers-substitute.nix @@ -121,8 +121,8 @@ path_info_json = substituter.succeed(f"nix path-info --json-format 2 --json {tarball_store_path}").strip() path_info_dict = json.loads(path_info_json)["info"] narHash_obj = path_info_dict[os.path.basename(tarball_store_path)]["narHash"] - # Convert from structured format {"algorithm": "sha256", "format": "base16", "hash": "..."} to SRI string - tarball_hash_sri = substituter.succeed(f"nix hash convert --to sri {narHash_obj['algorithm']}:{narHash_obj['hash']}").strip() + # Convert to SRI string + tarball_hash_sri = substituter.succeed(f"nix hash convert --to sri '{json.dumps(narHash_obj)}'").strip() print(f"Tarball NAR hash (SRI): {tarball_hash_sri}") # Also get the old format hash for fetchTarball (which uses sha256 parameter)