libutil: Use Boost.URL for URI parsing

doc/manual/rl-next/rfc4007-zone-id-in-uri-rfc6874.md

@@ -0,0 +1,6 @@
+---
+synopsis: "Represent IPv6 RFC4007 ZoneId literals in conformance with RFC6874"
+prs: [13445]
+---
+
+Prior versions of Nix since [#4646](https://github.com/NixOS/nix/pull/4646) accepted [IPv6 scoped addresses](https://datatracker.ietf.org/doc/html/rfc4007) in URIs like [store references](@docroot@/store/types/index.md#store-url-format) in the textual representation with a literal percent character: `[fe80::1%18]`. This was ambiguous, because the the percent literal `%` is reserved by [RFC3986](https://datatracker.ietf.org/doc/html/rfc3986), since it's used to indicate percent encoding. Nix now requires that the percent `%` symbol is percent-encoded as `%25`. This implements [RFC6874](https://datatracker.ietf.org/doc/html/rfc6874), which defines the representation of zone identifiers in URIs. The example from above now has to be specified as `[fe80::1%2518]`.

packaging/dependencies.nix

@@ -62,6 +62,7 @@ scope: {
         "--with-context"
         "--with-coroutine"
         "--with-iostreams"
+        "--with-url"
       ];
       enableIcu = false;
     }).overrideAttrs

src/libflake-tests/flakeref.cc

@@ -48,6 +48,13 @@ TEST(parseFlakeRef, path)
         ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?revCount=123");
         ASSERT_EQ(fragment, "bla");
     }
+
+    {
+        auto s = "/foo bar/baz?dir=bla space";
+        auto flakeref = parseFlakeRef(fetchSettings, s);
+        ASSERT_EQ(flakeref.to_string(), "path:/foo%20bar/baz?dir=bla%20space");
+        ASSERT_EQ(flakeref.toAttrs().at("dir"), fetchers::Attr("bla space"));
+    }
 }
 
 TEST(to_string, doesntReencodeUrl)

src/libstore-tests/data/store-reference/local_3.txt

@@ -0,0 +1 @@
+local://?root=/foo bar/baz

src/libstore-tests/store-reference.cc

@@ -85,10 +85,24 @@ static StoreReference localExample_2{
         },
 };
 
+static StoreReference localExample_3{
+    .variant =
+        StoreReference::Specified{
+            .scheme = "local",
+        },
+    .params =
+        {
+            {"root", "/foo bar/baz"},
+        },
+};
+
 URI_TEST(local_1, localExample_1)
 
 URI_TEST(local_2, localExample_2)
 
+/* Test path with spaces */
+URI_TEST(local_3, localExample_3)
+
 URI_TEST_READ(local_shorthand_1, localExample_1)
 
 URI_TEST_READ(local_shorthand_2, localExample_2)

src/libutil-test-support/include/nix/util/tests/gmock-matchers.hh

@@ -0,0 +1,56 @@
+#pragma once
+///@file
+
+#include "nix/util/terminal.hh"
+#include <gmock/gmock.h>
+
+namespace nix::testing {
+
+namespace internal {
+
+/**
+ * GMock matcher that matches substring while stripping off all ANSI escapes.
+ * Useful for checking exceptions messages in unit tests.
+ */
+class HasSubstrIgnoreANSIMatcher
+{
+public:
+    explicit HasSubstrIgnoreANSIMatcher(std::string substring)
+        : substring(std::move(substring))
+    {
+    }
+
+    bool MatchAndExplain(const char * s, ::testing::MatchResultListener * listener) const
+    {
+        return s != nullptr && MatchAndExplain(std::string(s), listener);
+    }
+
+    template<typename MatcheeStringType>
+    bool MatchAndExplain(const MatcheeStringType & s, [[maybe_unused]] ::testing::MatchResultListener * listener) const
+    {
+        return filterANSIEscapes(s, /*filterAll=*/true).find(substring) != substring.npos;
+    }
+
+    void DescribeTo(::std::ostream * os) const
+    {
+        *os << "has substring " << substring;
+    }
+
+    void DescribeNegationTo(::std::ostream * os) const
+    {
+        *os << "has no substring " << substring;
+    }
+
+private:
+    std::string substring;
+};
+
+} // namespace internal
+
+inline ::testing::PolymorphicMatcher<internal::HasSubstrIgnoreANSIMatcher>
+HasSubstrIgnoreANSIMatcher(const std::string & substring)
+{
+    return ::testing::MakePolymorphicMatcher(internal::HasSubstrIgnoreANSIMatcher(substring));
+}
+
+} // namespace nix::testing

src/libutil-test-support/include/nix/util/tests/meson.build

@@ -4,6 +4,7 @@ include_dirs = [include_directories('../../..')]
 
 headers = files(
   'characterization.hh',
+  'gmock-matchers.hh',
   'gtest-with-params.hh',
   'hash.hh',
   'nix_api_util.hh',

src/libutil-tests/url.cc

@@ -1,5 +1,7 @@
 #include "nix/util/url.hh"
+#include "nix/util/tests/gmock-matchers.hh"
 #include <gtest/gtest.h>
+#include <gmock/gmock.h>
 
 namespace nix {
 
@@ -122,9 +124,9 @@ TEST(parseURL, parseIPv4Address)
     ASSERT_EQ(parsed, expected);
 }
 
-TEST(parseURL, parseScopedRFC4007IPv6Address)
+TEST(parseURL, parseScopedRFC6874IPv6Address)
 {
-    auto s = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080";
+    auto s = "http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080";
     auto parsed = parseURL(s);
 
     ParsedURL expected{
@@ -289,6 +291,14 @@ TEST(percentDecode, trailingPercent)
     ASSERT_EQ(d, s);
 }
 
+TEST(percentDecode, incompleteEncoding)
+{
+    ASSERT_THAT(
+        []() { percentDecode("%1"); },
+        ::testing::ThrowsMessage<BadURL>(
+            testing::HasSubstrIgnoreANSIMatcher("error: invalid URI parameter '%1': incomplete pct-encoding")));
+}
+
 /* ----------------------------------------------------------------------------
  * percentEncode
  * --------------------------------------------------------------------------*/

src/libutil/include/nix/util/url-parts.hh

@@ -8,21 +8,10 @@ namespace nix {
 
 // URI stuff.
 const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
-const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)";
-const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?";
-const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
 const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
 const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
-const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
-const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
-const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
-const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
 const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
-const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
 const static std::string fragmentRegex = "(?:" + pcharRegex + "|[/? \"^])*";
-const static std::string segmentRegex = "(?:" + pcharRegex + "*)";
-const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
-const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
 
 /// A Git ref (i.e. branch or tag name).
 /// \todo check that this is correct.

src/libutil/include/nix/util/url.hh

@@ -15,7 +15,7 @@ struct ParsedURL
 
     std::string to_string() const;
 
-    bool operator==(const ParsedURL & other) const noexcept;
+    bool operator==(const ParsedURL & other) const noexcept = default;
 
     /**
      * Remove `.` and `..` path elements.
@@ -34,6 +34,17 @@ StringMap decodeQuery(const std::string & query);
 
 std::string encodeQuery(const StringMap & query);
 
+/**
+ * Parse a Nix URL into a ParsedURL.
+ *
+ * Nix URI is mostly compliant with RFC3986, but with some deviations:
+ * - Literal spaces are allowed and don't have to be percent encoded.
+ *   This is mostly done for backward compatibility.
+ *
+ * @note IPv6 ZoneId literals (RFC4007) are represented in URIs according to RFC6874.
+ *
+ * @throws BadURL
+ */
 ParsedURL parseURL(const std::string & url);
 
 /**

src/libutil/meson.build

@@ -57,7 +57,7 @@ deps_private += blake3
 
 boost = dependency(
   'boost',
-  modules : ['context', 'coroutine', 'iostreams'],
+  modules : ['context', 'coroutine', 'iostreams', 'url'],
   include_type: 'system',
   version: '>=1.82.0'
 )