diff --git a/configure.ac b/configure.ac index 64fa12fc7c7e..79a3303ed8e9 100644 --- a/configure.ac +++ b/configure.ac @@ -279,6 +279,7 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) +PKG_CHECK_MODULES([PCRE2], [libpcre2-8 >= 10.39]) # documentation generation switch AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), diff --git a/flake.nix b/flake.nix index cc2a48d9c247..7595a21bf089 100644 --- a/flake.nix +++ b/flake.nix @@ -115,6 +115,7 @@ boost lowdown-nix gtest + pcre2 ] ++ lib.optionals stdenv.isLinux [libseccomp] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 563f24e48065..8de8685a2c13 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2478,6 +2478,7 @@ void EvalState::printStats() sizes.attr("Bindings", sizeof(Bindings)); sizes.attr("Attr", sizeof(Attr)); } + topObj.attr("regexCache", regexCacheSize(regexCache)); topObj.attr("nrOpUpdates", nrOpUpdates); topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); topObj.attr("nrThunks", nrThunks); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f07f15d43672..9aa7a4fd81ee 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -76,6 +76,7 @@ void initGC(); struct RegexCache; std::shared_ptr makeRegexCache(); +size_t regexCacheSize(std::shared_ptr cache); struct DebugTrace { std::optional pos; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 016631647cf1..11dcf11fe4eb 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -pthread +libexpr_LDFLAGS += -lboost_context -pthread -lpcre2-8 ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 22f6ad3cc859..c21792e1961f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -21,12 +21,10 @@ #include #include -#include #include #include - namespace nix { @@ -3497,209 +3495,6 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); -struct RegexCache -{ - // TODO use C++20 transparent comparison when available - std::unordered_map cache; - std::list keys; - - std::regex get(std::string_view re) - { - auto it = cache.find(re); - if (it != cache.end()) - return it->second; - keys.emplace_back(re); - return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; - } -}; - -std::shared_ptr makeRegexCache() -{ - return std::make_shared(); -} - -void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) -{ - auto re = state.forceStringNoCtx(*args[0], pos); - - try { - - auto regex = state.regexCache->get(re); - - PathSet context; - const auto str = state.forceString(*args[1], context, pos); - - std::cmatch match; - if (!std::regex_match(str.begin(), str.end(), match, regex)) { - v.mkNull(); - return; - } - - // the first match is the whole string - const size_t len = match.size() - 1; - state.mkList(v, len); - for (size_t i = 0; i < len; ++i) { - if (!match[i+1].matched) - (v.listElems()[i] = state.allocValue())->mkNull(); - else - (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); - } - - } catch (std::regex_error & e) { - if (e.code() == std::regex_constants::error_space) { - // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = state.positions[pos] - })); - } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = state.positions[pos] - })); - } -} - -static RegisterPrimOp primop_match({ - .name = "__match", - .args = {"regex", "str"}, - .doc = R"s( - Returns a list if the [extended POSIX regular - expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04) - *regex* matches *str* precisely, otherwise returns `null`. Each item - in the list is a regex group. - - ```nix - builtins.match "ab" "abc" - ``` - - Evaluates to `null`. - - ```nix - builtins.match "abc" "abc" - ``` - - Evaluates to `[ ]`. - - ```nix - builtins.match "a(b)(c)" "abc" - ``` - - Evaluates to `[ "b" "c" ]`. - - ```nix - builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " - ``` - - Evaluates to `[ "FOO" ]`. - )s", - .fun = prim_match, -}); - -/* Split a string with a regular expression, and return a list of the - non-matching parts interleaved by the lists of the matching groups. */ -void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) -{ - auto re = state.forceStringNoCtx(*args[0], pos); - - try { - - auto regex = state.regexCache->get(re); - - PathSet context; - const auto str = state.forceString(*args[1], context, pos); - - auto begin = std::cregex_iterator(str.begin(), str.end(), regex); - auto end = std::cregex_iterator(); - - // Any matches results are surrounded by non-matching results. - const size_t len = std::distance(begin, end); - state.mkList(v, 2 * len + 1); - size_t idx = 0; - - if (len == 0) { - v.listElems()[idx++] = args[1]; - return; - } - - for (auto i = begin; i != end; ++i) { - assert(idx <= 2 * len + 1 - 3); - auto match = *i; - - // Add a string for non-matched characters. - (v.listElems()[idx++] = state.allocValue())->mkString(match.prefix().str()); - - // Add a list for matched substrings. - const size_t slen = match.size() - 1; - auto elem = v.listElems()[idx++] = state.allocValue(); - - // Start at 1, beacause the first match is the whole string. - state.mkList(*elem, slen); - for (size_t si = 0; si < slen; ++si) { - if (!match[si + 1].matched) - (elem->listElems()[si] = state.allocValue())->mkNull(); - else - (elem->listElems()[si] = state.allocValue())->mkString(match[si + 1].str()); - } - - // Add a string for non-matched suffix characters. - if (idx == 2 * len) - (v.listElems()[idx++] = state.allocValue())->mkString(match.suffix().str()); - } - - assert(idx == 2 * len + 1); - - } catch (std::regex_error & e) { - if (e.code() == std::regex_constants::error_space) { - // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = state.positions[pos] - })); - } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = state.positions[pos] - })); - } -} - -static RegisterPrimOp primop_split({ - .name = "__split", - .args = {"regex", "str"}, - .doc = R"s( - Returns a list composed of non matched strings interleaved with the - lists of the [extended POSIX regular - expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04) - *regex* matches of *str*. Each item in the lists of matched - sequences is a regex group. - - ```nix - builtins.split "(a)b" "abc" - ``` - - Evaluates to `[ "" [ "a" ] "c" ]`. - - ```nix - builtins.split "([ac])" "abc" - ``` - - Evaluates to `[ "" [ "a" ] "b" [ "c" ] "" ]`. - - ```nix - builtins.split "(a)|(c)" "abc" - ``` - - Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`. - - ```nix - builtins.split "([[:upper:]]+)" " FOO " - ``` - - Evaluates to `[ " " [ "FOO" ] " " ]`. - )s", - .fun = prim_split, -}); static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * args, Value & v) { diff --git a/src/libexpr/primops/regex.cc b/src/libexpr/primops/regex.cc new file mode 100644 index 000000000000..343442b9007b --- /dev/null +++ b/src/libexpr/primops/regex.cc @@ -0,0 +1,360 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "derivations.hh" +#include "store-api.hh" +#define PCRE2_CODE_UNIT_WIDTH 8 +#include + + +namespace nix { + +MakeError(RegexError, Error); +namespace PCRE { + +class MatchData; + +class Regex +{ + friend class MatchData; +protected: + pcre2_code* code; + size_t usage = 0; + std::vector> name_table; +public: + Regex(std::string_view re) + { + int errorcode; + PCRE2_SIZE erroffset; + + code = pcre2_compile((const unsigned char*)re.data(), re.length(), 0, &errorcode, &erroffset, nullptr); + + if (code == nullptr) { + unsigned char err[256]; + pcre2_get_error_message(errorcode, err, sizeof(err)); + throw RegexError("unable to compile regex: %1% at offset %2%", err, erroffset); + } + // parse nametable + uint32_t namecount; + pcre2_pattern_info(code, PCRE2_INFO_NAMECOUNT, &namecount); + if (namecount) { + PCRE2_SPTR tabptr; + uint32_t name_entry_size; + pcre2_pattern_info(code, PCRE2_INFO_NAMETABLE, &tabptr); + pcre2_pattern_info(code, PCRE2_INFO_NAMEENTRYSIZE, &name_entry_size); + name_table.reserve(namecount); + for (size_t i = 0; i < namecount; i++) { + int n = tabptr[0] << 8 | tabptr[1]; + name_table.emplace_back((const char*)(tabptr+2), n); + tabptr += name_entry_size; + } + } + } + Regex(const Regex&) = delete; + Regex(Regex&&) = delete; // move not implemented + ~Regex() + { + pcre2_code_free(code); + }; + + const decltype(name_table)& nameTable() const { + return name_table; + }; + + int match(const std::string_view str, MatchData& match, PCRE2_SIZE startoffset = 0, uint32_t options = 0); + uint32_t captureCount() noexcept + { + uint32_t len; + pcre2_pattern_info(code, PCRE2_INFO_CAPTURECOUNT, &len); + return len; + } + + void compile() + { + assert(pcre2_jit_compile(code, PCRE2_JIT_COMPLETE) == 0); + } +}; + +class MatchData +{ + friend class Regex; +protected: + pcre2_match_data* match; + std::string_view str; + uint32_t size_; + PCRE2_SIZE* ovector; + Regex& re; +public: + MatchData(Regex& re) noexcept + : re(re) + { + match = pcre2_match_data_create_from_pattern(re.code, NULL); + size_ = pcre2_get_ovector_count(match); + ovector = pcre2_get_ovector_pointer(match); + }; + MatchData(const MatchData&) = delete; + MatchData(MatchData&&) = delete; + ~MatchData() + { + pcre2_match_data_free(match); + }; + + const Regex& regex() const noexcept + { + return re; + }; + + uint32_t size() const noexcept + { + return size_; + }; + + std::optional operator[](std::size_t i) const + { + assert(i < size()); + if (ovector[2*i] == PCRE2_UNSET) return {}; + return str.substr(ovector[2*i], ovector[2*i+1] - ovector[2*i]); + }; + + uint32_t startPos() const noexcept + { + return pcre2_get_startchar(match); + }; +}; + +int Regex::match(std::string_view str, MatchData& match, PCRE2_SIZE startoffset, uint32_t options) +{ + // Cache the string in the match data for match[] to work. + match.str = str; + // compile if we're using this regex more than once + if (this->usage++) this->compile(); + + int rc = pcre2_match(code, (const unsigned char*)str.data(), str.length(), + startoffset, options, match.match, NULL); + + assert(rc != 0); // match data too small, shouldn't happen + + if (rc < 0 && rc != PCRE2_ERROR_NOMATCH) { + unsigned char err[256]; + pcre2_get_error_message(rc, err, sizeof(err)); + throw RegexError("unable to match regex: %1%", err); + } + return rc; +}; + +}; + +struct RegexCache +{ + // TODO use C++20 transparent comparison when available + std::unordered_map cache; + std::list keys; + + PCRE::Regex& get(std::string_view re) + { + auto it = cache.find(re); + if (it != cache.end()) + return it->second; + keys.emplace_back(re); + return cache.emplace(keys.back(), keys.back()).first->second; + } +}; + +std::shared_ptr makeRegexCache() +{ + return std::make_shared(); +} + +size_t regexCacheSize(std::shared_ptr cache) +{ + return cache->keys.size(); +} + +static void extract_matches(EvalState & state, const PCRE::MatchData& match, Value & v) +{ + auto nameTable = match.regex().nameTable(); + if (nameTable.size()) { + // try to extract named bindings + auto bindings = state.buildBindings(nameTable.size()); + for (auto i : nameTable) { + Value & elem = bindings.alloc(i.first); + if (!match[i.second].has_value()) elem.mkNull(); + else elem.mkString(*match[i.second]); + } + v.mkAttrs(bindings); + } else { + // the first match is the whole string + const size_t len = match.size() - 1; + state.mkList(v, len); + for (size_t i = 0; i < len; ++i) { + if (!match[i+1].has_value()) + (v.listElems()[i] = state.allocValue())->mkNull(); + else + (v.listElems()[i] = state.allocValue())->mkString(*match[i + 1]); + } + } +} + +void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + + auto& regex = state.regexCache->get(re); + + PathSet context; + const auto str = state.forceString(*args[1], context, pos); + + PCRE::MatchData match(regex); + int rc = regex.match(str, match, 0, PCRE2_ANCHORED | PCRE2_ENDANCHORED); + if (rc == PCRE2_ERROR_NOMATCH) { + v.mkNull(); + return; + } + extract_matches(state, match, v); + + } catch (RegexError & e) { + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("error while evaluating regex '%s': ", re, e.what()), + .errPos = state.positions[pos] + })); + } +} + +static RegisterPrimOp primop_match({ + .name = "__match", + .args = {"regex", "str"}, + .doc = R"s( + Returns a list if the [extended POSIX regular + expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04) + *regex* matches *str* precisely, otherwise returns `null`. Each item + in the list is a regex group. + + ```nix + builtins.match "ab" "abc" + ``` + + Evaluates to `null`. + + ```nix + builtins.match "abc" "abc" + ``` + + Evaluates to `[ ]`. + + ```nix + builtins.match "a(b)(c)" "abc" + ``` + + Evaluates to `[ "b" "c" ]`. + + ```nix + builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " + ``` + + Evaluates to `[ "FOO" ]`. + )s", + .fun = prim_match, +}); + +/* Split a string with a regular expression, and return a list of the + non-matching parts interleaved by the lists of the matching groups. */ +void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + auto& regex = state.regexCache->get(re); + + // We're going to use this regex many times, JIT it. + regex.compile(); + + PathSet context; + const auto str = state.forceString(*args[1], context, pos); + + + PCRE::MatchData match(regex); + int rc = regex.match(str, match); + + if (rc == PCRE2_ERROR_NOMATCH) { + state.mkList(v, 1); + v.listElems()[0] = args[1]; + return; + } + ValueVector result; + result.reserve(3); + size_t lastmatchend = 0; + size_t newmatchstart = 0; + while (rc != PCRE2_ERROR_NOMATCH && newmatchstart <= str.length()) { + // Add a string for non-matched characters. + Value * prefix = state.allocValue(); + prefix->mkString(str.substr(lastmatchend, match.startPos() - lastmatchend)); + result.push_back(prefix); + + // Add a list for matched substrings. + auto elem = state.allocValue(); + result.push_back(elem); + + extract_matches(state, match, *elem); + + lastmatchend = match.startPos() + match[0]->length(); + newmatchstart = lastmatchend + (match[0]->length() == 0 ? 1 : 0); + if (newmatchstart <= str.length()) + rc = regex.match(str, match, newmatchstart); + } + + // Add a string for non-matched suffix characters. + Value * rest = state.allocValue(); + rest->mkString(str.substr(lastmatchend, -1)); + result.push_back(rest); + + state.mkList(v, result.size()); + for (size_t n = 0; n < result.size(); ++n) + v.listElems()[n] = result[n]; + + } catch (RegexError & e) { + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("error while evaluating regex '%s': ", re, e.what()), + .errPos = state.positions[pos] + })); + } +} + +static RegisterPrimOp primop_split({ + .name = "__split", + .args = {"regex", "str"}, + .doc = R"s( + Returns a list composed of non matched strings interleaved with the + lists of the [extended POSIX regular + expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04) + *regex* matches of *str*. Each item in the lists of matched + sequences is a regex group. + + ```nix + builtins.split "(a)b" "abc" + ``` + + Evaluates to `[ "" [ "a" ] "c" ]`. + + ```nix + builtins.split "([ac])" "abc" + ``` + + Evaluates to `[ "" [ "a" ] "b" [ "c" ] "" ]`. + + ```nix + builtins.split "(a)|(c)" "abc" + ``` + + Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`. + + ```nix + builtins.split "([[:upper:]]+)" " FOO " + ``` + + Evaluates to `[ " " [ "FOO" ] " " ]`. + )s", + .fun = prim_split, +}); + +}