From 7b91e2d0ead41e9fb44919575d0752e36ffa667d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 16 Dec 2022 12:07:39 +0100 Subject: [PATCH 1/9] Implement `Regex` engine on PCRE2 (#12840) --- spec/std/regex_spec.cr | 20 +++-- src/regex/engine.cr | 13 ++- src/regex/lib_pcre2.cr | 89 +++++++++++++++++++++ src/regex/pcre2.cr | 176 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 290 insertions(+), 8 deletions(-) create mode 100644 src/regex/lib_pcre2.cr create mode 100644 src/regex/pcre2.cr diff --git a/spec/std/regex_spec.cr b/spec/std/regex_spec.cr index c38194bc8785..5935e85060fd 100644 --- a/spec/std/regex_spec.cr +++ b/spec/std/regex_spec.cr @@ -200,12 +200,16 @@ describe "Regex" do /foo/.matches?("foo", options: Regex::Options::ANCHORED).should be_true end - it "matches a large single line string" do - LibPCRE.config LibPCRE::CONFIG_JIT, out jit_enabled - pending! "PCRE JIT mode not available." unless 1 == jit_enabled + it "doesn't crash with a large single line string" do + {% if Regex::Engine.resolve.name == "Regex::PCRE" %} + LibPCRE.config LibPCRE::CONFIG_JIT, out jit_enabled + pending! "PCRE JIT mode not available." unless 1 == jit_enabled + {% end %} str = File.read(datapath("large_single_line_string.txt")) - str.matches?(/^(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?$/).should be_false + str.matches?(/^(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?$/) + # We don't care whether this actually matches or not, it's just to make + # sure the engine does not stack overflow with a large string. end end @@ -422,6 +426,12 @@ describe "Regex" do it ".error?" do Regex.error?("(foo|bar)").should be_nil - Regex.error?("(foo|bar").should eq "missing ) at 8" + Regex.error?("(foo|bar").should eq( + if Regex::Engine.to_s == "Regex::PCRE2" + "missing closing parenthesis at 8" + else + "missing ) at 8" + end + ) end end diff --git a/src/regex/engine.cr b/src/regex/engine.cr index ad69e5d034bf..766917f87dd2 100644 --- a/src/regex/engine.cr +++ b/src/regex/engine.cr @@ -1,4 +1,11 @@ -require "./pcre" +{% if flag?(:use_pcre2) || (!flag?(:use_pcre) && !flag?(:win32) && `hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libpcre2-8 || printf %s false` != "false") %} + require "./pcre2" -# :nodoc: -alias Regex::Engine = PCRE + # :nodoc: + alias Regex::Engine = PCRE2 +{% else %} + require "./pcre" + + # :nodoc: + alias Regex::Engine = PCRE +{% end %} diff --git a/src/regex/lib_pcre2.cr b/src/regex/lib_pcre2.cr new file mode 100644 index 000000000000..922c492b7e1a --- /dev/null +++ b/src/regex/lib_pcre2.cr @@ -0,0 +1,89 @@ +@[Link("pcre2-8")] +lib LibPCRE2 + alias Int = LibC::Int + + UNSET = ~LibC::SizeT.new(0) + + ANCHORED = 0x80000000 + NO_UTF_CHECK = 0x40000000 + ENDANCHORED = 0x20000000 + + ALLOW_EMPTY_CLASS = 0x00000001 + ALT_BSUX = 0x00000002 + AUTO_CALLOUT = 0x00000004 + CASELESS = 0x00000008 + DOLLAR_ENDONLY = 0x00000010 + DOTALL = 0x00000020 + DUPNAMES = 0x00000040 + EXTENDED = 0x00000080 + FIRSTLINE = 0x00000100 + MATCH_UNSET_BACKREF = 0x00000200 + MULTILINE = 0x00000400 + NEVER_UCP = 0x00000800 + NEVER_UTF = 0x00001000 + NO_AUTO_CAPTURE = 0x00002000 + NO_AUTO_POSSESS = 0x00004000 + NO_DOTSTAR_ANCHOR = 0x00008000 + NO_START_OPTIMIZE = 0x00010000 + UCP = 0x00020000 + UNGREEDY = 0x00040000 + UTF = 0x00080000 + NEVER_BACKSLASH_C = 0x00100000 + ALT_CIRCUMFLEX = 0x00200000 + ALT_VERBNAMES = 0x00400000 + USE_OFFSET_LIMIT = 0x00800000 + EXTENDED_MORE = 0x01000000 + LITERAL = 0x02000000 + MATCH_INVALID_UTF = 0x04000000 + + ERROR_NOMATCH = -1 + + INFO_ALLOPTIONS = 0 + INFO_ARGOPTIONS = 1 + INFO_BACKREFMAX = 2 + INFO_BSR = 3 + INFO_CAPTURECOUNT = 4 + INFO_FIRSTCODEUNIT = 5 + INFO_FIRSTCODETYPE = 6 + INFO_FIRSTBITMAP = 7 + INFO_HASCRORLF = 8 + INFO_JCHANGED = 9 + INFO_JITSIZE = 10 + INFO_LASTCODEUNIT = 11 + INFO_LASTCODETYPE = 12 + INFO_MATCHEMPTY = 13 + INFO_MATCHLIMIT = 14 + INFO_MAXLOOKBEHIND = 15 + INFO_MINLENGTH = 16 + INFO_NAMECOUNT = 17 + INFO_NAMEENTRYSIZE = 18 + INFO_NAMETABLE = 19 + INFO_NEWLINE = 20 + INFO_DEPTHLIMIT = 21 + INFO_RECURSIONLIMIT = 21 # Obsolete synonym + INFO_SIZE = 22 + INFO_HASBACKSLASHC = 23 + INFO_FRAMESIZE = 24 + INFO_HEAPLIMIT = 25 + INFO_EXTRAOPTIONS = 26 + + type Code = Void* + type CompileContext = Void* + type MatchData = Void* + + fun get_error_message = pcre2_get_error_message_8(errorcode : Int, buffer : UInt8*, bufflen : LibC::SizeT) : Int + + fun compile = pcre2_compile_8(pattern : UInt8*, length : LibC::SizeT, options : UInt32, errorcode : LibC::SizeT*, erroroffset : Int*, ccontext : CompileContext*) : Code* + fun code_free = pcre2_code_free_8(code : Code*) : Void + + fun pattern_info = pcre2_pattern_info_8(code : Code*, what : UInt32, where : Void*) : Int + + fun match = pcre2_match_8(code : Code*, subject : UInt8*, length : LibC::SizeT, startoffset : LibC::SizeT, options : UInt32, match_data : MatchData*, mcontext : Void*) : Int + fun match_data_create_from_pattern = pcre2_match_data_create_from_pattern_8(code : Code*, gcontext : Void*) : MatchData* + fun match_data_free = pcre2_match_data_free_8(match_data : MatchData*) : Void + + fun substring_nametable_scan = pcre2_substring_nametable_scan_8(code : Code*, name : UInt8*, first : UInt8*, last : UInt8*) : Int + + fun get_ovector_pointer = pcre2_get_ovector_pointer_8(match_data : MatchData*) : LibC::SizeT* + fun get_ovector_count = pcre2_get_ovector_count_8(match_data : MatchData*) : UInt32 +end diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr new file mode 100644 index 000000000000..3eea20280268 --- /dev/null +++ b/src/regex/pcre2.cr @@ -0,0 +1,176 @@ +require "./lib_pcre2" + +# :nodoc: +module Regex::PCRE2 + @re : LibPCRE2::Code* + + # :nodoc: + def initialize(*, _source @source : String, _options @options) + @re = PCRE2.compile(source, pcre2_options(options) | LibPCRE2::UTF | LibPCRE2::NO_UTF_CHECK | LibPCRE2::DUPNAMES | LibPCRE2::UCP) do |error_message| + raise ArgumentError.new(error_message) + end + end + + protected def self.compile(source, options) + if res = LibPCRE2.compile(source, source.bytesize, options, out errorcode, out erroroffset, nil) + res + else + message = String.new(256) do |buffer| + bytesize = LibPCRE2.get_error_message(errorcode, buffer, 256) + {bytesize, 0} + end + yield "#{message} at #{erroroffset}" + end + end + + private def pcre2_options(options) + flag = 0 + options.each do |option| + flag |= case option + when .ignore_case? then LibPCRE2::CASELESS + when .multiline? then LibPCRE2::DOTALL | LibPCRE2::MULTILINE + when .extended? then LibPCRE2::EXTENDED + when .anchored? then LibPCRE2::ANCHORED + when .utf_8? then LibPCRE2::UTF + when .no_utf8_check? then LibPCRE2::NO_UTF_CHECK + when .dupnames? then LibPCRE2::DUPNAMES + when .ucp? then LibPCRE2::UCP + else + raise "unreachable" + end + end + flag + end + + def finalize + {% unless flag?(:interpreted) %} + LibPCRE2.code_free @re + {% end %} + end + + protected def self.error_impl(source) + code = PCRE2.compile(source, LibPCRE2::UTF | LibPCRE2::NO_UTF_CHECK | LibPCRE2::DUPNAMES | LibPCRE2::UCP) do |error_message| + return error_message + end + + LibPCRE2.code_free code + + nil + end + + private def pattern_info(what) + value = uninitialized UInt32 + pattern_info(what, pointerof(value)) + value + end + + private def pattern_info(what, where) + ret = LibPCRE2.pattern_info(@re, what, where) + if ret != 0 + raise "error pattern_info #{what}: #{ret}" + end + end + + private def name_table_impl + lookup = Hash(Int32, String).new + + each_capture_group do |capture_number, name_entry| + lookup[capture_number] = String.new(name_entry.to_unsafe + 2) + end + + lookup + end + + # :nodoc: + def each_capture_group + name_table = uninitialized UInt8* + pattern_info(LibPCRE2::INFO_NAMETABLE, pointerof(name_table)) + + name_entry_size = pattern_info(LibPCRE2::INFO_NAMEENTRYSIZE) + + name_count = pattern_info(LibPCRE2::INFO_NAMECOUNT) + name_count.times do + capture_number = (name_table[0] << 8) | name_table[1] + + yield capture_number, Slice.new(name_table, name_entry_size) + + name_table += name_entry_size + end + end + + private def capture_count_impl + pattern_info(LibPCRE2::INFO_CAPTURECOUNT).to_i32 + end + + private def match_impl(str, byte_index, options) + match_data = match_data(str, byte_index, options) || return + + ovector = LibPCRE2.get_ovector_pointer(match_data) + ovector_count = LibPCRE2.get_ovector_count(match_data) + LibPCRE2.match_data_free(match_data) + + ::Regex::MatchData.new(self, @re, str, byte_index, ovector, ovector_count.to_i32 - 1) + end + + private def matches_impl(str, byte_index, options) + if match_data = match_data(str, byte_index, options) + LibPCRE2.match_data_free(match_data) + true + else + false + end + end + + private def match_data(str, byte_index, options) + match_data = LibPCRE2.match_data_create_from_pattern(@re, nil) + match_count = LibPCRE2.match(@re, str, str.bytesize, byte_index, pcre2_options(options) | LibPCRE2::NO_UTF_CHECK, match_data, nil) + + if match_count < 0 + LibPCRE2.match_data_free(match_data) + case match_count + when LibPCRE2::ERROR_NOMATCH + return + else + raise "error!" + end + end + + match_data + end + + module MatchData + # :nodoc: + def initialize(@regex : Regex, @code : LibPCRE2::Code*, @string : String, @pos : Int32, @ovector : UInt64*, @group_size : Int32) + end + + private def byte_range(n, &) + n += size if n < 0 + range = Range.new(@ovector[n * 2].to_i32!, @ovector[n * 2 + 1].to_i32!, exclusive: true) + if range.begin < 0 || range.end < 0 + yield n + else + range + end + end + + private def fetch_impl(group_name : String) + selected_range = nil + exists = false + @regex.each_capture_group do |number, name_entry| + if name_entry[2, group_name.bytesize] == group_name.to_slice + exists = true + range = byte_range(number) { nil } + if (range && selected_range && range.begin > selected_range.begin) || !selected_range + selected_range = range + end + end + end + + if selected_range + @string.byte_slice(selected_range.begin, selected_range.end - selected_range.begin) + else + yield exists + end + end + end +end From f1fb7aad77b6839aecd3cc11164ca1c3fda1654f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Fri, 16 Dec 2022 15:16:42 +0100 Subject: [PATCH 2/9] Fix OOB in Regex::Engine:PCRE2 capture group look --- spec/std/regex/match_data_spec.cr | 3 +++ src/regex/pcre2.cr | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/spec/std/regex/match_data_spec.cr b/spec/std/regex/match_data_spec.cr index 7fc9c3219faa..70b11b1104a8 100644 --- a/spec/std/regex/match_data_spec.cr +++ b/spec/std/regex/match_data_spec.cr @@ -216,6 +216,8 @@ describe "Regex::MatchData" do it "raises exception when named group doesn't exist" do md = matchdata(/foo/, "foo") expect_raises(KeyError, "Capture group 'group' does not exist") { md["group"] } + + expect_raises(KeyError, "Capture group 'groupwithlongname' does not exist") { md["groupwithlongname"] } end it "captures empty group" do @@ -302,6 +304,7 @@ describe "Regex::MatchData" do it "returns nil exception when named group doesn't exist" do md = matchdata(/foo/, "foo") md["group"]?.should be_nil + md["groupwithlongname"]?.should be_nil end it "capture empty group" do diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index 3eea20280268..5cd0e6771d3c 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -157,7 +157,7 @@ module Regex::PCRE2 selected_range = nil exists = false @regex.each_capture_group do |number, name_entry| - if name_entry[2, group_name.bytesize] == group_name.to_slice + if name_entry[2, group_name.bytesize]? == group_name.to_slice exists = true range = byte_range(number) { nil } if (range && selected_range && range.begin > selected_range.begin) || !selected_range From c4ebd4c424478fcb254449d58e778f44c7b7cb74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 17 Dec 2022 14:27:14 +0100 Subject: [PATCH 3/9] Allocate matchdata via GC --- src/regex/lib_pcre2.cr | 7 ++++++- src/regex/pcre2.cr | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/regex/lib_pcre2.cr b/src/regex/lib_pcre2.cr index 922c492b7e1a..ea5226a4817c 100644 --- a/src/regex/lib_pcre2.cr +++ b/src/regex/lib_pcre2.cr @@ -70,6 +70,7 @@ lib LibPCRE2 type Code = Void* type CompileContext = Void* type MatchData = Void* + type GeneralContext = Void* fun get_error_message = pcre2_get_error_message_8(errorcode : Int, buffer : UInt8*, bufflen : LibC::SizeT) : Int @@ -79,11 +80,15 @@ lib LibPCRE2 fun pattern_info = pcre2_pattern_info_8(code : Code*, what : UInt32, where : Void*) : Int fun match = pcre2_match_8(code : Code*, subject : UInt8*, length : LibC::SizeT, startoffset : LibC::SizeT, options : UInt32, match_data : MatchData*, mcontext : Void*) : Int - fun match_data_create_from_pattern = pcre2_match_data_create_from_pattern_8(code : Code*, gcontext : Void*) : MatchData* + fun match_data_create_from_pattern = pcre2_match_data_create_from_pattern_8(code : Code*, gcontext : GeneralContext) : MatchData* fun match_data_free = pcre2_match_data_free_8(match_data : MatchData*) : Void fun substring_nametable_scan = pcre2_substring_nametable_scan_8(code : Code*, name : UInt8*, first : UInt8*, last : UInt8*) : Int fun get_ovector_pointer = pcre2_get_ovector_pointer_8(match_data : MatchData*) : LibC::SizeT* fun get_ovector_count = pcre2_get_ovector_count_8(match_data : MatchData*) : UInt32 + + # void *private_malloc(Int, void *); + # void private_free(void *, void *); + fun general_context_create = pcre2_general_context_create_8(private_malloc : Void*, private_free : Void*, memory_data : Void*) : GeneralContext end diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index 5cd0e6771d3c..613ff7e988fa 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -107,26 +107,27 @@ module Regex::PCRE2 ovector = LibPCRE2.get_ovector_pointer(match_data) ovector_count = LibPCRE2.get_ovector_count(match_data) - LibPCRE2.match_data_free(match_data) ::Regex::MatchData.new(self, @re, str, byte_index, ovector, ovector_count.to_i32 - 1) end private def matches_impl(str, byte_index, options) if match_data = match_data(str, byte_index, options) - LibPCRE2.match_data_free(match_data) true else false end end + class_getter general_context do + LibPCRE2.general_context_create(->(size : LibC::Int, data : Void*) { GC.malloc(size) }.pointer, ->(pointer : Void*, data : Void*) { GC.free(pointer) }.pointer, nil) + end + private def match_data(str, byte_index, options) - match_data = LibPCRE2.match_data_create_from_pattern(@re, nil) + match_data = LibPCRE2.match_data_create_from_pattern(@re, Regex::PCRE2.general_context) match_count = LibPCRE2.match(@re, str, str.bytesize, byte_index, pcre2_options(options) | LibPCRE2::NO_UTF_CHECK, match_data, nil) if match_count < 0 - LibPCRE2.match_data_free(match_data) case match_count when LibPCRE2::ERROR_NOMATCH return From ac92ddf27d3255e8db5a31396bb74ae7993b5ced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 17 Dec 2022 14:41:41 +0100 Subject: [PATCH 4/9] [CI] Add workflow for testing regex engine implementations --- .github/workflows/regex-engine.yml | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/regex-engine.yml diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml new file mode 100644 index 000000000000..06d51fda6c93 --- /dev/null +++ b/.github/workflows/regex-engine.yml @@ -0,0 +1,31 @@ +name: Regex Engine CI + +on: [push, pull_request] + +jobs: + pcre: + runs-on: ubuntu-latest + name: "PCRE" + container: crystallang/crystal:1.6.2-alpine + steps: + - name: Download Crystal source + uses: actions/checkout@v3 + - name: Assert using PCRE + run: bin/crystal eval 'abort unless Regex::Engine == Regex::PCRE' + - name: Run Regex specs + run: bin/crystal spec --order=random spec/std/regex* + pcre2: + runs-on: ubuntu-latest + name: "PCRE2" + container: crystallang/crystal:1.6.2-alpine + steps: + - name: Download Crystal source + uses: actions/checkout@v3 + - name: Install PCRE2 + run: apk add pcre2-dev + - name: Assert using PCRE2 + run: bin/crystal eval 'abort unless Regex::Engine == Regex::PCRE2' + - name: Assert select PCRE + run: bin/crystal eval -Duse_pcre 'abort unless Regex::Engine == Regex::PCRE' + - name: Run Regex specs + run: bin/crystal spec --order=random spec/std/regex* From 6c9d1b10c91857ba8314e186d512615f94667ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 17 Dec 2022 17:44:13 +0100 Subject: [PATCH 5/9] Improve error reporting for match error --- src/regex/lib_pcre2.cr | 87 +++++++++++++++++++++++++++++++++++++++++- src/regex/pcre2.cr | 6 +-- 2 files changed, 89 insertions(+), 4 deletions(-) diff --git a/src/regex/lib_pcre2.cr b/src/regex/lib_pcre2.cr index ea5226a4817c..22dcb9238787 100644 --- a/src/regex/lib_pcre2.cr +++ b/src/regex/lib_pcre2.cr @@ -36,7 +36,92 @@ lib LibPCRE2 LITERAL = 0x02000000 MATCH_INVALID_UTF = 0x04000000 - ERROR_NOMATCH = -1 + enum Error + # "Expected" matching error codes: no match and partial match. + + NOMATCH = -1 + PARTIAL = -2 + + # Error codes for UTF-8 validity checks + + UTF8_ERR1 = -3 + UTF8_ERR2 = -4 + UTF8_ERR3 = -5 + UTF8_ERR4 = -6 + UTF8_ERR5 = -7 + UTF8_ERR6 = -8 + UTF8_ERR7 = -9 + UTF8_ERR8 = -10 + UTF8_ERR9 = -11 + UTF8_ERR10 = -12 + UTF8_ERR11 = -13 + UTF8_ERR12 = -14 + UTF8_ERR13 = -15 + UTF8_ERR14 = -16 + UTF8_ERR15 = -17 + UTF8_ERR16 = -18 + UTF8_ERR17 = -19 + UTF8_ERR18 = -20 + UTF8_ERR19 = -21 + UTF8_ERR20 = -22 + UTF8_ERR21 = -23 + + # Error codes for UTF-16 validity checks + + UTF16_ERR1 = -24 + UTF16_ERR2 = -25 + UTF16_ERR3 = -26 + + # Error codes for UTF-32 validity checks + + UTF32_ERR1 = -27 + UTF32_ERR2 = -28 + + # Miscellaneous error codes for pcre2[_dfa]_match(), substring extraction + # functions, context functions, and serializing functions. They are in numerical + # order. Originally they were in alphabetical order too, but now that PCRE2 is + # released, the numbers must not be changed. + + BADDATA = -29 + MIXEDTABLES = -30 # Name was changed + BADMAGIC = -31 + BADMODE = -32 + BADOFFSET = -33 + BADOPTION = -34 + BADREPLACEMENT = -35 + BADUTFOFFSET = -36 + CALLOUT = -37 # Never used by PCRE2 itself + DFA_BADRESTART = -38 + DFA_RECURSE = -39 + DFA_UCOND = -40 + DFA_UFUNC = -41 + DFA_UITEM = -42 + DFA_WSSIZE = -43 + INTERNAL = -44 + JIT_BADOPTION = -45 + JIT_STACKLIMIT = -46 + MATCHLIMIT = -47 + NOMEMORY = -48 + NOSUBSTRING = -49 + NOUNIQUESUBSTRING = -50 + NULL = -51 + RECURSELOOP = -52 + DEPTHLIMIT = -53 + RECURSIONLIMIT = -53 # Obsolete synonym + UNAVAILABLE = -54 + UNSET = -55 + BADOFFSETLIMIT = -56 + BADREPESCAPE = -57 + REPMISSINGBRACE = -58 + BADSUBSTITUTION = -59 + BADSUBSPATTERN = -60 + TOOMANYREPLACE = -61 + BADSERIALIZEDDATA = -62 + HEAPLIMIT = -63 + CONVERT_SYNTAX = -64 + INTERNAL_DUPMATCH = -65 + DFA_UINVALID_UTF = -66 + end INFO_ALLOPTIONS = 0 INFO_ARGOPTIONS = 1 diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index 613ff7e988fa..ba7d67265d4c 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -128,11 +128,11 @@ module Regex::PCRE2 match_count = LibPCRE2.match(@re, str, str.bytesize, byte_index, pcre2_options(options) | LibPCRE2::NO_UTF_CHECK, match_data, nil) if match_count < 0 - case match_count - when LibPCRE2::ERROR_NOMATCH + case error = LibPCRE2::Error.new(match_count) + when .nomatch? return else - raise "error!" + raise Exception.new("Regex match error: #{error}") end end From 33f64792513bdeb56992530a13343a1631a5e23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 17 Dec 2022 17:44:36 +0100 Subject: [PATCH 6/9] Skip larg single line string spec when depth limit is insufficient --- spec/std/regex_spec.cr | 4 ++++ src/regex/lib_pcre2.cr | 21 +++++++++++++++++++++ src/regex/pcre2.cr | 6 ++++++ 3 files changed, 31 insertions(+) diff --git a/spec/std/regex_spec.cr b/spec/std/regex_spec.cr index 5935e85060fd..d3830124aa79 100644 --- a/spec/std/regex_spec.cr +++ b/spec/std/regex_spec.cr @@ -204,6 +204,10 @@ describe "Regex" do {% if Regex::Engine.resolve.name == "Regex::PCRE" %} LibPCRE.config LibPCRE::CONFIG_JIT, out jit_enabled pending! "PCRE JIT mode not available." unless 1 == jit_enabled + {% else %} + # This spec requires a fairly large depth limit. Some package builds + # have a more restrictive value which would make this test fail. + pending! "PCRE2 depth limit too low" unless Regex::PCRE2.config(LibPCRE2::CONFIG_DEPTHLIMIT, UInt32) > 8192 {% end %} str = File.read(datapath("large_single_line_string.txt")) diff --git a/src/regex/lib_pcre2.cr b/src/regex/lib_pcre2.cr index 22dcb9238787..32e3c9afa767 100644 --- a/src/regex/lib_pcre2.cr +++ b/src/regex/lib_pcre2.cr @@ -152,6 +152,26 @@ lib LibPCRE2 INFO_HEAPLIMIT = 25 INFO_EXTRAOPTIONS = 26 + # Request types for pcre2_config(). + + CONFIG_BSR = 0 + CONFIG_JIT = 1 + CONFIG_JITTARGET = 2 + CONFIG_LINKSIZE = 3 + CONFIG_MATCHLIMIT = 4 + CONFIG_NEWLINE = 5 + CONFIG_PARENSLIMIT = 6 + CONFIG_DEPTHLIMIT = 7 + CONFIG_RECURSIONLIMIT = 7 # Obsolete synonym + CONFIG_STACKRECURSE = 8 # Obsolete + CONFIG_UNICODE = 9 + CONFIG_UNICODE_VERSION = 10 + CONFIG_VERSION = 11 + CONFIG_HEAPLIMIT = 12 + CONFIG_NEVER_BACKSLASH_C = 13 + CONFIG_COMPILED_WIDTHS = 14 + CONFIG_TABLES_LENGTH = 15 + type Code = Void* type CompileContext = Void* type MatchData = Void* @@ -176,4 +196,5 @@ lib LibPCRE2 # void *private_malloc(Int, void *); # void private_free(void *, void *); fun general_context_create = pcre2_general_context_create_8(private_malloc : Void*, private_free : Void*, memory_data : Void*) : GeneralContext + fun config = pcre2_config_8(what : UInt32, where : Void*) : Int end diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index ba7d67265d4c..dffd2369ba11 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -139,6 +139,12 @@ module Regex::PCRE2 match_data end + def self.config(what, type : T.class) : T forall T + value = uninitialized T + LibPCRE2.config(what, pointerof(value)) + value + end + module MatchData # :nodoc: def initialize(@regex : Regex, @code : LibPCRE2::Code*, @string : String, @pos : Int32, @ovector : UInt64*, @group_size : Int32) From 979f01c397b992a2e09b94c7922ef4d4f81b1222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 19 Dec 2022 23:04:52 +0100 Subject: [PATCH 7/9] Make PCRE2 opt-in for now --- .github/workflows/regex-engine.yml | 4 ++-- src/regex/engine.cr | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/regex-engine.yml b/.github/workflows/regex-engine.yml index 06d51fda6c93..6bdd09a6b533 100644 --- a/.github/workflows/regex-engine.yml +++ b/.github/workflows/regex-engine.yml @@ -24,8 +24,8 @@ jobs: - name: Install PCRE2 run: apk add pcre2-dev - name: Assert using PCRE2 - run: bin/crystal eval 'abort unless Regex::Engine == Regex::PCRE2' + run: bin/crystal eval -Duse_pcre2 'abort unless Regex::Engine == Regex::PCRE2' - name: Assert select PCRE run: bin/crystal eval -Duse_pcre 'abort unless Regex::Engine == Regex::PCRE' - name: Run Regex specs - run: bin/crystal spec --order=random spec/std/regex* + run: bin/crystal spec -Duse_pcre2 --order=random spec/std/regex* diff --git a/src/regex/engine.cr b/src/regex/engine.cr index 766917f87dd2..c2a336accd0d 100644 --- a/src/regex/engine.cr +++ b/src/regex/engine.cr @@ -1,4 +1,4 @@ -{% if flag?(:use_pcre2) || (!flag?(:use_pcre) && !flag?(:win32) && `hash pkg-config 2> /dev/null && pkg-config --silence-errors --modversion libpcre2-8 || printf %s false` != "false") %} +{% if flag?(:use_pcre2) %} require "./pcre2" # :nodoc: From 626ad017da2ac6f5fa7712f5a8054744ee12cb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Thu, 1 Dec 2022 01:41:38 +0100 Subject: [PATCH 8/9] Implement PCRE2 JIT compilation --- src/regex/lib_pcre2.cr | 16 +++++++++++++++- src/regex/pcre2.cr | 31 ++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/regex/lib_pcre2.cr b/src/regex/lib_pcre2.cr index 32e3c9afa767..0df60a05499b 100644 --- a/src/regex/lib_pcre2.cr +++ b/src/regex/lib_pcre2.cr @@ -182,9 +182,23 @@ lib LibPCRE2 fun compile = pcre2_compile_8(pattern : UInt8*, length : LibC::SizeT, options : UInt32, errorcode : LibC::SizeT*, erroroffset : Int*, ccontext : CompileContext*) : Code* fun code_free = pcre2_code_free_8(code : Code*) : Void + type MatchContext = Void* + fun match_context_create = pcre2_match_context_create_8(gcontext : Void*) : MatchContext + + JIT_COMPLETE = 0x00000001_u32 # For full matching + JIT_PARTIAL_SOFT = 0x00000002_u32 + JIT_PARTIAL_HARD = 0x00000004_u32 + JIT_INVALID_UTF = 0x00000100_u32 + fun jit_compile = pcre2_jit_compile_8(code : Code*, options : UInt32) : Int + + type JITStack = Void* + + fun jit_stack_create = pcre2_jit_stack_create_8(startsize : LibC::SizeT, maxsize : LibC::SizeT, gcontext : GeneralContext) : JITStack + fun jit_stack_assign = pcre2_jit_stack_assign_8(mcontext : MatchContext, callable_function : Void*, callable_data : Void*) : Void + fun pattern_info = pcre2_pattern_info_8(code : Code*, what : UInt32, where : Void*) : Int - fun match = pcre2_match_8(code : Code*, subject : UInt8*, length : LibC::SizeT, startoffset : LibC::SizeT, options : UInt32, match_data : MatchData*, mcontext : Void*) : Int + fun match = pcre2_match_8(code : Code*, subject : UInt8*, length : LibC::SizeT, startoffset : LibC::SizeT, options : UInt32, match_data : MatchData*, mcontext : MatchContext) : Int fun match_data_create_from_pattern = pcre2_match_data_create_from_pattern_8(code : Code*, gcontext : GeneralContext) : MatchData* fun match_data_free = pcre2_match_data_free_8(match_data : MatchData*) : Void diff --git a/src/regex/pcre2.cr b/src/regex/pcre2.cr index dffd2369ba11..176a479bbb57 100644 --- a/src/regex/pcre2.cr +++ b/src/regex/pcre2.cr @@ -9,6 +9,20 @@ module Regex::PCRE2 @re = PCRE2.compile(source, pcre2_options(options) | LibPCRE2::UTF | LibPCRE2::NO_UTF_CHECK | LibPCRE2::DUPNAMES | LibPCRE2::UCP) do |error_message| raise ArgumentError.new(error_message) end + + jit_compile + end + + private def jit_compile : Nil + ret = LibPCRE2.jit_compile(@re, LibPCRE2::JIT_COMPLETE) + if ret < 0 + case error = LibPCRE2::Error.new(ret) + when .jit_badoption? + # okay + else + raise ArgumentError.new("Regex JIT compile error: #{error}") + end + end end protected def self.compile(source, options) @@ -123,9 +137,24 @@ module Regex::PCRE2 LibPCRE2.general_context_create(->(size : LibC::Int, data : Void*) { GC.malloc(size) }.pointer, ->(pointer : Void*, data : Void*) { GC.free(pointer) }.pointer, nil) end + # Returns a JIT stack that's shared in the current thread. + # + # Only a single `match` function can run per thread at any given time, so there + # can't be any concurrent access to the JIT stack. + @[ThreadLocal] + class_getter jit_stack : LibPCRE2::JITStack do + jit_stack = LibPCRE2.jit_stack_create(32_768, 1_048_576, Regex::PCRE2.general_context) + if jit_stack.null? + raise "Error allocating JIT stack" + end + jit_stack + end + private def match_data(str, byte_index, options) match_data = LibPCRE2.match_data_create_from_pattern(@re, Regex::PCRE2.general_context) - match_count = LibPCRE2.match(@re, str, str.bytesize, byte_index, pcre2_options(options) | LibPCRE2::NO_UTF_CHECK, match_data, nil) + match_context = LibPCRE2.match_context_create(nil) + LibPCRE2.jit_stack_assign(match_context, nil, Regex::PCRE2.jit_stack.as(Void*)) + match_count = LibPCRE2.match(@re, str, str.bytesize, byte_index, pcre2_options(options) | LibPCRE2::NO_UTF_CHECK, match_data, match_context) if match_count < 0 case error = LibPCRE2::Error.new(match_count) From 5dd4ddf635109b5b4d7541deb7569023539ef761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Mon, 19 Dec 2022 22:37:57 +0100 Subject: [PATCH 9/9] Ensure depth limit in large string spec --- spec/std/regex_spec.cr | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/spec/std/regex_spec.cr b/spec/std/regex_spec.cr index d3830124aa79..72f62a3463cd 100644 --- a/spec/std/regex_spec.cr +++ b/spec/std/regex_spec.cr @@ -201,17 +201,18 @@ describe "Regex" do end it "doesn't crash with a large single line string" do + str = File.read(datapath("large_single_line_string.txt")) + {% if Regex::Engine.resolve.name == "Regex::PCRE" %} LibPCRE.config LibPCRE::CONFIG_JIT, out jit_enabled pending! "PCRE JIT mode not available." unless 1 == jit_enabled + + str.matches?(/^(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?$/) {% else %} - # This spec requires a fairly large depth limit. Some package builds - # have a more restrictive value which would make this test fail. - pending! "PCRE2 depth limit too low" unless Regex::PCRE2.config(LibPCRE2::CONFIG_DEPTHLIMIT, UInt32) > 8192 + # Can't use regex literal because the *LIMIT_DEPTH verb is not supported in libpcre (only libpcre2) + # and thus the compiler doesn't recognize it. + str.matches?(Regex.new("(*LIMIT_DEPTH=8192)^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$")) {% end %} - - str = File.read(datapath("large_single_line_string.txt")) - str.matches?(/^(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?$/) # We don't care whether this actually matches or not, it's just to make # sure the engine does not stack overflow with a large string. end