[VAULT-21456] Add a drop in replacement for the Go regexp package that interns regular expressions

.github/workflows/go.yml

@@ -26,6 +26,7 @@ jobs:
           - "plugincontainer"
           - "pluginutil"
           - "random"
+          - "regexp"
           - "reloadutil"
           - "strutil"
           - "temperror"

regexp/go.mod

@@ -0,0 +1,11 @@
+module github.com/hashicorp/go-secure-stdlib/regexp
+
+go 1.24
+
+require github.com/stretchr/testify v1.10.0
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

regexp/go.sum

@@ -0,0 +1,10 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

regexp/regexp.go

@@ -0,0 +1,98 @@
+package regexp
+
+import (
+	"regexp"
+	"runtime"
+	"sync"
+	"weak"
+)
+
+// Interns the compilation of Regular Expressions.  If two regexs with the same pattern are compiled, the result
+// is the same *regexp.Regexp.  This avoids the compilation cost but more importantly the memory usage.
+//
+// Regular expressions produced from this package are backed by a form of weak-valued map, upon a regexp becoming
+// unreachable, they will be eventually removed from the map and memory reclaimed.
+
+var (
+	weakMap      = make(map[string]weak.Pointer[regexp.Regexp])
+	posixWeakMap = make(map[string]weak.Pointer[regexp.Regexp])
+	reverseMap   = make(map[weak.Pointer[regexp.Regexp]]string)
+	l            sync.Mutex
+)
+
+// CompileInterned compiles and interns a regular expression and returns a
+// pointer to it or an error.
+func CompileInterned(pattern string) (*regexp.Regexp, error) {
+	return compile(pattern, regexp.Compile, weakMap)
+}
+
+// CompilePOSIXInterned compiles and interns a regular expression using POSIX
+// syntax and returns a pointer to it or an error.
+func CompilePOSIXInterned(pattern string) (*regexp.Regexp, error) {
+	return compile(pattern, regexp.CompilePOSIX, posixWeakMap)
+}
+
+// MustCompileInterned compiles and interns a regular expression and returns a pointer to it or panics.
+func MustCompileInterned(pattern string) *regexp.Regexp {
+	return mustCompile(pattern, regexp.MustCompile, weakMap)
+}
+
+// MustCompilePOSIXInterned compiles and interns a regular expression using
+// POSIX syntax and returns a pointer to it or panics.
+func MustCompilePOSIXInterned(pattern string) *regexp.Regexp {
+	return mustCompile(pattern, regexp.MustCompilePOSIX, posixWeakMap)
+}
+
+// compile handles compiling and interning regular expressions. If the regexp is
+// already interned, a pointer to it is returned from the map. If the regexp is
+// not interned or is nil (since it's a weak pointer), it is compiled and stored
+// in the maps. The regexp is stored in the maps as a weak pointer, so that it
+// can be garbage collected.
+func compile(pattern string, compileFunc func(string) (*regexp.Regexp, error), m map[string]weak.Pointer[regexp.Regexp]) (*regexp.Regexp, error) {
+	l.Lock()
+	defer l.Unlock()
+	if itemPtr, ok := m[pattern]; ok {
+		ptr := itemPtr.Value()
+		if ptr != nil {
+			return ptr, nil
+		}
+		delete(m, pattern)
+		delete(posixWeakMap, pattern)
+		delete(reverseMap, itemPtr)
+	}
+	r, err := compileFunc(pattern)
+	if err != nil {
+		return nil, err
+	}
+	weakPointer := weak.Make(r)
+	m[pattern] = weakPointer
+	reverseMap[weakPointer] = pattern
+	runtime.AddCleanup(r, cleanup, weakPointer)
+	return r, nil
+}
+
+// mustCompile handles compiling and interning regular expressions just like
+// compile, but it will panic instead of returning an error. If the regexp is
+// already interned, a pointer to it is returned from the map. If the regexp is
+// not interned or is nil (since it's a weak pointer), it is compiled and stored
+// in the maps. The regexp is stored in the maps as a weak pointer, so that it
+// can be garbage collected.
+func mustCompile(pattern string, mustCompileFunc func(string) *regexp.Regexp, weakMap map[string]weak.Pointer[regexp.Regexp]) *regexp.Regexp {
+	compileFunc := func(string) (*regexp.Regexp, error) {
+		return mustCompileFunc(pattern), nil
+	}
+	res, _ := compile(pattern, compileFunc, weakMap)
+	return res
+}
+
+// cleanup is a cleanup function for *regexp.Regexp. It removes the entries from the
+// weak maps when the regexp object they point to is garbage collected.
+func cleanup(ptr weak.Pointer[regexp.Regexp]) {
+	l.Lock()
+	defer l.Unlock()
+	if s, ok := reverseMap[ptr]; ok {
+		delete(weakMap, s)
+		delete(posixWeakMap, s)
+		delete(reverseMap, ptr)
+	}
+}

regexp/regexp_test.go

@@ -0,0 +1,125 @@
+package regexp
+
+import (
+	"github.com/stretchr/testify/require"
+	"regexp"
+	"runtime"
+	"strconv"
+	"sync"
+	"testing"
+)
+
+// TestInternedRegexps tests that the regular expressions are compiled correctly,
+// are interned, and that the cleanup of interned regexps works as expected.
+//
+// Since this test depends on the garbage collector, it is not really
+// deterministic and might flake in the future. If that happens, the calls to
+// the garbage collector and the rest of the test should be removed.
+func TestInternedRegexps(t *testing.T) {
+	testCases := map[string]struct {
+		compileFunc     func(string) (*regexp.Regexp, error)
+		mustCompileFunc func(string) *regexp.Regexp
+		mustCompile     bool
+	}{
+		"CompileInterned": {
+			compileFunc:     CompileInterned,
+			mustCompileFunc: nil,
+			mustCompile:     false,
+		},
+		"MustCompileInterned": {
+			compileFunc:     nil,
+			mustCompileFunc: MustCompileInterned,
+			mustCompile:     true,
+		},
+		"CompilePOSIXInterned": {
+			compileFunc:     CompilePOSIXInterned,
+			mustCompileFunc: nil,
+			mustCompile:     false,
+		},
+		"MustCompilePOSIXInterned": {
+			compileFunc:     nil,
+			mustCompileFunc: MustCompilePOSIXInterned,
+			mustCompile:     true,
+		},
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			// Compile two identical regular expressions, their pointers should be the same
+			var r1, r2 *regexp.Regexp
+			var err error
+			if tc.mustCompile {
+				r1 = tc.mustCompileFunc(".*")
+				r2 = tc.mustCompileFunc(".*")
+			} else {
+				r1, err = tc.compileFunc(".*")
+				require.NoError(t, err)
+				r2, err = tc.compileFunc(".*")
+				require.NoError(t, err)
+				require.True(t, r1 == r2)
+
+				// While we're here, check that errors work as expected
+				_, err = tc.compileFunc("(")
+				require.Error(t, err)
+			}
+			require.True(t, r1 == r2)
+			// Remove references to the regexps and run the garbage collector
+			r1 = nil
+			r2 = nil
+
+			// Run the garbage collector twice to increase chances of the cleanup happening.
+			// This still doesn't make it deterministic, but in local testing it was enough
+			// to not flake a single time in over two million runs, so it should be good enough.
+			// A single call to runtime.GC() was flaky very frequently in local testing.
+			runtime.GC()
+			runtime.GC()
+
+			// Ensure that the cleanup happened and the maps used for interning regexp are empty
+			l.Lock()
+			require.Len(t, weakMap, 0)
+			require.Len(t, reverseMap, 0)
+			l.Unlock()
+		})
+	}
+}
+
+func BenchmarkRegexps(b *testing.B) {
+	s := make([]*regexp.Regexp, b.N)
+	for i := 0; i < b.N; i++ {
+		s[i] = regexp.MustCompile(`https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)`)
+	}
+}
+
+func BenchmarkInternedRegexps(b *testing.B) {
+	s := make([]*regexp.Regexp, b.N)
+	for i := 0; i < b.N; i++ {
+		s[i] = MustCompileInterned(`https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)`)
+	}
+}
+
+func BenchmarkConcurrentRegexps(b *testing.B) {
+	var wg sync.WaitGroup
+	for j := 0; j < runtime.NumCPU(); j++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < b.N; i++ {
+				regexp.MustCompile(`https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)` + strconv.Itoa(i) + "-" + strconv.Itoa(j))
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func BenchmarkConcurrentInternedRegexps(b *testing.B) {
+	var wg sync.WaitGroup
+	for j := 0; j < runtime.NumCPU(); j++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < b.N; i++ {
+				MustCompileInterned(`https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)` + strconv.Itoa(i) + "-" + strconv.Itoa(j))
+			}
+		}()
+	}
+	wg.Wait()
+}