From 3a5fe10a8df4bb029845a6df7772aa084a18255d Mon Sep 17 00:00:00 2001 From: "zainnadeem(RedOpsCell)" Date: Mon, 15 Jun 2026 11:48:51 +0500 Subject: [PATCH 1/2] fix: track lexer byte offsets for invalid UTF-8 --- lexerStream.go | 30 ++++++++++++++++++++---------- parsing_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/lexerStream.go b/lexerStream.go index 758682b..9dfb749 100644 --- a/lexerStream.go +++ b/lexerStream.go @@ -36,27 +36,37 @@ func newLexerStream(source string) *lexerStream { func (this *lexerStream) readCharacter() rune { character := this.source[this.position] + width := utf8.RuneLen(character) + if character == utf8.RuneError { + _, width = utf8.DecodeRuneInString(this.sourceString[this.strPosition:]) + } this.position += 1 - this.strPosition += utf8.RuneLen(character) + this.strPosition += width return character } func (this *lexerStream) rewind(amount int) { if amount < 0 { - this.position -= amount - this.strPosition -= amount + for i := 0; i > amount; i-- { + character := this.source[this.position] + width := utf8.RuneLen(character) + if character == utf8.RuneError { + _, width = utf8.DecodeRuneInString(this.sourceString[this.strPosition:]) + } + this.position += 1 + this.strPosition += width + } + return } - strAmount := 0 for i := 0; i < amount; i++ { - if this.position >= this.length { - strAmount += 1 - this.position -= 1 - continue + character := this.source[this.position-1] + width := utf8.RuneLen(character) + if character == utf8.RuneError { + _, width = utf8.DecodeLastRuneInString(this.sourceString[:this.strPosition]) } - strAmount += utf8.RuneLen(this.source[this.position-1]) this.position -= 1 + this.strPosition -= width } - this.strPosition -= strAmount } func (this lexerStream) canRead() bool { diff --git a/parsing_test.go b/parsing_test.go index f78740b..1b1a513 100644 --- a/parsing_test.go +++ b/parsing_test.go @@ -456,6 +456,38 @@ func TestConstantParsing(test *testing.T) { runTokenParsingTest(tokenParsingTests, test) } +func TestUTF8Parsing(test *testing.T) { + + testCases := []struct { + name string + input []byte + }{ + {"Multiple invalid UTF-8 bytes (nuclei#7462)", bytes.Repeat([]byte{0xff}, 7)}, + {"Single invalid UTF-8 byte", []byte{0xff}}, + {"Valid multibyte UTF-8 characters", []byte("界é\uFFFD")}, + {"Truncated UTF-8 sequence", []byte{0xe2, 0x82}}, + } + + for _, testCase := range testCases { + test.Run(testCase.name, func(test *testing.T) { + expressionString := "\"" + string(testCase.input) + "\"+aaaaaaaaaa" + expression, err := NewEvaluableExpressionWithFunctions(expressionString, nil) + if err != nil { + test.Fatalf("Failed to parse expression: %v", err) + } + + expected := []ExpressionToken{ + {Kind: STRING, Value: string(bytes.Runes(testCase.input))}, + {Kind: MODIFIER, Value: "+"}, + {Kind: VARIABLE, Value: "aaaaaaaaaa"}, + } + if tokens := expression.Tokens(); !reflect.DeepEqual(tokens, expected) { + test.Errorf("Expected tokens %#v, found %#v", expected, tokens) + } + }) + } +} + func TestLogicalOperatorParsing(test *testing.T) { tokenParsingTests := []TokenParsingTest{ From f0abd736e5d0dc628bdf19c4e52f1c06089d74e1 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 15 Jun 2026 12:05:33 +0200 Subject: [PATCH 2/2] harden rewind --- lexerStream.go | 8 ++ lexerStream_test.go | 248 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 lexerStream_test.go diff --git a/lexerStream.go b/lexerStream.go index 9dfb749..63d795e 100644 --- a/lexerStream.go +++ b/lexerStream.go @@ -46,8 +46,13 @@ func (this *lexerStream) readCharacter() rune { } func (this *lexerStream) rewind(amount int) { + // A negative amount seeks forward, re-consuming runes. Decode each rune's + // real byte width so the byte cursor stays in sync with invalid UTF-8. if amount < 0 { for i := 0; i > amount; i-- { + if this.position >= this.length { + break + } character := this.source[this.position] width := utf8.RuneLen(character) if character == utf8.RuneError { @@ -59,6 +64,9 @@ func (this *lexerStream) rewind(amount int) { return } for i := 0; i < amount; i++ { + if this.position <= 0 { + break + } character := this.source[this.position-1] width := utf8.RuneLen(character) if character == utf8.RuneError { diff --git a/lexerStream_test.go b/lexerStream_test.go new file mode 100644 index 0000000..e3bb39b --- /dev/null +++ b/lexerStream_test.go @@ -0,0 +1,248 @@ +package govaluate + +import ( + "reflect" + "testing" + "unicode/utf8" +) + +// lexerStreamCases collects inputs that stress byte-vs-rune cursor accounting: +// pure ASCII, valid multibyte, a real (3-byte) U+FFFD, single and multiple +// invalid bytes, a truncated sequence, and mixtures of all of the above. +var lexerStreamCases = []struct { + name string + source string +}{ + {"empty", ""}, + {"ascii", "abc123"}, + {"multibyte", "café界"}, + {"real replacement rune", "a\uFFFDb"}, + {"single invalid byte", "a\xffb"}, + {"multiple invalid bytes", "\xff\xff\xff\xff\xff\xff\xff"}, + {"truncated sequence", "a\xe2\x82"}, + {"mixed valid and invalid", "界\xffé\xff\xffz"}, +} + +// readAll drains the stream via readCharacter and returns the runes read. +func readAll(stream *lexerStream) []rune { + var runes []rune + for stream.canRead() { + runes = append(runes, stream.readCharacter()) + } + return runes +} + +// TestLexerStreamByteCursorNoDrift is the core regression guard for the +// invalid-UTF-8 fix: after reading the entire stream the byte cursor must land +// exactly on the end of the source string. Before the fix, each invalid byte +// advanced strPosition by 3 (utf8.RuneLen(RuneError)) instead of 1, so the +// cursor overshot and later slices panicked. +func TestLexerStreamByteCursorNoDrift(t *testing.T) { + for _, tc := range lexerStreamCases { + t.Run(tc.name, func(t *testing.T) { + stream := newLexerStream(tc.source) + defer stream.close() + + // strPosition must never run past the source while reading, and + // position/strPosition must advance together within bounds. + for stream.canRead() { + if stream.strPosition < 0 || stream.strPosition > len(tc.source) { + t.Fatalf("strPosition %d out of bounds [0,%d]", stream.strPosition, len(tc.source)) + } + _ = stream.readCharacter() + } + + if stream.strPosition != len(tc.source) { + t.Fatalf("after draining: strPosition = %d, want %d (byte cursor drifted)", stream.strPosition, len(tc.source)) + } + if stream.position != stream.length { + t.Fatalf("after draining: position = %d, want %d", stream.position, stream.length) + } + }) + } +} + +// TestLexerStreamReuseSliceMatchesRunes verifies the reuse-string fast path +// (sourceString[start:strPosition]) stays valid and consistent for every +// prefix length, which is exactly the slice that panicked on drift. +func TestLexerStreamReuseSliceMatchesRunes(t *testing.T) { + for _, tc := range lexerStreamCases { + t.Run(tc.name, func(t *testing.T) { + stream := newLexerStream(tc.source) + defer stream.close() + + start := stream.strPosition + for stream.canRead() { + _ = stream.readCharacter() + // Must never index out of range, and the slice must be a real + // substring ending where the cursor says it does. + slice := stream.sourceString[start:stream.strPosition] + if len(slice) > len(tc.source) { + t.Fatalf("reuse slice longer than source: %d > %d", len(slice), len(tc.source)) + } + } + }) + } +} + +// TestLexerStreamRewindRoundTrip ensures rewind(1) exactly undoes a single +// readCharacter for every position, including across invalid UTF-8 bytes. +func TestLexerStreamRewindRoundTrip(t *testing.T) { + for _, tc := range lexerStreamCases { + t.Run(tc.name, func(t *testing.T) { + stream := newLexerStream(tc.source) + defer stream.close() + + for stream.canRead() { + beforePos, beforeStr := stream.position, stream.strPosition + r := stream.readCharacter() + stream.rewind(1) + if stream.position != beforePos || stream.strPosition != beforeStr { + t.Fatalf("rewind(1) after reading %q did not restore cursor: pos %d->%d, str %d->%d", + r, beforePos, stream.position, beforeStr, stream.strPosition) + } + // re-read to make forward progress + _ = stream.readCharacter() + } + }) + } +} + +// TestLexerStreamRewindForwardRoundTrip ensures rewind(-1) (forward seek) is +// the inverse of rewind(1) (backward) for every position. +func TestLexerStreamRewindForwardRoundTrip(t *testing.T) { + for _, tc := range lexerStreamCases { + t.Run(tc.name, func(t *testing.T) { + stream := newLexerStream(tc.source) + defer stream.close() + + for stream.canRead() { + _ = stream.readCharacter() + afterPos, afterStr := stream.position, stream.strPosition + stream.rewind(1) // step back + stream.rewind(-1) // step forward again + if stream.position != afterPos || stream.strPosition != afterStr { + t.Fatalf("rewind(1)+rewind(-1) not identity: pos %d->%d, str %d->%d", + afterPos, stream.position, afterStr, stream.strPosition) + } + } + }) + } +} + +// TestLexerStreamRewindBounds covers the hardening from issue #5: rewind must +// not panic when asked to move past either end, and must leave the cursor at +// the boundary rather than going out of range. +func TestLexerStreamRewindBounds(t *testing.T) { + t.Run("backward past start", func(t *testing.T) { + stream := newLexerStream("ab") + defer stream.close() + // position is already 0; a backward rewind must clamp, not underflow. + stream.rewind(5) + if stream.position != 0 || stream.strPosition != 0 { + t.Fatalf("expected clamp at start, got pos=%d str=%d", stream.position, stream.strPosition) + } + }) + + t.Run("forward past end", func(t *testing.T) { + stream := newLexerStream("ab") + defer stream.close() + readAll(stream) // position == length now + posAtEnd, strAtEnd := stream.position, stream.strPosition + // a forward rewind at EOF must clamp, not read source[length]. + stream.rewind(-5) + if stream.position != posAtEnd || stream.strPosition != strAtEnd { + t.Fatalf("expected clamp at end, got pos=%d str=%d (was %d/%d)", stream.position, stream.strPosition, posAtEnd, strAtEnd) + } + }) + + t.Run("forward past end with invalid utf8", func(t *testing.T) { + stream := newLexerStream("\xff\xff") + defer stream.close() + readAll(stream) + posAtEnd, strAtEnd := stream.position, stream.strPosition + stream.rewind(-3) + if stream.position != posAtEnd || stream.strPosition != strAtEnd { + t.Fatalf("expected clamp at end, got pos=%d str=%d (was %d/%d)", stream.position, stream.strPosition, posAtEnd, strAtEnd) + } + }) +} + +// TestLexerStreamRuneRoundTrip sanity-checks that the runes read back match +// ranging over the source string (Go's canonical invalid-UTF-8 decoding), so +// tokenization semantics are unchanged. +func TestLexerStreamRuneRoundTrip(t *testing.T) { + for _, tc := range lexerStreamCases { + t.Run(tc.name, func(t *testing.T) { + stream := newLexerStream(tc.source) + defer stream.close() + got := readAll(stream) + + var want []rune + for _, r := range tc.source { + want = append(want, r) + } + if len(got) != len(want) { + t.Fatalf("rune count = %d, want %d", len(got), len(want)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("rune[%d] = %U, want %U", i, got[i], want[i]) + } + } + // invalid bytes must surface as RuneError, never corrupt the stream + _ = utf8.RuneError + }) + } +} + +// TestUTF8ExpressionTokenization locks in end-to-end tokenization behavior when +// invalid UTF-8 appears in different token contexts, ensuring the byte-cursor +// fix neither panics nor shifts the tokens that follow the malformed input. +func TestUTF8ExpressionTokenization(t *testing.T) { + cases := []struct { + name string + expression string + expected []ExpressionToken + }{ + { + name: "invalid bytes in string then operator and variable", + expression: "\"\xff\xff\" + foo", + expected: []ExpressionToken{ + {Kind: STRING, Value: "\uFFFD\uFFFD"}, + {Kind: MODIFIER, Value: "+"}, + {Kind: VARIABLE, Value: "foo"}, + }, + }, + { + name: "invalid bytes between two strings", + expression: "\"a\xffb\" == \"c\"", + expected: []ExpressionToken{ + {Kind: STRING, Value: "a\uFFFDb"}, + {Kind: COMPARATOR, Value: "=="}, + {Kind: STRING, Value: "c"}, + }, + }, + { + name: "invalid bytes then numeric", + expression: "\"\xe2\x82\" + 42", + expected: []ExpressionToken{ + {Kind: STRING, Value: "\uFFFD\uFFFD"}, + {Kind: MODIFIER, Value: "+"}, + {Kind: NUMERIC, Value: 42.0}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + expression, err := NewEvaluableExpressionWithFunctions(tc.expression, nil) + if err != nil { + t.Fatalf("failed to parse %q: %v", tc.expression, err) + } + if tokens := expression.Tokens(); !reflect.DeepEqual(tokens, tc.expected) { + t.Errorf("tokens for %q:\n got %#v\nwant %#v", tc.expression, tokens, tc.expected) + } + }) + } +}