feat: enhance KQL parser with escaped character support and performance tests

.github/dependabot.yml

@@ -0,0 +1,25 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10
+    labels:
+      - "dependencies"
+      - "go"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10
+    labels:
+      - "dependencies"
+      - "github-actions"
+    commit-message:
+      prefix: "chore"
+      include: "scope"

.gitignore

@@ -2,3 +2,4 @@
 .vscode
 *.iml
 __debug_bin*
+.DS_Store

.golangci.yaml

@@ -19,6 +19,7 @@ output:
 linters:
   disable:
     - godot
+    - gci
   presets:
     - bugs
     - comment
@@ -74,6 +75,7 @@ issues:
         - lll
         - gocognit
         - maintidx
+        - dupl
       test: "cognitive complexity 55 of func `Test_parseMatchExpr` is high (> 30)"
 
     - path: parser/token.go

README.md

@@ -1,37 +1,142 @@
-# KQL(kibana query language) Parser
-![GitHub CI](https://github.com/laojianzi/kql-go/actions/workflows/ci.yaml/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/laojianzi/kql-go)](https://goreportcard.com/report/github.com/laojianzi/kql-go) [![LICENSE](https://img.shields.io/github/license/laojianzi/kql-go.svg)](https://github.com/laojianzi/kql-go/blob/master/LICENSE) [![GoDoc](https://img.shields.io/badge/Godoc-reference-blue.svg)](https://pkg.go.dev/github.com/laojianzi/kql-go) [![DeepSource](https://app.deepsource.com/gh/laojianzi/kql-go.svg/?label=code+coverage&show_trend=false&token=BgPgeWYICSssJGgLh2UosQw7)](https://app.deepsource.com/gh/laojianzi/kql-go/)
+# KQL (Kibana Query Language) Parser
 
-The goal of this project is to build a KQL(kibana query language) parser in Go with the following key features:
+![GitHub CI](https://github.com/laojianzi/kql-go/actions/workflows/ci.yaml/badge.svg)
+[![Go Report Card](https://goreportcard.com/badge/github.com/laojianzi/kql-go)](https://goreportcard.com/report/github.com/laojianzi/kql-go)
+[![LICENSE](https://img.shields.io/github/license/laojianzi/kql-go.svg)](https://github.com/laojianzi/kql-go/blob/master/LICENSE)
+[![GoDoc](https://img.shields.io/badge/Godoc-reference-blue.svg)](https://pkg.go.dev/github.com/laojianzi/kql-go)
+[![DeepSource](https://app.deepsource.com/gh/laojianzi/kql-go.svg/?label=code+coverage&show_trend=false&token=BgPgeWYICSssJGgLh2UosQw7)](https://app.deepsource.com/gh/laojianzi/kql-go/)
 
-- Parse KQL(kibana query language) query into AST
-- output AST to KQL(kibana query language) query
+A Kibana Query Language (KQL) parser implemented in Go.
 
-This project is inspired by [github.com/AfterShip/clickhouse-sql-parser] and [https://github.com/cloudspannerecosystem/memefish]. Both of these are SQL parsers implemented in Go.
+## Features
 
-## How to use
+- Escaped character handling
+- Wildcard patterns
+- Parentheses grouping
+- AND/OR/NOT operators
+- Field:value pairs
+- String literals with quotes
 
-Playground: https://go.dev/play/p/m36hkz43PQL
+## Installation
 
-```Go
+```bash
+go get github.com/laojianzi/kql-go
+```
+
+## Quick Start
+
+```go
 package main
 
 import (
+    "fmt"
+
     "github.com/laojianzi/kql-go/parser"
 )
 
-query := `(service_name: "redis" OR service_name: "mysql") AND level: ("error" OR "warn") and start_time > 1723286863 anD latency >= 1.5`
-// Parse query into AST
-stmt, err := parser.New(query).Stmt()
-if err != nil {
-    panic(err)
+func main() {
+    query := `(service_name: "redis" OR service_name: "mysql") AND level: ("error" OR "warn") and start_time > 1723286863 anD latency >= 1.5`
+    // Parse query into AST
+    stmt, err := parser.New(query).Stmt()
+    if err != nil {
+        panic(err)
+    }
+
+    // output AST to KQL(kibana query language) query
+    fmt.Println(stmt.String())
+    // output:
+    // (service_name: "redis" OR service_name: "mysql") AND level: ("error" OR "warn") AND start_time > 1723286863 AND latency >= 1.5
 }
+```
+
+## Performance
+
+Recent benchmark results:
 
-// output AST to KQL(kibana query language) query
-fmt.Println(stmt.String())
-// output:
-// (service_name: "redis" OR service_name: "mysql") AND level: ("error" OR "warn") AND start_time > 1723286863 AND latency >= 1.5
 ```
+goos: darwin
+goarch: amd64
+cpu: Intel(R) Core(TM) i5-10500 CPU @ 3.10GHz
+
+BenchmarkParser/simple_field-12                           459882              2500 ns/op            1280 B/op         34 allocs/op
+BenchmarkParser/numeric_comparison-12                     728577              1646 ns/op             688 B/op         19 allocs/op
+BenchmarkParser/multiple_conditions-12                    211783              5966 ns/op            2385 B/op         62 allocs/op
+BenchmarkParser/complex_query-12                           63580             18675 ns/op            7235 B/op        168 allocs/op
+BenchmarkParser/escaped_chars-12                          108622             10926 ns/op            5416 B/op        131 allocs/op
+BenchmarkParser/many_conditions-12                         35870             34985 ns/op           12454 B/op        257 allocs/op
+BenchmarkParserParallel/simple_field-12                  1582999             773.8 ns/op            1280 B/op         34 allocs/op
+BenchmarkParserParallel/numeric_comparison-12            2465758             468.9 ns/op             688 B/op         19 allocs/op
+BenchmarkParserParallel/multiple_conditions-12            743210              1661 ns/op            2386 B/op         62 allocs/op
+BenchmarkParserParallel/complex_query-12                  219790              5692 ns/op            7238 B/op        168 allocs/op
+BenchmarkParserParallel/escaped_chars-12                  331581              3735 ns/op            5416 B/op        131 allocs/op
+BenchmarkParserParallel/many_conditions-12                125736              9812 ns/op           12459 B/op        257 allocs/op
+BenchmarkLexer/simple_field-12                            572068              1947 ns/op             832 B/op         25 allocs/op
+BenchmarkLexer/numeric_comparison-12                     1000000              1082 ns/op             264 B/op         11 allocs/op
+BenchmarkLexer/multiple_conditions-12                     278456              4342 ns/op            1360 B/op         42 allocs/op
+BenchmarkLexer/complex_query-12                            77738             16504 ns/op            4768 B/op        119 allocs/op
+BenchmarkLexer/escaped_chars-12                           129708              8450 ns/op            3768 B/op         96 allocs/op
+BenchmarkLexer/many_conditions-12                          39974             29785 ns/op            8944 B/op        192 allocs/op
+BenchmarkEscapeSequence/no_escape-12                      581481              2017 ns/op             720 B/op         26 allocs/op
+BenchmarkEscapeSequence/single_escape-12                  487568              2400 ns/op             936 B/op         32 allocs/op
+BenchmarkEscapeSequence/multiple_escapes-12               432496              2645 ns/op            1152 B/op         38 allocs/op
+BenchmarkEscapeSequence/mixed_escapes-12                  129600              9215 ns/op            3672 B/op        100 allocs/op
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+### Development Requirements
+
+- Go 1.16 or higher
+- golangci-lint for code quality checks
+
+### Running Tests
+
+```bash
+# Run unit tests
+go test -v -count=1 -race ./...
+
+# Run benchmarks
+go test -bench=. -benchmem ./...
+```
+
+## Examples
+
+### Basic Queries
+```go
+// Simple field value query
+query := `status: "active"`
+
+// Numeric comparison
+query := `age >= 18`
+
+// Multiple conditions
+query := `status: "active" AND age >= 18`
+```
+
+### Advanced Queries
+```go
+// Complex grouping with wildcards
+query := `(status: "active" OR status: "pending") AND name: "john*"`
+
+// Escaped characters
+query := `message: "Hello \"World\"" AND path: "C:\\Program Files\\*"`
+
+// Multiple conditions with various operators
+query := `status: "active" AND age >= 18 AND name: "john*" AND city: "New York"`
+```
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Acknowledgments
+
+This project is inspired by:
+- [github.com/AfterShip/clickhouse-sql-parser](https://github.com/AfterShip/clickhouse-sql-parser)
+- [github.com/cloudspannerecosystem/memefish](https://github.com/cloudspannerecosystem/memefish)
 
-## Contact us
+## References
 
-Feel free to open an issue or discussion if you have any issues or questions.
+- [Kibana Query Language Documentation](https://www.elastic.co/guide/en/kibana/current/kuery-query.html)

ast/binary_test.go

@@ -3,9 +3,10 @@ package ast_test
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/laojianzi/kql-go/ast"
 	"github.com/laojianzi/kql-go/token"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestBinaryExpr(t *testing.T) {
@@ -28,7 +29,7 @@ func TestBinaryExpr(t *testing.T) {
 			name: `"v1"`,
 			args: args{
 				pos:    0,
-				value:  ast.NewLiteral(0, 4, token.TokenKindString, "v1"),
+				value:  ast.NewLiteral(0, 4, token.TokenKindString, "v1", nil),
 				hasNot: false,
 			},
 			wantEnd:    4,
@@ -38,7 +39,7 @@ func TestBinaryExpr(t *testing.T) {
 			name: `NOT "v1"`,
 			args: args{
 				pos:    0,
-				value:  ast.NewLiteral(4, 8, token.TokenKindString, "v1"),
+				value:  ast.NewLiteral(4, 8, token.TokenKindString, "v1", nil),
 				hasNot: true,
 			},
 			wantEnd:    8,
@@ -49,7 +50,7 @@ func TestBinaryExpr(t *testing.T) {
 			args: args{
 				field:    "f1",
 				operator: token.TokenKindOperatorEql,
-				value:    ast.NewLiteral(4, 8, token.TokenKindString, "v1"),
+				value:    ast.NewLiteral(4, 8, token.TokenKindString, "v1", nil),
 				hasNot:   false,
 			},
 			wantEnd:    8,
@@ -61,7 +62,7 @@ func TestBinaryExpr(t *testing.T) {
 				pos:      0,
 				field:    "f1",
 				operator: token.TokenKindOperatorEql,
-				value:    ast.NewLiteral(8, 12, token.TokenKindString, "v1"),
+				value:    ast.NewLiteral(8, 12, token.TokenKindString, "v1", nil),
 				hasNot:   true,
 			},
 			wantEnd:    12,

ast/combine_test.go

@@ -3,9 +3,10 @@ package ast_test
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/laojianzi/kql-go/ast"
 	"github.com/laojianzi/kql-go/token"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestCombineExpr(t *testing.T) {
@@ -25,9 +26,9 @@ func TestCombineExpr(t *testing.T) {
 		{
 			name: `f1: "v1" OR NOT f1: "v2"`,
 			args: args{
-				leftExpr:  ast.NewBinaryExpr(0, "f1", token.TokenKindOperatorEql, ast.NewLiteral(4, 8, token.TokenKindString, "v1"), false),
+				leftExpr:  ast.NewBinaryExpr(0, "f1", token.TokenKindOperatorEql, ast.NewLiteral(4, 8, token.TokenKindString, "v1", nil), false),
 				keyword:   token.TokenKindKeywordOr,
-				rightExpr: ast.NewBinaryExpr(12, "f1", token.TokenKindOperatorEql, ast.NewLiteral(20, 24, token.TokenKindString, "v2"), true),
+				rightExpr: ast.NewBinaryExpr(12, "f1", token.TokenKindOperatorEql, ast.NewLiteral(20, 24, token.TokenKindString, "v2", nil), true),
 			},
 			wantEnd:    24,
 			wantString: `f1: "v1" OR NOT f1: "v2"`,
@@ -43,15 +44,15 @@ func TestCombineExpr(t *testing.T) {
 						8,
 						22,
 						ast.NewCombineExpr(
-							ast.NewBinaryExpr(9, "", 0, ast.NewLiteral(9, 13, token.TokenKindString, "v1"), false),
+							ast.NewBinaryExpr(9, "", 0, ast.NewLiteral(9, 13, token.TokenKindString, "v1", nil), false),
 							token.TokenKindKeywordOr,
-							ast.NewBinaryExpr(17, "", 0, ast.NewLiteral(17, 21, token.TokenKindString, "v2"), false),
+							ast.NewBinaryExpr(17, "", 0, ast.NewLiteral(17, 21, token.TokenKindString, "v2", nil), false),
 						),
 					),
 					true,
 				),
 				keyword:   token.TokenKindKeywordAnd,
-				rightExpr: ast.NewBinaryExpr(27, "f3", token.TokenKindOperatorEql, ast.NewLiteral(31, 35, token.TokenKindString, "v3"), false),
+				rightExpr: ast.NewBinaryExpr(27, "f3", token.TokenKindOperatorEql, ast.NewLiteral(31, 35, token.TokenKindString, "v3", nil), false),
 			},
 			wantEnd:    35,
 			wantString: `NOT f1: ("v1" OR "v2") AND f3: "v3"`,

ast/literal.go

@@ -4,21 +4,24 @@ import "github.com/laojianzi/kql-go/token"
 
 // Literal is a literal(int, float, string or identifier) value.
 type Literal struct {
-	pos             int
-	end             int
+	pos           int
+	end           int
+	escapeIndexes []int
+
 	Kind            token.Kind // int, float, string or identifier
 	Value           string
 	WithDoubleQuote bool
 }
 
 // NewLiteral creates a new literal value.
-func NewLiteral(pos, end int, kind token.Kind, value string) *Literal {
+func NewLiteral(pos, end int, kind token.Kind, value string, escapeIndexes []int) *Literal {
 	return &Literal{
 		pos:             pos,
 		end:             end,
 		Kind:            kind,
 		Value:           value,
 		WithDoubleQuote: kind == token.TokenKindString,
+		escapeIndexes:   escapeIndexes,
 	}
 }
 
@@ -34,9 +37,27 @@ func (e *Literal) End() int {
 
 // String returns the string representation of the literal value.
 func (e *Literal) String() string {
+	value := e.Value
+
+	if len(e.escapeIndexes) > 0 {
+		var (
+			runes     = []rune(value)
+			newValue  []rune
+			lastIndex int
+		)
+
+		for _, escapeIndex := range e.escapeIndexes {
+			newValue = append(newValue, runes[lastIndex:escapeIndex]...)
+			newValue = append(newValue, '\\')
+			lastIndex = escapeIndex
+		}
+
+		value = string(append(newValue, runes[lastIndex:]...))
+	}
+
 	if e.WithDoubleQuote {
-		return `"` + e.Value + `"`
+		return `"` + value + `"`
 	}
 
-	return e.Value
+	return value
 }

ast/literal_test.go

@@ -3,9 +3,10 @@ package ast_test
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/laojianzi/kql-go/ast"
 	"github.com/laojianzi/kql-go/token"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestLiteral(t *testing.T) {
@@ -73,7 +74,7 @@ func TestLiteral(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			expr := ast.NewLiteral(c.args.pos, c.args.end, c.args.kind, c.args.value)
+			expr := ast.NewLiteral(c.args.pos, c.args.end, c.args.kind, c.args.value, nil)
 			assert.Equal(t, c.wantPos, expr.Pos())
 			assert.Equal(t, c.wantEnd, expr.End())
 			assert.Equal(t, c.wantString, expr.String())