feat: enhance KQL parser with escaped character support and performance tests

.github/dependabot.yml

@@ -0,0 +1,25 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10
+    labels:
+      - "dependencies"
+      - "go"
+    commit-message:
+      prefix: "chore"
+      include: "scope"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10
+    labels:
+      - "dependencies"
+      - "github-actions"
+    commit-message:
+      prefix: "chore"
+      include: "scope"

.github/workflows/benchmark.yml

@@ -0,0 +1,38 @@
+name: Performance Benchmarks
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  benchmark:
+    name: Performance regression check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.16"
+
+      - name: Run benchmarks
+        run: |
+          go test -bench=. -benchmem ./parser | tee benchmark.txt
+
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'go'
+          output-file-path: benchmark.txt
+          external-data-json-path: ./cache/benchmark-data.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          comment-on-alert: true
+          fail-on-alert: true

.gitignore

@@ -2,3 +2,4 @@
 .vscode
 *.iml
 __debug_bin*
+.DS_Store

.golangci.yaml

@@ -19,6 +19,7 @@ output:
 linters:
   disable:
     - godot
+    - gci
   presets:
     - bugs
     - comment
@@ -74,6 +75,7 @@ issues:
         - lll
         - gocognit
         - maintidx
+        - dupl
       test: "cognitive complexity 55 of func `Test_parseMatchExpr` is high (> 30)"
 
     - path: parser/token.go

README.md

@@ -1,37 +1,127 @@
-# KQL(kibana query language) Parser
-![GitHub CI](https://github.com/laojianzi/kql-go/actions/workflows/ci.yaml/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/laojianzi/kql-go)](https://goreportcard.com/report/github.com/laojianzi/kql-go) [![LICENSE](https://img.shields.io/github/license/laojianzi/kql-go.svg)](https://github.com/laojianzi/kql-go/blob/master/LICENSE) [![GoDoc](https://img.shields.io/badge/Godoc-reference-blue.svg)](https://pkg.go.dev/github.com/laojianzi/kql-go) [![DeepSource](https://app.deepsource.com/gh/laojianzi/kql-go.svg/?label=code+coverage&show_trend=false&token=BgPgeWYICSssJGgLh2UosQw7)](https://app.deepsource.com/gh/laojianzi/kql-go/)
+# KQL (Kibana Query Language) Parser
 
-The goal of this project is to build a KQL(kibana query language) parser in Go with the following key features:
+![GitHub CI](https://github.com/laojianzi/kql-go/actions/workflows/ci.yaml/badge.svg)
+[![Go Report Card](https://goreportcard.com/badge/github.com/laojianzi/kql-go)](https://goreportcard.com/report/github.com/laojianzi/kql-go)
+[![LICENSE](https://img.shields.io/github/license/laojianzi/kql-go.svg)](https://github.com/laojianzi/kql-go/blob/master/LICENSE)
+[![GoDoc](https://img.shields.io/badge/Godoc-reference-blue.svg)](https://pkg.go.dev/github.com/laojianzi/kql-go)
+[![DeepSource](https://app.deepsource.com/gh/laojianzi/kql-go.svg/?label=code+coverage&show_trend=false&token=BgPgeWYICSssJGgLh2UosQw7)](https://app.deepsource.com/gh/laojianzi/kql-go/)
 
-- Parse KQL(kibana query language) query into AST
-- output AST to KQL(kibana query language) query
+A high-performance Kibana Query Language (KQL) parser implemented in Go.
 
-This project is inspired by [github.com/AfterShip/clickhouse-sql-parser] and [https://github.com/cloudspannerecosystem/memefish]. Both of these are SQL parsers implemented in Go.
+## Features
 
-## How to use
+- Full KQL syntax support
+- Escaped character handling
+- Wildcard patterns
+- Parentheses grouping
+- AND/OR/NOT operators
+- Field:value pairs
+- String literals with quotes
+- Detailed error messages
+- Thread-safe design
+- High performance with minimal allocations
 
-Playground: https://go.dev/play/p/m36hkz43PQL
+## Installation
 
-```Go
+```bash
+go get github.com/laojianzi/kql-go
+```
+
+## Quick Start
+
+```go
 package main
 
 import (
+    "fmt"
+
     "github.com/laojianzi/kql-go/parser"
 )
 
-query := `(service_name: "redis" OR service_name: "mysql") AND level: ("error" OR "warn") and start_time > 1723286863 anD latency >= 1.5`
-// Parse query into AST
-stmt, err := parser.New(query).Stmt()
-if err != nil {
-    panic(err)
+func main() {
+    query := `(service_name: "redis" OR service_name: "mysql") AND level: ("error" OR "warn") and start_time > 1723286863 anD latency >= 1.5`
+    // Parse query into AST
+    stmt, err := parser.New(query).Stmt()
+    if err != nil {
+        panic(err)
+    }
+
+    // output AST to KQL(kibana query language) query
+    fmt.Println(stmt.String())
+    // output:
+    // (service_name: "redis" OR service_name: "mysql") AND level: ("error" OR "warn") AND start_time > 1723286863 AND latency >= 1.5
 }
+```
+
+## Performance
+
+Recent benchmark results:
 
-// output AST to KQL(kibana query language) query
-fmt.Println(stmt.String())
-// output:
-// (service_name: "redis" OR service_name: "mysql") AND level: ("error" OR "warn") AND start_time > 1723286863 AND latency >= 1.5
 ```
+BenchmarkParser/simple         	  749059	      1543 ns/op	     576 B/op	      15 allocs/op
+BenchmarkParser/with_escape   	  653020	      1845 ns/op	     624 B/op	      19 allocs/op
+BenchmarkParser/complex       	  103722	     11127 ns/op	    2848 B/op	      68 allocs/op
+BenchmarkLexer/simple        	218364832	      5.514 ns/op	       0 B/op	       0 allocs/op
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+### Development Requirements
+
+- Go 1.16 or higher
+- golangci-lint for code quality checks
+
+### Running Tests
+
+```bash
+# Run unit tests
+go test -v -count=1 -race ./...
+
+# Run benchmarks
+go test -bench=. -benchmem ./...
+
+# Run fuzzing tests
+go test -fuzz=. ./...
+```
+
+## Examples
+
+### Basic Queries
+```go
+// Simple field value query
+query := `status: "active"`
+
+// Numeric comparison
+query := `age >= 18`
+
+// Multiple conditions
+query := `status: "active" AND age >= 18`
+```
+
+### Advanced Queries
+```go
+// Complex grouping with wildcards
+query := `(status: "active" OR status: "pending") AND name: "john*"`
+
+// Escaped characters
+query := `message: "Hello \"World\"" AND path: "C:\\Program Files\\*"`
+
+// Multiple conditions with various operators
+query := `status: "active" AND age >= 18 AND name: "john*" AND city: "New York"`
+```
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Acknowledgments
+
+This project is inspired by:
+- [github.com/AfterShip/clickhouse-sql-parser](https://github.com/AfterShip/clickhouse-sql-parser)
+- [github.com/cloudspannerecosystem/memefish](https://github.com/cloudspannerecosystem/memefish)
 
-## Contact us
+## References
 
-Feel free to open an issue or discussion if you have any issues or questions.
+- [Kibana Query Language Documentation](https://www.elastic.co/guide/en/kibana/current/kuery-query.html)

ast/binary_test.go

@@ -3,9 +3,10 @@ package ast_test
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/laojianzi/kql-go/ast"
 	"github.com/laojianzi/kql-go/token"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestBinaryExpr(t *testing.T) {
@@ -28,7 +29,7 @@ func TestBinaryExpr(t *testing.T) {
 			name: `"v1"`,
 			args: args{
 				pos:    0,
-				value:  ast.NewLiteral(0, 4, token.TokenKindString, "v1"),
+				value:  ast.NewLiteral(0, 4, token.TokenKindString, "v1", nil),
 				hasNot: false,
 			},
 			wantEnd:    4,
@@ -38,7 +39,7 @@ func TestBinaryExpr(t *testing.T) {
 			name: `NOT "v1"`,
 			args: args{
 				pos:    0,
-				value:  ast.NewLiteral(4, 8, token.TokenKindString, "v1"),
+				value:  ast.NewLiteral(4, 8, token.TokenKindString, "v1", nil),
 				hasNot: true,
 			},
 			wantEnd:    8,
@@ -49,7 +50,7 @@ func TestBinaryExpr(t *testing.T) {
 			args: args{
 				field:    "f1",
 				operator: token.TokenKindOperatorEql,
-				value:    ast.NewLiteral(4, 8, token.TokenKindString, "v1"),
+				value:    ast.NewLiteral(4, 8, token.TokenKindString, "v1", nil),
 				hasNot:   false,
 			},
 			wantEnd:    8,
@@ -61,7 +62,7 @@ func TestBinaryExpr(t *testing.T) {
 				pos:      0,
 				field:    "f1",
 				operator: token.TokenKindOperatorEql,
-				value:    ast.NewLiteral(8, 12, token.TokenKindString, "v1"),
+				value:    ast.NewLiteral(8, 12, token.TokenKindString, "v1", nil),
 				hasNot:   true,
 			},
 			wantEnd:    12,

ast/combine_test.go

@@ -3,9 +3,10 @@ package ast_test
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/laojianzi/kql-go/ast"
 	"github.com/laojianzi/kql-go/token"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestCombineExpr(t *testing.T) {
@@ -25,9 +26,9 @@ func TestCombineExpr(t *testing.T) {
 		{
 			name: `f1: "v1" OR NOT f1: "v2"`,
 			args: args{
-				leftExpr:  ast.NewBinaryExpr(0, "f1", token.TokenKindOperatorEql, ast.NewLiteral(4, 8, token.TokenKindString, "v1"), false),
+				leftExpr:  ast.NewBinaryExpr(0, "f1", token.TokenKindOperatorEql, ast.NewLiteral(4, 8, token.TokenKindString, "v1", nil), false),
 				keyword:   token.TokenKindKeywordOr,
-				rightExpr: ast.NewBinaryExpr(12, "f1", token.TokenKindOperatorEql, ast.NewLiteral(20, 24, token.TokenKindString, "v2"), true),
+				rightExpr: ast.NewBinaryExpr(12, "f1", token.TokenKindOperatorEql, ast.NewLiteral(20, 24, token.TokenKindString, "v2", nil), true),
 			},
 			wantEnd:    24,
 			wantString: `f1: "v1" OR NOT f1: "v2"`,
@@ -43,15 +44,15 @@ func TestCombineExpr(t *testing.T) {
 						8,
 						22,
 						ast.NewCombineExpr(
-							ast.NewBinaryExpr(9, "", 0, ast.NewLiteral(9, 13, token.TokenKindString, "v1"), false),
+							ast.NewBinaryExpr(9, "", 0, ast.NewLiteral(9, 13, token.TokenKindString, "v1", nil), false),
 							token.TokenKindKeywordOr,
-							ast.NewBinaryExpr(17, "", 0, ast.NewLiteral(17, 21, token.TokenKindString, "v2"), false),
+							ast.NewBinaryExpr(17, "", 0, ast.NewLiteral(17, 21, token.TokenKindString, "v2", nil), false),
 						),
 					),
 					true,
 				),
 				keyword:   token.TokenKindKeywordAnd,
-				rightExpr: ast.NewBinaryExpr(27, "f3", token.TokenKindOperatorEql, ast.NewLiteral(31, 35, token.TokenKindString, "v3"), false),
+				rightExpr: ast.NewBinaryExpr(27, "f3", token.TokenKindOperatorEql, ast.NewLiteral(31, 35, token.TokenKindString, "v3", nil), false),
 			},
 			wantEnd:    35,
 			wantString: `NOT f1: ("v1" OR "v2") AND f3: "v3"`,

ast/literal.go

@@ -4,21 +4,24 @@ import "github.com/laojianzi/kql-go/token"
 
 // Literal is a literal(int, float, string or identifier) value.
 type Literal struct {
-	pos             int
-	end             int
+	pos           int
+	end           int
+	escapeIndexes []int
+
 	Kind            token.Kind // int, float, string or identifier
 	Value           string
 	WithDoubleQuote bool
 }
 
 // NewLiteral creates a new literal value.
-func NewLiteral(pos, end int, kind token.Kind, value string) *Literal {
+func NewLiteral(pos, end int, kind token.Kind, value string, escapeIndexes []int) *Literal {
 	return &Literal{
 		pos:             pos,
 		end:             end,
 		Kind:            kind,
 		Value:           value,
 		WithDoubleQuote: kind == token.TokenKindString,
+		escapeIndexes:   escapeIndexes,
 	}
 }
 
@@ -34,9 +37,21 @@ func (e *Literal) End() int {
 
 // String returns the string representation of the literal value.
 func (e *Literal) String() string {
+	value := e.Value
+	if len(e.escapeIndexes) > 0 {
+		runes, newValue, lastIndex := []rune(value), []rune{}, 0
+		for _, escapeIndex := range e.escapeIndexes {
+			newValue = append(newValue, runes[lastIndex:escapeIndex]...)
+			newValue = append(newValue, '\\')
+			lastIndex = escapeIndex
+		}
+
+		value = string(append(newValue, runes[lastIndex:]...))
+	}
+
 	if e.WithDoubleQuote {
-		return `"` + e.Value + `"`
+		return `"` + value + `"`
 	}
 
-	return e.Value
+	return value
 }

ast/literal_test.go

@@ -3,9 +3,10 @@ package ast_test
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/laojianzi/kql-go/ast"
 	"github.com/laojianzi/kql-go/token"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestLiteral(t *testing.T) {
@@ -73,7 +74,7 @@ func TestLiteral(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			expr := ast.NewLiteral(c.args.pos, c.args.end, c.args.kind, c.args.value)
+			expr := ast.NewLiteral(c.args.pos, c.args.end, c.args.kind, c.args.value, nil)
 			assert.Equal(t, c.wantPos, expr.Pos())
 			assert.Equal(t, c.wantEnd, expr.End())
 			assert.Equal(t, c.wantString, expr.String())