diff --git a/conf/conf.go b/conf/conf.go index 2e2f4920..bf8154ed 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -54,7 +54,7 @@ func Get() *Conf { // Schedule is the schedule. type Schedule int -// +// Schedule enum const ( Hourly Schedule = iota Daily diff --git a/conf/log.go b/conf/log.go index 62aa55e2..df32a0c9 100644 --- a/conf/log.go +++ b/conf/log.go @@ -178,7 +178,7 @@ func (l *Log) GetWriter() io.Writer { return (io.Writer)(l.Writer) } -//Rotate rotate the log file +// Rotate rotate the log file func (l *Log) Rotate() { if l.Writer == nil || l.IsStdout == true { return diff --git a/eval/eval.go b/eval/eval.go new file mode 100644 index 00000000..e4b45289 --- /dev/null +++ b/eval/eval.go @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package eval + +import ( + "fmt" + "time" + + "github.com/Knetic/govaluate" + log "github.com/sirupsen/logrus" +) + +// Variable is the variable type +type Variable struct { + Name string `yaml:"name"` + Type VarType `yaml:"type"` + Query string `yaml:"query"` + Value interface{} +} + +// NewVariable is the function to create a variable +func NewVariable(name string, t VarType, query string) *Variable { + return &Variable{ + Name: name, + Type: t, + Query: query, + Value: nil, + } +} + +// Evaluator is the structure of evaluator +type Evaluator struct { + Variables []Variable `yaml:"variables"` + DocType DocType `yaml:"doc"` + Expression string `yaml:"expression"` + Document string `yaml:"-"` + Extractor Extractor `yaml:"-"` + EvalFuncs map[string]govaluate.ExpressionFunction `yaml:"-"` +} + +// NewEvaluator is the function to create a evaluator +func NewEvaluator(doc string, t DocType, exp string) *Evaluator { + e := &Evaluator{ + Variables: make([]Variable, 0), + DocType: t, + Expression: exp, + Document: doc, + } + e.Config() + return e +} + +// Config is the function to config the evaluator +func (e *Evaluator) Config() error { + e.configExtractor() + e.configEvalFunctions() + return nil +} + +func (e *Evaluator) configExtractor() { + switch e.DocType { + case HTML: + e.Extractor = NewHTMLExtractor(e.Document) + case XML: + e.Extractor = NewXMLExtractor(e.Document) + case JSON: + e.Extractor = NewJSONExtractor(e.Document) + case TEXT: + e.Extractor = NewRegexExtractor(e.Document) + default: + e.Extractor = nil + log.Errorf("Unsupported document type: %s", e.DocType) + } +} + +func (e *Evaluator) configEvalFunctions() { + + extract := func(t VarType, query string, failed interface{}) (interface{}, error) { + v := Variable{ + Type: t, + Query: query, + } + if err := e.ExtractValue(&v); err != nil { + return failed, err + } + return v.Value, nil + } + + e.EvalFuncs = map[string]govaluate.ExpressionFunction{ + + // Extract value by XPath/Regex Expression + "x_str": func(args ...interface{}) (interface{}, error) { + return extract(String, args[0].(string), "") + }, + "x_float": func(args ...interface{}) (interface{}, error) { + return extract(Float, args[0].(string), 0.0) + }, + "x_int": func(args ...interface{}) (interface{}, error) { + v, e := extract(Int, args[0].(string), 0) + return float64(v.(int)), e + }, + "x_bool": func(args ...interface{}) (interface{}, error) { + return extract(Bool, args[0].(string), false) + }, + "x_time": func(args ...interface{}) (interface{}, error) { + v := Variable{ + Type: Time, + Query: args[0].(string), + } + + if err := e.ExtractValue(&v); err != nil { + return (time.Time{}), err + } + return (float64)(v.Value.(int64)), nil + }, + "x_duration": func(args ...interface{}) (interface{}, error) { + v, e := extract(Duration, args[0].(string), 0) + return (float64)(v.(time.Duration)), e + }, + + // Functional functions + "strlen": func(args ...interface{}) (interface{}, error) { + length := len(args[0].(string)) + return (float64)(length), nil + }, + "now": func(args ...interface{}) (interface{}, error) { + return (float64)(time.Now().Unix()), nil + }, + "duration": func(args ...interface{}) (interface{}, error) { + str := args[0].(string) + d, err := time.ParseDuration(str) + if err != nil { + return nil, err + } + return (float64)(d), nil + }, + } +} + +// SetDocument is the function to set the document +func (e *Evaluator) SetDocument(t DocType, doc string) { + if e.DocType != t { + e.DocType = t + e.Document = doc + e.configExtractor() + } else { + e.Document = doc + e.Extractor.SetDocument(doc) + } +} + +// AddVariable is the function to add a variable +func (e *Evaluator) AddVariable(v *Variable) { + e.Variables = append(e.Variables, *v) +} + +// CleanVariable is the function to clean the variable +func (e *Evaluator) CleanVariable() { + e.Variables = make([]Variable, 0) +} + +// Evaluate is the function to evaluate the expression +func (e *Evaluator) Evaluate() (bool, error) { + + if err := e.Extract(); err != nil { + return false, err + } + + expression, err := govaluate.NewEvaluableExpressionWithFunctions(e.Expression, e.EvalFuncs) + if err != nil { + return false, err + } + + variables := make(map[string]interface{}) + for _, v := range e.Variables { + variables[v.Name] = v.Value + } + + result, err := expression.Evaluate(variables) + if err != nil { + return false, err + } + switch result.(type) { + case bool: + return result.(bool), nil + case float64: + return result.(float64) != 0, nil + case string: + return result.(string) != "", nil + } + return false, fmt.Errorf("Unsupported type: %T", result) +} + +// Extract is the function to extract the value from the document +func (e *Evaluator) Extract() error { + for i := 0; i < len(e.Variables); i++ { + if err := e.ExtractValue(&e.Variables[i]); err != nil { + return err + } + } + return nil +} + +// ExtractValue is the function to extract the value from the document +func (e *Evaluator) ExtractValue(v *Variable) error { + if e.DocType == Unsupported || e.Extractor == nil { + return fmt.Errorf("Unsupported document type: %s", e.DocType) + } + e.Extractor.SetQuery(v.Query) + e.Extractor.SetVarType(v.Type) + value, err := e.Extractor.Extract() + if err != nil { + return err + } + if v.Type == Time { + v.Value = value.(time.Time).Local().Unix() + } else { + v.Value = value + } + return nil +} diff --git a/eval/eval_test.go b/eval/eval_test.go new file mode 100644 index 00000000..5b898971 --- /dev/null +++ b/eval/eval_test.go @@ -0,0 +1,383 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package eval + +import ( + "errors" + "fmt" + "reflect" + "testing" + "time" + + "bou.ke/monkey" + "github.com/Knetic/govaluate" + "github.com/stretchr/testify/assert" +) + +func assertResult(t *testing.T, eval *Evaluator, success bool) { + result, err := eval.Evaluate() + if success { + assert.Nil(t, err) + assert.True(t, result) + } else { + assert.NotNil(t, err) + assert.False(t, result) + } +} + +func TestHTMLEval(t *testing.T) { + htmlTemp := ` + + + Hello World + + +

Hello World

+

This is a simple example of a HTML document.

+
%s
+
service is ok
+
512
+
1024
+
500ms
+
false
+ + ` + htmlDoc := fmt.Sprintf(htmlTemp, time.Now().Format(time.RFC3339)) + + // ---- test message ---- + eval := NewEvaluator(htmlDoc, HTML, "message == 'service is ok'") + v := NewVariable("message", String, `//div[@id="message"]`) + eval.AddVariable(v) + assertResult(t, eval, true) + + eval = NewEvaluator(htmlDoc, HTML, "x_str('//div[@id=\\'message\\']') == 'service is ok'") + assertResult(t, eval, true) + + // ---- test title ---- + eval.CleanVariable() + eval.AddVariable(NewVariable("title", String, "//h1")) + eval.Expression = "title =~ 'World'" + assertResult(t, eval, true) + + eval = NewEvaluator(htmlDoc, HTML, "x_str('//h1') =~ 'World'") + assertResult(t, eval, true) + + // ---- test memory ---- + eval = NewEvaluator(htmlDoc, HTML, "(mem_used / mem_total) < 0.8") + eval.AddVariable(NewVariable("mem_used", Int, "//div[@id='mem_used']")) + eval.AddVariable(NewVariable("mem_total", Int, "//div[@id='mem_total']")) + assertResult(t, eval, true) + + eval = NewEvaluator(htmlDoc, HTML, "x_int('//div[@id=\\'mem_used\\']') / x_int('//div[@id=\\'mem_total\\']') < 0.8") + assertResult(t, eval, true) + + // ---- test time ---- + pass := time.Now().Add(-10 * time.Second) + eval = NewEvaluator(htmlDoc, HTML, "time > '"+pass.Format(time.RFC3339)+"'") + eval.AddVariable(NewVariable("time", Time, "//div[@id='time']")) + assertResult(t, eval, true) + + eval = NewEvaluator(htmlDoc, HTML, "x_time('//div[@id=\\'time\\']') > '"+pass.Format(time.RFC3339)+"'") + assertResult(t, eval, true) + + // ---- test live ---- + eval = NewEvaluator(htmlDoc, HTML, "!live") + eval.AddVariable(NewVariable("live", Bool, "//div[@id='live']")) + assertResult(t, eval, true) + + eval = NewEvaluator(htmlDoc, HTML, "!x_bool('//div[@id=\\'live\\']')") + assertResult(t, eval, true) + + // test strlen() function + eval = NewEvaluator(htmlDoc, HTML, "strlen(title) > 10") + eval.AddVariable(NewVariable("title", String, "//title")) + assertResult(t, eval, true) + + eval = NewEvaluator(htmlDoc, HTML, "strlen(x_str('//title')) > 10") + assertResult(t, eval, true) + + // test now() function + htmlDoc = fmt.Sprintf(htmlTemp, pass.Format(time.RFC3339)) + eval = NewEvaluator(htmlDoc, HTML, "now() - time > 5") + eval.AddVariable(NewVariable("time", Time, "//div[@id='time']")) + assertResult(t, eval, true) + + eval = NewEvaluator(htmlDoc, HTML, "now() - x_time('//div[@id=\\'time\\']') > 5") + assertResult(t, eval, true) + + // test duration() function + eval = NewEvaluator(htmlDoc, HTML, "duration(rt) < duration('1s')") + eval.AddVariable(NewVariable("rt", String, "//div[@id='resp_time']")) + assertResult(t, eval, true) + + eval = NewEvaluator(htmlDoc, HTML, "x_duration('//div[@id=\\'resp_time\\']') < duration('1s')") + assertResult(t, eval, true) + + // test duration() function error + eval = NewEvaluator(htmlDoc, HTML, "duration(rt) < '1000'") + eval.AddVariable(NewVariable("rt", String, "//div[@id='time']")) + assertResult(t, eval, false) + + eval = NewEvaluator(htmlDoc, HTML, "duration(x_str('//div[@id=\\'time\\']')) < '1000'") + assertResult(t, eval, false) +} + +func TestJSONEval(t *testing.T) { + json := `{ + "name": "Server", + "time": "` + time.Now().Format(time.RFC3339) + `", + "mem_used": 512, + "mem_total": 1024, + "resp_time": "500ms" + }` + + // ---- test name ---- + eval := NewEvaluator(json, JSON, "name == 'Server'") + v := NewVariable("name", String, "//name") + eval.AddVariable(v) + assertResult(t, eval, true) + + eval = NewEvaluator(json, JSON, "x_str('//name') == 'Server'") + assertResult(t, eval, true) + + // ---- test time ---- + eval = NewEvaluator(json, JSON, `time > '`+time.Now().Add(-10*time.Second).Format(time.RFC3339)+`'`) + eval.AddVariable(NewVariable("time", Time, "//time")) + assertResult(t, eval, true) + + eval = NewEvaluator(json, JSON, "x_time('//time') > '"+time.Now().Add(-10*time.Second).Format(time.RFC3339)+"'") + assertResult(t, eval, true) + + // ---- test memory ---- + eval = NewEvaluator(json, JSON, "(mem_used / mem_total) < 0.8") + eval.AddVariable(NewVariable("mem_used", Int, "//mem_used")) + eval.AddVariable(NewVariable("mem_total", Int, "//mem_total")) + assertResult(t, eval, true) + + eval = NewEvaluator(json, JSON, "x_int('//mem_used') / x_int('//mem_total') < 0.8") + assertResult(t, eval, true) + + // ---- test resp_time ---- + eval = NewEvaluator(json, JSON, "duration(rt) ") + eval.AddVariable(NewVariable("rt", String, "//resp_time")) + assertResult(t, eval, true) + + eval = NewEvaluator(json, JSON, "x_duration('//resp_time') < duration('1s')") + assertResult(t, eval, true) + + // ---- test string concat ---- + eval = NewEvaluator(json, JSON, "name + ' ' + time") + eval.AddVariable(NewVariable("name", String, "//name")) + eval.AddVariable(NewVariable("time", String, "//time")) + assertResult(t, eval, true) + + eval = NewEvaluator(json, JSON, "strlen(x_str('//name') + ' ' + x_str('//time')) > 10") + assertResult(t, eval, true) + + // ----- test minus ---- + eval = NewEvaluator(json, JSON, "mem_total - mem_used") + eval.AddVariable(NewVariable("mem_used", Int, "//mem_used")) + eval.AddVariable(NewVariable("mem_total", Int, "//mem_total")) + assertResult(t, eval, true) + + eval = NewEvaluator(json, JSON, "x_int('//mem_total') - x_int('//mem_used')") + assertResult(t, eval, true) +} + +func TestXMLEval(t *testing.T) { + now := time.Now().Format(time.RFC3339) + xmlDoc := ` + + Server + + 0.88 + 512 + 1024 + 500ms + true + ` + + // ---- test name ---- + eval := NewEvaluator(xmlDoc, XML, "name == 'Server'") + v := NewVariable("name", String, "//name") + eval.AddVariable(v) + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_str('//name') == 'Server'") + assertResult(t, eval, true) + + eval.CleanVariable() + eval.Expression = "x_str('//name') == 'Server'" + assertResult(t, eval, true) + + // ---- test time ---- + eval = NewEvaluator(xmlDoc, XML, `t == '`+now+`'`) + eval.AddVariable(NewVariable("t", Time, "//time")) + assertResult(t, eval, true) + + eval.CleanVariable() + eval.Expression = "x_time('//time') == '" + now + "'" + assertResult(t, eval, true) + + // ---- test cpu , memory and live ---- + eval = NewEvaluator(xmlDoc, XML, "live && (mem_used / mem_total) < 0.8 && cpu < 0.9") + eval.AddVariable(NewVariable("mem_used", Int, "//mem_used")) + eval.AddVariable(NewVariable("mem_total", Int, "//mem_total")) + eval.AddVariable(NewVariable("cpu", Float, "//cpu")) + eval.AddVariable(NewVariable("live", Bool, "//live")) + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_bool('//live') && x_float('//cpu') < 0.9 && x_int('//mem_used') / x_int('//mem_total') < 0.8") + assertResult(t, eval, true) +} + +func TestRegexEval(t *testing.T) { + text := `name: Server, cpu: 0.8, mem_used: 512, mem_total: 1024, resp_time: 256ms, live: true` + + // ---- test name ---- + eval := NewEvaluator(text, TEXT, "name == 'Server'") + v := NewVariable("name", String, "name: (?P[a-zA-Z0-9 ]*)") + eval.AddVariable(v) + assertResult(t, eval, true) + + eval = NewEvaluator(text, TEXT, "x_str('name: (?P[a-zA-Z0-9 ]*)') == 'Server'") + assertResult(t, eval, true) + + // ---- test live memory cpu ---- + eval = NewEvaluator(text, TEXT, "live && (mem_used / mem_total) < 0.8 && cpu < 0.9") + eval.AddVariable(NewVariable("mem_used", Int, "mem_used: (?P[0-9]*)")) + eval.AddVariable(NewVariable("mem_total", Int, "mem_total: (?P[0-9]*)")) + eval.AddVariable(NewVariable("cpu", Float, "cpu: (?P[0-9.]*)")) + eval.AddVariable(NewVariable("live", Bool, "live: (?Ptrue|false)")) + assertResult(t, eval, true) + + eval = NewEvaluator(text, TEXT, "x_bool('live: (?Ptrue|false)') && x_float('cpu: (?P[0-9.]*)') < 0.9 && x_int('mem_used: (?P[0-9]*)') / x_int('mem_total: (?P[0-9]*)') < 0.8") + assertResult(t, eval, true) + + // ---- test mix usage ---- + // - retrieve name and mem_used from extract function + // - set live, cpu and mem_total as the variables + eval = NewEvaluator(text, TEXT, "x_str('name: (?P[a-zA-Z0-9 ]*)') == 'Server' && live && (x_int('mem_used: (?P[0-9]*)') / mem_total) < 0.8 && cpu < 0.9") + eval.AddVariable(NewVariable("mem_total", Int, "mem_total: (?P[0-9]*)")) + eval.AddVariable(NewVariable("cpu", Float, "cpu: (?P[0-9.]*)")) + eval.AddVariable(NewVariable("live", Bool, "live: (?Ptrue|false)")) + assertResult(t, eval, true) +} + +func TestFailure(t *testing.T) { + htmlDoc := `` + eval := NewEvaluator(htmlDoc, HTML, "name == 'Server'") + v := NewVariable("name", String, "///name") + eval.AddVariable(v) + assertResult(t, eval, false) + + eval = NewEvaluator("", Unsupported, "") + v = NewVariable("name", String, "//name") + eval.AddVariable(v) + assertResult(t, eval, false) + result, err := eval.Evaluate() + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "Unsupported") + assert.False(t, result) + + eval = NewEvaluator(htmlDoc, HTML, "name == 'Server'") + v = NewVariable("name", String, "//name") + eval.AddVariable(v) + + var expression govaluate.EvaluableExpression + monkey.PatchInstanceMethod(reflect.TypeOf(expression), "Evaluate", func(govaluate.EvaluableExpression, map[string]interface{}) (interface{}, error) { + n := 10 + return n, nil + }) + + result, err = eval.Evaluate() + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "Unsupported") + assert.False(t, result) + + monkey.Patch(govaluate.NewEvaluableExpressionWithFunctions, func(expression string, functions map[string]govaluate.ExpressionFunction) (*govaluate.EvaluableExpression, error) { + return nil, errors.New("error") + }) + + result, err = eval.Evaluate() + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "error") + assert.False(t, result) + + monkey.UnpatchAll() +} + +func TestExtractFunc(t *testing.T) { + now := time.Now().Format(time.RFC3339) + xmlDoc := ` + + Server + + 0.88 + 512 + 1024 + 500ms + true + 2022-08-11 10:10:10 + ` + + eval := NewEvaluator(xmlDoc, XML, "x_str('//name') == 'Server'") + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_time('//time') == '"+now+"'") + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_float('//cpu') < 0.9") + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_int('//mem_used') < 800") + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_int('//mem_used') / x_int('//mem_total') < 0.8") + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_duration('//resp_time') < duration('1000ms')") + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_bool('//live')") + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_time('//date') == '2022-08-11 10:10:10'") + assertResult(t, eval, true) + + eval = NewEvaluator(xmlDoc, XML, "x_time('//error') == '2022-08-11 10:10:11'") + assertResult(t, eval, false) + + eval = NewEvaluator(xmlDoc, XML, "x_int('//error') < 10") + assertResult(t, eval, false) +} + +func TestSetDocument(t *testing.T) { + text := `name: Server, cpu: 0.8, mem_used: 512, mem_total: 1024, resp_time: 256ms, live: true` + + eval := NewEvaluator(text, TEXT, "x_str('name: (?P[a-zA-Z0-9 ]*)') == 'Server'") + assertResult(t, eval, true) + + eval.SetDocument(TEXT, "name: Server, cpu: 0.5, mem_used: 512, mem_total: 1024, resp_time: 256ms, live: true") + eval.Expression = "x_float('cpu: (?P[0-9.]*)') < 0.6" + assertResult(t, eval, true) + + eval.SetDocument(JSON, `{"name": "Server", "cpu": 0.3, "mem_used": 512, "mem_total": 1024, "resp_time": "256ms", "live": true}`) + eval.Expression = "x_float('//cpu') < 0.4" + assertResult(t, eval, true) +} diff --git a/eval/extract.go b/eval/extract.go new file mode 100644 index 00000000..d61a5f30 --- /dev/null +++ b/eval/extract.go @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package eval + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" + + hq "github.com/antchfx/htmlquery" + jq "github.com/antchfx/jsonquery" + xq "github.com/antchfx/xmlquery" + "golang.org/x/net/html" +) + +// Extractor is the interface for all extractors +type Extractor interface { + SetQuery(string) + SetVarType(VarType) + SetDocument(string) + Extract() (interface{}, error) +} + +// BaseExtractor is the base extractor +type BaseExtractor struct { + Name string `yaml:"name"` // variable name + VarType VarType `yaml:"type"` // variable type + Document string `yaml:"-"` + ExtractStrFn func() (string, error) +} + +// SetVarType sets the variable type +func (x *BaseExtractor) SetVarType(t VarType) { + x.VarType = t +} + +// SetDocument sets the document +func (x *BaseExtractor) SetDocument(doc string) { + x.Document = doc +} + +// Extract extracts the value from the document by xpath expression +func (x *BaseExtractor) Extract() (interface{}, error) { + switch x.VarType { + case String: + return x.ExtractStrFn() + case Int: + return x.ExtractInt() + case Float: + return x.ExtractFloat() + case Bool: + return x.ExtractBool() + case Time: + return x.ExtractTime() + case Duration: + return x.ExtractDuration() + } + return nil, fmt.Errorf("unknown type: %s", x.VarType) +} + +// ExtractInt extracts the value from the document by xpath expression +func (x *BaseExtractor) ExtractInt() (int, error) { + s, err := x.ExtractStrFn() + if err != nil { + return 0, err + } + return strconv.Atoi(s) +} + +// ExtractFloat extracts the value from the document by xpath expression +func (x *BaseExtractor) ExtractFloat() (float64, error) { + s, err := x.ExtractStrFn() + if err != nil { + return 0, err + } + return strconv.ParseFloat(s, 64) +} + +// ExtractBool extracts the value from the document by xpath expression +func (x *BaseExtractor) ExtractBool() (bool, error) { + s, err := x.ExtractStrFn() + if err != nil { + return false, err + } + return strconv.ParseBool(s) +} + +// ExtractTime extracts the value from the document by xpath expression +func (x *BaseExtractor) ExtractTime() (time.Time, error) { + s, err := x.ExtractStrFn() + if err != nil { + return time.Time{}, err + } + return tryParseTime(s) +} + +// copy from: https://github.com/Knetic/govaluate/blob/master/parsing.go#L473 +func tryParseTime(str string) (time.Time, error) { + + timeFormats := [...]string{ + time.ANSIC, + time.UnixDate, + time.RubyDate, + time.Kitchen, + time.RFC3339, + time.RFC3339Nano, + "2006-01-02", // RFC 3339 + "2006-01-02 15:04", // RFC 3339 with minutes + "2006-01-02 15:04:05", // RFC 3339 with seconds + "2006-01-02 15:04:05-07:00", // RFC 3339 with seconds and timezone + "2006-01-02T15Z0700", // ISO8601 with hour + "2006-01-02T15:04Z0700", // ISO8601 with minutes + "2006-01-02T15:04:05Z0700", // ISO8601 with seconds + "2006-01-02T15:04:05.999999999Z0700", // ISO8601 with nanoseconds + } + + for _, format := range timeFormats { + ret, err := tryParseExactTime(str, format) + if err == nil { + return ret, nil + } + } + + return time.Time{}, fmt.Errorf("Cannot parse the time: %s", str) +} + +func tryParseExactTime(candidate string, format string) (time.Time, error) { + var ret time.Time + var err error + + ret, err = time.ParseInLocation(format, candidate, time.Local) + if err != nil { + return time.Time{}, err + } + return ret, nil +} + +// ExtractDuration extracts the value from the document by xpath expression +func (x *BaseExtractor) ExtractDuration() (time.Duration, error) { + s, err := x.ExtractStrFn() + if err != nil { + return 0, err + } + return time.ParseDuration(s) +} + +// ----------------------------------------------------------------------------- + +// XPathNode is the generic type for xpath node +type XPathNode interface { + jq.Node | xq.Node | html.Node +} + +// XPathExtractor is a struct for extracting values from a html/xml/json string +type XPathExtractor[T XPathNode] struct { + BaseExtractor + XPath string `yaml:"xpath"` // xpath expression + Parser func(string) (*T, error) + Query func(*T, string) (*T, error) + Inner func(*T) string +} + +// SetQuery sets the xpath expression +func (x *XPathExtractor[T]) SetQuery(q string) { + x.XPath = q +} + +// Query query the string from the document by xpath expression +func Query[T XPathNode](document, xpath string, + parser func(string) (*T, error), + query func(*T, string) (*T, error), + inner func(*T) string) (string, error) { + doc, err := parser(document) + if err != nil { + return "", err + } + n, err := query(doc, xpath) + if err != nil { + return "", err + } + if n == nil { + return "", nil + } + return inner(n), nil +} + +// ExtractString extracts the value from the document by xpath expression +func (x *XPathExtractor[T]) ExtractStr() (string, error) { + return Query(x.Document, x.XPath, x.Parser, x.Query, x.Inner) +} + +// NewJSONExtractor creates a new JSONExtractor +func NewJSONExtractor(document string) *XPathExtractor[jq.Node] { + x := &XPathExtractor[jq.Node]{ + BaseExtractor: BaseExtractor{ + VarType: String, + Document: document, + }, + Parser: func(document string) (*jq.Node, error) { + return jq.Parse(strings.NewReader(document)) + }, + Query: func(doc *jq.Node, xpath string) (*jq.Node, error) { + return jq.Query(doc, xpath) + }, + Inner: func(n *jq.Node) string { + return n.InnerText() + }, + } + x.ExtractStrFn = x.ExtractStr + return x +} + +// NewXMLExtractor creates a new XMLExtractor +func NewXMLExtractor(document string) *XPathExtractor[xq.Node] { + x := &XPathExtractor[xq.Node]{ + BaseExtractor: BaseExtractor{ + VarType: String, + Document: document, + }, + Parser: func(document string) (*xq.Node, error) { + return xq.Parse(strings.NewReader(document)) + }, + Query: func(doc *xq.Node, xpath string) (*xq.Node, error) { + return xq.Query(doc, xpath) + }, + Inner: func(n *xq.Node) string { + return n.InnerText() + }, + } + x.ExtractStrFn = x.ExtractStr + return x +} + +// NewHTMLExtractor creates a new HTMLExtractor +func NewHTMLExtractor(document string) *XPathExtractor[html.Node] { + x := &XPathExtractor[html.Node]{ + BaseExtractor: BaseExtractor{ + VarType: String, + Document: document, + }, + Parser: func(document string) (*html.Node, error) { + return html.Parse(strings.NewReader(document)) + }, + Query: func(doc *html.Node, xpath string) (*html.Node, error) { + return hq.Query(doc, xpath) + }, + Inner: func(n *html.Node) string { + return hq.InnerText(n) + }, + } + x.Document = document + x.ExtractStrFn = x.ExtractStr + return x +} + +//------------------------------------------------------------------------------ + +// RegexExtractor is a struct for extracting values from a plain string +type RegexExtractor struct { + BaseExtractor + Regex string `yaml:"regex"` // regex expression +} + +// SetQuery sets the regex expression +func (r *RegexExtractor) SetQuery(q string) { + r.Regex = q +} + +// MatchStr matches the string with the regex expression +func (r *RegexExtractor) MatchStr() (string, error) { + re := regexp.MustCompile(r.Regex) + match := re.FindStringSubmatch(r.Document) + if match == nil { + return "", fmt.Errorf("no match found for - %s", r.Regex) + } + for i, name := range re.SubexpNames() { + if i > 0 && i <= len(match) { + if len(name) > 0 { + r.Name = name + } + return match[i], nil + } + } + return match[0], nil +} + +// NewRegexExtractor creates a new RegexExtractor +func NewRegexExtractor(document string) *RegexExtractor { + x := &RegexExtractor{ + BaseExtractor: BaseExtractor{ + VarType: String, + Document: document, + }, + } + x.ExtractStrFn = x.MatchStr + return x +} diff --git a/eval/extract_test.go b/eval/extract_test.go new file mode 100644 index 00000000..6374db17 --- /dev/null +++ b/eval/extract_test.go @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package eval + +import ( + "errors" + "io" + "testing" + "time" + + "bou.ke/monkey" + "github.com/stretchr/testify/assert" + "golang.org/x/net/html" +) + +func assertExtractor(t *testing.T, extractor Extractor, query string, vt VarType, expected interface{}, success bool) { + extractor.SetQuery(query) + extractor.SetVarType(vt) + result, err := extractor.Extract() + if success { + assert.Nil(t, err) + } else { + assert.NotNil(t, err) + } + assert.Equal(t, expected, result) +} +func assertExtractorSucc(t *testing.T, extractor Extractor, query string, vt VarType, expected interface{}) { + assertExtractor(t, extractor, query, vt, expected, true) +} +func assertExtractorFail(t *testing.T, extractor Extractor, query string, vt VarType, expected interface{}) { + assertExtractor(t, extractor, query, vt, expected, false) +} + +func TestHTMLExtractor(t *testing.T) { + var htmlDoc = ` + + + Hello World Example + + +

Hello World Example

+

+ EaseProbe Github is a simple, standalone, and lightWeight tool that can do health/status checking, written in Go. +

+
+
1
+
2
+
+
+
Bob
+ +
35000.12
+
1984-10-12
+
40h
+
true
+
+ + + ` + extractor := NewHTMLExtractor(htmlDoc) + + assertExtractorSucc(t, extractor, "//div[@id='number']/div[@class='one']", Int, 1) + assertExtractorSucc(t, extractor, "//title", String, "Hello World Example") + assertExtractorSucc(t, extractor, "//div[@id='person']/div[@class='name']", String, "Bob") + assertExtractorSucc(t, extractor, "//div[@id='person']/div[@class='email']", String, "bob@example.com") + assertExtractorSucc(t, extractor, "//div[@id='person']/div[@class='salary']", Float, 35000.12) + expected, _ := tryParseTime("1984-10-12") + assertExtractorSucc(t, extractor, "//div[@id='person']/div[@class='birth']", Time, expected) + assertExtractorSucc(t, extractor, "//div[@id='person']/div[@class='work']", Duration, 40*time.Hour) + assertExtractorSucc(t, extractor, "//div[@id='person']/div[@class='fulltime']", Bool, true) + // multiple results only return the first one + assertExtractorSucc(t, extractor, "//div[@id='person']/div", String, "Bob") + // empty result + assertExtractorSucc(t, extractor, "//div[@id='person']/div[@class='none']", String, "") + // invalid xpath + assertExtractorFail(t, extractor, "///asdf", String, "") +} + +func TestJSONExtractor(t *testing.T) { + jsonDoc := ` + { + "company": { + "name": "MegaEase", + "person": [{ + "name": "Bob", + "email": "bob@example.com", + "age": 35, + "salary": 35000.12, + "birth": "1984-10-12", + "work": "40h", + "fulltime": true + }, + { + "name": "Alice", + "email": "alice@example.com", + "age": 25, + "salary": 25000.12, + "birth": "1985-10-12", + "work": "30h", + "fulltime": false + } + ] + } + }` + extractor := NewJSONExtractor(jsonDoc) + + assertExtractorSucc(t, extractor, "//name", String, "MegaEase") + assertExtractorSucc(t, extractor, "//company/name", String, "MegaEase") + assertExtractorSucc(t, extractor, "//email", String, "bob@example.com") + assertExtractorSucc(t, extractor, "//company/person/*[1]/name", String, "Bob") + assertExtractorSucc(t, extractor, "//company/person/*[2]/email", String, "alice@example.com") + assertExtractorSucc(t, extractor, "//company/person/*[last()]/name", String, "Alice") + assertExtractorSucc(t, extractor, "//company/person/*[last()]/age", Int, 25) + assertExtractorSucc(t, extractor, "//company/person/*[salary=25000.12]/salary", Float, 25000.12) + expected, _ := tryParseTime("1984-10-12") + assertExtractorSucc(t, extractor, "//company/person/*[name='Bob']/birth", Time, expected) + assertExtractorSucc(t, extractor, "//company/person/*[name='Alice']/work", Duration, 30*time.Hour) + assertExtractorSucc(t, extractor, "//*/email[contains(.,'bob')]", String, "bob@example.com") + assertExtractorSucc(t, extractor, "//work", Duration, 40*time.Hour) + assertExtractorSucc(t, extractor, "//person/*[2]/fulltime", Bool, false) +} + +func TestXMLExtractor(t *testing.T) { + xmlDoc := ` + + MegaEase + + Bob + bob@example.com + 35 + 35000.12 + 1984-10-12 + 40h + true + + + Alice + alice@example.com + 25 + 25000.12 + 1985-10-12 + 30h + false + + ` + extractor := NewXMLExtractor(xmlDoc) + + assertExtractorSucc(t, extractor, "//name", String, "MegaEase") + assertExtractorSucc(t, extractor, "//company/name", String, "MegaEase") + assertExtractorSucc(t, extractor, "//company/person[1]/name", String, "Bob") + assertExtractorSucc(t, extractor, "//company/person[last()]/name", String, "Alice") + assertExtractorSucc(t, extractor, "//person[@id='emp1002']/age", Int, 25) + assertExtractorSucc(t, extractor, "//company/person[salary=35000.12]/salary", Float, 35000.12) + assertExtractorSucc(t, extractor, "//person[salary<30000]/salary", Float, 25000.12) + expected, _ := tryParseTime("1984-10-12") + assertExtractorSucc(t, extractor, "//company/person[name='Bob']/birth", Time, expected) + assertExtractorSucc(t, extractor, "//company/person[name='Alice']/work", Duration, 30*time.Hour) + assertExtractorSucc(t, extractor, "//company/person[name='Bob']/work", Duration, 40*time.Hour) +} + +func TestRegexExtractor(t *testing.T) { + regexDoc := `name: Bob, email: bob@example.com, age: 35, salary: 35000.12, birth: 1984-10-12, work: 40h, fulltime: true` + + extractor := NewRegexExtractor(regexDoc) + + assertExtractorSucc(t, extractor, "name: (?P[a-zA-Z0-9 ]*)", String, "Bob") + assertExtractorSucc(t, extractor, "email: (?P[a-zA-Z0-9@.]*)", String, "bob@example.com") + assertExtractorSucc(t, extractor, "age: (?P[0-9]*)", Int, 35) + assertExtractorSucc(t, extractor, "age: (?P\\d+)", Int, 35) + assertExtractorSucc(t, extractor, "salary: (?P[0-9.]*)", Float, 35000.12) + assertExtractorSucc(t, extractor, "salary: (?P\\d+\\.\\d+)", Float, 35000.12) + expected, _ := tryParseTime("1984-10-12") + assertExtractorSucc(t, extractor, "birth: (?P[0-9-]*)", Time, expected) + assertExtractorSucc(t, extractor, "birth: (?P\\d{4}-\\d{2}-\\d{2})", Time, expected) + assertExtractorSucc(t, extractor, "work: (?P\\d+[hms])", Duration, 40*time.Hour) + assertExtractorSucc(t, extractor, "fulltime: (?Ptrue|false)", Bool, true) + // no Submatch + assertExtractorSucc(t, extractor, "name: ", String, "name: ") + // no match + assertExtractorFail(t, extractor, "mismatch", String, "") +} + +func TestFailed(t *testing.T) { + doc := "
hello world
" + extractor := NewHTMLExtractor(doc) + invalid := "///div" + assertExtractorFail(t, extractor, invalid, Int, 0) + assertExtractorFail(t, extractor, invalid, Float, 0.0) + assertExtractorFail(t, extractor, invalid, Bool, false) + assertExtractorFail(t, extractor, invalid, Time, time.Time{}) + assertExtractorFail(t, extractor, invalid, Duration, time.Duration(0)) + assertExtractorFail(t, extractor, invalid, Unknown, nil) + + monkey.Patch(html.Parse, func(io.Reader) (*html.Node, error) { + return nil, errors.New("parse error") + }) + + extractor.VarType = String + result, err := extractor.Extract() + assert.NotNil(t, err) + assert.Equal(t, "parse error", err.Error()) + assert.Equal(t, "", result) + + monkey.Unpatch(html.Parse) +} diff --git a/eval/types.go b/eval/types.go new file mode 100644 index 00000000..0e29cf80 --- /dev/null +++ b/eval/types.go @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package eval + +import ( + "strings" + + "github.com/megaease/easeprobe/global" +) + +// ----------------------------------------------------------------------------- + +// DocType is the different type of document +type DocType int + +// The Document Type +const ( + Unsupported DocType = iota + HTML + XML + JSON + TEXT +) + +var docTypeToStr = map[DocType]string{ + Unsupported: "unsupported", + HTML: "html", + XML: "xml", + JSON: "json", + TEXT: "text", +} + +var strToDocType = global.ReverseMap(docTypeToStr) + +// String covert the DocType to string +func (t DocType) String() string { + return docTypeToStr[t] +} + +// Type covert the string to Type +func (t *DocType) Type(s string) { + *t = strToDocType[strings.ToLower(s)] +} + +// MarshalYAML is marshal the type +func (t DocType) MarshalYAML() (interface{}, error) { + return t.String(), nil +} + +// UnmarshalYAML is unmarshal the type +func (t *DocType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + t.Type(str) + return nil +} + +// ----------------------------------------------------------------------------- + +// VarType is an enum for the different types of values +type VarType int + +// The value types +const ( + Unknown VarType = iota + Int + Float + String + Bool + Time + Duration +) + +var varTypeToStr = map[VarType]string{ + Unknown: "unknown", + Int: "int", + Float: "float", + String: "string", + Bool: "bool", + Time: "time", + Duration: "duration", +} + +var strToVarType = global.ReverseMap(varTypeToStr) + +// String covert the Type to string +func (t VarType) String() string { + return varTypeToStr[t] +} + +// Type covert the string to Type +func (t *VarType) Type(s string) { + *t = strToVarType[strings.ToLower(s)] +} + +// MarshalYAML is marshal the type +func (t VarType) MarshalYAML() (interface{}, error) { + return t.String(), nil +} + +// UnmarshalYAML is unmarshal the type +func (t *VarType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var str string + if err := unmarshal(&str); err != nil { + return err + } + t.Type(str) + return nil +} diff --git a/eval/types_test.go b/eval/types_test.go new file mode 100644 index 00000000..30338fa6 --- /dev/null +++ b/eval/types_test.go @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2022, MegaEase + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package eval + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" +) + +func testVarType(t *testing.T, expect VarType, str string) { + var result VarType + result.Type(str) + assert.Equal(t, expect, result) +} + +func TestType(t *testing.T) { + testVarType(t, Unknown, "unknown") + testVarType(t, Int, "int") + testVarType(t, Float, "float") + testVarType(t, String, "string") + testVarType(t, Bool, "bool") + testVarType(t, Time, "time") + testVarType(t, Duration, "duration") + testVarType(t, Unknown, "unrecognized") +} + +func testVarTypeYAML(t *testing.T, expect VarType, str string) { + var result VarType + buf, err := yaml.Marshal(expect) + assert.Nil(t, err) + assert.Equal(t, str, string(buf)) + + assert.Nil(t, yaml.Unmarshal(buf, &result)) + assert.Equal(t, expect, result) +} + +func TestVarTypeYAML(t *testing.T) { + testVarTypeYAML(t, Unknown, "unknown\n") + testVarTypeYAML(t, Int, "int\n") + testVarTypeYAML(t, Float, "float\n") + testVarTypeYAML(t, String, "string\n") + testVarTypeYAML(t, Bool, "bool\n") + testVarTypeYAML(t, Time, "time\n") + testVarTypeYAML(t, Duration, "duration\n") + + str := "-name:: value\n" + var result VarType + assert.NotNil(t, yaml.Unmarshal([]byte(str), &result)) +} + +//------------------------------------------------------------------------------ + +func testDocType(t *testing.T, expect DocType, str string) { + var result DocType + result.Type(str) + assert.Equal(t, expect, result) +} + +func TestDocType(t *testing.T) { + testDocType(t, Unsupported, "unsupported") + testDocType(t, HTML, "html") + testDocType(t, XML, "xml") + testDocType(t, JSON, "json") + testDocType(t, TEXT, "text") +} + +func testDocTypeYAML(t *testing.T, expect DocType, str string) { + var result DocType + buf, err := yaml.Marshal(expect) + assert.Nil(t, err) + assert.Equal(t, str, string(buf)) + + assert.Nil(t, yaml.Unmarshal(buf, &result)) + assert.Equal(t, expect, result) +} + +func TestDocTypeYAML(t *testing.T) { + testDocTypeYAML(t, Unsupported, "unsupported\n") + testDocTypeYAML(t, HTML, "html\n") + testDocTypeYAML(t, XML, "xml\n") + testDocTypeYAML(t, JSON, "json\n") + testDocTypeYAML(t, TEXT, "text\n") + + str := "-name:: value\n" + var result DocType + assert.NotNil(t, yaml.Unmarshal([]byte(str), &result)) +} diff --git a/global/easeprobe_test.go b/global/easeprobe_test.go index b5a9ec6a..d78468c9 100644 --- a/global/easeprobe_test.go +++ b/global/easeprobe_test.go @@ -51,7 +51,8 @@ func TestEaseProbe(t *testing.T) { // If you use VSCode run the test, // make sure add the following test flag in settings.json -// "go.testFlags": ["-gcflags=-l"], +// +// "go.testFlags": ["-gcflags=-l"], func TestEaseProbeFail(t *testing.T) { monkey.Patch(os.Hostname, func() (string, error) { return "", fmt.Errorf("error") diff --git a/go.mod b/go.mod index b59903c3..2fcd6cec 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,9 @@ go 1.18 require ( bou.ke/monkey v1.0.2 + github.com/antchfx/htmlquery v1.2.5 + github.com/antchfx/jsonquery v1.3.0 + github.com/antchfx/xmlquery v1.3.12 github.com/aws/aws-sdk-go v1.44.67 github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d github.com/go-chi/chi v4.1.2+incompatible @@ -27,10 +30,13 @@ require ( require ( github.com/BurntSushi/toml v1.1.0 // indirect + github.com/Knetic/govaluate v3.0.0+incompatible // indirect + github.com/antchfx/xpath v1.2.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.1 // indirect github.com/google/go-cmp v0.5.7 // indirect @@ -48,6 +54,7 @@ require ( github.com/prometheus/procfs v0.7.3 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rogpeppe/go-internal v1.8.1 // indirect + github.com/sergi/go-diff v1.1.0 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/uptrace/bun v1.1.7 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect @@ -56,9 +63,18 @@ require ( github.com/xdg-go/scram v1.1.1 // indirect github.com/xdg-go/stringprep v1.0.3 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/net v0.0.0-20220809012201-f428fae20770 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/text v0.3.7 // indirect + golang.org/x/tools v0.1.12-0.20220713141851-7464a5a40219 // indirect + golang.org/x/tools/gopls v0.9.1 // indirect + golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/protobuf v1.26.0 // indirect + honnef.co/go/tools v0.3.2 // indirect mellium.im/sasl v0.2.1 // indirect + mvdan.cc/gofumpt v0.3.0 // indirect + mvdan.cc/xurls/v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 6fc349b1..847f7b78 100644 --- a/go.sum +++ b/go.sum @@ -37,11 +37,21 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= +github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4= +github.com/antchfx/htmlquery v1.2.5/go.mod h1:2MCVBzYVafPBmKbrmwB9F5xdd+IEgRY61ci2oOsOQVw= +github.com/antchfx/jsonquery v1.3.0 h1:rftVBKEXpj8C9WVu+4mbqL5hd6nLz7/AbIvAQlq3D7o= +github.com/antchfx/jsonquery v1.3.0/go.mod h1:fZ88NWso7HlXESJ2hrNKnYx+xyT6pmvV1N6KMIg7FHo= +github.com/antchfx/xmlquery v1.3.12 h1:6TMGpdjpO/P8VhjnaYPXuqT3qyJ/VsqoyNTmJzNBTQ4= +github.com/antchfx/xmlquery v1.3.12/go.mod h1:3w2RvQvTz+DaT5fSgsELkSJcdNgkmg6vuXDEuhdwsPQ= +github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8= +github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/aws/aws-sdk-go v1.44.67 h1:+nxfXbMe8QUB6svLsuLYsp+WhZBKM26w62Zidir739A= github.com/aws/aws-sdk-go v1.44.67/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -69,6 +79,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= @@ -96,6 +107,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -235,6 +247,8 @@ github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XF github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/segmentio/kafka-go v0.4.33 h1:XHYuEifMYFVCU9A2p1wJprd7xHQKS+Sn6xgBr11+30k= github.com/segmentio/kafka-go v0.4.33/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -278,6 +292,7 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.mongodb.org/mongo-driver v1.10.0 h1:UtV6N5k14upNp4LTduX0QCufG124fSu25Wz9tu94GLg= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -307,6 +322,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220328175248-053ad81199eb h1:pC9Okm6BVmxEw76PUu0XUbOTQ92JX11hfvqTjAV3qxM= golang.org/x/exp v0.0.0-20220328175248-053ad81199eb/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e h1:qyrTQ++p1afMkO4DPEeLGq/3oTsdlvdH4vqZUBWzUKM= +golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -327,6 +344,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -348,6 +368,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -356,10 +377,13 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60 h1:8NSylCMxLW4JvserAndSgFL7aPli6A68yf0bYFTcWCM= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220809012201-f428fae20770 h1:dIi4qVdvjZEjiMDv7vhokAZNGnz3kepwuXqFKYDdDMs= +golang.org/x/net v0.0.0-20220809012201-f428fae20770/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -413,8 +437,10 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= @@ -473,6 +499,13 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12-0.20220713141851-7464a5a40219 h1:Ljlba2fVWOA1049JjsKii44g8nZN2GjpxMlzVc8AnQM= +golang.org/x/tools v0.1.12-0.20220713141851-7464a5a40219/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= +golang.org/x/tools/gopls v0.9.1 h1:SigsTL4Hpv3a6b/a7oPCLRv5QUeSM6PZNdta1oKY4B0= +golang.org/x/tools/gopls v0.9.1/go.mod h1:fkgIFE0meep1WkUekyJZAgyUIVrbjkTNtc1JEZxZYg8= +golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c h1:r5bbIROBQtRRgoutV8Q3sFY58VzHW6jMBYl48ANSyS4= +golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c/go.mod h1:UZshlUPxXeGUM9I14UOawXQg6yosDE9cr1vKY/DzgWo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -582,8 +615,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.3.2 h1:ytYb4rOqyp1TSa2EPvNVwtPQJctSELKaMyLfqNP4+34= +honnef.co/go/tools v0.3.2/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw= mellium.im/sasl v0.2.1 h1:nspKSRg7/SyO0cRGY71OkfHab8tf9kCts6a6oTDut0w= mellium.im/sasl v0.2.1/go.mod h1:ROaEDLQNuf9vjKqE1SrAfnsobm2YKXT1gnN1uDp1PjQ= +mvdan.cc/gofumpt v0.3.0 h1:kTojdZo9AcEYbQYhGuLf/zszYthRdhDNDUi2JKTxas4= +mvdan.cc/gofumpt v0.3.0/go.mod h1:0+VyGZWleeIj5oostkOex+nDBA0eyavuDnDusAJ8ylo= +mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= +mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/notify/discord/discord.go b/notify/discord/discord.go index ed39d562..31964cae 100644 --- a/notify/discord/discord.go +++ b/notify/discord/discord.go @@ -62,8 +62,8 @@ type Fields struct { } // Footer allows you to add footer to embed. footer is an object which includes two values: -// - text - sets name for author object. Markdown is disabled here!!! -// - icon_url - sets icon for author object. Requires text value. +// - text - sets name for author object. Markdown is disabled here!!! +// - icon_url - sets icon for author object. Requires text value. type Footer struct { Text string `json:"text"` IconURL string `json:"icon_url"` diff --git a/notify/log/format.go b/notify/log/format.go index 38c14788..c657da68 100644 --- a/notify/log/format.go +++ b/notify/log/format.go @@ -26,12 +26,12 @@ import ( log "github.com/sirupsen/logrus" ) -//SysLogFormatter is log custom format +// SysLogFormatter is log custom format type SysLogFormatter struct { Type Type `yaml:"-"` } -//Format details +// Format details func (s *SysLogFormatter) Format(entry *log.Entry) ([]byte, error) { if s.Type == SysLog { return []byte(fmt.Sprintf("%s\n", entry.Message)), nil diff --git a/notify/notify.go b/notify/notify.go index 249bc281..5ad49de4 100644 --- a/notify/notify.go +++ b/notify/notify.go @@ -34,7 +34,7 @@ import ( "github.com/megaease/easeprobe/probe" ) -//Config is the notify configuration +// Config is the notify configuration type Config struct { Log []log.NotifyConfig `yaml:"log"` Email []email.NotifyConfig `yaml:"email"` diff --git a/notify/sms/sms.go b/notify/sms/sms.go index ff55399e..bb5ed394 100644 --- a/notify/sms/sms.go +++ b/notify/sms/sms.go @@ -29,7 +29,7 @@ import ( log "github.com/sirupsen/logrus" ) -//NotifyConfig implements the structure of Sms +// NotifyConfig implements the structure of Sms type NotifyConfig struct { //Embed structure conf.Options `yaml:",inline"` diff --git a/probe/data.go b/probe/data.go index e14f5a36..344b4231 100644 --- a/probe/data.go +++ b/probe/data.go @@ -61,7 +61,8 @@ func GetMetaData() *MetaData { // SetResultData set the result of probe // Note: this function would be called by status update goroutine -// int saveData() in cmd/easeprobe/report.go +// +// int saveData() in cmd/easeprobe/report.go func SetResultData(name string, result *Result) { r := result.Clone() mutex.Lock() diff --git a/probe/http/http.go b/probe/http/http.go index d2d25193..1d0ba796 100644 --- a/probe/http/http.go +++ b/probe/http/http.go @@ -29,6 +29,7 @@ import ( "strconv" "strings" + "github.com/megaease/easeprobe/eval" "github.com/megaease/easeprobe/global" "github.com/megaease/easeprobe/probe" "github.com/megaease/easeprobe/probe/base" @@ -50,6 +51,9 @@ type HTTP struct { // Output Text Checker probe.TextChecker `yaml:",inline"` + // Evaluator + Evaluator eval.Evaluator `yaml:"eval,omitempty"` + // Option - HTTP Basic Auth Credentials User string `yaml:"username,omitempty"` Pass string `yaml:"password,omitempty"` @@ -158,6 +162,10 @@ func (h *HTTP) Config(gConf global.ProbeSettings) error { return err } + if err := h.Evaluator.Config(); err != nil { + return err + } + h.metrics = newMetrics(kind, tag) log.Debugf("[%s / %s] configuration: %+v", h.ProbeKind, h.ProbeName, h) @@ -226,6 +234,25 @@ func (h *HTTP) DoProbe() (bool, string) { return false, message } + if h.Evaluator.DocType != eval.Unsupported && h.Evaluator.Extractor != nil && + len(strings.TrimSpace(h.Evaluator.Expression)) > 0 { + + log.Debugf("[%s / %s] - Evaluator expression: %s", h.ProbeKind, h.ProbeName, h.Evaluator.Expression) + h.Evaluator.SetDocument(h.Evaluator.DocType, string(response)) + result, err := h.Evaluator.Evaluate() + if err != nil { + log.Errorf("[%s / %s] - %v", h.ProbeKind, h.ProbeName, err) + message += fmt.Sprintf(". Evaluation Error: %v", err) + return false, message + } + if !result { + log.Errorf("[%s / %s] - expression is evaluated to false!", h.ProbeKind, h.ProbeName) + message += fmt.Sprintf(". Expression is evaluated to false!") + return false, message + } + log.Debugf("[%s / %s] - expression is evaluated to true!", h.ProbeKind, h.ProbeName) + } + return true, message } diff --git a/probe/http/http_test.go b/probe/http/http_test.go index dca9a8a6..7f963f4f 100644 --- a/probe/http/http_test.go +++ b/probe/http/http_test.go @@ -29,6 +29,7 @@ import ( "testing" "bou.ke/monkey" + "github.com/megaease/easeprobe/eval" "github.com/megaease/easeprobe/global" "github.com/megaease/easeprobe/probe" "github.com/megaease/easeprobe/probe/base" @@ -46,6 +47,18 @@ func createHTTP() *HTTP { Contain: "good", NotContain: "bad", }, + Evaluator: eval.Evaluator{ + Variables: []eval.Variable{ + { + Name: "name", + Type: eval.String, + Query: "//name", + Value: nil, + }, + }, + DocType: eval.JSON, + Expression: "name == 'EaseProbe'", + }, User: "user", Pass: "pass", TLS: global.TLS{ @@ -134,13 +147,24 @@ func TestHTTPDoProbe(t *testing.T) { }, nil }) monkey.Patch(ioutil.ReadAll, func(r io.Reader) ([]byte, error) { - return []byte("good"), nil + return []byte(`{ "name": "EaseProbe", "status": "good"}`), nil }) s, m := h.DoProbe() assert.True(t, s) assert.Contains(t, m, "200") + // evaluate error + h.Evaluator.Expression = "name == 'N/A'" + s, m = h.DoProbe() + assert.False(t, s) + assert.Contains(t, m, "Expression is evaluated to false") + + h.Evaluator.Variables[0].Query = "///name" + s, m = h.DoProbe() + assert.False(t, s) + assert.Contains(t, m, "Evaluation Error") + // response does not contain good string monkey.Patch(ioutil.ReadAll, func(r io.Reader) ([]byte, error) { return []byte("bad"), nil diff --git a/probe/status.go b/probe/status.go index 63418d28..f0aaf640 100644 --- a/probe/status.go +++ b/probe/status.go @@ -64,7 +64,7 @@ func (s Status) String() string { return "unknown" } -//Status convert the string to Status +// Status convert the string to Status func (s *Status) Status(status string) { if val, ok := toStatus[strings.ToLower(status)]; ok { *s = val diff --git a/report/common.go b/report/common.go index 1ac144a9..298c2e72 100644 --- a/report/common.go +++ b/report/common.go @@ -124,7 +124,7 @@ func SlackTimeFormation(t time.Time, act string, format string) string { t.Unix(), act, act, t.UTC().Format(format)) } -//JSONEscape escape the string +// JSONEscape escape the string func JSONEscape(str string) string { b, err := json.Marshal(str) if err != nil {