feat: support regexp output check for http/shell/ssh probe (megaease#169

)

* feat: support regexp output check for http/shell/ssh probe

* rename oc to tc

* compile the regexp during the startup

* add the golang regexp link in README.md

README.md

@@ -568,10 +568,17 @@ http:
     # Response Checking
     contain: "success" # response body must contain this string, if not the probe is considered failed.
     not_contain: "failure" # response body must NOT contain this string, if it does the probe is considered failed.
+    regex: false # if true, the contain and not_contain will be treated as regular expression. default: false
     # configuration
     timeout: 10s # default is 30 seconds
 
 ```
+
+> **Note**:
+>
+> The Regular Expression supported refer to https://github.com/google/re2/wiki/Syntax
+
+
 ### 3.2 TCP Probe Configuration
 
 ```YAML
@@ -616,6 +623,8 @@ shell:
       - "REDISCLI_AUTH=abc123"
     # check the command output, if does not contain the PONG, mark the status down
     contain : "PONG"
+    not_contain: "failure" # response body must NOT contain this string, if it does the probe is considered failed.
+    regex: false # if true, the `contain` and `not_contain` will be treated as regular expression. default: false
 
   # Run Zookeeper command `stat` to check the zookeeper status
   - name: Zookeeper (Local)
@@ -626,6 +635,10 @@ shell:
     contain: "Mode:"
 ```
 
+> **Note**:
+>
+> The Regular Expression supported refer to https://github.com/google/re2/wiki/Syntax
+
 ### 3.4 SSH Command Probe Configuration
 
 SSH probe is similar to Shell probe.
@@ -670,6 +683,8 @@ ssh:
         - "REDISCLI_AUTH=abc123"
       # check the command output, if does not contain the PONG, mark the status down
       contain : "PONG"
+      not_contain: "failure" # response body must NOT contain this string, if it does the probe is considered failed.
+      regex: false # if true, the contain and not_contain will be treated as regular expression. default: false
 
     # Check the process status of `Kafka`
     - name:  Kafka (GCP)
@@ -679,6 +694,9 @@ ssh:
       key: /path/to/private.key
       cmd: "ps -ef | grep kafka"
 ```
+> **Note**:
+>
+> The Regular Expression supported refer to https://github.com/google/re2/wiki/Syntax
 
 ### 3.5 TLS Probe Configuration
 

probe/common.go

@@ -19,20 +19,84 @@ package probe
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 )
 
-// CheckOutput checks the output text,
+// TextChecker is the struct to check the output
+type TextChecker struct {
+	Contain    string `yaml:"contain,omitempty"`
+	NotContain string `yaml:"not_contain,omitempty"`
+	RegExp     bool   `yaml:"regex,omitempty"`
+
+	containReg    *regexp.Regexp `yaml:"-"`
+	notContainReg *regexp.Regexp `yaml:"-"`
+}
+
+// Config the text checker initialize the regexp
+func (tc *TextChecker) Config() (err error) {
+	if !tc.RegExp {
+		return nil
+	}
+
+	if len(tc.Contain) == 0 {
+		tc.containReg = nil
+	} else if tc.containReg, err = regexp.Compile(tc.Contain); err != nil {
+		tc.containReg = nil
+		return err
+	}
+
+	if len(tc.NotContain) == 0 {
+		tc.notContainReg = nil
+	} else if tc.notContainReg, err = regexp.Compile(tc.NotContain); err != nil {
+		tc.notContainReg = nil
+		return err
+	}
+
+	return nil
+}
+
+// Check the text
+func (tc *TextChecker) Check(Text string) error {
+	if tc.RegExp {
+		return tc.CheckRegExp(Text)
+	}
+	return tc.CheckText(Text)
+}
+
+func (tc *TextChecker) String() string {
+	if tc.RegExp {
+		return fmt.Sprintf("RegExp Mode - Contain:[%s], NotContain:[%s]", tc.Contain, tc.NotContain)
+	}
+	return fmt.Sprintf("Text Mode - Contain:[%s], NotContain:[%s]", tc.Contain, tc.NotContain)
+}
+
+// CheckText checks the output text,
 // - if it contains a configured string then return nil
 // - if it does not contain a configured string then return nil
-func CheckOutput(Contain, NotContain, Output string) error {
+func (tc *TextChecker) CheckText(Output string) error {
+
+	if len(tc.Contain) > 0 && !strings.Contains(Output, tc.Contain) {
+		return fmt.Errorf("the output does not contain [%s]", tc.Contain)
+	}
+
+	if len(tc.NotContain) > 0 && strings.Contains(Output, tc.NotContain) {
+		return fmt.Errorf("the output contains [%s]", tc.NotContain)
+	}
+	return nil
+}
+
+// CheckRegExp checks the output text,
+// - if it contains a configured pattern then return nil
+// - if it does not contain a configured pattern then return nil
+func (tc *TextChecker) CheckRegExp(Output string) error {
 
-	if len(Contain) > 0 && !strings.Contains(Output, Contain) {
-		return fmt.Errorf("the output does not contain [%s]", Contain)
+	if len(tc.Contain) > 0 && tc.containReg != nil && !tc.containReg.MatchString(Output) {
+		return fmt.Errorf("the output does not match the pattern [%s]", tc.Contain)
 	}
 
-	if len(NotContain) > 0 && strings.Contains(Output, NotContain) {
-		return fmt.Errorf("the output contains [%s]", NotContain)
+	if len(tc.NotContain) > 0 && tc.notContainReg != nil && tc.notContainReg.MatchString(Output) {
+		return fmt.Errorf("the output match the pattern [%s]", tc.NotContain)
 	}
 	return nil
 }

probe/common_test.go

@@ -23,20 +23,130 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestCheckOutput(t *testing.T) {
-	err := CheckOutput("hello", "good", "easeprobe hello world")
+func TestCheckText(t *testing.T) {
+	tc := TextChecker{}
+
+	tc.Contain = "hello"
+	tc.NotContain = "bad"
+	err := tc.Check("easeprobe hello world")
 	assert.Nil(t, err)
 
-	err = CheckOutput("hello", "world", "easeprobe hello world")
+	tc.Contain = "hello"
+	tc.NotContain = "world"
+	err = tc.Check("easeprobe hello world")
 	assert.NotNil(t, err)
 
-	err = CheckOutput("hello", "world", "easeprobe hello world")
+	tc.Contain = ""
+	tc.NotContain = "world"
+	err = tc.Check("easeprobe hello world")
 	assert.NotNil(t, err)
 
-	err = CheckOutput("good", "bad", "easeprobe hello world")
+	tc.Contain = "hello"
+	tc.NotContain = ""
+	err = tc.Check("easeprobe hello world")
+	assert.Nil(t, err)
+
+	tc.Contain = "good"
+	tc.NotContain = ""
+	err = tc.Check("easeprobe hello world")
+	assert.NotNil(t, err)
+
+	tc.Contain = ""
+	tc.NotContain = "bad"
+	err = tc.Check("easeprobe hello world")
+	assert.Nil(t, err)
+
+	tc.Contain = "good"
+	tc.NotContain = "bad"
+	err = tc.Check("easeprobe hello world")
 	assert.NotNil(t, err)
 }
 
+func testRegExpHelper(t *testing.T, regExp string, str string, match bool) {
+	tc := TextChecker{RegExp: true}
+	tc.Contain = regExp
+	tc.Config()
+	if match {
+		assert.Nil(t, tc.CheckRegExp(str))
+	} else {
+		assert.NotNil(t, tc.CheckRegExp(str))
+	}
+
+	tc.Contain = ""
+	tc.NotContain = regExp
+	tc.Config()
+	if match {
+		assert.NotNil(t, tc.CheckRegExp(str))
+	} else {
+		assert.Nil(t, tc.CheckRegExp(str))
+	}
+}
+
+func TestCheckRegExp(t *testing.T) {
+
+	word := `word[0-9]+`
+	testRegExpHelper(t, word, "word word10 word", true)
+	testRegExpHelper(t, word, "word word word", false)
+
+	time := "[0-9]?[0-9]:[0-9][0-9]"
+	testRegExpHelper(t, time, "easeprobe hello world 12:34", true)
+	testRegExpHelper(t, time, "easeprobe hello world 1234", false)
+
+	html := `<\/?[\w\s]*>|<.+[\W]>`
+	testRegExpHelper(t, html, "<p>test hello world </p>", true)
+	testRegExpHelper(t, html, "test hello world", false)
+
+	or := `word1|word2`
+	testRegExpHelper(t, or, "word1 easeprobe word2", true)
+	testRegExpHelper(t, or, "word2 easeprobe word1", true)
+	testRegExpHelper(t, or, "word3 easeprobe word1", true)
+	testRegExpHelper(t, or, "word2 easeprobe word3", true)
+	testRegExpHelper(t, or, "word easeprobe word3", false)
+	testRegExpHelper(t, or, "word easeprobe hello world", false)
+
+	unsupported := "(?=.*word1)(?=.*word2)"
+	tc := TextChecker{RegExp: true}
+	tc.Contain = unsupported
+	err := tc.Config()
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), "invalid or unsupported Perl syntax")
+
+	tc.Contain = ""
+	tc.NotContain = unsupported
+	err = tc.Config()
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), "invalid or unsupported Perl syntax")
+
+}
+
+func TestTextChecker(t *testing.T) {
+	checker := TextChecker{
+		Contain:    "hello",
+		NotContain: "",
+		RegExp:     false,
+	}
+	checker.Config()
+	assert.Nil(t, checker.Check("hello world"))
+	assert.Contains(t, checker.String(), "Text Mode")
+
+	checker = TextChecker{
+		Contain:    "[0-9]+$",
+		NotContain: "",
+		RegExp:     true,
+	}
+	checker.Config()
+	assert.Nil(t, checker.Check("hello world 2022"))
+	assert.Contains(t, checker.String(), "RegExp Mode")
+
+	checker = TextChecker{
+		Contain:    "",
+		NotContain: `<\/?[\w\s]*>|<.+[\W]>`,
+		RegExp:     true,
+	}
+	checker.Config()
+	assert.NotNil(t, checker.Check("<p>test hello world </p>"))
+}
+
 func TestCheckEmpty(t *testing.T) {
 	assert.Equal(t, "a", CheckEmpty("a"))
 	assert.Equal(t, "empty", CheckEmpty("    "))

probe/http/http.go

@@ -45,8 +45,9 @@ type HTTP struct {
 	Method            string            `yaml:"method,omitempty"`
 	Headers           map[string]string `yaml:"headers,omitempty"`
 	Body              string            `yaml:"body,omitempty"`
-	Contain           string            `yaml:"contain,omitempty"`
-	NotContain        string            `yaml:"not_contain,omitempty"`
+
+	// Output Text Checker
+	probe.TextChecker `yaml:",inline"`
 
 	// Option - HTTP Basic Auth Credentials
 	User string `yaml:"username,omitempty"`
@@ -136,6 +137,10 @@ func (h *HTTP) Config(gConf global.ProbeSettings) error {
 	}
 	h.SuccessCode = codeRange
 
+	if err := h.TextChecker.Config(); err != nil {
+		return err
+	}
+
 	h.metrics = newMetrics(kind, tag)
 
 	log.Debugf("[%s / %s] configuration: %+v", h.ProbeKind, h.ProbeName, h)
@@ -197,7 +202,9 @@ func (h *HTTP) DoProbe() (bool, string) {
 	}
 
 	message := fmt.Sprintf("HTTP Status Code is %d", resp.StatusCode)
-	if err := probe.CheckOutput(h.Contain, h.NotContain, string(response)); err != nil {
+
+	log.Debugf("[%s / %s] - %s", h.ProbeKind, h.ProbeName, h.TextChecker.String())
+	if err := h.Check(string(response)); err != nil {
 		log.Errorf("[%s / %s] - %v", h.ProbeKind, h.ProbeName, err)
 		message += fmt.Sprintf(". Error: %v", err)
 		return false, message

probe/http/http_test.go

@@ -30,6 +30,7 @@ import (
 
 	"bou.ke/monkey"
 	"github.com/megaease/easeprobe/global"
+	"github.com/megaease/easeprobe/probe"
 	"github.com/megaease/easeprobe/probe/base"
 	"github.com/stretchr/testify/assert"
 )
@@ -41,10 +42,12 @@ func createHTTP() *HTTP {
 		ContentEncoding: "text/json",
 		Headers:         map[string]string{"header1": "value1", "header2": "value2"},
 		Body:            "{ \"key1\": \"value1\", \"key2\": \"value2\" }",
-		Contain:         "good",
-		NotContain:      "bad",
-		User:            "user",
-		Pass:            "pass",
+		TextChecker: probe.TextChecker{
+			Contain:    "good",
+			NotContain: "bad",
+		},
+		User: "user",
+		Pass: "pass",
 		TLS: global.TLS{
 			CA:   "ca.crt",
 			Cert: "cert.crt",
@@ -82,6 +85,29 @@ func TestHTTPConfig(t *testing.T) {
 	monkey.UnpatchAll()
 }
 
+func TestTextCheckerConfig(t *testing.T) {
+	h := createHTTP()
+	h.TextChecker = probe.TextChecker{
+		Contain:    "",
+		NotContain: "",
+		RegExp:     true,
+	}
+	h.TLS = global.TLS{}
+
+	err := h.Config(global.ProbeSettings{})
+	assert.NoError(t, err)
+
+	h.Contain = `[a-zA-z]\d+`
+	err = h.Config(global.ProbeSettings{})
+	assert.NoError(t, err)
+	assert.Equal(t, `[a-zA-z]\d+`, h.TextChecker.Contain)
+
+	h.NotContain = `(?=.*word1)(?=.*word2)`
+	err = h.Config(global.ProbeSettings{})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "invalid or unsupported Perl syntax")
+}
+
 func TestHTTPDoProbe(t *testing.T) {
 	// clear request
 	h := createHTTP()