Skip to content

Commit

Permalink
feat: Add a command to generate JSON Schema (megaease#213)
Browse files Browse the repository at this point in the history
* add json schema generator

* go fmt

* remove the comma in the description

* escape the comma

* more fix

* documatation

* fix the title error

* fix the format enum  error

* add the jsonschema issue in comment

* Apply suggestions from code review

Co-authored-by: Pantelis Roditis <[email protected]>

* Update docs/Manual.md

Co-authored-by: Pantelis Roditis <[email protected]>

Co-authored-by: Pantelis Roditis <[email protected]>
Co-authored-by: Kun Zhao <[email protected]>
  • Loading branch information
3 people authored Sep 13, 2022
1 parent 2aed1a1 commit 6c3c65f
Show file tree
Hide file tree
Showing 46 changed files with 2,302 additions and 220 deletions.
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,8 @@ settings:
interval: 1m # probe every minute for all probes
```
You can check the [EaseProbe JSON Schema](./docs/Manual.md#81-easeprobe-json-schema) section to use a JSON Scheme file to make your life easier when you edit the configuration file.
## 2.3 Run
You can run the following command to start EaseProbe once built
Expand Down
10 changes: 10 additions & 0 deletions cmd/easeprobe/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ func main() {

dryNotify := flag.Bool("d", os.Getenv("PROBE_DRY") == "true", "dry notification mode")
yamlFile := flag.String("f", getEnvOrDefault("PROBE_CONFIG", "config.yaml"), "configuration file")
jsonSchema := flag.Bool("j", false, "show JSON schema")
version := flag.Bool("v", false, "prints version")
flag.Parse()

Expand All @@ -84,6 +85,15 @@ func main() {
os.Exit(0)
}

if *jsonSchema {
schema, err := conf.JSONSchema()
if err != nil {
log.Fatalf("failed to show JSON schema: %v", err)
}
fmt.Println(schema)
os.Exit(0)
}

c, err := conf.New(yamlFile)
if err != nil {
log.Errorln("Fatal: Cannot read the YAML configuration file!")
Expand Down
101 changes: 67 additions & 34 deletions conf/conf.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package conf

import (
"encoding/json"
"io/ioutil"
httpClient "net/http"
netUrl "net/url"
Expand All @@ -41,6 +42,7 @@ import (
"github.com/megaease/easeprobe/probe/tcp"
"github.com/megaease/easeprobe/probe/tls"

"github.com/invopop/jsonschema"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)
Expand Down Expand Up @@ -86,60 +88,91 @@ func (s *Schedule) UnmarshalYAML(unmarshal func(interface{}) error) error {

// Notify is the settings of notification
type Notify struct {
Retry global.Retry `yaml:"retry"`
Dry bool `yaml:"dry"`
Retry global.Retry `yaml:"retry" json:"retry,omitempty" jsonschema:"title=retry,description=the retry settings"`
Dry bool `yaml:"dry" json:"dry,omitempty" jsonschema:"title=dry,description=set true to make the notification dry run and will not be sent the message,default=false"`
}

// Probe is the settings of prober
type Probe struct {
Interval time.Duration `yaml:"interval"`
Timeout time.Duration `yaml:"timeout"`
Interval time.Duration `yaml:"interval" json:"interval,omitempty" jsonschema:"type=string,format=duration,title=Probe Interval,description=the interval of probe,default=1m"`
Timeout time.Duration `yaml:"timeout" json:"timeout,omitempty" jsonschema:"type=string,format=duration,title=Probe Timeout,description=the timeout of probe,default=30s"`
}

// SLAReport is the settings for SLA report
type SLAReport struct {
Schedule Schedule `yaml:"schedule"`
Time string `yaml:"time"`
Debug bool `yaml:"debug"`
DataFile string `yaml:"data"`
Backups int `yaml:"backups"`
Channels []string `yaml:"channels"`
Schedule Schedule `yaml:"schedule" json:"schedule" jsonschema:"type=string,enum=none,enum=hourly,enum=daily,enum=weekly,enum=monthly,title=Schedule,description=the schedule of SLA report"`
Time string `yaml:"time" json:"time,omitempty" jsonschema:"format=time,title=Time,description=the time of SLA report need to send out,example=23:59:59+08:00"`
Debug bool `yaml:"debug" json:"debug,omitempty" jsonschema:"title=Debug,description=if true the SLA report will be printed to stdout,default=false"`
DataFile string `yaml:"data" json:"data,omitempty" jsonschema:"title=Data File,description=the data file of SLA report, absolute path"`
Backups int `yaml:"backups" json:"backups,omitempty" jsonschema:"title=Backups,description=the number of backups of SLA report,default=5"`
Channels []string `yaml:"channels" json:"channels,omitempty" jsonschema:"title=Channels,description=the channels of SLA report"`
}

// HTTPServer is the settings of http server
type HTTPServer struct {
IP string `yaml:"ip"`
Port string `yaml:"port"`
AutoRefreshTime time.Duration `yaml:"refresh"`
AccessLog Log `yaml:"log"`
IP string `yaml:"ip" json:"ip" jsonschema:"title=Web Server IP,description=the local ip address of the http server need to listen on,example=0.0.0.0"`
Port string `yaml:"port" json:"port" jsonschema:"type=integer,title=Web Server Port,description=port of the http server,default=8181"`
AutoRefreshTime time.Duration `yaml:"refresh" json:"refresh,omitempty" jsonschema:"type=string,title=Auto Refresh Time,description=auto refresh time of the http server,example=5s"`
AccessLog Log `yaml:"log" json:"log,omitempty" jsonschema:"title=Access Log,description=access log of the http server"`
}

// Settings is the EaseProbe configuration
type Settings struct {
Name string `yaml:"name"`
IconURL string `yaml:"icon"`
PIDFile string `yaml:"pid"`
Log Log `yaml:"log"`
TimeFormat string `yaml:"timeformat"`
TimeZone string `yaml:"timezone"`
Probe Probe `yaml:"probe"`
Notify Notify `yaml:"notify"`
SLAReport SLAReport `yaml:"sla"`
HTTPServer HTTPServer `yaml:"http"`
Name string `yaml:"name" json:"name,omitempty" jsonschema:"title=EaseProbe Name,description=The name of the EaseProbe instance,default=EaseProbe"`
IconURL string `yaml:"icon" json:"icon,omitempty" jsonschema:"title=Icon URL,description=The URL of the icon of the EaseProbe instance"`
PIDFile string `yaml:"pid" json:"pid,omitempty" jsonschema:"title=PID File,description=The PID file of the EaseProbe instance ('-' means no PID file)"`
Log Log `yaml:"log" json:"log,omitempty" jsonschema:"title=EaseProbe Log,description=The log settings of the EaseProbe instance"`
TimeFormat string `yaml:"timeformat" json:"timeformat,omitempty" jsonschema:"title=Time Format,description=The time format of the EaseProbe instance,default=2006-01-02 15:04:05Z07:00"`
TimeZone string `yaml:"timezone" json:"timezone,omitempty" jsonschema:"title=Time Zone,description=The time zone of the EaseProbe instance,example=Asia/Shanghai,example=Europe/Berlin,default=UTC"`
Probe Probe `yaml:"probe" json:"probe,omitempty" jsonschema:"title=Probe Settings,description=The global probe settings of the EaseProbe instance"`
Notify Notify `yaml:"notify" json:"notify,omitempty" jsonschema:"title=Notify Settings,description=The global notify settings of the EaseProbe instance"`
SLAReport SLAReport `yaml:"sla" json:"sla,omitempty" jsonschema:"title=SLA Report Settings,description=The SLA report settings of the EaseProbe instance"`
HTTPServer HTTPServer `yaml:"http" json:"http,omitempty" jsonschema:"title=HTTP Server Settings,description=The HTTP server settings of the EaseProbe instance"`
}

// Conf is Probe configuration
type Conf struct {
Version string `yaml:"version"`
HTTP []http.HTTP `yaml:"http"`
TCP []tcp.TCP `yaml:"tcp"`
Shell []shell.Shell `yaml:"shell"`
Client []client.Client `yaml:"client"`
SSH ssh.SSH `yaml:"ssh"`
TLS []tls.TLS `yaml:"tls"`
Host host.Host `yaml:"host"`
Notify notify.Config `yaml:"notify"`
Settings Settings `yaml:"settings"`
Version string `yaml:"version" json:"version,omitempty" jsonschema:"title=Version,description=Version of the EaseProbe configuration"`
HTTP []http.HTTP `yaml:"http" json:"http,omitempty" jsonschema:"title=HTTP Probe,description=HTTP Probe Configuration"`
TCP []tcp.TCP `yaml:"tcp" json:"tcp,omitempty" jsonschema:"title=TCP Probe,description=TCP Probe Configuration"`
Shell []shell.Shell `yaml:"shell" json:"shell,omitempty" jsonschema:"title=Shell Probe,description=Shell Probe Configuration"`
Client []client.Client `yaml:"client" json:"client,omitempty" jsonschema:"title=Native Client Probe,description=Native Client Probe Configuration"`
SSH ssh.SSH `yaml:"ssh" json:"ssh,omitempty" jsonschema:"title=SSH Probe,description=SSH Probe Configuration"`
TLS []tls.TLS `yaml:"tls" json:"tls,omitempty" jsonschema:"title=TLS Probe,description=TLS Probe Configuration"`
Host host.Host `yaml:"host" json:"host,omitempty" jsonschema:"title=Host Probe,description=Host Probe Configuration"`
Notify notify.Config `yaml:"notify" json:"notify,omitempty" jsonschema:"title=Notification,description=Notification Configuration"`
Settings Settings `yaml:"settings" json:"settings,omitempty" jsonschema:"title=Global Settings,description=EaseProbe Global configuration"`
}

// JSONSchema return the json schema of the configuration
func JSONSchema() (string, error) {
r := new(jsonschema.Reflector)

// The Struct name could be same, but the package name is different
// For example, all of the notification plugins have the same struct name - `NotifyConfig`
// This would cause the json schema to be wrong `$ref` to the same name.
// the following code is to fix this issue by adding the package name to the struct name
// p.s. this issue has been reported in: https://github.com/invopop/jsonschema/issues/42
r.Namer = func(t reflect.Type) string {
name := t.Name()
if t.Kind() == reflect.Struct {
v := reflect.New(t)
vt := v.Elem().Type()
name = vt.PkgPath() + "/" + vt.Name()
name = strings.TrimPrefix(name, "github.com/megaease/easeprobe/")
name = strings.ReplaceAll(name, "/", "_")
log.Debugf("The struct name has been replaced [%s ==> %s]", t.Name(), name)
}
return name
}

schema := r.Reflect(&Conf{})

resBytes, err := json.MarshalIndent(schema, "", " ")
if err != nil {
return "", err
}
return string(resBytes), nil
}

// Check if string is a url
Expand Down
15 changes: 15 additions & 0 deletions conf/conf_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package conf

import (
"encoding/json"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -735,3 +736,17 @@ func TestEmptyProbes(t *testing.T) {
os.RemoveAll(file)
os.RemoveAll("data")
}

func TestJSONSchema(t *testing.T) {
schema, err := JSONSchema()
assert.Nil(t, err)
assert.NotEmpty(t, schema)

monkey.Patch(json.MarshalIndent, func(v interface{}, prefix, indent string) ([]byte, error) {
return nil, fmt.Errorf("error")
})
schema, err = JSONSchema()
assert.NotNil(t, err)
assert.Empty(t, schema)
monkey.UnpatchAll()
}
20 changes: 10 additions & 10 deletions conf/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,16 @@ func (l *LogLevel) GetLevel() log.Level {

// Log is the log settings
type Log struct {
Level LogLevel `yaml:"level"`
File string `yaml:"file"`
SelfRotate bool `yaml:"self_rotate"`
MaxSize int `yaml:"size"`
MaxAge int `yaml:"age"`
MaxBackups int `yaml:"backups"`
Compress bool `yaml:"compress"`
Writer io.Writer `yaml:"-"`
Logger *log.Logger `yaml:"-"`
IsStdout bool `yaml:"-"`
Level LogLevel `yaml:"level" json:"level,omitempty" jsonschema:"type=string,enum=debug,enum=info,enum=warn,enum=error,enum=fatal,enum=panic,title=Log Level,description=Log Level"`
File string `yaml:"file" json:"file,omitempty" jsonschema:"title=Log File,description=the file to save the log"`
SelfRotate bool `yaml:"self_rotate" json:"self_rotate,omitempty" jsonschema:"title=Self Rotate,description=whether to rotate the log file by self"`
MaxSize int `yaml:"size" json:"size,omitempty" jsonschema:"title=Max Size,description=the max size of the log file. the log file will be rotated if the size is larger than this value"`
MaxAge int `yaml:"age" json:"age,omitempty" jsonschema:"title=Max Age,description=the max age of the log file. the log file will be rotated if the age is larger than this value"`
MaxBackups int `yaml:"backups" json:"backups,omitempty" jsonschema:"title=Max Backups,description=the max backups of the log file. the rotated log file will be deleted if the backups is larger than this value"`
Compress bool `yaml:"compress" json:"compress,omitempty" jsonschema:"title=Compress,description=whether to compress the rotated log file"`
Writer io.Writer `yaml:"-" json:"-"`
Logger *log.Logger `yaml:"-" json:"-"`
IsStdout bool `yaml:"-" json:"-"`
}

// NewLog create a new Log
Expand Down
27 changes: 26 additions & 1 deletion docs/Manual.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ EaseProbe is a simple, standalone, and lightweight tool that can do health/statu
- [7.7 Native Client Probe Configuration](#77-native-client-probe-configuration)
- [7.8 Notification Configuration](#78-notification-configuration)
- [7.9 Global Setting Configuration](#79-global-setting-configuration)
- [8. Tools](#8-tools)
- [8.1 EaseProbe JSON Schema](#81-easeprobe-json-schema)



Expand Down Expand Up @@ -1329,4 +1331,27 @@ settings:
# check the following link to see the time zone list
# https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
timezone: "America/New_York" # default: UTC
```
```
# 8. Tools
## 8.1 EaseProbe JSON Schema
We have a JSON schema that can be used to validate your EaseProbe configuration. The schema can be found at [resources/schema.json](https://raw.githubusercontent.com/megaease/easeprobe/main/resources/schema.json).
The schema file can be generated at any time by running the following command:
```bash
$ easeprobe -j > schema.json
```
In order to use the schema with VSCode for validating your configuration, you need to install the [YAML extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) and add the following configuration to your `settings.json` file:
```json
{
"yaml.schemas": {
"https://raw.githubusercontent.com/megaease/easeprobe/main/resources/schema.json": [
"/path/to/config.yaml"
]
}
}
```
20 changes: 10 additions & 10 deletions eval/eval.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ import (

// Variable is the variable type
type Variable struct {
Name string `yaml:"name"`
Type VarType `yaml:"type"`
Query string `yaml:"query"`
Value interface{}
Name string `yaml:"name" json:"name" jsonschema:"required,title=Variable Name,description=Variable Name"`
Type VarType `yaml:"type" json:"type" jsonschema:"required,type=string,enum=int,enum=string,enum=bool,enum=float,enum=bool,enum=time,enum=duration,title=Variable Type,description=Variable Type"`
Query string `yaml:"query" json:"query" jsonschema:"required,title=Query,description=XPath/Regex Expression to extract the value"`
Value interface{} `yaml:"-" json:"-"`
}

// NewVariable is the function to create a variable
Expand All @@ -46,12 +46,12 @@ func NewVariable(name string, t VarType, query string) *Variable {

// Evaluator is the structure of evaluator
type Evaluator struct {
Variables []Variable `yaml:"variables"`
DocType DocType `yaml:"doc"`
Expression string `yaml:"expression"`
Document string `yaml:"-"`
Extractor Extractor `yaml:"-"`
EvalFuncs map[string]govaluate.ExpressionFunction `yaml:"-"`
Variables []Variable `yaml:"variables,omitempty" json:"variables,omitempty" jsonschema:"title=Variables Definition,description=define the variables used in the expression"`
DocType DocType `yaml:"doc" json:"doc" jsonschema:"required,type=string,enum=html,enum=xml,enum=json,enum=text,title=Document Type,description=Document Type"`
Expression string `yaml:"expression" json:"expression" jsonschema:"required,title=Expression,description=Expression need to be evaluated"`
Document string `yaml:"-" json:"-"`
Extractor Extractor `yaml:"-" json:"-"`
EvalFuncs map[string]govaluate.ExpressionFunction `yaml:"-" json:"-"`
}

// NewEvaluator is the function to create a evaluator
Expand Down
12 changes: 6 additions & 6 deletions global/global.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,16 +94,16 @@ const (

// Retry is the settings of retry
type Retry struct {
Times int `yaml:"times"`
Interval time.Duration `yaml:"interval"`
Times int `yaml:"times" json:"times,omitempty" jsonschema:"title=Retry Times,description=how many times need to retry,minimum=1"`
Interval time.Duration `yaml:"interval" json:"interval,omitempty" jsonschema:"type=string,format=duration,title=Retry Interval,description=the interval between each retry"`
}

// TLS is the configuration for TLS files
type TLS struct {
CA string `yaml:"ca"`
Cert string `yaml:"cert"`
Key string `yaml:"key"`
Insecure bool `yaml:"insecure"`
CA string `yaml:"ca" json:"ca,omitempty" jsonschema:"title=CA File,description=the CA file path"`
Cert string `yaml:"cert" json:"cert,omitempty" jsonschema:"title=Cert File,description=the Cert file path"`
Key string `yaml:"key" json:"key,omitempty" jsonschema:"title=Key File,description=the Key file path"`
Insecure bool `yaml:"insecure" json:"insecure,omitempty" jsonschema:"title=Insecure,description=whether to skip the TLS verification"`
}

// The normalize() function logic as below:
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ require (
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect
github.com/invopop/jsonschema v0.6.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.15.7 // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,11 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/invopop/jsonschema v0.6.0 h1:8e+xY8ZEn8gDHUYylSlLHy22P+SLeIRIHv3nM3hCbmY=
github.com/invopop/jsonschema v0.6.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
Expand Down Expand Up @@ -260,6 +264,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
Expand Down
Loading

0 comments on commit 6c3c65f

Please sign in to comment.