refactor: replace logger, update config and implement require model

builtin/frame_defaults.js

@@ -1,5 +1,7 @@
 const name = "frame-defaults";
 const description = "Validates consistency of DefaultLocale inside FrameDefaults element (if present)";
+const xpath = require('xpath');
+const time = require('time');
 const countryCodes = [ // ISO 639-1 country codes
   "aa", "ab", "ae", "af", "ak", "am", "an", "ar", "as", "av",
   "ay", "az", "ba", "be", "bg", "bh", "bi", "bm", "bn", "bo",

builtin/stop_point_names.js

@@ -1,6 +1,7 @@
 const name = "stop-point-names";
 const description = "Make sure every ScheduledStopPoint have a name";
-const framesPath = xpath.join(".", "PublicationDelivery", "dataObjects", "CompositeFrame", "frames")
+const xpath = require('xpath');
+const framesPath = xpath.join(".", "PublicationDelivery", "dataObjects", "CompositeFrame", "frames");
 const scheduledStopPointsPath = xpath.join(framesPath, "ServiceFrame", "scheduledStopPoints", "ScheduledStopPoint");
 const namePath = xpath.join("Name");
 

cmd/validate.go

@@ -14,13 +14,14 @@ import (
 	"time"
 
 	"github.com/concreteit/greenlight"
-	"github.com/concreteit/greenlight/logger"
 	"github.com/influxdata/influxdb-client-go/v2"
 	"github.com/influxdata/influxdb-client-go/v2/api/write"
 	"github.com/shirou/gopsutil/cpu"
 	"github.com/shirou/gopsutil/mem"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 )
 
 const (
@@ -61,22 +62,19 @@ func stringsJoin(v string, o []string, joinHandler func(elem ...string) string)
 
 func init() {
 	validateCmd.Flags().StringP("builtin-scripts", "", "true", "Whether to use built in validation rules")
-	validateCmd.Flags().StringSliceP("input", "i", []string{}, "XML file, dir or archive to validate")
+	validateCmd.Flags().StringP("input", "i", "", "XML file, dir or archive to validate")
 	validateCmd.Flags().StringP("log-level", "l", "debug", "Set logger level")
-	validateCmd.Flags().StringP("log-path", "", ".", "Log file location")
+	validateCmd.Flags().BoolP("no-log", "", false, "Whether log output should be disabled")
 	validateCmd.Flags().BoolP("no-constraint", "", false, "Use lite schema validation (w/o constraint)")
-	validateCmd.Flags().StringP("report-format", "", "json", "Detailed validation report format")
-	validateCmd.Flags().StringP("report-path", "", ".", "Detail validation report file location")
+	validateCmd.Flags().StringP("result-format", "", "json", "Detailed validation result format")
+	validateCmd.Flags().BoolP("no-result", "", false, "Whether result output should be disabled")
 	validateCmd.Flags().StringP("schema", "s", "xsd/NeTEx_publication.xsd", "Use XML Schema file for validation")
 	validateCmd.Flags().StringP("scripts", "", "", "Directory or file path to look for scripts")
 	validateCmd.Flags().BoolP("telemetry", "", true, "Whether to collect and send information about execution time")
 
 	// default script paths
 	viper.SetDefault("scripts", stringsJoin("scripts", configPaths, path.Join))
 
-	// default `input` paths
-	viper.SetDefault("input", stringsJoin("documents", configPaths, path.Join))
-
 	// set paths to look for configuration file (first come, first serve)
 	for _, p := range configPaths {
 		viper.AddConfigPath(p)
@@ -95,13 +93,13 @@ func init() {
 
 	// bind from cli input
 	viper.BindPFlag("input", validateCmd.Flags().Lookup("input"))
-	viper.BindPFlag("logLevel", validateCmd.Flags().Lookup("log-level"))
+	viper.BindPFlag("output.log.level", validateCmd.Flags().Lookup("log-level"))
+	viper.BindPFlag("output.log.disabled", validateCmd.Flags().Lookup("no-log"))
+	viper.BindPFlag("output.result.format", validateCmd.Flags().Lookup("result-format"))
+	viper.BindPFlag("output.result.disabled", validateCmd.Flags().Lookup("no-result"))
 	viper.BindPFlag("schema", validateCmd.Flags().Lookup("schema"))
 	viper.BindPFlag("builtin", validateCmd.Flags().Lookup("builtin-scripts"))
 	viper.BindPFlag("scripts", validateCmd.Flags().Lookup("scripts"))
-	viper.BindPFlag("output.log.path", validateCmd.Flags().Lookup("log-path"))
-	viper.BindPFlag("output.report.format", validateCmd.Flags().Lookup("report-format"))
-	viper.BindPFlag("output.report.path", validateCmd.Flags().Lookup("report-path"))
 	viper.BindPFlag("telemetry", validateCmd.Flags().Lookup("telemetry"))
 
 	rootCmd.AddCommand(validateCmd)
@@ -160,76 +158,81 @@ func createValidationContext(input string) (*greenlight.ValidationContext, error
 	return ctx, nil
 }
 
-type Details struct {
-	Path    string                         `json:"path" xml:"path,attr"`
-	Results []*greenlight.ValidationResult `json:"results" xml:"Result"`
-}
-
 func validate(cmd *cobra.Command, args []string) {
-	client := influxdb2.NewClient(influxURL, influxToken)
-	defer client.Close()
-
-	writeAPI := client.WriteAPI(influxOrg, influxBucket)
-	logFilePath := fmt.Sprintf("%s/log-%s",
-		viper.GetString("output.log.path"),
-		time.Now().Format("20060102150405"),
-	)
-	fileOutput, err := logger.NewFileOutput(greenlight.EnvPath(logFilePath))
+	logger, err := newLogger()
 	if err != nil {
 		log.Fatal(err)
 	}
-
-	l := logger.New()
-	l.AddOutput(fileOutput)
-	if viper.GetString("logLevel") != "" {
-		l.SetLogLevel(logger.LogLevel(viper.GetString("logLevel")))
-	}
+	defer logger.Sync()
 
 	schema := viper.GetString("schema")
 	if nc, err := cmd.Flags().GetBool("no-constraint"); nc && err == nil {
 		schema = "xsd/NeTEx_publication-NoConstraint.xsd"
 	}
 	validator, err := greenlight.NewValidator(
 		greenlight.WithSchemaFile(schema),
-		greenlight.WithLogger(l),
+		greenlight.WithLogger(logger),
 		greenlight.WithBuiltinScripts(viper.GetBool("builtin")),
 		greenlight.WithScriptingPaths(viper.GetStringSlice("scripts")),
 	)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	input := viper.GetStringSlice("input")
-	if len(input) == 0 {
-		log.Fatal("no input paths defined")
+	input := viper.GetString("input")
+	if input == "" {
+		log.Fatal("no input provided")
 	}
 
-	var ctx *greenlight.ValidationContext
-	for _, path := range input {
-		c, err := createValidationContext(greenlight.EnvPath(path))
-		if err != nil {
-			fmt.Println(err)
-			return
-		}
-
-		ctx = c
-		break
+	ctx, err := createValidationContext(greenlight.EnvPath(input))
+	if err != nil {
+		log.Fatal(err)
 	}
 
-	details := []Details{}
 	validator.Validate(ctx)
 
-	buf, err := json.MarshalIndent(ctx.Results(), "", " ")
-	if err != nil {
-		fmt.Println(err)
-	} else {
-		fmt.Println(string(buf))
+	if !viper.GetBool("output.result.disabled") {
+		switch viper.GetString("output.result.format") {
+		case "json":
+			buf, err := json.MarshalIndent(ctx.Results(), "", " ")
+			if err != nil {
+				fmt.Println(err)
+			} else {
+				fmt.Println(string(buf))
+			}
+		case "plain":
+			results := ctx.Results()
+			for _, result := range results {
+				rows := result.CsvRecords(true)
+				for _, row := range rows {
+					fmt.Println(strings.Join(row, ","))
+				}
+			}
+		case "xml":
+			buf, err := xml.MarshalIndent(ctx.Results(), "", " ")
+			if err != nil {
+				fmt.Println(err)
+			} else {
+				fmt.Println(string(buf))
+			}
+		}
 	}
 
-	for _, r := range ctx.Results() {
+	if viper.GetBool("telemetry") {
+		logTelemetry(validator, ctx.Results())
+	}
+}
+
+func logTelemetry(validator *greenlight.Validator, results []*greenlight.ValidationResult) {
+	client := influxdb2.NewClient(influxURL, influxToken)
+	defer client.Close()
+
+	writeAPI := client.WriteAPI(influxOrg, influxBucket)
+
+	for _, r := range results {
 		if viper.GetBool("telemetry") {
 			p := newPoint("document")
-			p.AddField("schema_name", schema)
+			p.AddField("schema_name", validator.SchemaPath())
 			p.AddField("schema_bytes", validator.SchemaSize())
 			p.AddField("execution_time_ms", r.ExecutionTime().Milliseconds())
 			p.AddField("name", r.Name)
@@ -238,7 +241,7 @@ func validate(cmd *cobra.Command, args []string) {
 
 			for _, rule := range r.ValidationRules {
 				p := newPoint("rule")
-				p.AddField("schema_name", schema)
+				p.AddField("schema_name", validator.SchemaPath())
 				p.AddField("schema_bytes", validator.SchemaSize())
 				p.AddField("execution_time_ms", rule.ExecutionTime().Milliseconds())
 				p.AddField("document_name", r.Name)
@@ -250,42 +253,7 @@ func validate(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	details = append(details, Details{
-		Path:    ctx.Name(),
-		Results: ctx.Results(),
-	})
-
 	writeAPI.Flush()
-
-	if viper.GetString("output.report.format") != "" && viper.GetString("output.report.path") != "" {
-		filePath := fmt.Sprintf("%s/report-%s.%s",
-			viper.GetString("output.report.path"),
-			time.Now().Format("20060102150405"),
-			viper.GetString("output.report.format"),
-		)
-		f, err := os.Create(greenlight.EnvPath(filePath))
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		var enc encoder
-		switch viper.GetString("output.report.format") {
-		case "json":
-			e := json.NewEncoder(f)
-			e.SetIndent("", "  ")
-			enc = e
-		case "xml":
-			e := xml.NewEncoder(f)
-			e.Indent("  ", "    ")
-			enc = e
-		default:
-			log.Fatalf("unsupport output file format '%s'\n", viper.GetString("output.report.format"))
-		}
-
-		if err := enc.Encode(details); err != nil {
-			log.Fatal(err)
-		}
-	}
 }
 
 func newPoint(measurement string) *write.Point {
@@ -314,3 +282,23 @@ func newPoint(measurement string) *write.Point {
 
 	return p
 }
+
+func newLogger() (*zap.Logger, error) {
+	loggerOptions := []zap.Option{}
+	if viper.GetBool("output.log.disabled") {
+		loggerOptions = append(loggerOptions, zap.IncreaseLevel(zapcore.FatalLevel))
+	} else if viper.GetString("output.log.level") != "" {
+		switch viper.GetString("output.log.level") {
+		case "debug":
+			loggerOptions = append(loggerOptions, zap.IncreaseLevel(zapcore.DebugLevel))
+		case "info":
+			loggerOptions = append(loggerOptions, zap.IncreaseLevel(zapcore.InfoLevel))
+		case "warn":
+			loggerOptions = append(loggerOptions, zap.IncreaseLevel(zapcore.WarnLevel))
+		case "error":
+			loggerOptions = append(loggerOptions, zap.IncreaseLevel(zapcore.ErrorLevel))
+		}
+	}
+
+	return zap.NewDevelopment(loggerOptions...)
+}

example.config.yaml

@@ -1,21 +1,12 @@
-schema: xsd/NeTEx_publication.xsd # schema to use for validation, comes shipped with the source/container image
-logLevel: debug # default is undefined, setting this parameter disables the fancy setting, regardless of its value
-input: # where to look for documents
-  - ~/.greenlight/documents
-  - /etc/greenlight/documents
-  - /documents
-  - /greenlight/documents
-  - ./documents
+input: testdata
 output:
   log:
+    level: debug
+    disabled: false
     path: ~/.greenlight
-  report:
+  result:
     format: json # formats available are: json or xml
-    path: ~/.greenlight # where to save the file (filename format is ${path}/report-${current_date_time}.${format}
-builtin: true # whether to use builtin scripts
-scripts: # where to look for custom scripts
-  - ~/.greenlight/scripts
-  - /etc/greenlight/scripts
-  - /scripts
-  - /greenlight/scripts
-  - ./scripts
+    disabled: false
+validation:
+  schema: xsd/NeTEx_publication.xsd # schema to use for validation, comes shipped with the source/container image
+  builtin: true # whether to use builtin scripts

go.mod

@@ -10,28 +10,25 @@ require (
 	github.com/dop251/goja v0.0.0-20211207134806-acd374ca9c94
 	github.com/eclipse/paho.mqtt.golang v1.3.5
 	github.com/fhmq/hmq v0.0.0-20220119031624-bf2b91c535a8
+	github.com/gorilla/websocket v1.4.2
 	github.com/h2non/filetype v1.1.3
 	github.com/influxdata/influxdb-client-go/v2 v2.6.0
+	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
 	github.com/labstack/echo/v4 v4.6.3
 	github.com/lestrrat-go/libxml2 v0.0.0-00010101000000-000000000000
 	github.com/matoous/go-nanoid v1.5.0
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.10.1
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	go.uber.org/zap v1.21.0
 )
 
 require (
 	github.com/Shopify/sarama v1.31.0 // indirect
 	github.com/bitly/go-simplejson v0.5.0 // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
-	github.com/dgraph-io/badger/v3 v3.2103.2 // indirect
-	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
@@ -43,15 +40,10 @@ require (
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-playground/validator/v10 v10.4.1 // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/google/uuid v1.1.2 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
@@ -63,7 +55,6 @@ require (
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect
-	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c // indirect
 	github.com/labstack/gommon v0.3.1 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
@@ -92,10 +83,8 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.20.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/net v0.0.0-20220105145211-5b0dc2dfae98 // indirect
 	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect