Implement static test runner

README.md

@@ -150,23 +150,29 @@ _Context: package_
 
 Use this command to run tests on a package. Currently, there are two types of tests available.
 
+#### Asset Loading Tests
+
+These tests ensure that all the Elasticsearch and Kibana assets defined by your package get loaded up as expected.
+
+For details on how to run asset loading tests for a package, see the [HOWTO guide](docs/howto/asset_testing.md).
+
 #### Pipeline Tests
 
 These tests allow you to exercise any Ingest Node Pipelines defined by your packages.
 
 For details on how to configure and run pipeline tests for a package, see the [HOWTO guide](docs/howto/pipeline_testing.md).
 
-#### System Tests
+#### Static Tests
 
-These tests allow you to test a package's ability to ingest data end-to-end. 
+These tests allow you to verify if all static resources of the package are valid, e.g. if all fields of the `sample_event.json` are documented.
 
-For details on how to configure and run system tests for a package, see the [HOWTO guide](docs/howto/system_testing.md).
+For details on how to run static tests for a package, see the [HOWTO guide](docs/howto/static_testing.md).
 
-#### Asset Loading Tests
+#### System Tests
 
-These tests ensure that all the Elasticsearch and Kibana assets defined by your package get loaded up as expected.
+These tests allow you to test a package's ability to ingest data end-to-end. 
 
-For details on how to run asset loading tests for a package, see the [HOWTO guide](docs/howto/asset_testing.md).
+For details on how to configure and run system tests for a package, see the [HOWTO guide](docs/howto/system_testing.md).
 
 
 ### `elastic-package uninstall`

cmd/testrunner.go

@@ -24,10 +24,16 @@ import (
 
 const testLongDescription = `Use this command to run tests on a package. Currently, the following types of tests are available:
 
+Asset Loading Tests
+These tests allow you to exercise installing a package to ensure that its assets are loaded into Elasticsearch and Kibana as expected.
+
 Pipeline Tests
 These tests allow you to exercise any Ingest Node Pipelines defined by your packages.
 For details on how to configure pipeline test for a package, review the HOWTO guide (see: https://github.com/elastic/elastic-package/blob/master/docs/howto/pipeline_testing.md).
 
+Static Tests
+These tests allow you to verify if all static resources of the package are valid, e.g. if all fields of the sample_event.json are documented.
+
 System Tests
 These tests allow you to test a package's ability to ingest data end-to-end.
 For details on how to configure amd run system tests, review the HOWTO guide (see: https://github.com/elastic/elastic-package/blob/master/docs/howto/system_testing.md).
@@ -126,9 +132,16 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command
 				}
 			}
 
-			testFolders, err = testrunner.FindTestFolders(packageRootPath, dataStreams, testType)
-			if err != nil {
-				return errors.Wrap(err, "unable to determine test folder paths")
+			if runner.TestFolderRequired() {
+				testFolders, err = testrunner.FindTestFolders(packageRootPath, dataStreams, testType)
+				if err != nil {
+					return errors.Wrap(err, "unable to determine test folder paths")
+				}
+			} else {
+				testFolders, err = testrunner.AssumeTestFolders(packageRootPath, dataStreams, testType)
+				if err != nil {
+					return errors.Wrap(err, "unable to assume test folder paths")
+				}
 			}
 
 			if failOnMissing && len(testFolders) == 0 {

docs/howto/asset_testing.md

@@ -1,4 +1,4 @@
-# HOWTO: Writing asset loading tests for a package
+# HOWTO: Running asset loading tests for a package
 
 ## Introduction
 

docs/howto/static_testing.md

@@ -0,0 +1,28 @@
+# HOWTO: Running static tests for a package
+
+## Introduction
+
+Static tests allow you to verify if all static resources of the package are valid, e.g. are all fields of the `sample_event.json` documented.
+They don't require any additional configuration (unless you would like to skip them).
+
+## Coverage
+
+Static tests cover the following resources:
+
+1. Sample event for a data stream - verification if the file uses only documented fields. 
+
+## Running static tests
+
+Static tests don't require the Elastic stack to be up and running. Simply navigate to the package's root folder
+(or any sub-folder under it) and run the following command.
+
+```
+elastic-package test static
+```
+
+If you want to run pipeline tests for **specific data streams** in a package, navigate to the package's root folder
+(or any sub-folder under it) and run the following command.
+
+```
+elastic-package test static --data-streams <data stream 1>[,<data stream 2>,...]
+```

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.9.0
 	github.com/elastic/go-licenser v0.3.1
 	github.com/elastic/go-ucfg v0.8.3
-	github.com/elastic/package-spec/code/go v0.0.0-20210222225810-541116689fca
+	github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.1.0
 	github.com/go-openapi/strfmt v0.19.6 // indirect

go.sum

@@ -87,8 +87,8 @@ github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrE
 github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ=
 github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo=
 github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
-github.com/elastic/package-spec/code/go v0.0.0-20210222225810-541116689fca h1:dEAP2TACrsICd/pMzFlEx2llqqWcB68CjJRfnMhQL8E=
-github.com/elastic/package-spec/code/go v0.0.0-20210222225810-541116689fca/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8=
+github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3 h1:SX1mNX3H7+NQ+88eBu8nsWJGCLUQnbGdxh/qpS/6Wq8=
+github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

internal/fields/validate.go

@@ -22,13 +22,23 @@ import (
 
 // Validator is responsible for fields validation.
 type Validator struct {
-	schema               []FieldDefinition
-	numericKeywordFields map[string]struct{}
+	schema []FieldDefinition
+
+	defaultNumericConversion bool
+	numericKeywordFields     map[string]struct{}
 }
 
 // ValidatorOption represents an optional flag that can be passed to  CreateValidatorForDataStream.
 type ValidatorOption func(*Validator) error
 
+// WithDefaultNumericConversion configures the validator to accept defined keyword (or constant_keyword) fields as numeric-type.
+func WithDefaultNumericConversion() ValidatorOption {
+	return func(v *Validator) error {
+		v.defaultNumericConversion = true
+		return nil
+	}
+}
+
 // WithNumericKeywordFields configures the validator to accept specific fields to have numeric-type
 // while defined as keyword or constant_keyword.
 func WithNumericKeywordFields(fields []string) ValidatorOption {
@@ -155,7 +165,8 @@ func (v *Validator) validateScalarElement(key string, val interface{}) error {
 	}
 
 	// Convert numeric keyword fields to string for validation.
-	if _, found := v.numericKeywordFields[key]; found && isNumericKeyword(*definition, val) {
+	_, found := v.numericKeywordFields[key]
+	if (found || v.defaultNumericConversion) && isNumericKeyword(*definition, val) {
 		val = fmt.Sprintf("%q", val)
 	}
 

internal/testrunner/runners/asset/runner.go

@@ -142,6 +142,10 @@ func (r *runner) TearDown() error {
 	return nil
 }
 
+func (r *runner) TestFolderRequired() bool {
+	return false
+}
+
 func findActualAsset(actualAssets []packages.Asset, expectedAsset packages.Asset) bool {
 	for _, a := range actualAssets {
 		if a.Type == expectedAsset.Type && a.ID == expectedAsset.ID {

internal/testrunner/runners/pipeline/runner.go

@@ -32,6 +32,10 @@ type runner struct {
 	options testrunner.TestOptions
 }
 
+func (r *runner) TestFolderRequired() bool {
+	return true
+}
+
 // Type returns the type of test that can be run by this test runner.
 func (r *runner) Type() testrunner.TestType {
 	return TestType

internal/testrunner/runners/runners.go

@@ -8,5 +8,6 @@ import (
 	// Registered test runners
 	_ "github.com/elastic/elastic-package/internal/testrunner/runners/asset"
 	_ "github.com/elastic/elastic-package/internal/testrunner/runners/pipeline"
+	_ "github.com/elastic/elastic-package/internal/testrunner/runners/static"
 	_ "github.com/elastic/elastic-package/internal/testrunner/runners/system"
 )

internal/testrunner/runners/static/runner.go

@@ -0,0 +1,130 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package static
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-package/internal/fields"
+	"github.com/elastic/elastic-package/internal/logger"
+	"github.com/elastic/elastic-package/internal/testrunner"
+)
+
+const sampleEventJSON = "sample_event.json"
+
+type runner struct {
+	options testrunner.TestOptions
+}
+
+var _ testrunner.TestRunner = new(runner)
+
+func init() {
+	testrunner.RegisterRunner(&runner{})
+}
+
+const (
+	// TestType defining asset loading tests
+	TestType testrunner.TestType = "static"
+)
+
+func (r runner) Type() testrunner.TestType {
+	return TestType
+}
+
+func (r runner) String() string {
+	return "static files"
+}
+
+func (r runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, error) {
+	r.options = options
+	return r.run()
+}
+
+func (r runner) run() ([]testrunner.TestResult, error) {
+	result := testrunner.NewResultComposer(testrunner.TestResult{
+		TestType:   TestType,
+		Package:    r.options.TestFolder.Package,
+		DataStream: r.options.TestFolder.DataStream,
+	})
+
+	testConfig, err := newConfig(r.options.TestFolder.Path)
+	if err != nil {
+		return result.WithError(errors.Wrap(err, "unable to load asset loading test config file"))
+	}
+
+	if testConfig != nil && testConfig.Skip != nil {
+		logger.Warnf("skipping %s test for %s: %s (details: %s)",
+			TestType, r.options.TestFolder.Package,
+			testConfig.Skip.Reason, testConfig.Skip.Link.String())
+		return result.WithSkip(testConfig.Skip)
+	}
+
+	var results []testrunner.TestResult
+	results = append(results, r.verifySampleEvent()...)
+	return results, nil
+}
+
+func (r runner) verifySampleEvent() []testrunner.TestResult {
+	dataStreamPath := filepath.Join(r.options.PackageRootPath, "data_stream", r.options.TestFolder.DataStream)
+	sampleEventPath := filepath.Join(dataStreamPath, sampleEventJSON)
+	_, err := os.Stat(sampleEventPath)
+	if os.IsNotExist(err) {
+		return []testrunner.TestResult{} // nothing to succeed, nothing to skip
+	}
+
+	resultComposer := testrunner.NewResultComposer(testrunner.TestResult{
+		Name:       "Verify " + sampleEventJSON,
+		TestType:   TestType,
+		Package:    r.options.TestFolder.Package,
+		DataStream: r.options.TestFolder.DataStream,
+	})
+
+	if err != nil {
+		results, _ := resultComposer.WithError(errors.Wrap(err, "stat file failed"))
+		return results
+	}
+
+	fieldsValidator, err := fields.CreateValidatorForDataStream(
+		dataStreamPath,
+		fields.WithDefaultNumericConversion())
+	if err != nil {
+		results, _ := resultComposer.WithError(errors.Wrap(err, "creating fields validator for data stream failed"))
+		return results
+	}
+
+	content, err := ioutil.ReadFile(sampleEventPath)
+	if err != nil {
+		results, _ := resultComposer.WithError(errors.Wrap(err, "can't read file"))
+		return results
+	}
+
+	multiErr := fieldsValidator.ValidateDocumentBody(content)
+	if len(multiErr) > 0 {
+		results, _ := resultComposer.WithError(testrunner.ErrTestCaseFailed{
+			Reason:  "one or more errors found in document",
+			Details: multiErr.Error(),
+		})
+		return results
+	}
+
+	results, _ := resultComposer.WithSuccess()
+	return results
+}
+
+func (r runner) TearDown() error {
+	return nil // it's a static test runner, no state is stored
+}
+
+func (r runner) CanRunPerDataStream() bool {
+	return true
+}
+
+func (r *runner) TestFolderRequired() bool {
+	return false
+}

internal/testrunner/runners/static/test_config.go

@@ -0,0 +1,47 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package static
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/elastic/go-ucfg"
+	"github.com/elastic/go-ucfg/yaml"
+	"github.com/pkg/errors"
+
+	"github.com/elastic/elastic-package/internal/testrunner"
+)
+
+type testConfig struct {
+	testrunner.SkippableConfig `config:",inline"`
+}
+
+func newConfig(staticTestFolderPath string) (*testConfig, error) {
+	configFilePath := filepath.Join(staticTestFolderPath, "config.yml")
+
+	// Test configuration file is optional for static loading tests. If it
+	// doesn't exist, we can return early.
+	if _, err := os.Stat(configFilePath); os.IsNotExist(err) {
+		return nil, nil
+	}
+
+	data, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not load static loading test configuration file: %s", configFilePath)
+	}
+
+	var c testConfig
+	cfg, err := yaml.NewConfig(data, ucfg.PathSep("."))
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to load static loading test configuration file: %s", configFilePath)
+	}
+	if err := cfg.Unpack(&c); err != nil {
+		return nil, errors.Wrapf(err, "unable to unpack static loading test configuration file: %s", configFilePath)
+	}
+
+	return &c, nil
+}

internal/testrunner/runners/system/runner.go

@@ -73,6 +73,10 @@ func (r *runner) CanRunPerDataStream() bool {
 	return true
 }
 
+func (r *runner) TestFolderRequired() bool {
+	return true
+}
+
 // Run runs the system tests defined under the given folder
 func (r *runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, error) {
 	r.options = options