diff --git a/README.md b/README.md index b8559a7e1d..8ae4226c9f 100644 --- a/README.md +++ b/README.md @@ -150,23 +150,29 @@ _Context: package_ Use this command to run tests on a package. Currently, there are two types of tests available. +#### Asset Loading Tests + +These tests ensure that all the Elasticsearch and Kibana assets defined by your package get loaded up as expected. + +For details on how to run asset loading tests for a package, see the [HOWTO guide](docs/howto/asset_testing.md). + #### Pipeline Tests These tests allow you to exercise any Ingest Node Pipelines defined by your packages. For details on how to configure and run pipeline tests for a package, see the [HOWTO guide](docs/howto/pipeline_testing.md). -#### System Tests +#### Static Tests -These tests allow you to test a package's ability to ingest data end-to-end. +These tests allow you to verify if all static resources of the package are valid, e.g. if all fields of the `sample_event.json` are documented. -For details on how to configure and run system tests for a package, see the [HOWTO guide](docs/howto/system_testing.md). +For details on how to run static tests for a package, see the [HOWTO guide](docs/howto/static_testing.md). -#### Asset Loading Tests +#### System Tests -These tests ensure that all the Elasticsearch and Kibana assets defined by your package get loaded up as expected. +These tests allow you to test a package's ability to ingest data end-to-end. -For details on how to run asset loading tests for a package, see the [HOWTO guide](docs/howto/asset_testing.md). +For details on how to configure and run system tests for a package, see the [HOWTO guide](docs/howto/system_testing.md). ### `elastic-package uninstall` diff --git a/cmd/testrunner.go b/cmd/testrunner.go index 6c7d80d7f7..0cb1132230 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -24,10 +24,16 @@ import ( const testLongDescription = `Use this command to run tests on a package. Currently, the following types of tests are available: +Asset Loading Tests +These tests allow you to exercise installing a package to ensure that its assets are loaded into Elasticsearch and Kibana as expected. + Pipeline Tests These tests allow you to exercise any Ingest Node Pipelines defined by your packages. For details on how to configure pipeline test for a package, review the HOWTO guide (see: https://github.com/elastic/elastic-package/blob/master/docs/howto/pipeline_testing.md). +Static Tests +These tests allow you to verify if all static resources of the package are valid, e.g. if all fields of the sample_event.json are documented. + System Tests These tests allow you to test a package's ability to ingest data end-to-end. For details on how to configure amd run system tests, review the HOWTO guide (see: https://github.com/elastic/elastic-package/blob/master/docs/howto/system_testing.md). @@ -126,9 +132,16 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command } } - testFolders, err = testrunner.FindTestFolders(packageRootPath, dataStreams, testType) - if err != nil { - return errors.Wrap(err, "unable to determine test folder paths") + if runner.TestFolderRequired() { + testFolders, err = testrunner.FindTestFolders(packageRootPath, dataStreams, testType) + if err != nil { + return errors.Wrap(err, "unable to determine test folder paths") + } + } else { + testFolders, err = testrunner.AssumeTestFolders(packageRootPath, dataStreams, testType) + if err != nil { + return errors.Wrap(err, "unable to assume test folder paths") + } } if failOnMissing && len(testFolders) == 0 { diff --git a/docs/howto/asset_testing.md b/docs/howto/asset_testing.md index 2d39752c19..cd16a8cdd9 100644 --- a/docs/howto/asset_testing.md +++ b/docs/howto/asset_testing.md @@ -1,4 +1,4 @@ -# HOWTO: Writing asset loading tests for a package +# HOWTO: Running asset loading tests for a package ## Introduction diff --git a/docs/howto/static_testing.md b/docs/howto/static_testing.md new file mode 100644 index 0000000000..185974c906 --- /dev/null +++ b/docs/howto/static_testing.md @@ -0,0 +1,28 @@ +# HOWTO: Running static tests for a package + +## Introduction + +Static tests allow you to verify if all static resources of the package are valid, e.g. are all fields of the `sample_event.json` documented. +They don't require any additional configuration (unless you would like to skip them). + +## Coverage + +Static tests cover the following resources: + +1. Sample event for a data stream - verification if the file uses only documented fields. + +## Running static tests + +Static tests don't require the Elastic stack to be up and running. Simply navigate to the package's root folder +(or any sub-folder under it) and run the following command. + +``` +elastic-package test static +``` + +If you want to run pipeline tests for **specific data streams** in a package, navigate to the package's root folder +(or any sub-folder under it) and run the following command. + +``` +elastic-package test static --data-streams [,,...] +``` diff --git a/go.mod b/go.mod index fee2659647..b02076324b 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/elastic/go-elasticsearch/v7 v7.9.0 github.com/elastic/go-licenser v0.3.1 github.com/elastic/go-ucfg v0.8.3 - github.com/elastic/package-spec/code/go v0.0.0-20210222225810-541116689fca + github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/go-openapi/strfmt v0.19.6 // indirect diff --git a/go.sum b/go.sum index 1671e7cd6d..c829914fbb 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,8 @@ github.com/elastic/go-licenser v0.3.1 h1:RmRukU/JUmts+rpexAw0Fvt2ly7VVu6mw8z4HrE github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ= github.com/elastic/go-ucfg v0.8.3 h1:leywnFjzr2QneZZWhE6uWd+QN/UpP0sdJRHYyuFvkeo= github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo= -github.com/elastic/package-spec/code/go v0.0.0-20210222225810-541116689fca h1:dEAP2TACrsICd/pMzFlEx2llqqWcB68CjJRfnMhQL8E= -github.com/elastic/package-spec/code/go v0.0.0-20210222225810-541116689fca/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8= +github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3 h1:SX1mNX3H7+NQ+88eBu8nsWJGCLUQnbGdxh/qpS/6Wq8= +github.com/elastic/package-spec/code/go v0.0.0-20210301084210-584b422597f3/go.mod h1:dog1l3e8NoRYxuB8yIbbOWglE6GSQuU6ZL75wT9pKL8= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= diff --git a/internal/fields/validate.go b/internal/fields/validate.go index b04a3f852a..8bfb8159de 100644 --- a/internal/fields/validate.go +++ b/internal/fields/validate.go @@ -22,13 +22,23 @@ import ( // Validator is responsible for fields validation. type Validator struct { - schema []FieldDefinition - numericKeywordFields map[string]struct{} + schema []FieldDefinition + + defaultNumericConversion bool + numericKeywordFields map[string]struct{} } // ValidatorOption represents an optional flag that can be passed to CreateValidatorForDataStream. type ValidatorOption func(*Validator) error +// WithDefaultNumericConversion configures the validator to accept defined keyword (or constant_keyword) fields as numeric-type. +func WithDefaultNumericConversion() ValidatorOption { + return func(v *Validator) error { + v.defaultNumericConversion = true + return nil + } +} + // WithNumericKeywordFields configures the validator to accept specific fields to have numeric-type // while defined as keyword or constant_keyword. func WithNumericKeywordFields(fields []string) ValidatorOption { @@ -155,7 +165,8 @@ func (v *Validator) validateScalarElement(key string, val interface{}) error { } // Convert numeric keyword fields to string for validation. - if _, found := v.numericKeywordFields[key]; found && isNumericKeyword(*definition, val) { + _, found := v.numericKeywordFields[key] + if (found || v.defaultNumericConversion) && isNumericKeyword(*definition, val) { val = fmt.Sprintf("%q", val) } diff --git a/internal/testrunner/runners/asset/runner.go b/internal/testrunner/runners/asset/runner.go index 8764b7ab9a..42f0f64534 100644 --- a/internal/testrunner/runners/asset/runner.go +++ b/internal/testrunner/runners/asset/runner.go @@ -142,6 +142,10 @@ func (r *runner) TearDown() error { return nil } +func (r *runner) TestFolderRequired() bool { + return false +} + func findActualAsset(actualAssets []packages.Asset, expectedAsset packages.Asset) bool { for _, a := range actualAssets { if a.Type == expectedAsset.Type && a.ID == expectedAsset.ID { diff --git a/internal/testrunner/runners/pipeline/runner.go b/internal/testrunner/runners/pipeline/runner.go index e3ab4b48f5..0d4947e4d9 100644 --- a/internal/testrunner/runners/pipeline/runner.go +++ b/internal/testrunner/runners/pipeline/runner.go @@ -32,6 +32,10 @@ type runner struct { options testrunner.TestOptions } +func (r *runner) TestFolderRequired() bool { + return true +} + // Type returns the type of test that can be run by this test runner. func (r *runner) Type() testrunner.TestType { return TestType diff --git a/internal/testrunner/runners/runners.go b/internal/testrunner/runners/runners.go index c1faaf31fb..c0e129dc96 100644 --- a/internal/testrunner/runners/runners.go +++ b/internal/testrunner/runners/runners.go @@ -8,5 +8,6 @@ import ( // Registered test runners _ "github.com/elastic/elastic-package/internal/testrunner/runners/asset" _ "github.com/elastic/elastic-package/internal/testrunner/runners/pipeline" + _ "github.com/elastic/elastic-package/internal/testrunner/runners/static" _ "github.com/elastic/elastic-package/internal/testrunner/runners/system" ) diff --git a/internal/testrunner/runners/static/runner.go b/internal/testrunner/runners/static/runner.go new file mode 100644 index 0000000000..ac87c608c9 --- /dev/null +++ b/internal/testrunner/runners/static/runner.go @@ -0,0 +1,130 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package static + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/fields" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/testrunner" +) + +const sampleEventJSON = "sample_event.json" + +type runner struct { + options testrunner.TestOptions +} + +var _ testrunner.TestRunner = new(runner) + +func init() { + testrunner.RegisterRunner(&runner{}) +} + +const ( + // TestType defining asset loading tests + TestType testrunner.TestType = "static" +) + +func (r runner) Type() testrunner.TestType { + return TestType +} + +func (r runner) String() string { + return "static files" +} + +func (r runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, error) { + r.options = options + return r.run() +} + +func (r runner) run() ([]testrunner.TestResult, error) { + result := testrunner.NewResultComposer(testrunner.TestResult{ + TestType: TestType, + Package: r.options.TestFolder.Package, + DataStream: r.options.TestFolder.DataStream, + }) + + testConfig, err := newConfig(r.options.TestFolder.Path) + if err != nil { + return result.WithError(errors.Wrap(err, "unable to load asset loading test config file")) + } + + if testConfig != nil && testConfig.Skip != nil { + logger.Warnf("skipping %s test for %s: %s (details: %s)", + TestType, r.options.TestFolder.Package, + testConfig.Skip.Reason, testConfig.Skip.Link.String()) + return result.WithSkip(testConfig.Skip) + } + + var results []testrunner.TestResult + results = append(results, r.verifySampleEvent()...) + return results, nil +} + +func (r runner) verifySampleEvent() []testrunner.TestResult { + dataStreamPath := filepath.Join(r.options.PackageRootPath, "data_stream", r.options.TestFolder.DataStream) + sampleEventPath := filepath.Join(dataStreamPath, sampleEventJSON) + _, err := os.Stat(sampleEventPath) + if os.IsNotExist(err) { + return []testrunner.TestResult{} // nothing to succeed, nothing to skip + } + + resultComposer := testrunner.NewResultComposer(testrunner.TestResult{ + Name: "Verify " + sampleEventJSON, + TestType: TestType, + Package: r.options.TestFolder.Package, + DataStream: r.options.TestFolder.DataStream, + }) + + if err != nil { + results, _ := resultComposer.WithError(errors.Wrap(err, "stat file failed")) + return results + } + + fieldsValidator, err := fields.CreateValidatorForDataStream( + dataStreamPath, + fields.WithDefaultNumericConversion()) + if err != nil { + results, _ := resultComposer.WithError(errors.Wrap(err, "creating fields validator for data stream failed")) + return results + } + + content, err := ioutil.ReadFile(sampleEventPath) + if err != nil { + results, _ := resultComposer.WithError(errors.Wrap(err, "can't read file")) + return results + } + + multiErr := fieldsValidator.ValidateDocumentBody(content) + if len(multiErr) > 0 { + results, _ := resultComposer.WithError(testrunner.ErrTestCaseFailed{ + Reason: "one or more errors found in document", + Details: multiErr.Error(), + }) + return results + } + + results, _ := resultComposer.WithSuccess() + return results +} + +func (r runner) TearDown() error { + return nil // it's a static test runner, no state is stored +} + +func (r runner) CanRunPerDataStream() bool { + return true +} + +func (r *runner) TestFolderRequired() bool { + return false +} diff --git a/internal/testrunner/runners/static/test_config.go b/internal/testrunner/runners/static/test_config.go new file mode 100644 index 0000000000..dbb896cc64 --- /dev/null +++ b/internal/testrunner/runners/static/test_config.go @@ -0,0 +1,47 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package static + +import ( + "io/ioutil" + "os" + "path/filepath" + + "github.com/elastic/go-ucfg" + "github.com/elastic/go-ucfg/yaml" + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/testrunner" +) + +type testConfig struct { + testrunner.SkippableConfig `config:",inline"` +} + +func newConfig(staticTestFolderPath string) (*testConfig, error) { + configFilePath := filepath.Join(staticTestFolderPath, "config.yml") + + // Test configuration file is optional for static loading tests. If it + // doesn't exist, we can return early. + if _, err := os.Stat(configFilePath); os.IsNotExist(err) { + return nil, nil + } + + data, err := ioutil.ReadFile(configFilePath) + if err != nil { + return nil, errors.Wrapf(err, "could not load static loading test configuration file: %s", configFilePath) + } + + var c testConfig + cfg, err := yaml.NewConfig(data, ucfg.PathSep(".")) + if err != nil { + return nil, errors.Wrapf(err, "unable to load static loading test configuration file: %s", configFilePath) + } + if err := cfg.Unpack(&c); err != nil { + return nil, errors.Wrapf(err, "unable to unpack static loading test configuration file: %s", configFilePath) + } + + return &c, nil +} diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index fb4850a624..6426895323 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -73,6 +73,10 @@ func (r *runner) CanRunPerDataStream() bool { return true } +func (r *runner) TestFolderRequired() bool { + return true +} + // Run runs the system tests defined under the given folder func (r *runner) Run(options testrunner.TestOptions) ([]testrunner.TestResult, error) { r.options = options diff --git a/internal/testrunner/testrunner.go b/internal/testrunner/testrunner.go index 3bca9cf423..9526b05056 100644 --- a/internal/testrunner/testrunner.go +++ b/internal/testrunner/testrunner.go @@ -6,6 +6,8 @@ package testrunner import ( "fmt" + "io/ioutil" + "os" "path/filepath" "sort" "strings" @@ -47,6 +49,8 @@ type TestRunner interface { TearDown() error CanRunPerDataStream() bool + + TestFolderRequired() bool } var runners = map[TestType]TestRunner{} @@ -138,6 +142,43 @@ type TestFolder struct { DataStream string } +// AssumeTestFolders assumes potential test folders for the given package, data streams and test types. +func AssumeTestFolders(packageRootPath string, dataStreams []string, testType TestType) ([]TestFolder, error) { + // Expected folder structure: + // / + // data_stream/ + // / + + dataStreamsPath := filepath.Join(packageRootPath, "data_stream") + + if dataStreams == nil || len(dataStreams) == 0 { + fileInfos, err := ioutil.ReadDir(dataStreamsPath) + if os.IsNotExist(err) { + return []TestFolder{}, nil // data streams defined + } + if err != nil { + return nil, errors.Wrapf(err, "can't read directory (path: %s)", dataStreamsPath) + } + + for _, fi := range fileInfos { + if !fi.IsDir() { + continue + } + dataStreams = append(dataStreams, fi.Name()) + } + } + + var folders []TestFolder + for _, dataStream := range dataStreams { + folders = append(folders, TestFolder{ + Path: filepath.Join(dataStreamsPath, dataStream, "_dev", "test", string(testType)), + Package: filepath.Base(packageRootPath), + DataStream: dataStream, + }) + } + return folders, nil +} + // FindTestFolders finds test folders for the given package and, optionally, test type and data streams func FindTestFolders(packageRootPath string, dataStreams []string, testType TestType) ([]TestFolder, error) { // Expected folder structure: diff --git a/test/packages/apache/data_stream/status/sample_event.json b/test/packages/apache/data_stream/status/sample_event.json index 2c3ff3c003..2088a8eb44 100644 --- a/test/packages/apache/data_stream/status/sample_event.json +++ b/test/packages/apache/data_stream/status/sample_event.json @@ -1,11 +1,5 @@ { "@timestamp": "2020-06-24T10:19:48.005Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0", - "raw_index": "metrics-apache.status-default" - }, "metricset": { "name": "status", "period": 10000 @@ -47,7 +41,6 @@ "idle": 74 }, "bytes_per_sec": 83.6986, - "hostname": "127.0.0.1:8088", "uptime": { "server_uptime": 1566, "uptime": 1566 @@ -71,16 +64,6 @@ "dataset": "apache.status", "module": "apache" }, - "dataset": { - "type": "metrics", - "name": "apache.status", - "namespace": "default" - }, - "stream": { - "dataset": "apache.status", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" }, diff --git a/test/packages/apache/docs/README.md b/test/packages/apache/docs/README.md index cc01c77126..4f5b03838d 100644 --- a/test/packages/apache/docs/README.md +++ b/test/packages/apache/docs/README.md @@ -108,12 +108,6 @@ An example event for `status` looks as following: ```$json { "@timestamp": "2020-06-24T10:19:48.005Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0", - "raw_index": "metrics-apache.status-default" - }, "metricset": { "name": "status", "period": 10000 @@ -155,7 +149,6 @@ An example event for `status` looks as following: "idle": 74 }, "bytes_per_sec": 83.6986, - "hostname": "127.0.0.1:8088", "uptime": { "server_uptime": 1566, "uptime": 1566 @@ -179,16 +172,6 @@ An example event for `status` looks as following: "dataset": "apache.status", "module": "apache" }, - "dataset": { - "type": "metrics", - "name": "apache.status", - "namespace": "default" - }, - "stream": { - "dataset": "apache.status", - "namespace": "default", - "type": "metrics" - }, "ecs": { "version": "1.5.0" }, diff --git a/test/packages/aws/data_stream/billing/fields/ecs.yml b/test/packages/aws/data_stream/billing/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/billing/fields/ecs.yml +++ b/test/packages/aws/data_stream/billing/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/billing/fields/fields.yml b/test/packages/aws/data_stream/billing/fields/fields.yml index 1473a38143..bbbd27d554 100644 --- a/test/packages/aws/data_stream/billing/fields/fields.yml +++ b/test/packages/aws/data_stream/billing/fields/fields.yml @@ -19,3 +19,9 @@ - name: EstimatedCharges.max type: long description: Maximum estimated charges for AWS acccount. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/billing/sample_event.json b/test/packages/aws/data_stream/billing/sample_event.json index 9c14baf14b..0a252492f0 100644 --- a/test/packages/aws/data_stream/billing/sample_event.json +++ b/test/packages/aws/data_stream/billing/sample_event.json @@ -1,67 +1,48 @@ { - "_index": "metrics-aws.billing-default-000001", - "_id": "IMxJXHIBpGMSUzkZo-s0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:06.212Z", - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.billing", - "module": "aws", - "duration": 1938760247 - }, - "metricset": { - "name": "billing", - "period": 43200000 - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "billing": { - "metrics": { - "EstimatedCharges": { - "max": 1625.41 - } + "@timestamp": "2020-05-28T17:17:06.212Z", + "cloud": { + "provider": "aws", + "region": "us-east-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "event": { + "dataset": "aws.billing", + "module": "aws", + "duration": 1938760247 + }, + "metricset": { + "name": "billing", + "period": 43200000 + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "billing": { + "metrics": { + "EstimatedCharges": { + "max": 1625.41 } - }, - "cloudwatch": { - "namespace": "AWS/Billing" - }, - "dimensions": { - "Currency": "USD" } }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.billing", - "namespace": "default" + "cloudwatch": { + "namespace": "AWS/Billing" }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + "dimensions": { + "Currency": "USD" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:06.212Z" - ] + "service": { + "type": "aws" }, - "sort": [ - 1590686226212 - ] + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml b/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/sample_event.json b/test/packages/aws/data_stream/cloudwatch_metrics/sample_event.json index b79ac10d01..431705cacd 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/sample_event.json +++ b/test/packages/aws/data_stream/cloudwatch_metrics/sample_event.json @@ -1,72 +1,53 @@ { - "_index": "metrics-aws.cloudwatch_metrics-default-000001", - "_id": "-sxJXHIBpGMSUzkZxex8", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:02.812Z", - "event": { - "duration": 14119105951, - "dataset": "aws.cloudwatch", - "module": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "stream": { - "dataset": "aws.cloudwatch_metrics", - "namespace": "default", - "type": "metrics" - }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } + "@timestamp": "2020-05-28T17:17:02.812Z", + "event": { + "duration": 14119105951, + "dataset": "aws.cloudwatch", + "module": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "dimensions": { + "InstanceId": "i-0830bfecfa7173cbe" }, - "aws": { - "dimensions": { - "InstanceId": "i-0830bfecfa7173cbe" - }, - "ec2": { - "metrics": { - "DiskWriteOps": { - "avg": 0, - "max": 0 - }, - "CPUUtilization": { - "avg": 0.7661943132361363, - "max": 0.833333333333333 - } + "ec2": { + "metrics": { + "DiskWriteOps": { + "avg": 0, + "max": 0 + }, + "CPUUtilization": { + "avg": 0.7661943132361363, + "max": 0.833333333333333 } - }, - "cloudwatch": { - "namespace": "AWS/EC2" } }, - "metricset": { - "period": 300000, - "name": "cloudwatch" + "cloudwatch": { + "namespace": "AWS/EC2" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:02.812Z" - ] - }, - "sort": [ - 1590686222812 - ] + "metricset": { + "period": 300000, + "name": "cloudwatch" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/dynamodb/fields/ecs.yml b/test/packages/aws/data_stream/dynamodb/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/dynamodb/fields/ecs.yml +++ b/test/packages/aws/data_stream/dynamodb/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/dynamodb/fields/fields.yml b/test/packages/aws/data_stream/dynamodb/fields/fields.yml index 892fc7edee..abd232950d 100644 --- a/test/packages/aws/data_stream/dynamodb/fields/fields.yml +++ b/test/packages/aws/data_stream/dynamodb/fields/fields.yml @@ -107,3 +107,9 @@ type: double description: | The percentage of provisioned write capacity utilized by the highest provisioned write table or global secondary index of an account. +- name: aws.cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/dynamodb/sample_event.json b/test/packages/aws/data_stream/dynamodb/sample_event.json index 8866a6ac1c..6973aa2c90 100644 --- a/test/packages/aws/data_stream/dynamodb/sample_event.json +++ b/test/packages/aws/data_stream/dynamodb/sample_event.json @@ -1,78 +1,59 @@ { - "_index": "metrics-aws.dynamodb-default-000001", - "_id": "YMxJXHIBpGMSUzkZzO0_", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "stream": { - "type": "metrics", - "dataset": "aws.dynamodb", - "namespace": "default" - }, - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "dimensions": { + "TableName": "TryDaxTable3" }, - "aws": { - "dimensions": { - "TableName": "TryDaxTable3" - }, - "dynamodb": { - "metrics": { - "ProvisionedWriteCapacityUnits": { - "avg": 1 - }, - "ProvisionedReadCapacityUnits": { - "avg": 1 - }, - "ConsumedWriteCapacityUnits": { - "avg": 0, - "sum": 0 - }, - "ConsumedReadCapacityUnits": { - "avg": 0, - "sum": 0 - } + "dynamodb": { + "metrics": { + "ProvisionedWriteCapacityUnits": { + "avg": 1 + }, + "ProvisionedReadCapacityUnits": { + "avg": 1 + }, + "ConsumedWriteCapacityUnits": { + "avg": 0, + "sum": 0 + }, + "ConsumedReadCapacityUnits": { + "avg": 0, + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/DynamoDB" } }, - "metricset": { - "name": "dynamodb", - "period": 300000 + "cloudwatch": { + "namespace": "AWS/DynamoDB" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:08.666Z" - ] - }, - "sort": [ - 1590686228666 - ] + "metricset": { + "name": "dynamodb", + "period": 300000 + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/ebs/fields/ecs.yml b/test/packages/aws/data_stream/ebs/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/ebs/fields/ecs.yml +++ b/test/packages/aws/data_stream/ebs/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/ebs/fields/fields.yml b/test/packages/aws/data_stream/ebs/fields/fields.yml index 371f83b07b..c230284e0d 100644 --- a/test/packages/aws/data_stream/ebs/fields/fields.yml +++ b/test/packages/aws/data_stream/ebs/fields/fields.yml @@ -46,3 +46,9 @@ - name: VolumeIdleTime.sum type: double description: The total number of seconds in a specified period of time when no read or write operations were submitted. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/ebs/sample_event.json b/test/packages/aws/data_stream/ebs/sample_event.json index e0daba4d00..ce81b383a5 100644 --- a/test/packages/aws/data_stream/ebs/sample_event.json +++ b/test/packages/aws/data_stream/ebs/sample_event.json @@ -1,90 +1,66 @@ { - "_index": "metrics-aws.ebs-default-000001", - "_id": "_89uXHIBpGMSUzkZoRoL", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:57:22.450Z", - "service": { - "type": "aws" - }, - "aws": { - "ebs": { - "metrics": { - "VolumeReadOps": { - "avg": 0 - }, - "VolumeQueueLength": { - "avg": 0.0000666666666666667 - }, - "VolumeWriteOps": { - "avg": 29 - }, - "VolumeTotalWriteTime": { - "sum": 0.02 - }, - "BurstBalance": { - "avg": 100 - }, - "VolumeWriteBytes": { - "avg": 14406.620689655172 - }, - "VolumeIdleTime": { - "sum": 299.98 - } + "@timestamp": "2020-05-28T17:57:22.450Z", + "service": { + "type": "aws" + }, + "aws": { + "ebs": { + "metrics": { + "VolumeReadOps": { + "avg": 0 + }, + "VolumeQueueLength": { + "avg": 0.0000666666666666667 + }, + "VolumeWriteOps": { + "avg": 29 + }, + "VolumeTotalWriteTime": { + "sum": 0.02 + }, + "BurstBalance": { + "avg": 100 + }, + "VolumeWriteBytes": { + "avg": 14406.620689655172 + }, + "VolumeIdleTime": { + "sum": 299.98 } - }, - "cloudwatch": { - "namespace": "AWS/EBS" - }, - "dimensions": { - "VolumeId": "vol-03370a204cc8b0a2f" - } - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" } }, - "event": { - "dataset": "aws.ebs", - "module": "aws", - "duration": 10488314037 - }, - "metricset": { - "period": 300000, - "name": "ebs" + "cloudwatch": { + "namespace": "AWS/EBS" }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ebs" + "dimensions": { + "VolumeId": "vol-03370a204cc8b0a2f" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:57:22.450Z" - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.ebs@/kibana-highlighted-field@" - ] + "event": { + "dataset": "aws.ebs", + "module": "aws", + "duration": 10488314037 }, - "sort": [ - 1590688642450 - ] + "metricset": { + "period": 300000, + "name": "ebs" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/ec2_metrics/sample_event.json b/test/packages/aws/data_stream/ec2_metrics/sample_event.json index 8351191696..ffdd822660 100644 --- a/test/packages/aws/data_stream/ec2_metrics/sample_event.json +++ b/test/packages/aws/data_stream/ec2_metrics/sample_event.json @@ -1,134 +1,110 @@ { - "_index": "metrics-aws.ec2_metrics-default-000001", - "_id": "b89uXHIBpGMSUzkZHxPP", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:56:37.255Z", - "aws": { - "ec2": { - "network": { - "in": { - "packets": 448.4, - "bytes_per_sec": 103.10266666666666, - "packets_per_sec": 1.4946666666666666, - "bytes": 30930.8 - }, - "out": { - "packets": 233.6, - "bytes_per_sec": 51.754666666666665, - "packets_per_sec": 0.7786666666666666, - "bytes": 15526.4 - } + "@timestamp": "2020-05-28T17:56:37.255Z", + "aws": { + "ec2": { + "network": { + "in": { + "packets": 448.4, + "bytes_per_sec": 103.10266666666666, + "packets_per_sec": 1.4946666666666666, + "bytes": 30930.8 }, - "status": { - "check_failed": 0, - "check_failed_instance": 0, - "check_failed_system": 0 + "out": { + "packets": 233.6, + "bytes_per_sec": 51.754666666666665, + "packets_per_sec": 0.7786666666666666, + "bytes": 15526.4 + } + }, + "status": { + "check_failed": 0, + "check_failed_instance": 0, + "check_failed_system": 0 + }, + "cpu": { + "credit_usage": 0.004566, + "credit_balance": 144, + "surplus_credit_balance": 0, + "surplus_credits_charged": 0, + "total": { + "pct": 0.0999999999997574 + } + }, + "diskio": { + "read": { + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0, + "count": 0 + }, + "write": { + "count": 0, + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0 + } + }, + "instance": { + "core": { + "count": 1 + }, + "threads_per_core": 1, + "public": { + "ip": "3.122.204.80", + "dns_name": "" + }, + "private": { + "ip": "10.0.0.122", + "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" }, - "cpu": { - "credit_usage": 0.004566, - "credit_balance": 144, - "surplus_credit_balance": 0, - "surplus_credits_charged": 0, - "total": { - "pct": 0.0999999999997574 - } + "image": { + "id": "ami-0b418580298265d5c" }, - "diskio": { - "read": { - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0, - "count": 0 - }, - "write": { - "count": 0, - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0 - } + "state": { + "name": "running", + "code": 16 }, - "instance": { - "core": { - "count": 1 - }, - "threads_per_core": 1, - "public": { - "ip": "3.122.204.80", - "dns_name": "" - }, - "private": { - "ip": "10.0.0.122", - "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" - }, - "image": { - "id": "ami-0b418580298265d5c" - }, - "state": { - "name": "running", - "code": 16 - }, - "monitoring": { - "state": "disabled" - } + "monitoring": { + "state": "disabled" } } - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "aws", - "duration": 23217499283, - "dataset": "aws.ec2" - }, - "metricset": { - "period": 300000, - "name": "ec2" - }, - "service": { - "type": "aws" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ec2_metrics" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "instance": { - "id": "i-04c1a32c2aace6b40" - }, - "machine": { - "type": "t2.micro" - }, - "availability_zone": "eu-central-1a" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:56:37.255Z" - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "aws", + "duration": 23217499283, + "dataset": "aws.ec2" + }, + "metricset": { + "period": 300000, + "name": "ec2" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.ec2@/kibana-highlighted-field@" - ] + "service": { + "type": "aws" }, - "sort": [ - 1590688597255 - ] + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "instance": { + "id": "i-04c1a32c2aace6b40" + }, + "machine": { + "type": "t2.micro" + }, + "availability_zone": "eu-central-1a" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml b/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/elb_metrics/fields/fields.yml b/test/packages/aws/data_stream/elb_metrics/fields/fields.yml index d1a7a32445..dd916b17f4 100644 --- a/test/packages/aws/data_stream/elb_metrics/fields/fields.yml +++ b/test/packages/aws/data_stream/elb_metrics/fields/fields.yml @@ -193,3 +193,9 @@ - name: TargetGroup type: keyword description: Filters the metric data by target group. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/elb_metrics/sample_event.json b/test/packages/aws/data_stream/elb_metrics/sample_event.json index 86515657dd..d187909719 100644 --- a/test/packages/aws/data_stream/elb_metrics/sample_event.json +++ b/test/packages/aws/data_stream/elb_metrics/sample_event.json @@ -1,87 +1,63 @@ { - "_index": "metrics-aws.elb_metrics-default-000001", - "_id": "i89vXHIBpGMSUzkZuSyO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:30.211Z", - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "aws": { - "elb": { - "metrics": { - "EstimatedALBNewConnectionCount": { - "avg": 32 - }, - "EstimatedALBConsumedLCUs": { - "avg": 0.00035000000000000005 - }, - "EstimatedProcessedBytes": { - "avg": 967 - }, - "EstimatedALBActiveConnectionCount": { - "avg": 5 - }, - "HealthyHostCount": { - "max": 2 - }, - "UnHealthyHostCount": { - "max": 0 - } + "@timestamp": "2020-05-28T17:58:30.211Z", + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "aws": { + "elb": { + "metrics": { + "EstimatedALBNewConnectionCount": { + "avg": 32 + }, + "EstimatedALBConsumedLCUs": { + "avg": 0.00035000000000000005 + }, + "EstimatedProcessedBytes": { + "avg": 967 + }, + "EstimatedALBActiveConnectionCount": { + "avg": 5 + }, + "HealthyHostCount": { + "max": 2 + }, + "UnHealthyHostCount": { + "max": 0 } - }, - "cloudwatch": { - "namespace": "AWS/ELB" - }, - "dimensions": { - "LoadBalancerName": "filebeat-aws-elb-test-elb" } }, - "metricset": { - "name": "elb", - "period": 60000 - }, - "event": { - "dataset": "aws.elb", - "module": "aws", - "duration": 15044430616 - }, - "service": { - "type": "aws" + "cloudwatch": { + "namespace": "AWS/ELB" }, - "stream": { - "type": "metrics", - "dataset": "aws.elb_metrics", - "namespace": "default" + "dimensions": { + "LoadBalancerName": "filebeat-aws-elb-test-elb" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:30.211Z" - ] + "metricset": { + "name": "elb", + "period": 60000 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.elb@/kibana-highlighted-field@" - ] + "event": { + "dataset": "aws.elb", + "module": "aws", + "duration": 15044430616 }, - "sort": [ - 1590688710211 - ] + "service": { + "type": "aws" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/lambda/fields/ecs.yml b/test/packages/aws/data_stream/lambda/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/lambda/fields/ecs.yml +++ b/test/packages/aws/data_stream/lambda/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/lambda/fields/fields.yml b/test/packages/aws/data_stream/lambda/fields/fields.yml index 82a59893af..5209e0d30e 100644 --- a/test/packages/aws/data_stream/lambda/fields/fields.yml +++ b/test/packages/aws/data_stream/lambda/fields/fields.yml @@ -58,3 +58,9 @@ - name: ProvisionedConcurrencySpilloverInvocations.sum type: long description: The number of times your function code is executed on standard concurrency when all provisioned concurrency is in use. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/lambda/sample_event.json b/test/packages/aws/data_stream/lambda/sample_event.json index 0a88bde60c..b1542233bd 100644 --- a/test/packages/aws/data_stream/lambda/sample_event.json +++ b/test/packages/aws/data_stream/lambda/sample_event.json @@ -1,77 +1,58 @@ { - "_index": "metrics-aws.lambda-default-000001", - "_id": "YMxJXHIBpGMSUzkZzO0_", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "stream": { - "type": "metrics", - "dataset": "aws.lambda", - "namespace": "default" - }, - "service": { - "type": "aws" + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" }, - "ecs": { - "version": "1.5.0" + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/Lambda" }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" + "dimensions": { + "FunctionName": "ec2-owner-tagger-serverless", + "Resource": "ec2-owner-tagger-serverless" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/Lambda" - }, - "dimensions": { - "FunctionName": "ec2-owner-tagger-serverless", - "Resource": "ec2-owner-tagger-serverless" - }, - "lambda": { - "metrics": { - "Duration": { - "avg": 8218.073333333334 - }, - "Errors": { - "avg": 1 - }, - "Invocations": { - "avg": 1 - }, - "Throttles": { - "avg": 0 - } + "lambda": { + "metrics": { + "Duration": { + "avg": 8218.073333333334 + }, + "Errors": { + "avg": 1 + }, + "Invocations": { + "avg": 1 + }, + "Throttles": { + "avg": 0 } } - }, - "metricset": { - "name": "dynamodb", - "period": 300000 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:08.666Z" - ] - }, - "sort": [ - 1590686228666 - ] + "metricset": { + "name": "dynamodb", + "period": 300000 + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/natgateway/fields/ecs.yml b/test/packages/aws/data_stream/natgateway/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/natgateway/fields/ecs.yml +++ b/test/packages/aws/data_stream/natgateway/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/natgateway/fields/fields.yml b/test/packages/aws/data_stream/natgateway/fields/fields.yml index 78ffdb3b5a..c3e7172455 100644 --- a/test/packages/aws/data_stream/natgateway/fields/fields.yml +++ b/test/packages/aws/data_stream/natgateway/fields/fields.yml @@ -55,3 +55,9 @@ - name: ActiveConnectionCount.max type: long description: The total number of concurrent active TCP connections through the NAT gateway. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/natgateway/sample_event.json b/test/packages/aws/data_stream/natgateway/sample_event.json index d6b2303b93..11f136cd63 100644 --- a/test/packages/aws/data_stream/natgateway/sample_event.json +++ b/test/packages/aws/data_stream/natgateway/sample_event.json @@ -1,108 +1,84 @@ { - "_index": "metrics-aws.natgateway-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.natgateway", - "namespace": "default", - "type": "metrics" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/NATGateway" }, - "ecs": { - "version": "1.5.0" + "dimensions": { + "NatGatewayId": "nat-0a5cb7b9807908cc0" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/NATGateway" - }, - "dimensions": { - "NatGatewayId": "nat-0a5cb7b9807908cc0" - }, - "natgateway": { - "metrics": { - "ActiveConnectionCount": { - "max": 0 - }, - "BytesInFromDestination": { - "sum": 0 - }, - "BytesInFromSource": { - "sum": 0 - }, - "BytesOutToDestination": { - "sum": 0 - }, - "BytesOutToSource": { - "sum": 0 - }, - "ConnectionAttemptCount": { - "sum": 0 - }, - "ConnectionEstablishedCount": { - "sum": 0 - }, - "ErrorPortAllocation": { - "sum": 0 - }, - "PacketsDropCount": { - "sum": 0 - }, - "PacketsInFromDestination": { - "sum": 0 - }, - "PacketsInFromSource": { - "sum": 0 - }, - "PacketsOutToDestination": { - "sum": 0 - }, - "PacketsOutToSource": { - "sum": 0 - } + "natgateway": { + "metrics": { + "ActiveConnectionCount": { + "max": 0 + }, + "BytesInFromDestination": { + "sum": 0 + }, + "BytesInFromSource": { + "sum": 0 + }, + "BytesOutToDestination": { + "sum": 0 + }, + "BytesOutToSource": { + "sum": 0 + }, + "ConnectionAttemptCount": { + "sum": 0 + }, + "ConnectionEstablishedCount": { + "sum": 0 + }, + "ErrorPortAllocation": { + "sum": 0 + }, + "PacketsDropCount": { + "sum": 0 + }, + "PacketsInFromDestination": { + "sum": 0 + }, + "PacketsInFromSource": { + "sum": 0 + }, + "PacketsOutToDestination": { + "sum": 0 + }, + "PacketsOutToSource": { + "sum": 0 } } - }, - "event": { - "dataset": "aws.natgateway", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "natgateway" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.natgateway", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.natgateway@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "natgateway" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/rds/fields/ecs.yml b/test/packages/aws/data_stream/rds/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/rds/fields/ecs.yml +++ b/test/packages/aws/data_stream/rds/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/rds/sample_event.json b/test/packages/aws/data_stream/rds/sample_event.json index cb71045ad1..27bfc3c0bf 100644 --- a/test/packages/aws/data_stream/rds/sample_event.json +++ b/test/packages/aws/data_stream/rds/sample_event.json @@ -1,113 +1,89 @@ { - "_index": "metrics-aws.rds-default-000001", - "_id": "k89vXHIBpGMSUzkZuSyO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:34.537Z", - "ecs": { - "version": "1.5.0" - }, - "service": { - "type": "aws" - }, - "aws": { - "rds": { - "latency": { - "dml": 0, - "insert": 0, - "update": 0, - "commit": 0, - "ddl": 0, - "delete": 0, - "select": 0.21927814569536422 - }, - "queries": 6.197934021992669, - "aurora_bin_log_replica_lag": 0, - "transactions": { - "blocked": 0, - "active": 0 - }, - "deadlocks": 0, - "login_failures": 0, - "throughput": { - "network": 1.399813358218904, - "insert": 0, - "ddl": 0, - "select": 2.5165408396246853, - "delete": 0, - "commit": 0, - "network_transmit": 0.699906679109452, - "update": 0, - "dml": 0, - "network_receive": 0.699906679109452 - }, - "cpu": { - "total": { - "pct": 0.03 - } - }, - "db_instance": { - "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", - "class": "db.r5.large", - "identifier": "database-1-instance-1-eu-west-1a", - "status": "available" - }, - "cache_hit_ratio.result_set": 0, - "aurora_replica.lag.ms": 19.576, - "free_local_storage.bytes": 32431271936, - "cache_hit_ratio.buffer": 100, - "disk_usage": { - "bin_log.bytes": 0 - }, - "db_instance.identifier": "database-1-instance-1-eu-west-1a", - "freeable_memory.bytes": 4436537344, - "engine_uptime.sec": 10463030, - "database_connections": 0 - } - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" + "@timestamp": "2020-05-28T17:58:34.537Z", + "ecs": { + "version": "1.5.0" + }, + "service": { + "type": "aws" + }, + "aws": { + "rds": { + "latency": { + "dml": 0, + "insert": 0, + "update": 0, + "commit": 0, + "ddl": 0, + "delete": 0, + "select": 0.21927814569536422 }, - "availability_zone": "eu-west-1a" - }, - "event": { - "dataset": "aws.rds", - "module": "aws", - "duration": 10777919184 - }, - "metricset": { - "name": "rds", - "period": 60000 - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.rds" - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + "queries": 6.197934021992669, + "aurora_bin_log_replica_lag": 0, + "transactions": { + "blocked": 0, + "active": 0 + }, + "deadlocks": 0, + "login_failures": 0, + "throughput": { + "network": 1.399813358218904, + "insert": 0, + "ddl": 0, + "select": 2.5165408396246853, + "delete": 0, + "commit": 0, + "network_transmit": 0.699906679109452, + "update": 0, + "dml": 0, + "network_receive": 0.699906679109452 + }, + "cpu": { + "total": { + "pct": 0.03 + } + }, + "db_instance": { + "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", + "class": "db.r5.large", + "identifier": "database-1-instance-1-eu-west-1a", + "status": "available" + }, + "cache_hit_ratio.result_set": 0, + "aurora_replica.lag.ms": 19.576, + "free_local_storage.bytes": 32431271936, + "cache_hit_ratio.buffer": 100, + "disk_usage": { + "bin_log.bytes": 0 + }, + "db_instance.identifier": "database-1-instance-1-eu-west-1a", + "freeable_memory.bytes": 4436537344, + "engine_uptime.sec": 10463030, + "database_connections": 0 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:34.537Z" - ] + "cloud": { + "provider": "aws", + "region": "eu-west-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "availability_zone": "eu-west-1a" + }, + "event": { + "dataset": "aws.rds", + "module": "aws", + "duration": 10777919184 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.rds@/kibana-highlighted-field@" - ] + "metricset": { + "name": "rds", + "period": 60000 }, - "sort": [ - 1590688714537 - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml b/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml +++ b/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/s3_daily_storage/sample_event.json b/test/packages/aws/data_stream/s3_daily_storage/sample_event.json index b288a47a79..f3e230ff06 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/sample_event.json +++ b/test/packages/aws/data_stream/s3_daily_storage/sample_event.json @@ -1,72 +1,48 @@ { - "_index": "metrics-aws.s3_daily_storage-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.s3_daily_storage", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_daily_storage": { - "bucket": { - "size": { - "bytes": 207372 - } - }, - "number_of_objects": 128 + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" } }, - "event": { - "dataset": "aws.s3_daily_storage", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_daily_storage" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "s3_daily_storage": { + "bucket": { + "size": { + "bytes": 207372 + } }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "number_of_objects": 128 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.s3_daily_storage", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "s3_daily_storage" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.s3_daily_storage@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3_request/fields/ecs.yml b/test/packages/aws/data_stream/s3_request/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/s3_request/fields/ecs.yml +++ b/test/packages/aws/data_stream/s3_request/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/s3_request/sample_event.json b/test/packages/aws/data_stream/s3_request/sample_event.json index 9a8d1d7013..3d1822e57c 100644 --- a/test/packages/aws/data_stream/s3_request/sample_event.json +++ b/test/packages/aws/data_stream/s3_request/sample_event.json @@ -1,85 +1,61 @@ { - "_index": "metrics-aws.s3_request-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.s3_request", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_request": { - "downloaded": { - "bytes": 534 - }, - "errors": { - "4xx": 0, - "5xx": 0 - }, - "latency": { - "first_byte.ms": 214, - "total_request.ms": 533 - }, - "requests": { - "list": 2, - "put": 10, - "total": 12 - }, - "uploaded": { - "bytes": 13572 - } + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" } }, - "event": { - "dataset": "aws.s3_request", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_request" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "s3_request": { + "downloaded": { + "bytes": 534 }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "errors": { + "4xx": 0, + "5xx": 0 + }, + "latency": { + "first_byte.ms": 214, + "total_request.ms": 533 + }, + "requests": { + "list": 2, + "put": 10, + "total": 12 + }, + "uploaded": { + "bytes": 13572 + } } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.s3_request", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "s3_request" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.s3_request@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/sqs/fields/ecs.yml b/test/packages/aws/data_stream/sqs/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/sqs/fields/ecs.yml +++ b/test/packages/aws/data_stream/sqs/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/sqs/sample_event.json b/test/packages/aws/data_stream/sqs/sample_event.json index f59384aed6..714ab645a4 100644 --- a/test/packages/aws/data_stream/sqs/sample_event.json +++ b/test/packages/aws/data_stream/sqs/sample_event.json @@ -1,77 +1,53 @@ { - "_index": "metrics-aws.sqs-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.sqs", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "sqs": { - "empty_receives": 0, - "messages": { - "delayed": 0, - "deleted": 0, - "not_visible": 0, - "received": 0, - "sent": 0, - "visible": 2 - }, - "oldest_message_age": { - "sec": 78494 - }, - "queue": { - "name": "test-s3-notification" - }, - "sent_message_size": {} - } - }, - "event": { - "dataset": "aws.sqs", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "sqs" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "sqs": { + "empty_receives": 0, + "messages": { + "delayed": 0, + "deleted": 0, + "not_visible": 0, + "received": 0, + "sent": 0, + "visible": 2 }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "oldest_message_age": { + "sec": 78494 + }, + "queue": { + "name": "test-s3-notification" + }, + "sent_message_size": {} } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.sqs", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "sqs" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.sqs@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/transitgateway/fields/ecs.yml b/test/packages/aws/data_stream/transitgateway/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/transitgateway/fields/ecs.yml +++ b/test/packages/aws/data_stream/transitgateway/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/transitgateway/fields/fields.yml b/test/packages/aws/data_stream/transitgateway/fields/fields.yml index f1c3361968..e89af40940 100644 --- a/test/packages/aws/data_stream/transitgateway/fields/fields.yml +++ b/test/packages/aws/data_stream/transitgateway/fields/fields.yml @@ -34,3 +34,9 @@ - name: PacketDropCountNoRoute.sum type: long description: The number of packets dropped because they did not match a route. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/transitgateway/sample_event.json b/test/packages/aws/data_stream/transitgateway/sample_event.json index d7c4ebda32..3f9d5b46ea 100644 --- a/test/packages/aws/data_stream/transitgateway/sample_event.json +++ b/test/packages/aws/data_stream/transitgateway/sample_event.json @@ -1,87 +1,63 @@ { - "_index": "metrics-aws.transitgateway-default-000001", - "_id": "WNToXHIBpGMSUzkZaeVh", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T20:10:20.953Z", - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "aws": { - "transitgateway": { - "metrics": { - "PacketsIn": { - "sum": 0 - }, - "BytesIn": { - "sum": 0 - }, - "BytesOut": { - "sum": 0 - }, - "PacketsOut": { - "sum": 0 - }, - "PacketDropCountBlackhole": { - "sum": 0 - }, - "PacketDropCountNoRoute": { - "sum": 0 - } + "@timestamp": "2020-05-28T20:10:20.953Z", + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "transitgateway": { + "metrics": { + "PacketsIn": { + "sum": 0 + }, + "BytesIn": { + "sum": 0 + }, + "BytesOut": { + "sum": 0 + }, + "PacketsOut": { + "sum": 0 + }, + "PacketDropCountBlackhole": { + "sum": 0 + }, + "PacketDropCountNoRoute": { + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/TransitGateway" - }, - "dimensions": { - "TransitGateway": "tgw-0630672a32f12808a" } }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + "cloudwatch": { + "namespace": "AWS/TransitGateway" }, - "event": { - "dataset": "aws.transitgateway", - "module": "aws", - "duration": 12762825681 - }, - "metricset": { - "period": 60000, - "name": "transitgateway" - }, - "service": { - "type": "aws" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.transitgateway" + "dimensions": { + "TransitGateway": "tgw-0630672a32f12808a" } }, - "fields": { - "@timestamp": [ - "2020-05-28T20:10:20.953Z" - ] + "ecs": { + "version": "1.5.0" + }, + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "event": { + "dataset": "aws.transitgateway", + "module": "aws", + "duration": 12762825681 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.transitgateway@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "transitgateway" }, - "sort": [ - 1590696620953 - ] + "service": { + "type": "aws" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/usage/fields/ecs.yml b/test/packages/aws/data_stream/usage/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/usage/fields/ecs.yml +++ b/test/packages/aws/data_stream/usage/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/usage/fields/fields.yml b/test/packages/aws/data_stream/usage/fields/fields.yml index bb75496faa..7cd5c5e37b 100644 --- a/test/packages/aws/data_stream/usage/fields/fields.yml +++ b/test/packages/aws/data_stream/usage/fields/fields.yml @@ -28,3 +28,9 @@ - name: ResourceCount.sum type: long description: The number of the specified resources running in your account. The resources are defined by the dimensions associated with the metric. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/usage/sample_event.json b/test/packages/aws/data_stream/usage/sample_event.json index da51bc4416..c67701e60f 100644 --- a/test/packages/aws/data_stream/usage/sample_event.json +++ b/test/packages/aws/data_stream/usage/sample_event.json @@ -1,75 +1,51 @@ { - "_index": "metrics-aws.usage-default-000001", - "_id": "YM9vXHIBpGMSUzkZiSlC", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:30.929Z", - "aws": { - "usage": { - "metrics": { - "CallCount": { - "sum": 1 - } + "@timestamp": "2020-05-28T17:58:30.929Z", + "aws": { + "usage": { + "metrics": { + "CallCount": { + "sum": 1 } - }, - "cloudwatch": { - "namespace": "AWS/Usage" - }, - "dimensions": { - "Type": "API", - "Resource": "GetMetricData", - "Service": "CloudWatch", - "Class": "None" } }, - "event": { - "duration": 1191329839, - "dataset": "aws.usage", - "module": "aws" + "cloudwatch": { + "namespace": "AWS/Usage" }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.usage", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-north-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "metricset": { - "name": "usage", - "period": 60000 - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" + "dimensions": { + "Type": "API", + "Resource": "GetMetricData", + "Service": "CloudWatch", + "Class": "None" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:30.929Z" - ] + "event": { + "duration": 1191329839, + "dataset": "aws.usage", + "module": "aws" + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-north-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.usage@/kibana-highlighted-field@" - ] + "metricset": { + "name": "usage", + "period": 60000 }, - "sort": [ - 1590688710929 - ] + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + } } \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpn/fields/ecs.yml b/test/packages/aws/data_stream/vpn/fields/ecs.yml index e49975bc27..432ee5f4d8 100644 --- a/test/packages/aws/data_stream/vpn/fields/ecs.yml +++ b/test/packages/aws/data_stream/vpn/fields/ecs.yml @@ -43,3 +43,7 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/aws/data_stream/vpn/fields/fields.yml b/test/packages/aws/data_stream/vpn/fields/fields.yml index f8cbeb71f6..5a5ff461f0 100644 --- a/test/packages/aws/data_stream/vpn/fields/fields.yml +++ b/test/packages/aws/data_stream/vpn/fields/fields.yml @@ -25,3 +25,9 @@ - name: TunnelIpAddress type: keyword description: Filters the metric data by the IP address of the tunnel for the virtual private gateway. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/vpn/sample_event.json b/test/packages/aws/data_stream/vpn/sample_event.json index 4a8aab803b..a5f331f9c5 100644 --- a/test/packages/aws/data_stream/vpn/sample_event.json +++ b/test/packages/aws/data_stream/vpn/sample_event.json @@ -1,75 +1,51 @@ { - "_index": "metrics-aws.vpn-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.vpn", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "vpn": { - "metrics": { - "TunnelState": { - "avg": 0 - }, - "TunnelDataIn": { - "sum": 0 - }, - "TunnelDataOut": { - "sum": 0 - } + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "vpn": { + "metrics": { + "TunnelState": { + "avg": 0 + }, + "TunnelDataIn": { + "sum": 0 + }, + "TunnelDataOut": { + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/VPN" } }, - "event": { - "dataset": "aws.vpn", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "vpn" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "cloudwatch": { + "namespace": "AWS/VPN" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.vpn", + "module": "aws", + "duration": 10418157072 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.vpn@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "vpn" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } \ No newline at end of file diff --git a/test/packages/aws/docs/README.md b/test/packages/aws/docs/README.md index a00fe0ef39..cbb773ac1e 100644 --- a/test/packages/aws/docs/README.md +++ b/test/packages/aws/docs/README.md @@ -604,71 +604,52 @@ An example event for `billing` looks as following: ```$json { - "_index": "metrics-aws.billing-default-000001", - "_id": "IMxJXHIBpGMSUzkZo-s0", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:06.212Z", - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.billing", - "module": "aws", - "duration": 1938760247 - }, - "metricset": { - "name": "billing", - "period": 43200000 - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "billing": { - "metrics": { - "EstimatedCharges": { - "max": 1625.41 - } + "@timestamp": "2020-05-28T17:17:06.212Z", + "cloud": { + "provider": "aws", + "region": "us-east-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "event": { + "dataset": "aws.billing", + "module": "aws", + "duration": 1938760247 + }, + "metricset": { + "name": "billing", + "period": 43200000 + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "billing": { + "metrics": { + "EstimatedCharges": { + "max": 1625.41 } - }, - "cloudwatch": { - "namespace": "AWS/Billing" - }, - "dimensions": { - "Currency": "USD" } }, - "service": { - "type": "aws" + "cloudwatch": { + "namespace": "AWS/Billing" }, - "stream": { - "type": "metrics", - "dataset": "aws.billing", - "namespace": "default" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + "dimensions": { + "Currency": "USD" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:06.212Z" - ] + "service": { + "type": "aws" }, - "sort": [ - 1590686226212 - ] + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + } } ``` @@ -679,6 +660,7 @@ An example event for `billing` looks as following: | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | | aws.billing.metrics.EstimatedCharges.max | Maximum estimated charges for AWS acccount. | long | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.Currency | Currency name. | keyword | | aws.dimensions.ServiceName | AWS service name. | keyword | @@ -701,6 +683,7 @@ An example event for `billing` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -717,6 +700,7 @@ An example event for `billing` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### cloudwatch @@ -725,76 +709,57 @@ An example event for `cloudwatch` looks as following: ```$json { - "_index": "metrics-aws.cloudwatch_metrics-default-000001", - "_id": "-sxJXHIBpGMSUzkZxex8", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:02.812Z", - "event": { - "duration": 14119105951, - "dataset": "aws.cloudwatch", - "module": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "stream": { - "dataset": "aws.cloudwatch_metrics", - "namespace": "default", - "type": "metrics" - }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } + "@timestamp": "2020-05-28T17:17:02.812Z", + "event": { + "duration": 14119105951, + "dataset": "aws.cloudwatch", + "module": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "dimensions": { + "InstanceId": "i-0830bfecfa7173cbe" }, - "aws": { - "dimensions": { - "InstanceId": "i-0830bfecfa7173cbe" - }, - "ec2": { - "metrics": { - "DiskWriteOps": { - "avg": 0, - "max": 0 - }, - "CPUUtilization": { - "avg": 0.7661943132361363, - "max": 0.833333333333333 - } + "ec2": { + "metrics": { + "DiskWriteOps": { + "avg": 0, + "max": 0 + }, + "CPUUtilization": { + "avg": 0.7661943132361363, + "max": 0.833333333333333 } - }, - "cloudwatch": { - "namespace": "AWS/EC2" } }, - "metricset": { - "period": 300000, - "name": "cloudwatch" + "cloudwatch": { + "namespace": "AWS/EC2" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:02.812Z" - ] - }, - "sort": [ - 1590686222812 - ] + "metricset": { + "period": 300000, + "name": "cloudwatch" + } } ``` @@ -825,6 +790,7 @@ An example event for `cloudwatch` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -841,6 +807,7 @@ An example event for `cloudwatch` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### dynamodb @@ -849,82 +816,63 @@ An example event for `dynamodb` looks as following: ```$json { - "_index": "metrics-aws.dynamodb-default-000001", - "_id": "YMxJXHIBpGMSUzkZzO0_", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "stream": { - "type": "metrics", - "dataset": "aws.dynamodb", - "namespace": "default" - }, - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "dimensions": { + "TableName": "TryDaxTable3" }, - "aws": { - "dimensions": { - "TableName": "TryDaxTable3" - }, - "dynamodb": { - "metrics": { - "ProvisionedWriteCapacityUnits": { - "avg": 1 - }, - "ProvisionedReadCapacityUnits": { - "avg": 1 - }, - "ConsumedWriteCapacityUnits": { - "avg": 0, - "sum": 0 - }, - "ConsumedReadCapacityUnits": { - "avg": 0, - "sum": 0 - } + "dynamodb": { + "metrics": { + "ProvisionedWriteCapacityUnits": { + "avg": 1 + }, + "ProvisionedReadCapacityUnits": { + "avg": 1 + }, + "ConsumedWriteCapacityUnits": { + "avg": 0, + "sum": 0 + }, + "ConsumedReadCapacityUnits": { + "avg": 0, + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/DynamoDB" } }, - "metricset": { - "name": "dynamodb", - "period": 300000 + "cloudwatch": { + "namespace": "AWS/DynamoDB" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:08.666Z" - ] - }, - "sort": [ - 1590686228666 - ] + "metricset": { + "name": "dynamodb", + "period": 300000 + } } ``` @@ -934,6 +882,7 @@ An example event for `dynamodb` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dynamodb.metrics.AccountMaxReads.max | The maximum number of read capacity units that can be used by an account. This limit does not apply to on-demand tables or global secondary indexes. | long | | aws.dynamodb.metrics.AccountMaxTableLevelReads.max | The maximum number of read capacity units that can be used by a table or global secondary index of an account. For on-demand tables this limit caps the maximum read request units a table or a global secondary index can use. | long | @@ -981,6 +930,7 @@ An example event for `dynamodb` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -997,6 +947,7 @@ An example event for `dynamodb` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### ebs @@ -1005,94 +956,70 @@ An example event for `ebs` looks as following: ```$json { - "_index": "metrics-aws.ebs-default-000001", - "_id": "_89uXHIBpGMSUzkZoRoL", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:57:22.450Z", - "service": { - "type": "aws" - }, - "aws": { - "ebs": { - "metrics": { - "VolumeReadOps": { - "avg": 0 - }, - "VolumeQueueLength": { - "avg": 0.0000666666666666667 - }, - "VolumeWriteOps": { - "avg": 29 - }, - "VolumeTotalWriteTime": { - "sum": 0.02 - }, - "BurstBalance": { - "avg": 100 - }, - "VolumeWriteBytes": { - "avg": 14406.620689655172 - }, - "VolumeIdleTime": { - "sum": 299.98 - } + "@timestamp": "2020-05-28T17:57:22.450Z", + "service": { + "type": "aws" + }, + "aws": { + "ebs": { + "metrics": { + "VolumeReadOps": { + "avg": 0 + }, + "VolumeQueueLength": { + "avg": 0.0000666666666666667 + }, + "VolumeWriteOps": { + "avg": 29 + }, + "VolumeTotalWriteTime": { + "sum": 0.02 + }, + "BurstBalance": { + "avg": 100 + }, + "VolumeWriteBytes": { + "avg": 14406.620689655172 + }, + "VolumeIdleTime": { + "sum": 299.98 } - }, - "cloudwatch": { - "namespace": "AWS/EBS" - }, - "dimensions": { - "VolumeId": "vol-03370a204cc8b0a2f" } }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + "cloudwatch": { + "namespace": "AWS/EBS" }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.ebs", - "module": "aws", - "duration": 10488314037 - }, - "metricset": { - "period": 300000, - "name": "ebs" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ebs" + "dimensions": { + "VolumeId": "vol-03370a204cc8b0a2f" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:57:22.450Z" - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.ebs@/kibana-highlighted-field@" - ] + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } }, - "sort": [ - 1590688642450 - ] + "event": { + "dataset": "aws.ebs", + "module": "aws", + "duration": 10488314037 + }, + "metricset": { + "period": 300000, + "name": "ebs" + } } ``` @@ -1102,6 +1029,7 @@ An example event for `ebs` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.VolumeId | Amazon EBS volume ID | keyword | | aws.ebs.metrics.BurstBalance.avg | Used with General Purpose SSD (gp2), Throughput Optimized HDD (st1), and Cold HDD (sc1) volumes only. Provides information about the percentage of I/O credits (for gp2) or throughput credits (for st1 and sc1) remaining in the burst bucket. | double | @@ -1134,6 +1062,7 @@ An example event for `ebs` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1150,6 +1079,7 @@ An example event for `ebs` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### ec2 @@ -1158,138 +1088,114 @@ An example event for `ec2` looks as following: ```$json { - "_index": "metrics-aws.ec2_metrics-default-000001", - "_id": "b89uXHIBpGMSUzkZHxPP", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:56:37.255Z", - "aws": { - "ec2": { - "network": { - "in": { - "packets": 448.4, - "bytes_per_sec": 103.10266666666666, - "packets_per_sec": 1.4946666666666666, - "bytes": 30930.8 - }, - "out": { - "packets": 233.6, - "bytes_per_sec": 51.754666666666665, - "packets_per_sec": 0.7786666666666666, - "bytes": 15526.4 - } + "@timestamp": "2020-05-28T17:56:37.255Z", + "aws": { + "ec2": { + "network": { + "in": { + "packets": 448.4, + "bytes_per_sec": 103.10266666666666, + "packets_per_sec": 1.4946666666666666, + "bytes": 30930.8 + }, + "out": { + "packets": 233.6, + "bytes_per_sec": 51.754666666666665, + "packets_per_sec": 0.7786666666666666, + "bytes": 15526.4 + } + }, + "status": { + "check_failed": 0, + "check_failed_instance": 0, + "check_failed_system": 0 + }, + "cpu": { + "credit_usage": 0.004566, + "credit_balance": 144, + "surplus_credit_balance": 0, + "surplus_credits_charged": 0, + "total": { + "pct": 0.0999999999997574 + } + }, + "diskio": { + "read": { + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0, + "count": 0 + }, + "write": { + "count": 0, + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0 + } + }, + "instance": { + "core": { + "count": 1 }, - "status": { - "check_failed": 0, - "check_failed_instance": 0, - "check_failed_system": 0 + "threads_per_core": 1, + "public": { + "ip": "3.122.204.80", + "dns_name": "" }, - "cpu": { - "credit_usage": 0.004566, - "credit_balance": 144, - "surplus_credit_balance": 0, - "surplus_credits_charged": 0, - "total": { - "pct": 0.0999999999997574 - } + "private": { + "ip": "10.0.0.122", + "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" }, - "diskio": { - "read": { - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0, - "count": 0 - }, - "write": { - "count": 0, - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0 - } + "image": { + "id": "ami-0b418580298265d5c" }, - "instance": { - "core": { - "count": 1 - }, - "threads_per_core": 1, - "public": { - "ip": "3.122.204.80", - "dns_name": "" - }, - "private": { - "ip": "10.0.0.122", - "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" - }, - "image": { - "id": "ami-0b418580298265d5c" - }, - "state": { - "name": "running", - "code": 16 - }, - "monitoring": { - "state": "disabled" - } + "state": { + "name": "running", + "code": 16 + }, + "monitoring": { + "state": "disabled" } } - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "aws", - "duration": 23217499283, - "dataset": "aws.ec2" - }, - "metricset": { - "period": 300000, - "name": "ec2" - }, - "service": { - "type": "aws" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ec2_metrics" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "instance": { - "id": "i-04c1a32c2aace6b40" - }, - "machine": { - "type": "t2.micro" - }, - "availability_zone": "eu-central-1a" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:56:37.255Z" - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.ec2@/kibana-highlighted-field@" - ] + "event": { + "module": "aws", + "duration": 23217499283, + "dataset": "aws.ec2" + }, + "metricset": { + "period": 300000, + "name": "ec2" }, - "sort": [ - 1590688597255 - ] + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "instance": { + "id": "i-04c1a32c2aace6b40" + }, + "machine": { + "type": "t2.micro" + }, + "availability_zone": "eu-central-1a" + } } ``` @@ -1394,91 +1300,67 @@ An example event for `elb` looks as following: ```$json { - "_index": "metrics-aws.elb_metrics-default-000001", - "_id": "i89vXHIBpGMSUzkZuSyO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:30.211Z", - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "aws": { - "elb": { - "metrics": { - "EstimatedALBNewConnectionCount": { - "avg": 32 - }, - "EstimatedALBConsumedLCUs": { - "avg": 0.00035000000000000005 - }, - "EstimatedProcessedBytes": { - "avg": 967 - }, - "EstimatedALBActiveConnectionCount": { - "avg": 5 - }, - "HealthyHostCount": { - "max": 2 - }, - "UnHealthyHostCount": { - "max": 0 - } + "@timestamp": "2020-05-28T17:58:30.211Z", + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "aws": { + "elb": { + "metrics": { + "EstimatedALBNewConnectionCount": { + "avg": 32 + }, + "EstimatedALBConsumedLCUs": { + "avg": 0.00035000000000000005 + }, + "EstimatedProcessedBytes": { + "avg": 967 + }, + "EstimatedALBActiveConnectionCount": { + "avg": 5 + }, + "HealthyHostCount": { + "max": 2 + }, + "UnHealthyHostCount": { + "max": 0 } - }, - "cloudwatch": { - "namespace": "AWS/ELB" - }, - "dimensions": { - "LoadBalancerName": "filebeat-aws-elb-test-elb" } }, - "metricset": { - "name": "elb", - "period": 60000 + "cloudwatch": { + "namespace": "AWS/ELB" }, - "event": { - "dataset": "aws.elb", - "module": "aws", - "duration": 15044430616 - }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.elb_metrics", - "namespace": "default" + "dimensions": { + "LoadBalancerName": "filebeat-aws-elb-test-elb" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:30.211Z" - ] + "metricset": { + "name": "elb", + "period": 60000 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.elb@/kibana-highlighted-field@" - ] + "event": { + "dataset": "aws.elb", + "module": "aws", + "duration": 15044430616 }, - "sort": [ - 1590688710211 - ] + "service": { + "type": "aws" + } } ``` @@ -1508,6 +1390,7 @@ An example event for `elb` looks as following: | aws.applicationelb.metrics.RejectedConnectionCount.sum | The number of connections that were rejected because the load balancer had reached its maximum number of connections. | long | | aws.applicationelb.metrics.RequestCount.sum | The number of requests processed over IPv4 and IPv6. | long | | aws.applicationelb.metrics.RuleEvaluations.sum | The number of rules processed by the load balancer given a request rate averaged over an hour. | long | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.AvailabilityZone | Filters the metric data by the specified Availability Zone. | keyword | | aws.dimensions.LoadBalancer | Filters the metric data by load balancer. | keyword | @@ -1565,6 +1448,7 @@ An example event for `elb` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1581,6 +1465,7 @@ An example event for `elb` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### lambda @@ -1589,81 +1474,62 @@ An example event for `lambda` looks as following: ```$json { - "_index": "metrics-aws.lambda-default-000001", - "_id": "YMxJXHIBpGMSUzkZzO0_", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "stream": { - "type": "metrics", - "dataset": "aws.lambda", - "namespace": "default" - }, - "service": { - "type": "aws" + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" }, - "ecs": { - "version": "1.5.0" + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/Lambda" }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" + "dimensions": { + "FunctionName": "ec2-owner-tagger-serverless", + "Resource": "ec2-owner-tagger-serverless" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/Lambda" - }, - "dimensions": { - "FunctionName": "ec2-owner-tagger-serverless", - "Resource": "ec2-owner-tagger-serverless" - }, - "lambda": { - "metrics": { - "Duration": { - "avg": 8218.073333333334 - }, - "Errors": { - "avg": 1 - }, - "Invocations": { - "avg": 1 - }, - "Throttles": { - "avg": 0 - } + "lambda": { + "metrics": { + "Duration": { + "avg": 8218.073333333334 + }, + "Errors": { + "avg": 1 + }, + "Invocations": { + "avg": 1 + }, + "Throttles": { + "avg": 0 } } - }, - "metricset": { - "name": "dynamodb", - "period": 300000 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:17:08.666Z" - ] - }, - "sort": [ - 1590686228666 - ] + "metricset": { + "name": "dynamodb", + "period": 300000 + } } ``` @@ -1673,6 +1539,7 @@ An example event for `lambda` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.ExecutedVersion | Use the ExecutedVersion dimension to compare error rates for two versions of a function that are both targets of a weighted alias. | keyword | | aws.dimensions.FunctionName | Lambda function name. | keyword | @@ -1709,6 +1576,7 @@ An example event for `lambda` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1725,6 +1593,7 @@ An example event for `lambda` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### natgateway @@ -1733,112 +1602,88 @@ An example event for `natgateway` looks as following: ```$json { - "_index": "metrics-aws.natgateway-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.natgateway", - "namespace": "default", - "type": "metrics" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/NATGateway" }, - "ecs": { - "version": "1.5.0" + "dimensions": { + "NatGatewayId": "nat-0a5cb7b9807908cc0" }, - "aws": { - "cloudwatch": { - "namespace": "AWS/NATGateway" - }, - "dimensions": { - "NatGatewayId": "nat-0a5cb7b9807908cc0" - }, - "natgateway": { - "metrics": { - "ActiveConnectionCount": { - "max": 0 - }, - "BytesInFromDestination": { - "sum": 0 - }, - "BytesInFromSource": { - "sum": 0 - }, - "BytesOutToDestination": { - "sum": 0 - }, - "BytesOutToSource": { - "sum": 0 - }, - "ConnectionAttemptCount": { - "sum": 0 - }, - "ConnectionEstablishedCount": { - "sum": 0 - }, - "ErrorPortAllocation": { - "sum": 0 - }, - "PacketsDropCount": { - "sum": 0 - }, - "PacketsInFromDestination": { - "sum": 0 - }, - "PacketsInFromSource": { - "sum": 0 - }, - "PacketsOutToDestination": { - "sum": 0 - }, - "PacketsOutToSource": { - "sum": 0 - } + "natgateway": { + "metrics": { + "ActiveConnectionCount": { + "max": 0 + }, + "BytesInFromDestination": { + "sum": 0 + }, + "BytesInFromSource": { + "sum": 0 + }, + "BytesOutToDestination": { + "sum": 0 + }, + "BytesOutToSource": { + "sum": 0 + }, + "ConnectionAttemptCount": { + "sum": 0 + }, + "ConnectionEstablishedCount": { + "sum": 0 + }, + "ErrorPortAllocation": { + "sum": 0 + }, + "PacketsDropCount": { + "sum": 0 + }, + "PacketsInFromDestination": { + "sum": 0 + }, + "PacketsInFromSource": { + "sum": 0 + }, + "PacketsOutToDestination": { + "sum": 0 + }, + "PacketsOutToSource": { + "sum": 0 } } - }, - "event": { - "dataset": "aws.natgateway", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "natgateway" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.natgateway", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "natgateway" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.natgateway@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -1848,6 +1693,7 @@ An example event for `natgateway` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.NatGatewayId | Filter the metric data by the NAT gateway ID. | keyword | | aws.natgateway.metrics.ActiveConnectionCount.max | The total number of concurrent active TCP connections through the NAT gateway. | long | @@ -1883,6 +1729,7 @@ An example event for `natgateway` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -1899,125 +1746,102 @@ An example event for `natgateway` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | -### rds - -An example event for `rds` looks as following: - -```$json -{ - "_index": "metrics-aws.rds-default-000001", - "_id": "k89vXHIBpGMSUzkZuSyO", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:34.537Z", - "ecs": { - "version": "1.5.0" - }, - "service": { - "type": "aws" - }, - "aws": { - "rds": { - "latency": { - "dml": 0, - "insert": 0, - "update": 0, - "commit": 0, - "ddl": 0, - "delete": 0, - "select": 0.21927814569536422 - }, - "queries": 6.197934021992669, - "aurora_bin_log_replica_lag": 0, - "transactions": { - "blocked": 0, - "active": 0 - }, - "deadlocks": 0, - "login_failures": 0, - "throughput": { - "network": 1.399813358218904, - "insert": 0, - "ddl": 0, - "select": 2.5165408396246853, - "delete": 0, - "commit": 0, - "network_transmit": 0.699906679109452, - "update": 0, - "dml": 0, - "network_receive": 0.699906679109452 - }, - "cpu": { - "total": { - "pct": 0.03 - } - }, - "db_instance": { - "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", - "class": "db.r5.large", - "identifier": "database-1-instance-1-eu-west-1a", - "status": "available" - }, - "cache_hit_ratio.result_set": 0, - "aurora_replica.lag.ms": 19.576, - "free_local_storage.bytes": 32431271936, - "cache_hit_ratio.buffer": 100, - "disk_usage": { - "bin_log.bytes": 0 - }, - "db_instance.identifier": "database-1-instance-1-eu-west-1a", - "freeable_memory.bytes": 4436537344, - "engine_uptime.sec": 10463030, - "database_connections": 0 - } - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" +### rds + +An example event for `rds` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:34.537Z", + "ecs": { + "version": "1.5.0" + }, + "service": { + "type": "aws" + }, + "aws": { + "rds": { + "latency": { + "dml": 0, + "insert": 0, + "update": 0, + "commit": 0, + "ddl": 0, + "delete": 0, + "select": 0.21927814569536422 }, - "availability_zone": "eu-west-1a" - }, - "event": { - "dataset": "aws.rds", - "module": "aws", - "duration": 10777919184 - }, - "metricset": { - "name": "rds", - "period": 60000 - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.rds" - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + "queries": 6.197934021992669, + "aurora_bin_log_replica_lag": 0, + "transactions": { + "blocked": 0, + "active": 0 + }, + "deadlocks": 0, + "login_failures": 0, + "throughput": { + "network": 1.399813358218904, + "insert": 0, + "ddl": 0, + "select": 2.5165408396246853, + "delete": 0, + "commit": 0, + "network_transmit": 0.699906679109452, + "update": 0, + "dml": 0, + "network_receive": 0.699906679109452 + }, + "cpu": { + "total": { + "pct": 0.03 + } + }, + "db_instance": { + "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", + "class": "db.r5.large", + "identifier": "database-1-instance-1-eu-west-1a", + "status": "available" + }, + "cache_hit_ratio.result_set": 0, + "aurora_replica.lag.ms": 19.576, + "free_local_storage.bytes": 32431271936, + "cache_hit_ratio.buffer": 100, + "disk_usage": { + "bin_log.bytes": 0 + }, + "db_instance.identifier": "database-1-instance-1-eu-west-1a", + "freeable_memory.bytes": 4436537344, + "engine_uptime.sec": 10463030, + "database_connections": 0 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:34.537Z" - ] + "cloud": { + "provider": "aws", + "region": "eu-west-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "availability_zone": "eu-west-1a" + }, + "event": { + "dataset": "aws.rds", + "module": "aws", + "duration": 10777919184 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.rds@/kibana-highlighted-field@" - ] + "metricset": { + "name": "rds", + "period": 60000 }, - "sort": [ - 1590688714537 - ] + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + } } ``` @@ -2128,6 +1952,7 @@ An example event for `rds` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -2144,6 +1969,7 @@ An example event for `rds` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### s3_daily_storage @@ -2152,76 +1978,52 @@ An example event for `s3_daily_storage` looks as following: ```$json { - "_index": "metrics-aws.s3_daily_storage-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.s3_daily_storage", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_daily_storage": { - "bucket": { - "size": { - "bytes": 207372 - } - }, - "number_of_objects": 128 + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" } }, - "event": { - "dataset": "aws.s3_daily_storage", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_daily_storage" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "s3_daily_storage": { + "bucket": { + "size": { + "bytes": 207372 + } }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "number_of_objects": 128 } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.s3_daily_storage", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "s3_daily_storage" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.s3_daily_storage@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -2256,6 +2058,7 @@ An example event for `s3_daily_storage` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -2272,6 +2075,7 @@ An example event for `s3_daily_storage` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### s3_request @@ -2280,89 +2084,65 @@ An example event for `s3_request` looks as following: ```$json { - "_index": "metrics-aws.s3_request-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.s3_request", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_request": { - "downloaded": { - "bytes": 534 - }, - "errors": { - "4xx": 0, - "5xx": 0 - }, - "latency": { - "first_byte.ms": 214, - "total_request.ms": 533 - }, - "requests": { - "list": 2, - "put": 10, - "total": 12 - }, - "uploaded": { - "bytes": 13572 - } + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" } }, - "event": { - "dataset": "aws.s3_request", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_request" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "s3_request": { + "downloaded": { + "bytes": 534 }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "errors": { + "4xx": 0, + "5xx": 0 + }, + "latency": { + "first_byte.ms": 214, + "total_request.ms": 533 + }, + "requests": { + "list": 2, + "put": 10, + "total": 12 + }, + "uploaded": { + "bytes": 13572 + } } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.s3_request", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "s3_request" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.s3_request@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -2411,6 +2191,7 @@ An example event for `s3_request` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -2427,6 +2208,7 @@ An example event for `s3_request` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### sns @@ -2556,81 +2338,57 @@ An example event for `sqs` looks as following: ```$json { - "_index": "metrics-aws.sqs-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.sqs", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "sqs": { - "empty_receives": 0, - "messages": { - "delayed": 0, - "deleted": 0, - "not_visible": 0, - "received": 0, - "sent": 0, - "visible": 2 - }, - "oldest_message_age": { - "sec": 78494 - }, - "queue": { - "name": "test-s3-notification" - }, - "sent_message_size": {} - } - }, - "event": { - "dataset": "aws.sqs", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "sqs" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "sqs": { + "empty_receives": 0, + "messages": { + "delayed": 0, + "deleted": 0, + "not_visible": 0, + "received": 0, + "sent": 0, + "visible": 2 }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "oldest_message_age": { + "sec": 78494 + }, + "queue": { + "name": "test-s3-notification" + }, + "sent_message_size": {} } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.sqs", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "sqs" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.sqs@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -2671,6 +2429,7 @@ An example event for `sqs` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -2687,6 +2446,7 @@ An example event for `sqs` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### transitgateway @@ -2695,91 +2455,67 @@ An example event for `transitgateway` looks as following: ```$json { - "_index": "metrics-aws.transitgateway-default-000001", - "_id": "WNToXHIBpGMSUzkZaeVh", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T20:10:20.953Z", - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "aws": { - "transitgateway": { - "metrics": { - "PacketsIn": { - "sum": 0 - }, - "BytesIn": { - "sum": 0 - }, - "BytesOut": { - "sum": 0 - }, - "PacketsOut": { - "sum": 0 - }, - "PacketDropCountBlackhole": { - "sum": 0 - }, - "PacketDropCountNoRoute": { - "sum": 0 - } + "@timestamp": "2020-05-28T20:10:20.953Z", + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "transitgateway": { + "metrics": { + "PacketsIn": { + "sum": 0 + }, + "BytesIn": { + "sum": 0 + }, + "BytesOut": { + "sum": 0 + }, + "PacketsOut": { + "sum": 0 + }, + "PacketDropCountBlackhole": { + "sum": 0 + }, + "PacketDropCountNoRoute": { + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/TransitGateway" - }, - "dimensions": { - "TransitGateway": "tgw-0630672a32f12808a" } }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - }, - "event": { - "dataset": "aws.transitgateway", - "module": "aws", - "duration": 12762825681 - }, - "metricset": { - "period": 60000, - "name": "transitgateway" + "cloudwatch": { + "namespace": "AWS/TransitGateway" }, - "service": { - "type": "aws" - }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.transitgateway" + "dimensions": { + "TransitGateway": "tgw-0630672a32f12808a" } }, - "fields": { - "@timestamp": [ - "2020-05-28T20:10:20.953Z" - ] + "ecs": { + "version": "1.5.0" + }, + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "event": { + "dataset": "aws.transitgateway", + "module": "aws", + "duration": 12762825681 }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.transitgateway@/kibana-highlighted-field@" - ] + "metricset": { + "period": 60000, + "name": "transitgateway" }, - "sort": [ - 1590696620953 - ] + "service": { + "type": "aws" + } } ``` @@ -2789,6 +2525,7 @@ An example event for `transitgateway` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.TransitGateway | Filters the metric data by transit gateway. | keyword | | aws.dimensions.TransitGatewayAttachment | Filters the metric data by transit gateway attachment. | keyword | @@ -2817,6 +2554,7 @@ An example event for `transitgateway` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -2833,6 +2571,7 @@ An example event for `transitgateway` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### usage @@ -2841,79 +2580,55 @@ An example event for `usage` looks as following: ```$json { - "_index": "metrics-aws.usage-default-000001", - "_id": "YM9vXHIBpGMSUzkZiSlC", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:30.929Z", - "aws": { - "usage": { - "metrics": { - "CallCount": { - "sum": 1 - } + "@timestamp": "2020-05-28T17:58:30.929Z", + "aws": { + "usage": { + "metrics": { + "CallCount": { + "sum": 1 } - }, - "cloudwatch": { - "namespace": "AWS/Usage" - }, - "dimensions": { - "Type": "API", - "Resource": "GetMetricData", - "Service": "CloudWatch", - "Class": "None" - } - }, - "event": { - "duration": 1191329839, - "dataset": "aws.usage", - "module": "aws" - }, - "service": { - "type": "aws" - }, - "stream": { - "type": "metrics", - "dataset": "aws.usage", - "namespace": "default" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-north-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" } }, - "metricset": { - "name": "usage", - "period": 60000 + "cloudwatch": { + "namespace": "AWS/Usage" }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" + "dimensions": { + "Type": "API", + "Resource": "GetMetricData", + "Service": "CloudWatch", + "Class": "None" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:30.929Z" - ] + "event": { + "duration": 1191329839, + "dataset": "aws.usage", + "module": "aws" + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-north-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.usage@/kibana-highlighted-field@" - ] + "metricset": { + "name": "usage", + "period": 60000 }, - "sort": [ - 1590688710929 - ] + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + } } ``` @@ -2923,6 +2638,7 @@ An example event for `usage` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.Class | The class of resource being tracked. | keyword | | aws.dimensions.Resource | The name of the API operation. | keyword | @@ -2949,6 +2665,7 @@ An example event for `usage` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -2965,6 +2682,7 @@ An example event for `usage` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | ### vpn @@ -2973,79 +2691,55 @@ An example event for `vpn` looks as following: ```$json { - "_index": "metrics-aws.vpn-default-000001", - "_id": "Ds9vXHIBpGMSUzkZmyod", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "stream": { - "dataset": "aws.vpn", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "vpn": { - "metrics": { - "TunnelState": { - "avg": 0 - }, - "TunnelDataIn": { - "sum": 0 - }, - "TunnelDataOut": { - "sum": 0 - } + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "vpn": { + "metrics": { + "TunnelState": { + "avg": 0 + }, + "TunnelDataIn": { + "sum": 0 + }, + "TunnelDataOut": { + "sum": 0 } - }, - "cloudwatch": { - "namespace": "AWS/VPN" } }, - "event": { - "dataset": "aws.vpn", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "vpn" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "cloudwatch": { + "namespace": "AWS/VPN" } }, - "fields": { - "@timestamp": [ - "2020-05-28T17:58:27.154Z" - ] + "event": { + "dataset": "aws.vpn", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "vpn" }, - "highlight": { - "event.dataset": [ - "@kibana-highlighted-field@aws.vpn@/kibana-highlighted-field@" - ] + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" }, - "sort": [ - 1590688707154 - ] + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } } ``` @@ -3055,6 +2749,7 @@ An example event for `vpn` looks as following: |---|---|---| | @timestamp | Event timestamp. | date | | aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | | aws.dimensions.* | Metric dimensions. | object | | aws.dimensions.TunnelIpAddress | Filters the metric data by the IP address of the tunnel for the virtual private gateway. | keyword | | aws.dimensions.VpnId | Filters the metric data by the Site-to-Site VPN connection ID. | keyword | @@ -3080,6 +2775,7 @@ An example event for `vpn` looks as following: | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| ecs.version | | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | | host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | @@ -3096,4 +2792,5 @@ An example event for `vpn` looks as following: | host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | | keyword | diff --git a/test/packages/kubernetes/data_stream/pod/sample_event.json b/test/packages/kubernetes/data_stream/pod/sample_event.json index 1ead86b4dd..62e44029da 100644 --- a/test/packages/kubernetes/data_stream/pod/sample_event.json +++ b/test/packages/kubernetes/data_stream/pod/sample_event.json @@ -1,152 +1,125 @@ { - "_index": ".ds-metrics-kubernetes.pod-default-000001", - "_id": "4Vl563IBolOt49UrYz6x", - "_version": 1, - "_score": null, - "_source": { - "@timestamp": "2020-06-25T12:34:59.729Z", - "kubernetes": { - "pod": { - "memory": { - "rss": { - "bytes": 7823360 - }, - "page_faults": 5742, - "major_page_faults": 0, - "usage": { - "limit": { - "pct": 0.0008033509820466402 - }, - "bytes": 13508608, - "node": { - "pct": 0.0008033509820466402 - } - }, - "available": { - "bytes": 0 - }, - "working_set": { - "bytes": 8556544 - } + "@timestamp": "2020-06-25T12:34:59.729Z", + "kubernetes": { + "pod": { + "memory": { + "rss": { + "bytes": 7823360 }, - "network": { - "rx": { - "bytes": 25671624, - "errors": 0 + "page_faults": 5742, + "major_page_faults": 0, + "usage": { + "limit": { + "pct": 0.0008033509820466402 }, - "tx": { - "errors": 0, - "bytes": 1092900259 + "bytes": 13508608, + "node": { + "pct": 0.0008033509820466402 } }, - "start_time": "2020-06-18T11:12:58Z", - "name": "kube-state-metrics-57cd6fdf9-hd959", - "uid": "a7c61334-dd52-4a12-bed5-4daee4c74139", - "cpu": { - "usage": { - "nanocores": 2811918, - "node": { - "pct": 0.0007029795 - }, - "limit": { - "pct": 0.0007029795 - } - } + "available": { + "bytes": 0 + }, + "working_set": { + "bytes": 8556544 } }, - "namespace": "kube-system", - "node": { - "name": "minikube" - } - }, - "event": { - "duration": 20735189, - "dataset": "kubernetes.pod", - "module": "kubernetes" - }, - "stream": { - "dataset": "kubernetes.pod", - "namespace": "default", - "type": "metrics" - }, - "ecs": { - "version": "1.5.0" - }, - "metricset": { - "period": 10000, - "name": "pod" - }, - "service": { - "type": "kubernetes", - "address": "minikube:10250" - }, - "dataset": { - "type": "metrics", - "name": "kubernetes.pod", - "namespace": "default" - }, - "host": { - "name": "minikube", - "hostname": "minikube", - "architecture": "x86_64", - "os": { - "kernel": "4.19.81", - "codename": "Core", - "platform": "centos", - "version": "7 (Core)", - "family": "redhat", - "name": "CentOS Linux" + "network": { + "rx": { + "bytes": 25671624, + "errors": 0 + }, + "tx": { + "errors": 0, + "bytes": 1092900259 + } }, - "id": "b0e83d397c054b8a99a431072fe4617b", - "containerized": false, - "ip": [ - "192.168.64.10", - "fe80::a883:2fff:fe7f:6b12", - "172.17.0.1", - "fe80::42:d4ff:fe8c:9493", - "fe80::2859:80ff:fe9e:fcd6", - "fe80::d83a:d9ff:fee9:7052", - "fe80::880a:b6ff:fe18:ba76", - "fe80::f447:faff:fe80:e88b", - "fe80::9cc3:ffff:fe95:e48e", - "fe80::6c1c:29ff:fe50:d40c", - "fe80::b4f3:11ff:fe60:14ed", - "fe80::20f2:2aff:fe96:1e7b", - "fe80::5434:baff:fede:5720", - "fe80::a878:91ff:fe29:81f7" - ], - "mac": [ - "aa:83:2f:7f:6b:12", - "02:42:d4:8c:94:93", - "2a:59:80:9e:fc:d6", - "da:3a:d9:e9:70:52", - "8a:0a:b6:18:ba:76", - "f6:47:fa:80:e8:8b", - "9e:c3:ff:95:e4:8e", - "6e:1c:29:50:d4:0c", - "b6:f3:11:60:14:ed", - "22:f2:2a:96:1e:7b", - "56:34:ba:de:57:20", - "aa:78:91:29:81:f7" - ] + "start_time": "2020-06-18T11:12:58Z", + "name": "kube-state-metrics-57cd6fdf9-hd959", + "uid": "a7c61334-dd52-4a12-bed5-4daee4c74139", + "cpu": { + "usage": { + "nanocores": 2811918, + "node": { + "pct": 0.0007029795 + }, + "limit": { + "pct": 0.0007029795 + } + } + } }, - "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", - "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "namespace": "kube-system", + "node": { "name": "minikube" } }, - "fields": { - "kubernetes.pod.start_time": [ - "2020-06-18T11:12:58.000Z" + "event": { + "duration": 20735189, + "dataset": "kubernetes.pod", + "module": "kubernetes" + }, + "ecs": { + "version": "1.5.0" + }, + "metricset": { + "period": 10000, + "name": "pod" + }, + "service": { + "type": "kubernetes", + "address": "minikube:10250" + }, + "host": { + "name": "minikube", + "hostname": "minikube", + "architecture": "x86_64", + "os": { + "kernel": "4.19.81", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "b0e83d397c054b8a99a431072fe4617b", + "containerized": false, + "ip": [ + "192.168.64.10", + "fe80::a883:2fff:fe7f:6b12", + "172.17.0.1", + "fe80::42:d4ff:fe8c:9493", + "fe80::2859:80ff:fe9e:fcd6", + "fe80::d83a:d9ff:fee9:7052", + "fe80::880a:b6ff:fe18:ba76", + "fe80::f447:faff:fe80:e88b", + "fe80::9cc3:ffff:fe95:e48e", + "fe80::6c1c:29ff:fe50:d40c", + "fe80::b4f3:11ff:fe60:14ed", + "fe80::20f2:2aff:fe96:1e7b", + "fe80::5434:baff:fede:5720", + "fe80::a878:91ff:fe29:81f7" ], - "@timestamp": [ - "2020-06-25T12:34:59.729Z" + "mac": [ + "aa:83:2f:7f:6b:12", + "02:42:d4:8c:94:93", + "2a:59:80:9e:fc:d6", + "da:3a:d9:e9:70:52", + "8a:0a:b6:18:ba:76", + "f6:47:fa:80:e8:8b", + "9e:c3:ff:95:e4:8e", + "6e:1c:29:50:d4:0c", + "b6:f3:11:60:14:ed", + "22:f2:2a:96:1e:7b", + "56:34:ba:de:57:20", + "aa:78:91:29:81:f7" ] }, - "sort": [ - 1593088499729 - ] + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "b964a246-96c0-456a-a5c2-8c8b1040ecaf", + "id": "f7ec69f9-4997-4e76-b6c7-0c75206b727a", + "name": "minikube" + } } \ No newline at end of file diff --git a/test/packages/nginx/data_stream/stubstatus/fields/fields.yml b/test/packages/nginx/data_stream/stubstatus/fields/fields.yml index 5ded312df6..eb62a74b75 100644 --- a/test/packages/nginx/data_stream/stubstatus/fields/fields.yml +++ b/test/packages/nginx/data_stream/stubstatus/fields/fields.yml @@ -41,3 +41,9 @@ type: long description: | The current number of idle client connections waiting for a request. +- name: ecs.version + type: keyword +- name: service.address + type: keyword +- name: service.type + type: keyword diff --git a/test/packages/nginx/data_stream/stubstatus/sample_event.json b/test/packages/nginx/data_stream/stubstatus/sample_event.json index 85e73a6619..27aded9671 100644 --- a/test/packages/nginx/data_stream/stubstatus/sample_event.json +++ b/test/packages/nginx/data_stream/stubstatus/sample_event.json @@ -18,11 +18,6 @@ "active": 1 } }, - "stream": { - "namespace": "default", - "type": "metrics", - "dataset": "nginx.stubstatus" - }, "ecs": { "version": "1.5.0" },