diff --git a/cmd/build.go b/cmd/build.go index c9daaf0ae4..e73b1c1bfa 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -5,6 +5,8 @@ package cmd import ( + "strings" + "github.com/pkg/errors" "github.com/spf13/cobra" @@ -36,15 +38,17 @@ func setupBuildCommand() *cobra.Command { func buildCommandAction(cmd *cobra.Command, args []string) error { cmd.Println("Build the package") - target, err := docs.UpdateReadme() + targets, err := docs.UpdateReadmes() if err != nil { - return errors.Wrapf(err, "updating %s file failed", docs.ReadmeFile) + return errors.Wrap(err, "updating files failed") } - if target != "" { - cmd.Printf("%s file rendered: %s\n", docs.ReadmeFile, target) + + for _, target := range targets { + splitTarget := strings.Split(target, "/") + cmd.Printf("%s file rendered: %s\n", splitTarget[len(splitTarget)-1], target) } - target, err = builder.BuildPackage() + target, err := builder.BuildPackage() if err != nil { return errors.Wrap(err, "building package failed") } diff --git a/cmd/lint.go b/cmd/lint.go index e7d30c4548..eac0578868 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -5,8 +5,6 @@ package cmd import ( - "fmt" - "github.com/pkg/errors" "github.com/spf13/cobra" @@ -44,12 +42,17 @@ func lintCommandAction(cmd *cobra.Command, args []string) error { return errors.Wrap(err, "locating package root failed") } - ok, err := docs.IsReadmeUpToDate() + readmeFiles, err := docs.AreReadmesUpToDate() if err != nil { - return errors.Wrapf(err, "can't check if %s file is up-to-date", docs.ReadmeFile) - } - if !ok { - return fmt.Errorf("%s file is outdated. Rebuild the package with 'elastic-package build'", docs.ReadmeFile) + for _, f := range readmeFiles { + if !f.UpToDate { + cmd.Printf("%s is outdated. Rebuild the package with 'elastic-package build'\n", f.FileName) + } + if f.Error != nil { + cmd.Printf("check if %s is up-to-date failed: %s\n", f.FileName, f.Error) + } + } + return errors.Wrap(err, "checking readme files are up-to-date failed") } err = validator.ValidateFromPath(packageRootPath) diff --git a/internal/docs/readme.go b/internal/docs/readme.go index b9caa298b7..013440c0b6 100644 --- a/internal/docs/readme.go +++ b/internal/docs/readme.go @@ -6,6 +6,7 @@ package docs import ( "bytes" + "fmt" "io/ioutil" "os" "path/filepath" @@ -17,27 +18,62 @@ import ( "github.com/elastic/elastic-package/internal/packages" ) -// ReadmeFile for the Elastic package -const ReadmeFile = "README.md" +// ReadmeFile contains file name and status of each readme file. +type ReadmeFile struct { + FileName string + UpToDate bool + Error error +} + +// AreReadmesUpToDate function checks if all the .md readme files are up-to-date. +func AreReadmesUpToDate() ([]ReadmeFile, error) { + packageRoot, err := packages.MustFindPackageRoot() + if err != nil { + return nil, errors.Wrap(err, "package root not found") + } + + files, err := ioutil.ReadDir(filepath.Join(packageRoot, "_dev", "build", "docs")) + if err != nil && !os.IsNotExist(err) { + return nil, errors.Wrap(err, "reading directory entries failed") + } + + var readmeFiles []ReadmeFile + for _, f := range files { + fileName := f.Name() + ok, err := isReadmeUpToDate(fileName, packageRoot) + if !ok || err != nil { + readmeFile := ReadmeFile{ + FileName: fileName, + UpToDate: ok, + Error: err, + } + readmeFiles = append(readmeFiles, readmeFile) + } + } + + if readmeFiles != nil { + return readmeFiles, fmt.Errorf("checking readme files are up-to-date failed") + } + return readmeFiles, nil +} -// IsReadmeUpToDate function checks if the README file is up-to-date. -func IsReadmeUpToDate() (bool, error) { - logger.Debugf("Check if %s is up-to-date", ReadmeFile) +func isReadmeUpToDate(fileName, packageRoot string) (bool, error) { + logger.Debugf("Check if %s is up-to-date", fileName) packageRoot, err := packages.MustFindPackageRoot() if err != nil { return false, errors.Wrap(err, "package root not found") } - rendered, shouldBeRendered, err := generateReadme(packageRoot) + rendered, shouldBeRendered, err := generateReadme(fileName, packageRoot) if err != nil { - return false, err + return false, errors.Wrap(err, "generating readme file failed") } if !shouldBeRendered { return true, nil // README file is static and doesn't use template. } - existing, found, err := readReadme(packageRoot) + existing, found, err := readReadme(fileName, packageRoot) if err != nil { return false, errors.Wrap(err, "reading README file failed") } @@ -47,17 +83,43 @@ func IsReadmeUpToDate() (bool, error) { return bytes.Equal(existing, rendered), nil } -// UpdateReadme function updates the README file using ą defined template file. The function doesn't perform any action -// if the template file is not present. -func UpdateReadme() (string, error) { - logger.Debugf("Update the %s file", ReadmeFile) +// UpdateReadmes function updates all .md readme files using a defined template +// files. The function doesn't perform any action if the template file is not present. +func UpdateReadmes() ([]string, error) { + packageRoot, err := packages.MustFindPackageRoot() + if err != nil { + return nil, errors.Wrap(err, "package root not found") + } + + readmeFiles, err := ioutil.ReadDir(filepath.Join(packageRoot, "_dev", "build", "docs")) + if err != nil && !os.IsNotExist(err) { + return nil, errors.Wrap(err, "reading directory entries failed") + } + + var targets []string + for _, readme := range readmeFiles { + fileName := readme.Name() + target, err := updateReadme(fileName, packageRoot) + if err != nil { + return nil, errors.Wrapf(err, "updating readme file %s failed", fileName) + } + + if target != "" { + targets = append(targets, target) + } + } + return targets, nil +} + +func updateReadme(fileName, packageRoot string) (string, error) { + logger.Debugf("Update the %s file", fileName) packageRoot, err := packages.MustFindPackageRoot() if err != nil { return "", errors.Wrap(err, "package root not found") } - rendered, shouldBeRendered, err := generateReadme(packageRoot) + rendered, shouldBeRendered, err := generateReadme(fileName, packageRoot) if err != nil { return "", err } @@ -65,48 +127,48 @@ func UpdateReadme() (string, error) { return "", nil } - target, err := writeReadme(packageRoot, rendered) + target, err := writeReadme(fileName, packageRoot, rendered) if err != nil { - return "", errors.Wrapf(err, "writing %s file failed", ReadmeFile) + return "", errors.Wrapf(err, "writing %s file failed", fileName) } return target, nil } -func generateReadme(packageRoot string) ([]byte, bool, error) { - logger.Debugf("Generate %s file (package: %s)", ReadmeFile, packageRoot) - templatePath, found, err := findReadmeTemplatePath(packageRoot) +func generateReadme(fileName, packageRoot string) ([]byte, bool, error) { + logger.Debugf("Generate %s file (package: %s)", fileName, packageRoot) + templatePath, found, err := findReadmeTemplatePath(fileName, packageRoot) if err != nil { - return nil, false, errors.Wrapf(err, "can't locate %s template file", ReadmeFile) + return nil, false, errors.Wrapf(err, "can't locate %s template file", fileName) } if !found { logger.Debug("README file is static, can't be generated from the template file") return nil, false, nil } - logger.Debugf("Template file for %s found: %s", ReadmeFile, templatePath) + logger.Debugf("Template file for %s found: %s", fileName, templatePath) - rendered, err := renderReadme(packageRoot, templatePath) + rendered, err := renderReadme(fileName, packageRoot, templatePath) if err != nil { return nil, true, errors.Wrap(err, "rendering Readme failed") } return rendered, true, nil } -func findReadmeTemplatePath(packageRoot string) (string, bool, error) { - templatePath := filepath.Join(packageRoot, "_dev", "build", "docs", ReadmeFile) +func findReadmeTemplatePath(fileName, packageRoot string) (string, bool, error) { + templatePath := filepath.Join(packageRoot, "_dev", "build", "docs", fileName) _, err := os.Stat(templatePath) if err != nil && os.IsNotExist(err) { return "", false, nil // README.md file not found } if err != nil { - return "", false, errors.Wrapf(err, "can't located the %s file", ReadmeFile) + return "", false, errors.Wrapf(err, "can't stat the %s file", fileName) } return templatePath, true, nil } -func renderReadme(packageRoot, templatePath string) ([]byte, error) { - logger.Debugf("Render %s file (package: %s, templatePath: %s)", ReadmeFile, packageRoot, templatePath) +func renderReadme(fileName, packageRoot, templatePath string) ([]byte, error) { + logger.Debugf("Render %s file (package: %s, templatePath: %s)", fileName, packageRoot, templatePath) - t := template.New(ReadmeFile) + t := template.New(fileName) t, err := t.Funcs(template.FuncMap{ "event": func(dataStreamName string) (string, error) { return renderSampleEvent(packageRoot, dataStreamName) @@ -127,10 +189,10 @@ func renderReadme(packageRoot, templatePath string) ([]byte, error) { return rendered.Bytes(), nil } -func readReadme(packageRoot string) ([]byte, bool, error) { - logger.Debugf("Read existing %s file (package: %s)", ReadmeFile, packageRoot) +func readReadme(fileName, packageRoot string) ([]byte, bool, error) { + logger.Debugf("Read existing %s file (package: %s)", fileName, packageRoot) - readmePath := filepath.Join(packageRoot, "docs", ReadmeFile) + readmePath := filepath.Join(packageRoot, "docs", fileName) b, err := ioutil.ReadFile(readmePath) if err != nil && os.IsNotExist(err) { return nil, false, nil @@ -141,8 +203,8 @@ func readReadme(packageRoot string) ([]byte, bool, error) { return b, true, err } -func writeReadme(packageRoot string, content []byte) (string, error) { - logger.Debugf("Write %s file (package: %s)", ReadmeFile, packageRoot) +func writeReadme(fileName, packageRoot string, content []byte) (string, error) { + logger.Debugf("Write %s file (package: %s)", fileName, packageRoot) docsPath := docsPath(packageRoot) logger.Debugf("Create directories: %s", docsPath) @@ -151,8 +213,8 @@ func writeReadme(packageRoot string, content []byte) (string, error) { return "", errors.Wrapf(err, "mkdir failed (path: %s)", docsPath) } - aReadmePath := readmePath(packageRoot) - logger.Debugf("Write %s file to: %s", ReadmeFile, aReadmePath) + aReadmePath := readmePath(fileName, packageRoot) + logger.Debugf("Write %s file to: %s", fileName, aReadmePath) err = ioutil.WriteFile(aReadmePath, content, 0644) if err != nil { @@ -161,8 +223,8 @@ func writeReadme(packageRoot string, content []byte) (string, error) { return aReadmePath, nil } -func readmePath(packageRoot string) string { - return filepath.Join(docsPath(packageRoot), ReadmeFile) +func readmePath(fileName, packageRoot string) string { + return filepath.Join(docsPath(packageRoot), fileName) } func docsPath(packageRoot string) string { diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 1e0dfa5032..4f8103d623 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -97,6 +97,7 @@ type PackageManifest struct { Version string `config:"version" json:"version" yaml:"version"` Conditions Conditions `config:"conditions" json:"conditions" yaml:"conditions"` PolicyTemplates []PolicyTemplate `config:"policy_templates" json:"policy_templates" yaml:"policy_templates"` + Vars []Variable `config:"vars" json:"vars" yaml:"vars"` } // DataStreamManifest represents the structure of a data stream's manifest diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index f33d98ed16..b1406e6025 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -488,6 +488,8 @@ func createPackageDatastream( pkgVars := kibana.Vars{} input := pkg.PolicyTemplates[0].FindInputByType(streamInput) if input != nil { + // copy package-level vars into each input + input.Vars = append(input.Vars, pkg.Vars...) for _, pkgVar := range input.Vars { val := pkgVar.Default diff --git a/test/packages/aws/_dev/build/docs/README.md b/test/packages/aws/_dev/build/docs/README.md index 940a312aab..e36b499695 100644 --- a/test/packages/aws/_dev/build/docs/README.md +++ b/test/packages/aws/_dev/build/docs/README.md @@ -14,7 +14,6 @@ AWS credentials are required for running AWS integration. * *shared_credential_file*: directory of the shared credentials file. * *endpoint*: URL of the entry point for an AWS web service. * *role_arn*: AWS IAM Role to assume. -* *aws_partition*: AWS region partition name, value is one of `aws, aws-cn, aws-us-gov`, default is `aws`. ### Credential Types There are three types of AWS credentials can be used: access keys, temporary @@ -92,164 +91,3 @@ In order to enable AWS integration, please make sure these permissions are given * sqs:ListQueues * sts:GetCallerIdentity * iam:ListAccountAliases - -## Logs - -### cloudtrail - -The `cloudtrail` dataset collects the AWS CloudTrail logs. CloudTrail monitors -events for the account. If user creates a trail, it delivers those events as log - files to a specific Amazon S3 bucket. The `cloudtrail` dataset does not read - the CloudTrail Digest files that are delivered to the S3 bucket when Log File - Integrity is turned on, it only reads the CloudTrail logs. - -{{fields "cloudtrail"}} - -### cloudwatch - -The `cloudwatch` dataset collects CloudWatch logs. Users can use Amazon -CloudWatch logs to monitor, store, and access log files from different sources. -Export logs from log groups to an Amazon S3 bucket which has SQS notification -setup already. - -{{fields "cloudwatch_logs"}} - -### ec2 - -The `ec2` dataset is specifically for EC2 logs stored in AWS CloudWatch. Export logs -from log groups to Amazon S3 bucket which has SQS notification setup already. -With this dataset, EC2 logs will be parsed into fields like `ip_address` -and `process.name`. For logs from other services, please use `cloudwatch` dataset. - -{{fields "ec2_logs"}} - -### elb - -The `elb` dataset collects logs from AWS ELBs. Elastic Load Balancing provides -access logs that capture detailed information about requests sent to the load -balancer. Each log contains information such as the time the request was -received, the client's IP address, latencies, request paths, and server -responses. Users can use these access logs to analyze traffic patterns and to -troubleshoot issues. - -Please follow [enable access logs for classic load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html) -for sending Classic ELB access logs to S3 bucket. -For application load balancer, please follow [enable access log for application load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html#enable-access-logging). -For network load balancer, please follow [enable access log for network load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest//network/load-balancer-access-logs.html). - -{{fields "elb_logs"}} - -### s3access - -The `s3access` dataset collects server access logs from AWS S3. Server access -logging provides detailed records for the requests that are made to a bucket. -Server access logs are useful for many applications. For example, access log -information can be useful in security and access audits. It can also help users -to learn about customer base and understand Amazon S3 bill. - -Please follow [how to enable server access logging](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html#server-access-logging-overview) -for sending server access logs to S3 bucket. - -{{fields "s3access"}} - -### vpcflow - -{{fields "vpcflow"}} - -## Metrics - -### billing - -{{event "billing"}} - -{{fields "billing"}} - -### cloudwatch - -{{event "cloudwatch_metrics"}} - -{{fields "cloudwatch_metrics"}} - -### dynamodb - -{{event "dynamodb"}} - -{{fields "dynamodb"}} - -### ebs - -{{event "ebs"}} - -{{fields "ebs"}} - -### ec2 - -{{event "ec2_metrics"}} - -{{fields "ec2_metrics"}} - -### elb - -{{event "elb_metrics"}} - -{{fields "elb_metrics"}} - -### lambda - -{{event "lambda"}} - -{{fields "lambda"}} - -### natgateway - -{{event "natgateway"}} - -{{fields "natgateway"}} - -### rds - -{{event "rds"}} - -{{fields "rds"}} - -### s3_daily_storage - -{{event "s3_daily_storage"}} - -{{fields "s3_daily_storage"}} - -### s3_request - -{{event "s3_request"}} - -{{fields "s3_request"}} - -### sns - -{{event "sns"}} - -{{fields "sns"}} - -### sqs - -{{event "sqs"}} - -{{fields "sqs"}} - -### transitgateway - -{{event "transitgateway"}} - -{{fields "transitgateway"}} - -### usage - -{{event "usage"}} - -{{fields "usage"}} - -### vpn - -{{event "vpn"}} - -{{fields "vpn"}} diff --git a/test/packages/aws/_dev/build/docs/billing.md b/test/packages/aws/_dev/build/docs/billing.md new file mode 100644 index 0000000000..054ce727bf --- /dev/null +++ b/test/packages/aws/_dev/build/docs/billing.md @@ -0,0 +1,7 @@ +# billing + +## Metrics + +{{event "billing"}} + +{{fields "billing"}} diff --git a/test/packages/aws/_dev/build/docs/cloudtrail.md b/test/packages/aws/_dev/build/docs/cloudtrail.md new file mode 100644 index 0000000000..e06cbe105b --- /dev/null +++ b/test/packages/aws/_dev/build/docs/cloudtrail.md @@ -0,0 +1,11 @@ +# cloudtrail + +## Logs + +The `cloudtrail` dataset collects the AWS CloudTrail logs. CloudTrail monitors +events for the account. If user creates a trail, it delivers those events as log + files to a specific Amazon S3 bucket. The `cloudtrail` dataset does not read + the CloudTrail Digest files that are delivered to the S3 bucket when Log File + Integrity is turned on, it only reads the CloudTrail logs. + +{{fields "cloudtrail"}} diff --git a/test/packages/aws/_dev/build/docs/cloudwatch.md b/test/packages/aws/_dev/build/docs/cloudwatch.md new file mode 100644 index 0000000000..50fe1a5a5f --- /dev/null +++ b/test/packages/aws/_dev/build/docs/cloudwatch.md @@ -0,0 +1,16 @@ +# cloudwatch + +## Logs + +The `cloudwatch` dataset collects CloudWatch logs. Users can use Amazon +CloudWatch logs to monitor, store, and access log files from different sources. +Export logs from log groups to an Amazon S3 bucket which has SQS notification +setup already. + +{{fields "cloudwatch_logs"}} + +## Metrics + +{{event "cloudwatch_metrics"}} + +{{fields "cloudwatch_metrics"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/dynamodb.md b/test/packages/aws/_dev/build/docs/dynamodb.md new file mode 100644 index 0000000000..eb0890de32 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/dynamodb.md @@ -0,0 +1,7 @@ +# dynamodb + +## Metrics + +{{event "dynamodb"}} + +{{fields "dynamodb"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/ebs.md b/test/packages/aws/_dev/build/docs/ebs.md new file mode 100644 index 0000000000..ebb1cfda09 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/ebs.md @@ -0,0 +1,7 @@ +# ebs + +## Metrics + +{{event "ebs"}} + +{{fields "ebs"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/ec2.md b/test/packages/aws/_dev/build/docs/ec2.md new file mode 100644 index 0000000000..9a9fe5e428 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/ec2.md @@ -0,0 +1,16 @@ +# ec2 + +## Logs + +The `ec2` dataset is specifically for EC2 logs stored in AWS CloudWatch. Export logs +from log groups to Amazon S3 bucket which has SQS notification setup already. +With this dataset, EC2 logs will be parsed into fields like `ip_address` +and `process.name`. For logs from other services, please use `cloudwatch` dataset. + +{{fields "ec2_logs"}} + +## Metrics + +{{event "ec2_metrics"}} + +{{fields "ec2_metrics"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/elb.md b/test/packages/aws/_dev/build/docs/elb.md new file mode 100644 index 0000000000..608ed45e66 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/elb.md @@ -0,0 +1,23 @@ +# elb + +## Logs + +The `elb` dataset collects logs from AWS ELBs. Elastic Load Balancing provides +access logs that capture detailed information about requests sent to the load +balancer. Each log contains information such as the time the request was +received, the client's IP address, latencies, request paths, and server +responses. Users can use these access logs to analyze traffic patterns and to +troubleshoot issues. + +Please follow [enable access logs for classic load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html) +for sending Classic ELB access logs to S3 bucket. +For application load balancer, please follow [enable access log for application load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html#enable-access-logging). +For network load balancer, please follow [enable access log for network load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest//network/load-balancer-access-logs.html). + +{{fields "elb_logs"}} + +## Metrics + +{{event "elb_metrics"}} + +{{fields "elb_metrics"}} diff --git a/test/packages/aws/_dev/build/docs/lambda.md b/test/packages/aws/_dev/build/docs/lambda.md new file mode 100644 index 0000000000..0a10140330 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/lambda.md @@ -0,0 +1,7 @@ +# lambda + +## Metrics + +{{event "lambda"}} + +{{fields "lambda"}} diff --git a/test/packages/aws/_dev/build/docs/natgateway.md b/test/packages/aws/_dev/build/docs/natgateway.md new file mode 100644 index 0000000000..aaa495290e --- /dev/null +++ b/test/packages/aws/_dev/build/docs/natgateway.md @@ -0,0 +1,7 @@ +# natgateway + +## Metrics + +{{event "natgateway"}} + +{{fields "natgateway"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/rds.md b/test/packages/aws/_dev/build/docs/rds.md new file mode 100644 index 0000000000..3d43dd4f30 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/rds.md @@ -0,0 +1,7 @@ +# rds + +## Metrics + +{{event "rds"}} + +{{fields "rds"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/s3.md b/test/packages/aws/_dev/build/docs/s3.md new file mode 100644 index 0000000000..cfa236aa13 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/s3.md @@ -0,0 +1,27 @@ +# S3 + +## Logs +The `s3access` dataset collects server access logs from AWS S3. Server access +logging provides detailed records for the requests that are made to a bucket. +Server access logs are useful for many applications. For example, access log +information can be useful in security and access audits. It can also help users +to learn about customer base and understand Amazon S3 bill. + +Please follow [how to enable server access logging](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html#server-access-logging-overview) +for sending server access logs to S3 bucket. + +{{fields "s3access"}} + +## Metrics + +### s3_daily_storage + +{{event "s3_daily_storage"}} + +{{fields "s3_daily_storage"}} + +### s3_request + +{{event "s3_request"}} + +{{fields "s3_request"}} diff --git a/test/packages/aws/_dev/build/docs/sns.md b/test/packages/aws/_dev/build/docs/sns.md new file mode 100644 index 0000000000..5d659b2a69 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/sns.md @@ -0,0 +1,7 @@ +# sns + +## Metrics + +{{event "sns"}} + +{{fields "sns"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/sqs.md b/test/packages/aws/_dev/build/docs/sqs.md new file mode 100644 index 0000000000..2caee81298 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/sqs.md @@ -0,0 +1,7 @@ +# sqs + +## Metrics + +{{event "sqs"}} + +{{fields "sqs"}} \ No newline at end of file diff --git a/test/packages/aws/_dev/build/docs/transitgateway.md b/test/packages/aws/_dev/build/docs/transitgateway.md new file mode 100644 index 0000000000..45bdb6ee19 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/transitgateway.md @@ -0,0 +1,7 @@ +# transitgateway + +## Metrics + +{{event "transitgateway"}} + +{{fields "transitgateway"}} diff --git a/test/packages/aws/_dev/build/docs/usage.md b/test/packages/aws/_dev/build/docs/usage.md new file mode 100644 index 0000000000..bdd27653e3 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/usage.md @@ -0,0 +1,7 @@ +# usage + +## Metrics + +{{event "usage"}} + +{{fields "usage"}} diff --git a/test/packages/aws/_dev/build/docs/vpcflow.md b/test/packages/aws/_dev/build/docs/vpcflow.md new file mode 100644 index 0000000000..f707db5822 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/vpcflow.md @@ -0,0 +1,5 @@ +# vpcflow + +## Logs + +{{fields "vpcflow"}} diff --git a/test/packages/aws/_dev/build/docs/vpn.md b/test/packages/aws/_dev/build/docs/vpn.md new file mode 100644 index 0000000000..7edfa52125 --- /dev/null +++ b/test/packages/aws/_dev/build/docs/vpn.md @@ -0,0 +1,7 @@ +# vpn + +## Metrics + +{{event "vpn"}} + +{{fields "vpn"}} diff --git a/test/packages/aws/changelog.yml b/test/packages/aws/changelog.yml index cb5f29cc61..d21a66b190 100644 --- a/test/packages/aws/changelog.yml +++ b/test/packages/aws/changelog.yml @@ -1,6 +1,21 @@ # newer versions go on top +- version: "0.4.2" + changes: + - description: Use input groups. + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/767 +- version: "0.4.1" + changes: + - description: Correct sample event file. + type: bugfix # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/754 +- version: "0.4.0" + changes: + - description: Add changes to use ECS 1.8 fields. + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/721 - version: "0.0.3" changes: - description: initial release type: enhancement # can be one of: enhancement, bugfix, breaking-change - link: https://github.com/elastic/elastic-package/pull/143 + link: https://github.com/elastic/integrations/pull/21 diff --git a/test/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs index b3893ee194..caae1156d6 100644 --- a/test/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs @@ -18,9 +18,18 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if regions}} -regions: {{regions}} +{{#if lantency}} +latency: {{latency}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} +{{#if cost_explorer_config.group_by_dimension_keys}} +cost_explorer_config.group_by_dimension_keys: +{{#each cost_explorer_config.group_by_dimension_keys as |dimension_key i|}} +- {{dimension_key}} +{{/each}} +{{/if}} +{{#if cost_explorer_config.group_by_tag_keys}} +cost_explorer_config.group_by_tag_keys: +{{#each cost_explorer_config.group_by_tag_keys as |tag_key i|}} +- {{tag_key}} +{{/each}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/billing/fields/ecs.yml b/test/packages/aws/data_stream/billing/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/billing/fields/ecs.yml +++ b/test/packages/aws/data_stream/billing/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/billing/fields/fields.yml b/test/packages/aws/data_stream/billing/fields/fields.yml index bbbd27d554..5b3ee582a6 100644 --- a/test/packages/aws/data_stream/billing/fields/fields.yml +++ b/test/packages/aws/data_stream/billing/fields/fields.yml @@ -1,24 +1,83 @@ - name: aws type: group fields: - - name: dimensions + - name: billing type: group fields: + - name: EstimatedCharges.max + type: long + description: Maximum estimated charges for AWS acccount. + - name: Currency + type: keyword + description: Currency name. - name: ServiceName type: keyword description: AWS service name. - - name: Currency + - name: AmortizedCost + type: group + fields: + - name: amount + type: double + description: Amortized cost amount. + - name: unit + type: keyword + description: Amortized cost unit. + - name: BlendedCost + type: group + fields: + - name: amount + type: double + description: Blended cost amount. + - name: unit + type: keyword + description: Blended cost unit. + - name: NormalizedUsageAmount + type: group + fields: + - name: amount + type: double + description: Normalized usage amount. + - name: unit + type: keyword + description: Normalized usage amount unit. + - name: UnblendedCost + type: group + fields: + - name: amount + type: double + description: Unblended cost amount. + - name: unit + type: keyword + description: Unblended cost unit. + - name: UsageQuantity + type: group + fields: + - name: amount + type: double + description: Usage quantity amount. + - name: unit + type: keyword + description: Usage quantity unit. + - name: start_date type: keyword - description: Currency name. - - name: billing - type: group - fields: - - name: metrics + description: Start date for retrieving AWS costs. + - name: end_date + type: keyword + description: End date for retrieving AWS costs. + - name: group_definition type: group fields: - - name: EstimatedCharges.max - type: long - description: Maximum estimated charges for AWS acccount. + - name: key + type: keyword + description: The string that represents a key for a specified group. + - name: type + type: keyword + description: The string that represents the type of group. + - name: group_by + type: object + object_type: keyword + object_type_mapping_type: "*" + description: Cost explorer group by key values. - name: cloudwatch type: group fields: diff --git a/test/packages/aws/data_stream/billing/manifest.yml b/test/packages/aws/data_stream/billing/manifest.yml index 4a88cead7a..e42030e46c 100644 --- a/test/packages/aws/data_stream/billing/manifest.yml +++ b/test/packages/aws/data_stream/billing/manifest.yml @@ -1,5 +1,5 @@ title: AWS billing metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics @@ -17,5 +17,23 @@ streams: multi: false required: false show_user: false + - name: cost_explorer_config.group_by_dimension_keys + type: text + title: Cost Explorer Group By Dimension Keys + multi: true + required: false + show_user: true + default: + - "AZ" + - "INSTANCE_TYPE" + - "SERVICE" + - name: cost_explorer_config.group_by_tag_keys + type: text + title: Cost Explorer Group By Tag Keys + multi: true + required: false + show_user: true + default: + - "aws:createdBy" title: AWS Billing metrics description: Collect AWS billing metrics diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json index 5687a2a15f..40bc2d2235 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2014-03-25T21:08:14.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.114840Z", + "ingested": "2021-03-18T12:21:57.668559300Z", "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-25T18:45:11Z\"}}},\"eventTime\":\"2014-03-25T21:08:14Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AddUserToGroup\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"AWSConsole\",\"requestParameters\":{\"userName\":\"Bob\",\"groupName\":\"admin\"},\"responseElements\":null}", "provider": "iam.amazonaws.com", + "created": "2014-03-25T21:08:14.000Z", "kind": "event", "action": "AddUserToGroup", "type": [ @@ -55,7 +57,10 @@ }, "user": { "name": "Alice", - "id": "EX_PRINCIPAL_ID" + "id": "EX_PRINCIPAL_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", @@ -63,6 +68,9 @@ "name": "Other" }, "original": "AWSConsole" + }, + "group": { + "name": "admin" } } ] diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json index 5feed30efb..f658bfe11e 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json @@ -30,9 +30,10 @@ "ip": "123.145.67.89" }, "event": { - "ingested": "2020-11-19T22:16:17.142969600Z", + "ingested": "2021-03-18T12:21:58.085681300Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE:JohnRole1\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1\",\"accountId\":\"111111111111\",\"accessKeyId\":\"AKIAI44QH8DHBEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-10-02T21:50:54Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE\",\"arn\":\"arn:aws:iam::111111111111:role/JohnRole1\",\"accountId\":\"111111111111\",\"userName\":\"JohnDoe\"}}},\"eventTime\":\"2019-10-02T22:12:29Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"123.145.67.89\",\"userAgent\":\"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239\",\"requestParameters\":{\"incomingTransitiveTags\":{\"Department\":\"Engineering\"},\"tags\":[{\"value\":\"johndoe@example.com\",\"key\":\"Email\"},{\"value\":\"12345\",\"key\":\"CostCenter\"}],\"roleArn\":\"arn:aws:iam::111111111111:role/JohnRole2\",\"roleSessionName\":\"Role2WithTags\",\"transitiveTagKeys\":[\"Email\",\"CostCenter\"],\"durationSeconds\":3600},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAWHOJDLGPOEXAMPLE\",\"expiration\":\"Oct 2, 2019 11:12:29 PM\",\"sessionToken\":\"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAIFR7WHDTSOYQYHFUE:Role2WithTags\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags\"}},\"requestID\":\"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE\",\"eventID\":\"1917948f-3042-46ec-98e2-62865EXAMPLE\",\"resources\":[{\"ARN\":\"arn:aws:iam::111122223333:role/JohnRole2\",\"accountId\":\"111111111111\",\"type\":\"AWS::IAM::Role\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"111111111111\"}", "provider": "sts.amazonaws.com", + "created": "2019-10-02T22:12:29.000Z", "kind": "event", "action": "AssumeRole", "id": "1917948f-3042-46ec-98e2-62865EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json index a888654d3e..1791580f97 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-09T00:09:33.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.187739500Z", + "ingested": "2021-03-18T12:21:58.316476700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:09:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"AccessDeniedException\",\"errorMessage\":\"An unknown error occurred\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE\",\"eventID\":\"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-09T00:09:33.000Z", "kind": "event", "action": "ChangePassword", "id": "EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE", @@ -32,13 +38,14 @@ "cloudtrail": { "event_version": "1.05", "error_message": "An unknown error occurred", + "flattened": {}, + "event_type": "AwsApiCall", "user_identity": { "access_key_id": "EXAMPLE_KEY", "type": "IAMUser", "arn": "arn:aws:iam::0123456789012:user/Alice" }, "error_code": "AccessDeniedException", - "event_type": "AwsApiCall", "recipient_account_id": "0123456789012" } }, @@ -63,14 +70,20 @@ } }, "@timestamp": "2020-01-09T00:03:36.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.187745300Z", + "ingested": "2021-03-18T12:21:58.316527700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:03:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5c16-4eda-9724-EXAMPLE\",\"eventID\":\"EXAMPLE-35a7-4c25-9fc7-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-09T00:03:36.000Z", "kind": "event", "action": "ChangePassword", "id": "EXAMPLE-35a7-4c25-9fc7-EXAMPLE", @@ -86,6 +99,7 @@ "aws": { "cloudtrail": { "event_version": "1.05", + "flattened": {}, "user_identity": { "access_key_id": "EXAMPLE_KEY", "type": "IAMUser", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json index 8207118c4c..ed90ce62f7 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json @@ -1,11 +1,142 @@ { "expected": [ { + "cloud": { + "account": { + "id": "123456789123" + } + }, + "file": { + "path": "AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz", + "hash": { + "sha256": "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987" + } + }, + "@timestamp": "2020-09-11T19:36:49.000Z", + "related": { + "hash": [ + "10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987" + ] + }, "event": { - "ingested": "2020-11-19T22:16:17.237931600Z", + "ingested": "2021-03-18T12:21:58.386071900Z", "original": "{\"awsAccountId\":\"123456789123\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"alice-bucket\",\"digestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"47aaa19f7eec22e9bd0b5e58cfade8cb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"alice-bucket\",\"previousDigestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987\",\"logFiles\":[{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", "type": "info", "kind": "event" + }, + "aws": { + "cloudtrail": { + "flattened": { + "digest": { + "start_time": "2020-09-11T18:36:49.000Z", + "previous_s3_bucket": "alice-bucket", + "log_files": [ + { + "newestEventTime": "2020-09-11T19:26:24Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz", + "oldestEventTime": "2020-09-11T19:26:24Z", + "s3Bucket": "alice-bucket", + "hashValue": "420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T19:11:18Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz", + "oldestEventTime": "2020-09-11T19:11:18Z", + "s3Bucket": "alice-bucket", + "hashValue": "4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T18:32:04Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz", + "oldestEventTime": "2020-09-11T18:32:04Z", + "s3Bucket": "alice-bucket", + "hashValue": "2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T19:21:28Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz", + "oldestEventTime": "2020-09-11T19:21:28Z", + "s3Bucket": "alice-bucket", + "hashValue": "45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T18:51:21Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz", + "oldestEventTime": "2020-09-11T18:51:21Z", + "s3Bucket": "alice-bucket", + "hashValue": "515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T18:46:45Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz", + "oldestEventTime": "2020-09-11T18:46:45Z", + "s3Bucket": "alice-bucket", + "hashValue": "18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T19:01:06Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz", + "oldestEventTime": "2020-09-11T19:01:06Z", + "s3Bucket": "alice-bucket", + "hashValue": "54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T19:16:12Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz", + "oldestEventTime": "2020-09-11T19:16:12Z", + "s3Bucket": "alice-bucket", + "hashValue": "6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T18:56:05Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz", + "oldestEventTime": "2020-09-11T18:56:05Z", + "s3Bucket": "alice-bucket", + "hashValue": "b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T19:06:31Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz", + "oldestEventTime": "2020-09-11T19:06:31Z", + "s3Bucket": "alice-bucket", + "hashValue": "4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T18:41:58Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz", + "oldestEventTime": "2020-09-11T18:41:58Z", + "s3Bucket": "alice-bucket", + "hashValue": "94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6", + "hashAlgorithm": "SHA-256" + }, + { + "newestEventTime": "2020-09-11T18:37:10Z", + "s3Object": "AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz", + "oldestEventTime": "2020-09-11T18:37:10Z", + "s3Bucket": "alice-bucket", + "hashValue": "9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771", + "hashAlgorithm": "SHA-256" + } + ], + "newest_event_time": "2020-09-11T19:26:24.000Z", + "previous_hash_algorithm": "SHA-256", + "end_time": "2020-09-11T19:36:49.000Z", + "signature_algorithm": "SHA256withRSA", + "s3_bucket": "alice-bucket", + "oldest_event_time": "2020-09-11T18:32:04.000Z" + } + } + } } } ] diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json index bce9f5b0e2..c26f762960 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2014-07-16T15:49:27.000Z", + "related": { + "user": [ + "JohnDoe" + ] + }, "source": { "address": "192.0.2.110", "ip": "192.0.2.110" }, "event": { - "ingested": "2020-11-19T22:16:17.251357800Z", + "ingested": "2021-03-18T12:21:58.413998700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JohnDoe\",\"accountId\":\"111122223333\",\"userName\":\"JohnDoe\"},\"eventTime\":\"2014-07-16T15:49:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.110\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Success\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/s3/\",\"MFAUsed\":\"No\"},\"eventID\":\"3fcfb182-98f8-4744-bd45-10aEXAMPLE\"}", "provider": "signin.amazonaws.com", + "created": "2014-07-16T15:49:27.000Z", "kind": "event", "action": "ConsoleLogin", "id": "3fcfb182-98f8-4744-bd45-10aEXAMPLE", @@ -81,14 +87,20 @@ } }, "@timestamp": "2014-07-08T17:35:27.000Z", + "related": { + "user": [ + "JaneDoe" + ] + }, "source": { "address": "192.0.2.100", "ip": "192.0.2.100" }, "event": { - "ingested": "2020-11-19T22:16:17.251366800Z", + "ingested": "2021-03-18T12:21:58.414010500Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JaneDoe\",\"accountId\":\"111122223333\",\"userName\":\"JaneDoe\"},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", "provider": "signin.amazonaws.com", + "created": "2014-07-08T17:35:27.000Z", "kind": "event", "action": "ConsoleLogin", "id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE", @@ -160,9 +172,10 @@ "ip": "192.0.2.100" }, "event": { - "ingested": "2020-11-19T22:16:17.251419800Z", + "ingested": "2021-03-18T12:21:58.414020600Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", "provider": "signin.amazonaws.com", + "created": "2014-07-08T17:35:27.000Z", "kind": "event", "action": "ConsoleLogin", "id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json index 5920c8034a..d419d5f197 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-08T20:43:06.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.351340900Z", + "ingested": "2021-03-18T12:21:58.546694300Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:43:06Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"accessKey\":{\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"status\":\"Active\",\"userName\":\"Bob\",\"createDate\":\"Jan 8, 2020 8:43:06 PM\"}},\"requestID\":\"EXAMPLE-823a-48dc-8fa9-EXAMPLE\",\"eventID\":\"EXAMPLE-3cab-40f8-938b-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-08T20:43:06.000Z", "kind": "event", "action": "CreateAccessKey", "id": "EXAMPLE-3cab-40f8-938b-EXAMPLE", @@ -67,7 +69,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json index 365d5ea0ed..68ecb5cc2d 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-09T01:48:44.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.399618200Z", + "ingested": "2021-03-18T12:21:58.589102800Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T01:48:44Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":{\"group\":{\"createDate\":\"Jan 9, 2020 1:48:44 AM\",\"path\":\"/\",\"arn\":\"arn:aws:iam::0123456789012:group/TEST-GROUP\",\"groupName\":\"TEST-GROUP\",\"groupId\":\"EXAMPLE_ID\"}},\"requestID\":\"EXAMPLE-769d-4a61-b731-EXAMPLE\",\"eventID\":\"EXAMPLE-37ec-425a-a7ef-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-09T01:48:44.000Z", "kind": "event", "action": "CreateGroup", "id": "EXAMPLE-37ec-425a-a7ef-EXAMPLE", @@ -71,6 +77,10 @@ "name": "Other" }, "original": "signin.amazonaws.com" + }, + "group": { + "name": "TEST-GROUP", + "id": "EXAMPLE_ID" } }, { @@ -81,14 +91,20 @@ } }, "@timestamp": "2020-01-09T02:22:03.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.399623700Z", + "ingested": "2021-03-18T12:21:58.589115700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:22:03Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"EntityAlreadyExistsException\",\"errorMessage\":\"Group with name TEST-GROUP already exists.\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-c8ae-44dc-8114-EXAMPLE\",\"eventID\":\"EXAMPLE-09c6-4745-af70-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-09T02:22:03.000Z", "kind": "event", "action": "CreateGroup", "id": "EXAMPLE-09c6-4745-af70-EXAMPLE", @@ -132,6 +148,9 @@ "name": "Spider" }, "version": "1.16.310" + }, + "group": { + "name": "TEST-GROUP" } } ] diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json index 87f5636a7a..7770d4db0b 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json @@ -8,6 +8,11 @@ } }, "@timestamp": "2014-03-06T17:10:34.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "geo": { "continent_name": "North America", @@ -31,9 +36,10 @@ "ip": "72.21.198.64" }, "event": { - "ingested": "2020-11-19T22:16:17.471316700Z", + "ingested": "2021-03-18T12:21:58.659862400Z", "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-06T15:15:06Z\"}}},\"eventTime\":\"2014-03-06T17:10:34Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"CreateKeyPair\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"72.21.198.64\",\"userAgent\":\"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx\",\"requestParameters\":{\"keyName\":\"mykeypair\"},\"responseElements\":{\"keyName\":\"mykeypair\",\"keyFingerprint\":\"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21\",\"keyMaterial\":\"\u003csensitiveDataRemoved\u003e\"}}", "provider": "ec2.amazonaws.com", + "created": "2014-03-06T17:10:34.000Z", "kind": "event", "action": "CreateKeyPair", "type": [ diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json index 27a8e54faf..658999a4de 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-08T15:30:25.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.524325600Z", + "ingested": "2021-03-18T12:21:58.713988600Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T15:30:25Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"CreateTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"TEST-cloudtrail-bucket\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"enableLogFileValidation\":true,\"kmsKeyId\":\"\",\"isOrganizationTrail\":false},\"responseElements\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"TEST-cloudtrail-bucket\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"trailARN\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"logFileValidationEnabled\":true,\"isOrganizationTrail\":false},\"requestID\":\"EXAMPLE-5149-4cf2-be99-EXAMPLE\",\"eventID\":\"EXAMPLE-d04b-4eff-833a-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", + "created": "2020-01-08T15:30:25.000Z", "kind": "event", "action": "CreateTrail", "id": "EXAMPLE-d04b-4eff-833a-EXAMPLE", @@ -27,13 +33,12 @@ "event_version": "1.05", "flattened": { "request_parameters": { - "isMultiRegionTrail": true, - "s3BucketName": "TEST-cloudtrail-bucket", "name": "TEST-trail", "enableLogFileValidation": true, - "kmsKeyId": "", "isOrganizationTrail": false, - "includeGlobalServiceEvents": true + "isMultiRegionTrail": true, + "includeGlobalServiceEvents": true, + "s3BucketName": "TEST-cloudtrail-bucket" }, "response_elements": { "logFileValidationEnabled": true, @@ -58,7 +63,7 @@ "arn": "arn:aws:iam::0123456789012:user/Alice" }, "recipient_account_id": "0123456789012", - "request_parameters": "{isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, enableLogFileValidation=true, kmsKeyId=, isOrganizationTrail=false, includeGlobalServiceEvents=true}", + "request_parameters": "{isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, enableLogFileValidation=true, isOrganizationTrail=false, includeGlobalServiceEvents=true}", "response_elements": "{logFileValidationEnabled=true, isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, trailARN=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, isOrganizationTrail=false, includeGlobalServiceEvents=true}" } }, diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json index a3822f1ea3..9f3e956868 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2014-03-24T21:11:59.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.562386700Z", + "ingested": "2021-03-18T12:21:58.753913100Z", "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2014-03-24T21:11:59Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateUser\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.3.2 Python/2.7.5 Windows/7\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"user\":{\"createDate\":\"Mar 24, 2014 9:11:59 PM\",\"userName\":\"Bob\",\"arn\":\"arn:aws:iam::123456789012:user/Bob\",\"path\":\"/\",\"userId\":\"EXAMPLEUSERID\"}}}", "provider": "iam.amazonaws.com", + "created": "2014-03-24T21:11:59.000Z", "kind": "event", "action": "CreateUser", "type": [ @@ -60,7 +62,11 @@ }, "user": { "name": "Alice", - "id": "EX_PRINCIPAL_ID" + "id": "EX_PRINCIPAL_ID", + "target": { + "name": "Bob", + "id": "EXAMPLEUSERID" + } }, "user_agent": { "name": "aws-cli", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json index 9c8a74abc3..e5abcb762d 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2019-11-27T15:10:15.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.596968500Z", + "ingested": "2021-03-18T12:21:58.793253Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-11-27T15:07:22Z\"}}},\"eventTime\":\"2019-11-27T15:10:15Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateVirtualMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"console.amazonaws.com\",\"requestParameters\":{\"virtualMFADeviceName\":\"Alice\",\"path\":\"/\"},\"responseElements\":{\"virtualMFADevice\":{\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"}},\"requestID\":\"EXAMPLE-303b-4b0e-a8c7-EXAMPLE\",\"eventID\":\"EXAMPLE-351c-472a-b089-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2019-11-27T15:10:15.000Z", "kind": "event", "action": "CreateVirtualMFADevice", "id": "EXAMPLE-351c-472a-b089-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json index bf3383711f..565eebeebc 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json @@ -18,9 +18,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.634634200Z", + "ingested": "2021-03-18T12:21:58.833608500Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T00:34:02Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeactivateMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Alice\",\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-801a-4624-8fa0-EXAMPLE\",\"eventID\":\"EXAMPLE-1889-416b-ace9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T00:34:02.000Z", "kind": "event", "action": "DeactivateMFADevice", "id": "EXAMPLE-1889-416b-ace9-EXAMPLE", @@ -59,7 +60,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Alice" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json index 960c466c20..0da9502e7b 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-08T19:09:36.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.666712100Z", + "ingested": "2021-03-18T12:21:58.871497900Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T19:09:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\",\"accessKeyId\":\"EXAMPLE_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-3bea-41fa-a0b4-EXAMPLE\",\"eventID\":\"EXAMPLE-0698-46bd-998d-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-08T19:09:36.000Z", "kind": "event", "action": "DeleteAccessKey", "id": "EXAMPLE-0698-46bd-998d-EXAMPLE", @@ -59,7 +61,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json index 17b6d73ff1..9c9ee51b65 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json @@ -13,9 +13,10 @@ "ip": "192.0.2.1" }, "event": { - "ingested": "2020-11-19T22:16:17.699662100Z", + "ingested": "2021-03-18T12:21:58.913519100Z", "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AIDAQRSTUVWXYZEXAMPLE:devdsk\",\"arn\":\"arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk\",\"accountId\":\"777788889999\",\"accessKeyId\":\"AKIAQRSTUVWXYZEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2016-11-14T17:25:26Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AIDAQRSTUVWXYZEXAMPLE\",\"arn\":\"arn:aws:iam::777788889999:role/AssumeNothing\",\"accountId\":\"777788889999\",\"userName\":\"AssumeNothing\"}}},\"eventTime\":\"2016-11-14T17:25:45Z\",\"eventSource\":\"s3.amazonaws.com\",\"eventName\":\"DeleteBucket\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.1\",\"userAgent\":\"[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]\",\"requestParameters\":{\"bucketName\":\"my-test-bucket-cross-account\"},\"responseElements\":null,\"requestID\":\"EXAMPLE463D56D4C\",\"eventID\":\"dEXAMPLE-265a-41e0-9352-4401bEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"777788889999\"}", "provider": "s3.amazonaws.com", + "created": "2016-11-14T17:25:45.000Z", "kind": "event", "action": "DeleteBucket", "id": "dEXAMPLE-265a-41e0-9352-4401bEXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json index ea8e9dd01b..45c0d54385 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-09T02:25:44.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.735300400Z", + "ingested": "2021-03-18T12:21:58.955587100Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T02:25:44Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-66cb-4775-a203-EXAMPLE\",\"eventID\":\"EXAMPLE-cbc2-4cc3-8bbc-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-09T02:25:44.000Z", "kind": "event", "action": "DeleteGroup", "id": "EXAMPLE-cbc2-4cc3-8bbc-EXAMPLE", @@ -61,6 +67,9 @@ "name": "Other" }, "original": "signin.amazonaws.com" + }, + "group": { + "name": "TEST-GROUP" } }, { @@ -71,14 +80,20 @@ } }, "@timestamp": "2020-01-09T02:25:11.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.735308800Z", + "ingested": "2021-03-18T12:21:58.955599400Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_PRINCIPLE\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:25:11Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"DeleteConflictException\",\"errorMessage\":\"Cannot delete entity, must detach all policies first.\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-2a3c-4a94-b24f-EXAMPLE\",\"eventID\":\"EXAMPLE-5aa2-4b5f-a52a-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-09T02:25:11.000Z", "kind": "event", "action": "DeleteGroup", "id": "EXAMPLE-5aa2-4b5f-a52a-EXAMPLE", @@ -122,6 +137,9 @@ "name": "Spider" }, "version": "1.16.310" + }, + "group": { + "name": "TEST-GROUP" } } ] diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json index 37dcc108d4..271a0184f1 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-10T16:07:08.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.790009400Z", + "ingested": "2021-03-18T12:21:59.027109400Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:07:08Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-7b34-44ae-a22f-EXAMPLE\",\"eventID\":\"EXAMPLE-72ff-4d4f-9a8d-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T16:07:08.000Z", "kind": "event", "action": "DeleteSSHPublicKey", "id": "EXAMPLE-72ff-4d4f-9a8d-EXAMPLE", @@ -59,7 +61,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json index 559f6a2848..3d38ed7747 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-09T20:09:51.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.823102300Z", + "ingested": "2021-03-18T12:21:59.064641200Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T20:09:51Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"DeleteTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/test-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-d44f-4a2a-966f-EXAMPLE\",\"eventID\":\"EXAMPLE-3f9d-4634-8ff1-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", + "created": "2020-01-09T20:09:51.000Z", "kind": "event", "action": "DeleteTrail", "id": "EXAMPLE-3f9d-4634-8ff1-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json index 30b528e220..84e55f2668 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-03T15:50:52.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.852239100Z", + "ingested": "2021-03-18T12:21:59.104465600Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-03T15:26:38Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-03T15:50:52Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"0e794d53-cdb5-4f7d-b7db-5EXAMPLE\",\"eventID\":\"b89eb34b-8fcb-4cba-8439-d4EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-03T15:50:52.000Z", "kind": "event", "action": "DeleteUser", "id": "b89eb34b-8fcb-4cba-8439-d4EXAMPLE", @@ -58,7 +60,10 @@ }, "user": { "name": "Alice", - "id": "EX_PRINCIPAL_ID" + "id": "EX_PRINCIPAL_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json index 90153a4250..f204386ac6 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-10T00:34:02.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.885251200Z", + "ingested": "2021-03-18T12:21:59.159465700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T00:34:02Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteVirtualMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-af91-4d1a-aaf2-EXAMPLE\",\"eventID\":\"EXAMPLE-f8e6-4d5f-8525-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T00:34:02.000Z", "kind": "event", "action": "DeleteVirtualMFADevice", "id": "EXAMPLE-f8e6-4d5f-8525-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json index d511908596..bf99776cec 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2019-11-27T15:11:09.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.916356700Z", + "ingested": "2021-03-18T12:21:59.209951700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-11-27T15:07:22Z\"}}},\"eventTime\":\"2019-11-27T15:11:09Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"EnableMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"console.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\",\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-adea-490a-a806-EXAMPLE\",\"eventID\":\"EXAMPLE-3fdc-4b2a-9885-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2019-11-27T15:11:09.000Z", "kind": "event", "action": "EnableMFADevice", "id": "EXAMPLE-3fdc-4b2a-9885-EXAMPLE", @@ -58,7 +60,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json index b3d5089d27..bacd55ffe9 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json @@ -6,18 +6,69 @@ }, "@timestamp": "2020-09-09T23:00:00.000Z", "event": { - "ingested": "2020-11-19T22:16:17.948374300Z", + "ingested": "2021-03-18T12:21:59.258939800Z", "original": "{\"eventVersion\":\"1.07\",\"eventTime\":\"2020-09-09T23:00:00Z\",\"awsRegion\":\"us-east-1\",\"eventID\":\"41ed77ca-d659-b45a-8e9a-74e504300007\",\"eventType\":\"AwsCloudTrailInsight\",\"recipientAccountId\":\"123456789012\",\"sharedEventID\":\"e672c2b1-e71a-4779-f96c-02da7bb30d2e\",\"insightDetails\":{\"state\":\"End\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AttachUserPolicy\",\"insightType\":\"ApiCallRateInsight\",\"insffightContext\":{\"statistics\":{\"baseline\":{\"average\":0.0},\"insight\":{\"average\":2.0},\"insightDuration\":1,\"baselineDuration\":11459},\"attributions\":[{\"attribute\":\"userIdentityArn\",\"insight\":[{\"value\":\"arn:aws:iam::123456789012:user/Alice\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"userAgent\",\"insight\":[{\"value\":\"console.amazonaws.com\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"errorCode\",\"insight\":[{\"value\":\"null\",\"average\":2.0}],\"baseline\":[]}]}},\"eventCategory\":\"Insight\"}", + "created": "2020-09-09T23:00:00.000Z", + "kind": "event", "id": "41ed77ca-d659-b45a-8e9a-74e504300007", "type": "info", - "kind": "event", "outcome": "success" }, "aws": { "cloudtrail": { "event_version": "1.07", + "flattened": { + "insight_details": { + "eventSource": "iam.amazonaws.com", + "eventName": "AttachUserPolicy", + "state": "End", + "insightType": "ApiCallRateInsight", + "insffightContext": { + "attributions": [ + { + "insight": [ + { + "average": 2.0, + "value": "arn:aws:iam::123456789012:user/Alice" + } + ], + "attribute": "userIdentityArn" + }, + { + "insight": [ + { + "average": 2.0, + "value": "console.amazonaws.com" + } + ], + "attribute": "userAgent" + }, + { + "insight": [ + { + "average": 2.0, + "value": "null" + } + ], + "attribute": "errorCode" + } + ], + "statistics": { + "baselineDuration": 11459, + "insight": { + "average": 2.0 + }, + "insightDuration": 1, + "baseline": { + "average": 0.0 + } + } + } + } + }, "event_type": "AwsCloudTrailInsight", - "recipient_account_id": "123456789012" + "recipient_account_id": "123456789012", + "event_category": "Insight" } } } diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json index 3a765ccd39..18e1a3d32f 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-06T15:19:50.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:17.967490300Z", + "ingested": "2021-03-18T12:21:59.283988300Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-06T14:36:28Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-06T15:19:50Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"RemoveUserFromGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"Admin\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-0bf0-47be-bc80-EXAMPLE\",\"eventID\":\"EXAMPLE-6e8b-431a-94f4-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-06T15:19:50.000Z", "kind": "event", "action": "RemoveUserFromGroup", "id": "EXAMPLE-6e8b-431a-94f4-EXAMPLE", @@ -59,7 +61,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", @@ -67,6 +72,9 @@ "name": "Other" }, "original": "signin.amazonaws.com" + }, + "group": { + "name": "Admin" } } ] diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json index f14857f5d5..61268e5fdf 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-08T15:30:25.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.000799400Z", + "ingested": "2021-03-18T12:21:59.327007400Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T15:30:25Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"StartLogging\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"TEST-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-1c30-4f43-9763-EXAMPLE\",\"eventID\":\"EXAMPLE-aa78-4a84-a27f-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", + "created": "2020-01-08T15:30:25.000Z", "kind": "event", "action": "StartLogging", "id": "EXAMPLE-aa78-4a84-a27f-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json index 6f833e9648..e8dd172cb3 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-09T16:46:16.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.031847400Z", + "ingested": "2021-03-18T12:21:59.380771100Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T16:46:16Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"StopLogging\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-869f-4fec-86f9-EXAMPLE\",\"eventID\":\"EXAMPLE-8cc3-42db-9a0d-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", + "created": "2020-01-09T16:46:16.000Z", "kind": "event", "action": "StopLogging", "id": "EXAMPLE-8cc3-42db-9a0d-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json index 6bd1a0c7ad..9a307206b6 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-10T15:01:23.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.065954100Z", + "ingested": "2021-03-18T12:21:59.428014700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T15:01:23Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-7d0c-45f4-b25b-EXAMPLE\",\"eventID\":\"EXAMPLE-0ef0-42cd-8551-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T15:01:23.000Z", "kind": "event", "action": "UpdateAccessKey", "id": "EXAMPLE-0ef0-42cd-8551-EXAMPLE", @@ -60,7 +62,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json index 12e1a7a347..1d9ab6c939 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-10T18:05:33.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.099377700Z", + "ingested": "2021-03-18T12:21:59.467118100Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T18:05:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateAccountPasswordPolicy\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"requireLowercaseCharacters\":true,\"requireSymbols\":true,\"requireNumbers\":true,\"minimumPasswordLength\":12,\"requireUppercaseCharacters\":true,\"allowUsersToChangePassword\":true},\"responseElements\":null,\"requestID\":\"EXAMPLE-5ebf-4bc3-a349-EXAMPLE\",\"eventID\":\"EXAMPLE-91f9-49f3-948c-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T18:05:33.000Z", "kind": "event", "action": "UpdateAccountPasswordPolicy", "id": "EXAMPLE-91f9-49f3-948c-EXAMPLE", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json index b1f9b9e484..67752d3525 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json @@ -8,14 +8,20 @@ } }, "@timestamp": "2020-01-09T02:23:11.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.133366700Z", + "ingested": "2021-03-18T12:21:59.507937900Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:23:11Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"newGroupName\":\"TEST-GROUP2\",\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-c22d-4fca-b40a-EXAMPLE\",\"eventID\":\"EXAMPLE-c3aa-487b-b05e-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-09T02:23:11.000Z", "kind": "event", "action": "UpdateGroup", "id": "EXAMPLE-c3aa-487b-b05e-EXAMPLE", @@ -58,6 +64,9 @@ "name": "Spider" }, "version": "1.16.310" + }, + "group": { + "name": "TEST-GROUP" } }, { @@ -68,14 +77,20 @@ } }, "@timestamp": "2020-01-09T02:24:35.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.133376Z", + "ingested": "2021-03-18T12:21:59.507950500Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:24:35Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"EntityAlreadyExistsException\",\"errorMessage\":\"Group with name TEST-GROUP already exists.\",\"requestParameters\":{\"newGroupName\":\"TEST-GROUP\",\"groupName\":\"TEST-GROUP2\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-f673-4ce7-8529-EXAMPLE\",\"eventID\":\"EXAMPLE-6a0b-475c-b5db-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-09T02:24:35.000Z", "kind": "event", "action": "UpdateGroup", "id": "EXAMPLE-6a0b-475c-b5db-EXAMPLE", @@ -120,6 +135,9 @@ "name": "Spider" }, "version": "1.16.310" + }, + "group": { + "name": "TEST-GROUP2" } } ] diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json index 8ff89bf890..49d6a1bc09 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-10T18:25:42.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.187902Z", + "ingested": "2021-03-18T12:21:59.578117900Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T18:25:42Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateLoginProfile\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-0dc6-447a-8859-EXAMPLE\",\"eventID\":\"EXAMPLE-c3b6-4498-b818-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T18:25:42.000Z", "kind": "event", "action": "UpdateLoginProfile", "id": "EXAMPLE-c3b6-4498-b818-EXAMPLE", @@ -58,7 +60,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json index 3945eeaf87..1d499ee6b8 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-10T16:06:54.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -18,9 +19,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.221679200Z", + "ingested": "2021-03-18T12:21:59.614623700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:54Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"userName\":\"Bob\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-32f3-4a92-82e1-EXAMPLE\",\"eventID\":\"EXAMPLE-5c88-4652-9ee9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T16:06:54.000Z", "kind": "event", "action": "UpdateSSHPublicKey", "id": "EXAMPLE-5c88-4652-9ee9-EXAMPLE", @@ -60,7 +62,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", @@ -80,6 +85,7 @@ "@timestamp": "2020-01-10T16:06:54.000Z", "related": { "user": [ + "Alice", "Bob" ] }, @@ -88,9 +94,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.221688200Z", + "ingested": "2021-03-18T12:21:59.614636Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:54Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"userName\":\"Bob\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-32f3-4a92-82e1-EXAMPLE\",\"eventID\":\"EXAMPLE-5c88-4652-9ee9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T16:06:54.000Z", "kind": "event", "action": "UpdateSSHPublicKey", "id": "EXAMPLE-5c88-4652-9ee9-EXAMPLE", @@ -130,7 +137,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json index 7797515e82..a175fbaccb 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json @@ -8,6 +8,11 @@ } }, "@timestamp": "2016-07-14T19:15:45.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "geo": { "continent_name": "North America", @@ -31,9 +36,10 @@ "ip": "205.251.233.182" }, "event": { - "ingested": "2020-11-19T22:16:18.282302Z", + "ingested": "2021-03-18T12:21:59.681616700Z", "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2016-07-14T19:15:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"205.251.233.182\",\"userAgent\":\"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22\",\"errorCode\":\"TrailNotFoundException\",\"errorMessage\":\"Unknown trail: myTrail2 for the user: 123456789012\",\"requestParameters\":{\"name\":\"myTrail2\"},\"responseElements\":null,\"requestID\":\"5d40662a-49f7-11e6-97e4-dEXAMPLE\",\"eventID\":\"b7d4398e-b2f0-4faa-9c76-e2EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "cloudtrail.amazonaws.com", + "created": "2016-07-14T19:15:45.000Z", "kind": "event", "action": "UpdateTrail", "id": "b7d4398e-b2f0-4faa-9c76-e2EXAMPLE", @@ -84,14 +90,20 @@ } }, "@timestamp": "2020-01-08T20:58:45.000Z", + "related": { + "user": [ + "Alice" + ] + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.282312Z", + "ingested": "2021-03-18T12:21:59.681629500Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:58:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"s3BucketName\":\"test-cloudtrail-bucket\",\"snsTopicName\":\"\",\"isMultiRegionTrail\":true,\"enableLogFileValidation\":false,\"kmsKeyId\":\"\"},\"responseElements\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"test-cloudtrail-bucket\",\"snsTopicName\":\"\",\"snsTopicARN\":\"\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"trailARN\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"logFileValidationEnabled\":false,\"isOrganizationTrail\":false},\"requestID\":\"EXAMPLE-f3da-42d1-84f5-EXAMPLE\",\"eventID\":\"EXAMPLE-b5e9-4846-8407-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "cloudtrail.amazonaws.com", + "created": "2020-01-08T20:58:45.000Z", "kind": "event", "action": "UpdateTrail", "id": "EXAMPLE-b5e9-4846-8407-EXAMPLE", @@ -105,17 +117,13 @@ "request_parameters": { "name": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail", "enableLogFileValidation": false, - "kmsKeyId": "", "isMultiRegionTrail": true, - "s3BucketName": "test-cloudtrail-bucket", - "snsTopicName": "" + "s3BucketName": "test-cloudtrail-bucket" }, "response_elements": { - "snsTopicARN": "", "logFileValidationEnabled": false, "isMultiRegionTrail": true, "s3BucketName": "test-cloudtrail-bucket", - "snsTopicName": "", "name": "TEST-trail", "trailARN": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail", "isOrganizationTrail": false, @@ -135,8 +143,8 @@ "arn": "arn:aws:iam::0123456789012:user/Alice" }, "recipient_account_id": "0123456789012", - "request_parameters": "{isMultiRegionTrail=true, s3BucketName=test-cloudtrail-bucket, snsTopicName=, name=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, enableLogFileValidation=false, kmsKeyId=}", - "response_elements": "{snsTopicARN=, logFileValidationEnabled=false, isMultiRegionTrail=true, s3BucketName=test-cloudtrail-bucket, snsTopicName=, name=TEST-trail, trailARN=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, isOrganizationTrail=false, includeGlobalServiceEvents=true}" + "request_parameters": "{isMultiRegionTrail=true, s3BucketName=test-cloudtrail-bucket, name=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, enableLogFileValidation=false}", + "response_elements": "{logFileValidationEnabled=false, isMultiRegionTrail=true, s3BucketName=test-cloudtrail-bucket, name=TEST-trail, trailARN=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, isOrganizationTrail=false, includeGlobalServiceEvents=true}" } }, "user": { diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json index 3c36d58521..3ab6433e75 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json @@ -10,6 +10,7 @@ "@timestamp": "2020-01-08T20:53:12.000Z", "related": { "user": [ + "Alice", "Bob", "Robert" ] @@ -19,9 +20,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.347266800Z", + "ingested": "2021-03-18T12:21:59.763970400Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-08T20:53:12Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"userName\":\"Bob\",\"newUserName\":\"Robert\"},\"responseElements\":null,\"requestID\":\"3a6b3260-739d-465e-9406-bcEXAMPLE\",\"eventID\":\"9150d546-3564-4262-8e62-110EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-08T20:53:12.000Z", "kind": "event", "action": "UpdateUser", "id": "9150d546-3564-4262-8e62-110EXAMPLE", @@ -55,7 +57,13 @@ }, "user": { "name": "Alice", - "id": "EX_PRINCIPAL_ID" + "changes": { + "name": "Robert" + }, + "id": "EX_PRINCIPAL_ID", + "target": { + "name": "Bob" + } }, "user_agent": { "name": "aws-cli", @@ -68,9 +76,8 @@ }, { "event": { - "ingested": "2020-11-19T22:16:18.347276300Z", - "original": "", "type": "info", + "ingested": "2021-03-18T12:21:59.763982300Z", "kind": "event" } } diff --git a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json index 6e797c4a2f..4facb4fed2 100644 --- a/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json +++ b/test/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json @@ -18,9 +18,10 @@ "ip": "127.0.0.1" }, "event": { - "ingested": "2020-11-19T22:16:18.393021500Z", + "ingested": "2021-03-18T12:21:59.805534700Z", "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:40Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UploadSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\",\"userName\":\"Alice\"},\"responseElements\":{\"sSHPublicKey\":{\"fingerprint\":\"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de\",\"status\":\"Active\",\"uploadDate\":\"Jan 10, 2020 4:06:40 PM\",\"userName\":\"Alice\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\"}},\"requestID\":\"EXAMPLE-44b9-41cd-90f2-EXAMPLE\",\"eventID\":\"EXAMPLE-9a9d-4da4-9998-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", "provider": "iam.amazonaws.com", + "created": "2020-01-10T16:06:40.000Z", "kind": "event", "action": "UploadSSHPublicKey", "id": "EXAMPLE-9a9d-4da4-9998-EXAMPLE", @@ -64,7 +65,10 @@ }, "user": { "name": "Alice", - "id": "EXAMPLE_ID" + "id": "EXAMPLE_ID", + "target": { + "name": "Alice" + } }, "user_agent": { "name": "Other", diff --git a/test/packages/aws/data_stream/cloudtrail/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/cloudtrail/agent/stream/s3.yml.hbs index 82f82f63ba..309c650769 100644 --- a/test/packages/aws/data_stream/cloudtrail/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/cloudtrail/agent/stream/s3.yml.hbs @@ -27,9 +27,6 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} -{{/if}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} @@ -37,4 +34,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 \ No newline at end of file + ecs.version: 1.8.0 \ No newline at end of file diff --git a/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml index 033e6387aa..3e0000d26f 100644 --- a/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml @@ -16,6 +16,26 @@ processors: ignore_failure: true formats: - ISO8601 + - set: + field: event.created + value: '{{@timestamp}}' + - script: + description: Drops null/empty values recursively + lang: painless + source: | + boolean drop(Object o) { + if (o == null || o == "") { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).length == 0); + } + return false; + } + drop(ctx); - rename: field: "json.eventVersion" target_field: "aws.cloudtrail.event_version" @@ -24,6 +44,11 @@ processors: field: "json.userIdentity.type" target_field: "aws.cloudtrail.user_identity.type" ignore_failure: true + - append: + field: related.user + value: '{{json.userIdentity.userName}}' + allow_duplicates: false + if: 'ctx.json?.userIdentity?.userName != null' - rename: field: "json.userIdentity.userName" target_field: "user.name" @@ -58,7 +83,7 @@ processors: field: "json.userIdentity.sessionContext.sessionIssuer.type" target_field: "aws.cloudtrail.user_identity.session_context.session_issuer.type" ignore_failure: true -# userIdentity.sessionIssuer.userName is only set with assumed roles. + # userIdentity.sessionIssuer.userName is only set with assumed roles. - rename: field: "json.userIdentity.sessionContext.sessionIssuer.userName" target_field: "user.name" @@ -88,6 +113,10 @@ processors: value: "{{json.eventName}}" ignore_failure: true ignore_empty_value: true + - rename: + field: "json.eventCategory" + target_field: "aws.cloudtrail.event_category" + ignore_failure: true - rename: field: "json.awsRegion" target_field: "cloud.region" @@ -119,9 +148,9 @@ processors: target_field: source.as.number ignore_missing: true - rename: - field: source.as.organization_name - target_field: source.as.organization.name - ignore_missing: true + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true - user_agent: field: "json.userAgent" target_field: "user_agent" @@ -138,37 +167,36 @@ processors: field: "json.errorMessage" target_field: "aws.cloudtrail.error_message" ignore_failure: true - - rename: - field: json.requestParameters - target_field: "aws.cloudtrail.flattened.request_parameters" - if: ctx.json?.requestParameters != null - script: lang: painless source: | - if (ctx.aws.cloudtrail.flattened.request_parameters != null) { - ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString(); + if (ctx.aws.cloudtrail?.flattened == null) { + Map map = new HashMap(); + ctx.aws.cloudtrail.put("flattened", map); + } + if (ctx.json?.requestParameters != null) { + ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString(); + if (ctx.aws.cloudtrail.request_parameters.length() < 32766) { + ctx.aws.cloudtrail.flattened.put("request_parameters", ctx.json.requestParameters); + } } - ignore_failure: true - - rename: - field: json.responseElements - target_field: "aws.cloudtrail.flattened.response_elements" - if: ctx.json?.responseElements != null - - script: - lang: painless - source: | - if (ctx.aws.cloudtrail.flattened.response_elements != null) { - ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString(); + if (ctx.json?.responseElements != null) { + ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString(); + if (ctx.aws.cloudtrail.response_elements.length() < 32766) { + ctx.aws.cloudtrail.flattened.put("response_elements", ctx.json.responseElements); + } } - ignore_failure: true - - rename: - field: json.additionalEventData - target_field: "aws.cloudtrail.flattened.additional_eventdata" - if: ctx?.json?.additionalEventData != null - - script: - lang: painless - source: | - if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) { - ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString(); + if (ctx.json?.additionalEventData != null) { + ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString(); + if (ctx.aws.cloudtrail.additional_eventdata.length() < 32766) { + ctx.aws.cloudtrail.flattened.put("additional_eventdata", ctx.json.additionalEventData); + } + } + if (ctx.json?.serviceEventDetails != null) { + ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString(); + if (ctx.aws.cloudtrail.service_event_details.length() < 32766) { + ctx.aws.cloudtrail.flattened.put("service_event_details", ctx.json.serviceEventDetails); + } } ignore_failure: true - rename: @@ -211,17 +239,6 @@ processors: field: "json.recipientAccountId" target_field: "aws.cloudtrail.recipient_account_id" ignore_failure: true - - rename: - field: json.serviceEventDetails - target_field: "aws.cloudtrail.flattened.service_event_details" - if: ctx.json?.serviceEventDetails != null - - script: - lang: painless - source: | - if (ctx.aws.cloudtrail.flattened.service_event_details != null) { - ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString(); - } - ignore_failure: true - rename: field: "json.sharedEventId" target_field: "aws.cloudtrail.shared_event_id" @@ -230,28 +247,16 @@ processors: field: "json.vpcEndpointId" target_field: "aws.cloudtrail.vpc_endpoint_id" ignore_failure: true - - script: - lang: painless - ignore_failure: true - source: >- - void addRelatedUser(def ctx, String userName) { - if (ctx.related == null) { - Map map = new HashMap(); - ctx.put("related", map); - } - if (ctx.related.user == null) { - ArrayList al = new ArrayList(); - ctx.related.put("user", al); - } - ctx.related.user.add(userName); - } - if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) { - addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName); - } - if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) { - addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName); - } - + - append: + field: related.user + value: '{{aws.cloudtrail.flattened.request_parameters.userName}}' + allow_duplicates: false + if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.userName != null' + - append: + field: related.user + value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}' + allow_duplicates: false + if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null' - script: lang: painless ignore_failure: true @@ -615,9 +620,116 @@ processors: def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); + - rename: + field: "json.awsAccountId" + target_field: "cloud.account.id" + ignore_failure: true + - rename: + field: "json.previousDigestS3Object" + target_field: "file.path" + ignore_failure: true + - rename: + field: "json.previousDigestSignature" + target_field: "file.hash.sha256" + if: >- + ctx?.json?.previousDigestHashAlgorithm != null && ctx.json.previousDigestHashAlgorithm == 'SHA-256' + - append: + field: "related.hash" + value: "{{file.hash.sha256}}" + if: "ctx?.file?.hash?.sha256 != null" + - rename: + field: "json.logFiles" + target_field: "aws.cloudtrail.digest.log_files" + ignore_failure: true + - date: + field: "json.digestStartTime" + target_field: "aws.cloudtrail.digest.start_time" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.digestEndTime" + target_field: "@timestamp" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.digestEndTime" + target_field: "aws.cloudtrail.digest.end_time" + ignore_failure: true + formats: + - ISO8601 + - rename: + field: "json.digestS3Bucket" + target_field: "aws.cloudtrail.digest.s3_bucket" + ignore_failure: true + - date: + field: "json.newestEventTime" + target_field: "aws.cloudtrail.digest.newest_event_time" + ignore_failure: true + formats: + - ISO8601 + - date: + field: "json.oldestEventTime" + target_field: "aws.cloudtrail.digest.oldest_event_time" + ignore_failure: true + formats: + - ISO8601 + - rename: + field: "json.previousDigestS3Bucket" + target_field: "aws.cloudtrail.digest.previous_s3_bucket" + ignore_failure: true + - rename: + field: "json.previousDigestHashAlgorithm" + target_field: "aws.cloudtrail.digest.previous_hash_algorithm" + ignore_failure: true + - rename: + field: "json.publicKeyFingerprint" + target_field: "aws.cloudtrail.digest.public_key_fingerprint" + ignore_failure: true + - rename: + field: "json.digestSignatureAlgorithm" + target_field: "aws.cloudtrail.digest.signature_algorithm" + ignore_failure: true + - rename: + field: "json.insightDetails" + target_field: "aws.cloudtrail.insight_details" + ignore_failure: true + - set: + field: group.id + value: '{{aws.cloudtrail.flattened.response_elements.group.groupId}}' + ignore_empty_value: true + ignore_failure: true + - set: + field: user.target.id + value: '{{aws.cloudtrail.flattened.response_elements.user.userId}}' + ignore_empty_value: true + ignore_failure: true + - set: + field: user.changes.name + value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}' + ignore_empty_value: true + ignore_failure: true + - set: + field: group.name + value: '{{aws.cloudtrail.flattened.request_parameters.groupName}}' + ignore_empty_value: true + ignore_failure: true + - set: + field: user.target.name + value: '{{aws.cloudtrail.flattened.request_parameters.userName}}' + ignore_empty_value: true + ignore_failure: true + - rename: + field: aws.cloudtrail.digest + target_field: aws.cloudtrail.flattened.digest + ignore_missing: true + - rename: + field: aws.cloudtrail.insight_details + target_field: aws.cloudtrail.flattened.insight_details + ignore_missing: true - remove: - field: - - "json" + field: json ignore_missing: true on_failure: - set: diff --git a/test/packages/aws/data_stream/cloudtrail/fields/ecs.yml b/test/packages/aws/data_stream/cloudtrail/fields/ecs.yml index 2a35577f37..81ba6c50b9 100644 --- a/test/packages/aws/data_stream/cloudtrail/fields/ecs.yml +++ b/test/packages/aws/data_stream/cloudtrail/fields/ecs.yml @@ -16,6 +16,50 @@ - name: user.id type: keyword description: Unique identifier of the user. +- name: user.target.name + type: keyword + description: Short name or login of the user. +- name: user.target.id + type: keyword + description: Unique identifier of the user. +- name: user.changes.name + type: keyword + description: Short name or login of the user. +- name: group.id + type: keyword + description: Unique identifier for the group on the system/platform. +- name: group.name + type: keyword + description: Name of the group. +- name: file + title: File + type: group + fields: + - name: path + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Full path to the file, including the file name. It should include the drive letter, when appropriate. + - name: hash.md5 + type: keyword + ignore_above: 1024 + description: MD5 hash. + - name: hash.sha1 + type: keyword + ignore_above: 1024 + description: SHA1 hash. + - name: hash.sha256 + type: keyword + ignore_above: 1024 + description: SHA256 hash. + - name: hash.sha512 + type: keyword + ignore_above: 1024 + description: SHA512 hash. - name: cloud.account.id type: keyword description: The cloud account or organization id used to identify different entities in a multi-tenant environment. @@ -55,6 +99,9 @@ - name: related.user type: keyword description: All the user names seen on your event. +- name: related.hash + type: keyword + description: All the hashes seen on your event. - name: event.kind type: keyword description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal) diff --git a/test/packages/aws/data_stream/cloudtrail/fields/fields.yml b/test/packages/aws/data_stream/cloudtrail/fields/fields.yml index 5a12696f9f..5b59153c9b 100644 --- a/test/packages/aws/data_stream/cloudtrail/fields/fields.yml +++ b/test/packages/aws/data_stream/cloudtrail/fields/fields.yml @@ -5,6 +5,10 @@ type: keyword description: | The CloudTrail version of the log event format. + - name: event_category + type: keyword + description: | + The CloudTrail event category. - name: user_identity type: group fields: @@ -156,3 +160,11 @@ type: flattened description: >- Identifies the service event, including what triggered the event and the result. + - name: digest + type: flattened + description: >- + Additional digest information. + - name: insight_details + type: flattened + description: >- + Additional insight details. diff --git a/test/packages/aws/data_stream/cloudtrail/manifest.yml b/test/packages/aws/data_stream/cloudtrail/manifest.yml index ec0616d79a..94b8c1d0e3 100644 --- a/test/packages/aws/data_stream/cloudtrail/manifest.yml +++ b/test/packages/aws/data_stream/cloudtrail/manifest.yml @@ -1,5 +1,5 @@ title: AWS CloudTrail logs -release: experimental +release: beta type: logs streams: - input: s3 diff --git a/test/packages/aws/data_stream/cloudwatch_logs/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/cloudwatch_logs/agent/stream/s3.yml.hbs index d80e585342..6a00835778 100644 --- a/test/packages/aws/data_stream/cloudwatch_logs/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/cloudwatch_logs/agent/stream/s3.yml.hbs @@ -26,9 +26,6 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} -{{/if}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} diff --git a/test/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/test/packages/aws/data_stream/cloudwatch_logs/manifest.yml index cf8f56be4f..6fc61a757b 100644 --- a/test/packages/aws/data_stream/cloudwatch_logs/manifest.yml +++ b/test/packages/aws/data_stream/cloudwatch_logs/manifest.yml @@ -1,5 +1,5 @@ title: AWS CloudWatch logs -release: experimental +release: beta type: logs streams: - input: s3 diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs index b453371d1d..40e4c2530b 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs @@ -18,8 +18,14 @@ shared_credential_file: {{shared_credential_file}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} +{{#if regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if metrics}} metrics: {{metrics}} diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml b/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/cloudwatch_metrics/manifest.yml b/test/packages/aws/data_stream/cloudwatch_metrics/manifest.yml index c4fd774fce..dae477ae67 100644 --- a/test/packages/aws/data_stream/cloudwatch_metrics/manifest.yml +++ b/test/packages/aws/data_stream/cloudwatch_metrics/manifest.yml @@ -1,5 +1,5 @@ title: AWS CloudWatch metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs index 66d3809e67..07e4a166ac 100644 --- a/test/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/dynamodb/fields/ecs.yml b/test/packages/aws/data_stream/dynamodb/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/dynamodb/fields/ecs.yml +++ b/test/packages/aws/data_stream/dynamodb/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/dynamodb/manifest.yml b/test/packages/aws/data_stream/dynamodb/manifest.yml index 13f890ff33..12bec6c6e8 100644 --- a/test/packages/aws/data_stream/dynamodb/manifest.yml +++ b/test/packages/aws/data_stream/dynamodb/manifest.yml @@ -1,5 +1,5 @@ title: AWS DynamoDB metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs index 8d4031cccf..b0d8e145fa 100644 --- a/test/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/ebs/fields/ecs.yml b/test/packages/aws/data_stream/ebs/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/ebs/fields/ecs.yml +++ b/test/packages/aws/data_stream/ebs/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/ebs/manifest.yml b/test/packages/aws/data_stream/ebs/manifest.yml index 5484bf219a..5d0cce0e85 100644 --- a/test/packages/aws/data_stream/ebs/manifest.yml +++ b/test/packages/aws/data_stream/ebs/manifest.yml @@ -1,5 +1,5 @@ title: AWS EBS metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/ec2_logs/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/ec2_logs/agent/stream/s3.yml.hbs index d80e585342..6a00835778 100644 --- a/test/packages/aws/data_stream/ec2_logs/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/ec2_logs/agent/stream/s3.yml.hbs @@ -26,9 +26,6 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} -{{/if}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} diff --git a/test/packages/aws/data_stream/ec2_logs/manifest.yml b/test/packages/aws/data_stream/ec2_logs/manifest.yml index 092f59bf31..aca6fb44b4 100644 --- a/test/packages/aws/data_stream/ec2_logs/manifest.yml +++ b/test/packages/aws/data_stream/ec2_logs/manifest.yml @@ -1,5 +1,5 @@ title: AWS EC2 logs -release: experimental +release: beta type: logs streams: - input: s3 diff --git a/test/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs index 7b33ea477a..5eb40ca78b 100644 --- a/test/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml b/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/ec2_metrics/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml b/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml index 833aba3ec9..b2f34d3973 100644 --- a/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml +++ b/test/packages/aws/data_stream/ec2_metrics/fields/fields.yml @@ -78,14 +78,6 @@ format: bytes description: | Bytes read from all instance store volumes available to the instance. - - name: diskio.read.count - type: long - format: bytes - description: The number of disk IO reads - - name: diskio.read.count_per_sec - type: long - format: bytes - description: The number of disk IO reads per second - name: diskio.read.bytes_per_sec type: long description: | @@ -99,27 +91,19 @@ type: long description: | Bytes written per second to all instance store volumes available to the instance. - - name: diskio.read.ops + - name: diskio.read.count type: long description: | Completed read operations from all instance store volumes available to the instance in a specified period of time. - - name: diskio.read.ops_per_sec + - name: diskio.read.count_per_sec type: long description: | Completed read operations per second from all instance store volumes available to the instance in a specified period of time. - name: diskio.write.count - type: long - format: bytes - description: The number of disk IO writes - - name: diskio.write.count_per_sec - type: long - format: bytes - description: The number of disk IO writes per second - - name: diskio.write.ops type: long description: | Completed write operations to all instance store volumes available to the instance in a specified period of time. - - name: diskio.write.ops_per_sec + - name: diskio.write.count_per_sec type: long description: | Completed write operations per second to all instance store volumes available to the instance in a specified period of time. diff --git a/test/packages/aws/data_stream/ec2_metrics/manifest.yml b/test/packages/aws/data_stream/ec2_metrics/manifest.yml index 990f14e4a1..8a3d5fb87f 100644 --- a/test/packages/aws/data_stream/ec2_metrics/manifest.yml +++ b/test/packages/aws/data_stream/ec2_metrics/manifest.yml @@ -1,5 +1,5 @@ title: AWS EC2 metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/elb_logs/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/elb_logs/agent/stream/s3.yml.hbs index d80e585342..6a00835778 100644 --- a/test/packages/aws/data_stream/elb_logs/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/elb_logs/agent/stream/s3.yml.hbs @@ -26,9 +26,6 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} -{{/if}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} diff --git a/test/packages/aws/data_stream/elb_logs/manifest.yml b/test/packages/aws/data_stream/elb_logs/manifest.yml index d72ead363f..4fcba786fc 100644 --- a/test/packages/aws/data_stream/elb_logs/manifest.yml +++ b/test/packages/aws/data_stream/elb_logs/manifest.yml @@ -1,5 +1,5 @@ title: AWS ELB logs -release: experimental +release: beta type: logs streams: - input: s3 diff --git a/test/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs index 7cc624a32b..57c5acdd4c 100644 --- a/test/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml b/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml +++ b/test/packages/aws/data_stream/elb_metrics/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/elb_metrics/manifest.yml b/test/packages/aws/data_stream/elb_metrics/manifest.yml index acb61e4a98..1e6ed4e207 100644 --- a/test/packages/aws/data_stream/elb_metrics/manifest.yml +++ b/test/packages/aws/data_stream/elb_metrics/manifest.yml @@ -1,5 +1,5 @@ title: AWS ELB metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs index 5934e221b3..cf17d23388 100644 --- a/test/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/lambda/fields/ecs.yml b/test/packages/aws/data_stream/lambda/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/lambda/fields/ecs.yml +++ b/test/packages/aws/data_stream/lambda/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/lambda/manifest.yml b/test/packages/aws/data_stream/lambda/manifest.yml index 18eaf0d9d2..5e0684218b 100644 --- a/test/packages/aws/data_stream/lambda/manifest.yml +++ b/test/packages/aws/data_stream/lambda/manifest.yml @@ -1,5 +1,5 @@ title: AWS Lambda metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs index 84bdfad45e..94bed66ae8 100644 --- a/test/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/natgateway/fields/ecs.yml b/test/packages/aws/data_stream/natgateway/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/natgateway/fields/ecs.yml +++ b/test/packages/aws/data_stream/natgateway/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/natgateway/manifest.yml b/test/packages/aws/data_stream/natgateway/manifest.yml index c64f31b025..fc6cf801c3 100644 --- a/test/packages/aws/data_stream/natgateway/manifest.yml +++ b/test/packages/aws/data_stream/natgateway/manifest.yml @@ -1,5 +1,5 @@ title: AWS NAT gateway metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs index 38c9f6f83a..bf6deefcf8 100644 --- a/test/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/rds/fields/ecs.yml b/test/packages/aws/data_stream/rds/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/rds/fields/ecs.yml +++ b/test/packages/aws/data_stream/rds/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/rds/manifest.yml b/test/packages/aws/data_stream/rds/manifest.yml index 56a031f235..c920727e99 100644 --- a/test/packages/aws/data_stream/rds/manifest.yml +++ b/test/packages/aws/data_stream/rds/manifest.yml @@ -1,5 +1,5 @@ title: AWS RDS metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs index eacf139814..cac1cae04a 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/s3_daily_storage/agent/stream/stream.yml.hbs @@ -19,8 +19,11 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} +{{#if latency}} +latency: {{latency}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml b/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml +++ b/test/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/s3_daily_storage/manifest.yml b/test/packages/aws/data_stream/s3_daily_storage/manifest.yml index e1c450b277..89473f0ebb 100644 --- a/test/packages/aws/data_stream/s3_daily_storage/manifest.yml +++ b/test/packages/aws/data_stream/s3_daily_storage/manifest.yml @@ -1,5 +1,5 @@ title: AWS S3 daily storage metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs index 3cea704f6b..6f53aab34d 100644 --- a/test/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/s3_request/agent/stream/stream.yml.hbs @@ -19,8 +19,11 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} +{{#if latency}} +latency: {{latency}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3_request/fields/ecs.yml b/test/packages/aws/data_stream/s3_request/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/s3_request/fields/ecs.yml +++ b/test/packages/aws/data_stream/s3_request/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/s3_request/manifest.yml b/test/packages/aws/data_stream/s3_request/manifest.yml index 172be859c5..472461c764 100644 --- a/test/packages/aws/data_stream/s3_request/manifest.yml +++ b/test/packages/aws/data_stream/s3_request/manifest.yml @@ -1,5 +1,5 @@ title: AWS S3 request metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log new file mode 100644 index 0000000000..f96091a767 --- /dev/null +++ b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log @@ -0,0 +1,6 @@ +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 17 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 3 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2 diff --git a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-config.yml b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-config.yml new file mode 100644 index 0000000000..c39dc38617 --- /dev/null +++ b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json new file mode 100644 index 0000000000..8ebd68bda6 --- /dev/null +++ b/test/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json @@ -0,0 +1,561 @@ +{ + "expected": [ + { + "url": { + "path": "/test-s3-ks/", + "original": "/test-s3-ks/?location\u0026aws-account=627959692251", + "query": "location\u0026aws-account=627959692251" + }, + "geo": { + "continent_name": "North America", + "region_iso_code": "US-VA", + "city_name": "Ashburn", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Virginia", + "location": { + "lon": -77.4728, + "lat": 39.0481 + } + }, + "cloud": { + "provider": "aws" + }, + "@timestamp": "2019-08-01T00:24:41.000Z", + "related": { + "user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], + "ip": [ + "72.21.217.31" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 142 + }, + "status_code": 200 + } + }, + "client": { + "user": { + "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" + }, + "address": "72.21.217.31", + "ip": "72.21.217.31" + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "duration": 17000000, + "ingested": "2021-03-18T12:22:00.354561800Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:41 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 44EE8651683CB4DA REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 17 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "kind": "event", + "action": "REST.GET.LOCATION", + "id": "44EE8651683CB4DA", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "bytes_sent": 142, + "authentication_type": "AuthHeader", + "request_uri": "GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1", + "host_id": "BsCfJedfuSnds2QFoxi+E/O7M6OEWzJnw4dUaes/2hyA363sONRJKzB7EOY+Bt9DTHYUn+HoHxI=", + "host_header": "s3.ap-southeast-1.amazonaws.com", + "bucket": "test-s3-ks", + "remote_ip": "72.21.217.31", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "http_status": 200, + "total_time": 17, + "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "operation": "REST.GET.LOCATION", + "request_id": "44EE8651683CB4DA", + "user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" + } + }, + "user_agent": { + "name": "aws-sdk-java", + "original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "os": { + "name": "Linux", + "version": "4.9.137", + "full": "Linux 4.9.137" + }, + "device": { + "name": "Other" + }, + "version": "1.11.590" + } + }, + { + "url": { + "path": "/test-s3-ks/", + "original": "/test-s3-ks/?location\u0026aws-account=627959692251", + "query": "location\u0026aws-account=627959692251" + }, + "geo": { + "continent_name": "North America", + "region_iso_code": "US-VA", + "city_name": "Ashburn", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Virginia", + "location": { + "lon": -77.4728, + "lat": 39.0481 + } + }, + "cloud": { + "provider": "aws" + }, + "@timestamp": "2019-08-01T00:24:42.000Z", + "related": { + "user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], + "ip": [ + "72.21.217.31" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 142 + }, + "status_code": 200 + } + }, + "client": { + "user": { + "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" + }, + "address": "72.21.217.31", + "ip": "72.21.217.31" + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "duration": 3000000, + "ingested": "2021-03-18T12:22:00.354582800Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:42 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 E26222010BCC32B6 REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 3 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "kind": "event", + "action": "REST.GET.LOCATION", + "id": "E26222010BCC32B6", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "bytes_sent": 142, + "authentication_type": "AuthHeader", + "request_uri": "GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1", + "host_id": "gNl/Q1IzY6nGTBygqI3rnMz/ZFOFwOTDpSMrNca+IcEmMAd6sCIs1ZRLYDekD8LB9lrj9UdQLWE=", + "host_header": "s3.ap-southeast-1.amazonaws.com", + "bucket": "test-s3-ks", + "remote_ip": "72.21.217.31", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "http_status": 200, + "total_time": 3, + "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "operation": "REST.GET.LOCATION", + "request_id": "E26222010BCC32B6", + "user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" + } + }, + "user_agent": { + "name": "aws-sdk-java", + "original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "os": { + "name": "Linux", + "version": "4.9.137", + "full": "Linux 4.9.137" + }, + "device": { + "name": "Other" + }, + "version": "1.11.590" + } + }, + { + "url": { + "path": "/test-s3-ks/", + "original": "/test-s3-ks/?max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251", + "query": "max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251" + }, + "geo": { + "continent_name": "North America", + "region_iso_code": "US-VA", + "city_name": "Ashburn", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Virginia", + "location": { + "lon": -77.4728, + "lat": 39.0481 + } + }, + "cloud": { + "provider": "aws" + }, + "@timestamp": "2019-08-01T00:24:43.000Z", + "related": { + "user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], + "ip": [ + "72.21.217.31" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 265 + }, + "status_code": 200 + } + }, + "client": { + "user": { + "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" + }, + "address": "72.21.217.31", + "ip": "72.21.217.31" + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "duration": 2000000, + "ingested": "2021-03-18T12:22:00.354597800Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - \"GET /test-s3-ks/?max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251 HTTP/1.1\" 200 - 265 - 2 1 \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "kind": "event", + "action": "REST.GET.BUCKET", + "id": "4DD6D17D1C5C401C", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "turn_around_time": 1, + "bytes_sent": 265, + "authentication_type": "AuthHeader", + "request_uri": "GET /test-s3-ks/?max-keys=0\u0026encoding-type=url\u0026aws-account=627959692251 HTTP/1.1", + "host_id": "KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE=", + "host_header": "s3.ap-southeast-1.amazonaws.com", + "bucket": "test-s3-ks", + "remote_ip": "72.21.217.31", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "http_status": 200, + "total_time": 2, + "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "operation": "REST.GET.BUCKET", + "request_id": "4DD6D17D1C5C401C", + "user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" + } + }, + "user_agent": { + "name": "aws-sdk-java", + "original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "os": { + "name": "Linux", + "version": "4.9.137", + "full": "Linux 4.9.137" + }, + "device": { + "name": "Other" + }, + "version": "1.11.590" + } + }, + { + "url": { + "path": "/test-s3-ks/", + "original": "/test-s3-ks/?location\u0026aws-account=627959692251", + "query": "location\u0026aws-account=627959692251" + }, + "geo": { + "continent_name": "North America", + "region_iso_code": "US-VA", + "city_name": "Ashburn", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Virginia", + "location": { + "lon": -77.4728, + "lat": 39.0481 + } + }, + "cloud": { + "provider": "aws" + }, + "@timestamp": "2019-08-01T00:24:43.000Z", + "related": { + "user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], + "ip": [ + "72.21.217.31" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "version": "1.1", + "response": { + "body": { + "bytes": 142 + }, + "status_code": 200 + } + }, + "client": { + "user": { + "id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9" + }, + "address": "72.21.217.31", + "ip": "72.21.217.31" + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "duration": 4000000, + "ingested": "2021-03-18T12:22:00.354741400Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - \"GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1\" 200 - 142 - 4 - \"-\" \"AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation\" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", + "kind": "event", + "action": "REST.GET.LOCATION", + "id": "706992E2F3CC3C3D", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "bytes_sent": 142, + "authentication_type": "AuthHeader", + "request_uri": "GET /test-s3-ks/?location\u0026aws-account=627959692251 HTTP/1.1", + "host_id": "cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg=", + "host_header": "s3.ap-southeast-1.amazonaws.com", + "bucket": "test-s3-ks", + "remote_ip": "72.21.217.31", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "http_status": 200, + "total_time": 4, + "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "operation": "REST.GET.LOCATION", + "request_id": "706992E2F3CC3C3D", + "user_agent": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" + } + }, + "user_agent": { + "name": "aws-sdk-java", + "original": "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation", + "os": { + "name": "Linux", + "version": "4.9.137", + "full": "Linux 4.9.137" + }, + "device": { + "name": "Other" + }, + "version": "1.11.590" + } + }, + { + "geo": { + "continent_name": "Europe", + "region_iso_code": "ES-TE", + "city_name": "Teruel", + "country_iso_code": "ES", + "country_name": "Spain", + "region_name": "Teruel", + "location": { + "lon": -1.1065, + "lat": 40.3456 + } + }, + "cloud": { + "provider": "aws" + }, + "@timestamp": "2019-09-10T15:11:07.000Z", + "related": { + "user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], + "ip": [ + "77.227.156.41" + ] + }, + "client": { + "user": { + "id": "arn:aws:iam::123456:user/test@elastic.co" + }, + "address": "77.227.156.41", + "ip": "77.227.156.41" + }, + "http": { + "response": { + "status_code": 204 + } + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "ingested": "2021-03-18T12:22:00.354756500Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2", + "kind": "event", + "action": "BATCH.DELETE.OBJECT", + "id": "8CD7A4A71E2E5C9E", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "arn:aws:iam::123456:user/test@elastic.co", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "authentication_type": "AuthHeader", + "host_id": "IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk=", + "host_header": "s3.eu-central-1.amazonaws.com", + "bucket": "jsoriano-s3-test", + "remote_ip": "77.227.156.41", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "http_status": 204, + "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "operation": "BATCH.DELETE.OBJECT", + "request_id": "8CD7A4A71E2E5C9E", + "key": "jolokia-war-1.5.0.war", + "object_size": 344017 + } + } + }, + { + "geo": { + "continent_name": "North America", + "region_iso_code": "US-CO", + "city_name": "Denver", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Colorado", + "location": { + "lon": -105.0023, + "lat": 39.7044 + } + }, + "cloud": { + "provider": "aws" + }, + "@timestamp": "2019-09-19T17:06:39.000Z", + "related": { + "user": [ + "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2" + ], + "ip": [ + "174.29.206.152" + ] + }, + "client": { + "user": { + "id": "arn:aws:iam::123456:user/test@elastic.co" + }, + "address": "174.29.206.152", + "ip": "174.29.206.152" + }, + "http": { + "response": { + "status_code": 204 + } + }, + "tls": { + "cipher": "ECDHE-RSA-AES128-SHA", + "version": "1.2", + "version_protocol": "tls" + }, + "event": { + "ingested": "2021-03-18T12:22:00.354771Z", + "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2", + "kind": "event", + "action": "BATCH.DELETE.OBJECT", + "id": "6CE38F1312D32BDD", + "category": "web", + "type": [ + "access" + ], + "outcome": "success" + }, + "aws": { + "s3access": { + "requester": "arn:aws:iam::123456:user/test@elastic.co", + "tls_version": "TLSv1.2", + "signature_version": "SigV4", + "authentication_type": "AuthHeader", + "host_id": "LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0=", + "host_header": "s3-ap-southeast-1.amazonaws.com", + "bucket": "test-s3-ks", + "remote_ip": "174.29.206.152", + "cipher_suite": "ECDHE-RSA-AES128-SHA", + "http_status": 204, + "bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", + "operation": "BATCH.DELETE.OBJECT", + "request_id": "6CE38F1312D32BDD", + "key": "Screen+Shot+2019-09-09+at+9.08.44+AM.png", + "object_size": 57138 + } + } + } + ] +} \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3access/agent/stream/log.yml.hbs b/test/packages/aws/data_stream/s3access/agent/stream/log.yml.hbs index 1a5e67ea6a..31a201bc45 100644 --- a/test/packages/aws/data_stream/s3access/agent/stream/log.yml.hbs +++ b/test/packages/aws/data_stream/s3access/agent/stream/log.yml.hbs @@ -7,4 +7,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 \ No newline at end of file + ecs.version: 1.8.0 \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3access/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/s3access/agent/stream/s3.yml.hbs index d80e585342..cea7960b97 100644 --- a/test/packages/aws/data_stream/s3access/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/s3access/agent/stream/s3.yml.hbs @@ -26,9 +26,6 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} -{{/if}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} @@ -36,4 +33,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 \ No newline at end of file + ecs.version: 1.8.0 \ No newline at end of file diff --git a/test/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml b/test/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml index f346b427c0..74a565e828 100644 --- a/test/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml +++ b/test/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml @@ -2,6 +2,15 @@ description: "Pipeline for s3 server access logs" processors: + - set: + field: event.category + value: web + - append: + field: event.type + value: access + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -22,6 +31,40 @@ processors: S3ID: "[a-zA-Z0-9\\/_\\.\\-%+=]+" S3VERSION: "[a-zA-Z0-9.]+" + - script: + description: Drops null/empty values recursively + lang: painless + source: | + boolean drop(Object o) { + if (o == null || o == "") { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).length == 0); + } + return false; + } + drop(ctx); + + - grok: + field: aws.s3access.request_uri + ignore_failure: true + patterns: + - '%{NOTSPACE:http.request.method} %{NOTSPACE:url.original} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' + + # + # Best-effort parse of url.original in the form /path?query" + # + - grok: + field: url.original + ignore_failure: true + patterns: + - '^%{ABS_PATH:url.path}(?:\?%{DATA:url.query})?$' + pattern_definitions: + ABS_PATH: '/[^?]*' - append: if: "ctx?.aws?.s3access?.bucket_owner != null" field: related.user @@ -38,9 +81,9 @@ processors: - "dd/MMM/yyyy:H:m:s Z" - set: - if: "ctx?.aws?.s3access?.remote_ip != null" field: client.ip value: "{{aws.s3access.remote_ip}}" + ignore_empty_value: true - append: if: "ctx?.aws?.s3access?.remote_ip != null" @@ -48,9 +91,9 @@ processors: value: "{{aws.s3access.remote_ip}}" - set: - if: "ctx?.aws?.s3access?.remote_ip != null" field: client.address value: "{{aws.s3access.remote_ip}}" + ignore_empty_value: true - geoip: if: "ctx?.aws?.s3access?.remote_ip != null" @@ -58,24 +101,24 @@ processors: target_field: geo - set: - if: "ctx?.aws?.s3access?.requester != null" field: client.user.id value: "{{aws.s3access.requester}}" + ignore_empty_value: true - set: - if: "ctx?.aws?.s3access?.request_id != null" field: event.id value: "{{aws.s3access.request_id}}" + ignore_empty_value: true - set: - if: "ctx?.aws?.s3access?.operation != null" field: event.action value: "{{aws.s3access.operation}}" + ignore_empty_value: true - set: - if: "ctx?.aws?.s3access?.http_status != null" field: http.response.status_code value: "{{aws.s3access.http_status}}" + ignore_empty_value: true - convert: if: "ctx?.http?.response?.status_code != null" @@ -88,24 +131,39 @@ processors: value: failure - set: - if: "ctx?.aws?.s3access?.error_code != null" field: event.code value: "{{aws.s3access.error_code}}" + ignore_empty_value: true - set: if: "ctx?.aws?.s3access?.error_code == null" field: event.outcome value: success - - set: - if: "ctx?.aws?.s3access?.total_time != null" - field: event.duration - value: "{{aws.s3access.total_time}}" + - convert: + field: aws.s3access.bytes_sent + target_field: http.response.body.bytes + type: long + ignore_failure: true + + - convert: + field: aws.s3access.total_time + target_field: event.duration + type: long + ignore_failure: true + + - script: + lang: painless + if: ctx.event?.duration != null + params: + MS_TO_NS: 1000000 + source: >- + ctx.event.duration *= params.MS_TO_NS; - set: - if: "ctx?.aws?.s3access?.referrer != null" field: http.request.referrer value: "{{aws.s3access.referrer}}" + ignore_empty_value: true - user_agent: if: "ctx?.aws?.s3access?.user_agent != null" @@ -114,7 +172,7 @@ processors: - set: field: tls.cipher value: '{{aws.s3access.cipher_suite}}' - if: ctx.aws?.s3access?.cipher_suite != null + ignore_empty_value: true - script: lang: painless @@ -135,13 +193,18 @@ processors: field: event.kind value: event + # + # Save original message into event.original + # + - rename: + field: "message" + target_field: "event.original" + # # Remove temporary fields # - remove: - field: - - message - - _temp_ + field: _temp_ ignore_missing: true on_failure: diff --git a/test/packages/aws/data_stream/s3access/fields/ecs.yml b/test/packages/aws/data_stream/s3access/fields/ecs.yml new file mode 100644 index 0000000000..0d3655b8c1 --- /dev/null +++ b/test/packages/aws/data_stream/s3access/fields/ecs.yml @@ -0,0 +1,131 @@ +- name: related.user + type: keyword + description: All the user names seen on your event. +- name: related.ip + type: ip + description: All of the IPs seen on your event. +- name: client.ip + type: ip + description: IP address of the client. +- name: client.address + type: keyword + description: Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. +- name: client.user.id + type: keyword + description: Unique identifiers of the user. +- name: event.id + type: keyword + description: Unique ID to describe the event. +- name: event.action + type: keyword + description: The action captured by the event. +- name: event.outcome + type: keyword + description: This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. +- name: event.code + type: keyword + description: Identification code for this event, if one exists. +- name: event.duration + type: long + description: Duration of the event in nanoseconds. +- name: http + title: HTTP + type: group + fields: + - name: request.method + type: keyword + ignore_above: 1024 + description: 'HTTP request method.' + - name: request.referrer + type: keyword + ignore_above: 1024 + description: Referrer for this HTTP request. + - name: response.body.bytes + type: long + format: bytes + description: Size in bytes of the response body. + - name: response.status_code + type: long + description: HTTP response status code. + - name: version + type: keyword + ignore_above: 1024 + description: HTTP version. +- name: url + title: URL + type: group + fields: + - name: original + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: 'Unmodified original url as seen in the event source.' + - name: path + type: keyword + ignore_above: 1024 + description: Path of the request, such as "/search". + - name: query + type: keyword + ignore_above: 1024 + description: 'The query field describes the query string of the request, such as "q=elasticsearch".' +- name: tls.cipher + type: keyword + description: String indicating the cipher used during the current connection. +- name: tls.version + type: keyword + description: Numeric part of the version parsed from the original string. +- name: tls.version_protocol + type: keyword + description: Normalized lowercase protocol name parsed from original string. +- name: cloud.provider + type: keyword + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. +- name: event.kind + type: keyword + description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal) +- name: geo.city_name + type: keyword + description: City name. +- name: geo.country_name + type: keyword + description: Country name. +- name: geo.continent_name + type: keyword + description: Name of the continent. +- name: geo.country_iso_code + type: keyword + description: Country ISO code. +- name: geo.location + type: geo_point + description: Longitude and latitude. +- name: geo.region_iso_code + type: keyword + description: Region ISO code. +- name: geo.region_name + type: keyword + description: Region name. +- name: user_agent.device.name + type: keyword + description: Name of the device. +- name: user_agent.name + type: keyword + description: Name of the user agent. +- name: user_agent.original + type: keyword + description: Unparsed user_agent string. +- name: user_agent.os.full + type: keyword + description: Operating system name, including the version or code name. +- name: user_agent.os.name + type: keyword + description: Operating system name, without the version. +- name: user_agent.os.version + type: keyword + description: Operating system version as a raw string. +- name: user_agent.version + type: keyword + description: Version of the user agent. diff --git a/test/packages/aws/data_stream/s3access/fields/fields.yml b/test/packages/aws/data_stream/s3access/fields/fields.yml index 32a70dbbaa..e4b8c951d4 100644 --- a/test/packages/aws/data_stream/s3access/fields/fields.yml +++ b/test/packages/aws/data_stream/s3access/fields/fields.yml @@ -93,93 +93,3 @@ type: keyword description: | The Transport Layer Security (TLS) version negotiated by the client. -- name: related.user - type: keyword - description: All the user names seen on your event. -- name: related.ip - type: ip - description: All of the IPs seen on your event. -- name: client.ip - type: ip - description: IP address of the client. -- name: client.address - type: keyword - description: Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. -- name: client.user.id - type: keyword - description: Unique identifiers of the user. -- name: event.id - type: keyword - description: Unique ID to describe the event. -- name: event.action - type: keyword - description: The action captured by the event. -- name: http.response.status_code - type: long - description: HTTP response status code. -- name: event.outcome - type: keyword - description: This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. -- name: event.code - type: keyword - description: Identification code for this event, if one exists. -- name: event.duration - type: long - description: Duration of the event in nanoseconds. -- name: http.request.referrer - type: keyword - description: Referrer for this HTTP request. -- name: tls.cipher - type: keyword - description: String indicating the cipher used during the current connection. -- name: tls.version - type: keyword - description: Numeric part of the version parsed from the original string. -- name: tls.version_protocol - type: keyword - description: Normalized lowercase protocol name parsed from original string. -- name: cloud.provider - type: keyword - description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. -- name: event.kind - type: keyword - description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal) -- name: geo.city_name - type: keyword - description: City name. -- name: geo.continent_name - type: keyword - description: Name of the continent. -- name: geo.country_iso_code - type: keyword - description: Country ISO code. -- name: geo.location - type: geo_point - description: Longitude and latitude. -- name: geo.region_iso_code - type: keyword - description: Region ISO code. -- name: geo.region_name - type: keyword - description: Region name. -- name: user_agent.device.name - type: keyword - description: Name of the device. -- name: user_agent.name - type: keyword - description: Name of the user agent. -- name: user_agent.original - type: keyword - description: Unparsed user_agent string. -- name: user_agent.os.full - type: keyword - description: Operating system name, including the version or code name. -- name: user_agent.os.name - type: keyword - description: Operating system name, without the version. -- name: user_agent.os.version - type: keyword - description: Operating system version as a raw string. -- name: user_agent.version - type: keyword - description: Version of the user agent. diff --git a/test/packages/aws/data_stream/s3access/manifest.yml b/test/packages/aws/data_stream/s3access/manifest.yml index 6afb1de8e6..648a1d7f6f 100644 --- a/test/packages/aws/data_stream/s3access/manifest.yml +++ b/test/packages/aws/data_stream/s3access/manifest.yml @@ -1,5 +1,5 @@ title: AWS s3access logs -release: experimental +release: beta type: logs streams: - input: s3 diff --git a/test/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs index baec9d6904..6c56e029fb 100644 --- a/test/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/sns/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/sns/fields/ecs.yml b/test/packages/aws/data_stream/sns/fields/ecs.yml index e49975bc27..a02d7269c5 100644 --- a/test/packages/aws/data_stream/sns/fields/ecs.yml +++ b/test/packages/aws/data_stream/sns/fields/ecs.yml @@ -43,3 +43,11 @@ type: keyword description: Region in which this host is running. ignore_above: 1024 +- name: ecs.version + type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 +- name: service.type + type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/sns/fields/fields.yml b/test/packages/aws/data_stream/sns/fields/fields.yml index b3dd600053..c07522553d 100644 --- a/test/packages/aws/data_stream/sns/fields/fields.yml +++ b/test/packages/aws/data_stream/sns/fields/fields.yml @@ -61,3 +61,9 @@ - name: SMSMonthToDateSpentUSD.sum type: long description: The charges you have accrued since the start of the current calendar month for sending SMS messages. + - name: cloudwatch + type: group + fields: + - name: namespace + type: keyword + description: The namespace specified when query cloudwatch api. diff --git a/test/packages/aws/data_stream/sns/manifest.yml b/test/packages/aws/data_stream/sns/manifest.yml index dc81d8a2d2..806a5e416b 100644 --- a/test/packages/aws/data_stream/sns/manifest.yml +++ b/test/packages/aws/data_stream/sns/manifest.yml @@ -1,5 +1,5 @@ title: AWS SNS metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/sns/sample_event.json b/test/packages/aws/data_stream/sns/sample_event.json index 802010cd7b..af48ae9aa4 100644 --- a/test/packages/aws/data_stream/sns/sample_event.json +++ b/test/packages/aws/data_stream/sns/sample_event.json @@ -1,11 +1,15 @@ { "@timestamp": "2020-05-28T17:58:27.154Z", - "data_stream": { - "dataset": "aws.sns", - "namespace": "default", - "type": "metrics" + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" }, "aws": { + "cloudwatch": { + "namespace": "AWS/SNS" + }, "dimensions": { "TopicName": "test-sns-ks" }, diff --git a/test/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs index 7c77393d6f..2e9f1a2d15 100644 --- a/test/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/sqs/agent/stream/stream.yml.hbs @@ -19,8 +19,11 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} +{{#if latency}} +latency: {{latency}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/sqs/fields/ecs.yml b/test/packages/aws/data_stream/sqs/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/sqs/fields/ecs.yml +++ b/test/packages/aws/data_stream/sqs/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/sqs/manifest.yml b/test/packages/aws/data_stream/sqs/manifest.yml index 5059824125..b1a57a9faf 100644 --- a/test/packages/aws/data_stream/sqs/manifest.yml +++ b/test/packages/aws/data_stream/sqs/manifest.yml @@ -1,5 +1,5 @@ title: AWS SQS metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs index 98ad40a604..b5530f1b2c 100644 --- a/test/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/transitgateway/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/transitgateway/fields/ecs.yml b/test/packages/aws/data_stream/transitgateway/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/transitgateway/fields/ecs.yml +++ b/test/packages/aws/data_stream/transitgateway/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/transitgateway/manifest.yml b/test/packages/aws/data_stream/transitgateway/manifest.yml index 35a85bccaf..36ed6e401b 100644 --- a/test/packages/aws/data_stream/transitgateway/manifest.yml +++ b/test/packages/aws/data_stream/transitgateway/manifest.yml @@ -1,5 +1,5 @@ title: AWS Transit Gateway metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs index d2806791da..24c082cd4d 100644 --- a/test/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/usage/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/usage/fields/ecs.yml b/test/packages/aws/data_stream/usage/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/usage/fields/ecs.yml +++ b/test/packages/aws/data_stream/usage/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/usage/manifest.yml b/test/packages/aws/data_stream/usage/manifest.yml index 210d01543b..ca2c781a65 100644 --- a/test/packages/aws/data_stream/usage/manifest.yml +++ b/test/packages/aws/data_stream/usage/manifest.yml @@ -1,5 +1,5 @@ title: AWS usage metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/data_stream/vpcflow/agent/stream/log.yml.hbs b/test/packages/aws/data_stream/vpcflow/agent/stream/log.yml.hbs deleted file mode 100644 index de98b5c1ed..0000000000 --- a/test/packages/aws/data_stream/vpcflow/agent/stream/log.yml.hbs +++ /dev/null @@ -1,113 +0,0 @@ -paths: - {{#each paths as |path i|}} - - {{path}} - {{/each}} -exclude_files: [".gz$"] -processors: - - drop_event: - when.regexp.message: "^version" - - drop_event: - when.regexp.message: "^instance-id" - - script: - lang: javascript - source: > - function process(event) { - var message = event.Get("message"); - var tokens = message.split(" ").length; - event.Put("@metadata.message_token_count", tokens); - } - # Default vpc flow log format - - dissect: - when: - equals: - '@metadata.message_token_count': 14 - field: message - target_prefix: aws.vpcflow - tokenizer: '%{version} %{account_id} %{interface_id} %{srcaddr} %{dstaddr} %{srcport} %{dstport} %{protocol} %{packets} %{bytes} %{start} %{end} %{action} %{log_status}' - # Custom flow log for traffic through a NAT gateway - - dissect: - when: - equals: - '@metadata.message_token_count': 6 - field: message - target_prefix: aws.vpcflow - tokenizer: '%{instance_id} %{interface_id} %{srcaddr} %{dstaddr} %{pkt_srcaddr} %{pkt_dstaddr}' - # Custom flow log for traffic through a transit gateway - - dissect: - when: - equals: - '@metadata.message_token_count': 17 - field: message - target_prefix: aws.vpcflow - tokenizer: '%{version} %{interface_id} %{account_id} %{vpc_id} %{subnet_id} %{instance_id} %{srcaddr} %{dstaddr} %{srcport} %{dstport} %{protocol} %{tcp_flags} %{type} %{pkt_srcaddr} %{pkt_dstaddr} %{action} %{log_status}' - # TCP Flag Sequence - - dissect: - when: - equals: - '@metadata.message_token_count': 21 - field: message - target_prefix: aws.vpcflow - tokenizer: '%{version} %{vpc_id} %{subnet_id} %{instance_id} %{interface_id} %{account_id} %{type} %{srcaddr} %{dstaddr} %{srcport} %{dstport} %{pkt_srcaddr} %{pkt_dstaddr} %{protocol} %{bytes} %{packets} %{start} %{end} %{action} %{tcp_flags} %{log_status}' - - convert: - ignore_missing: true - fields: - - {from: aws.vpcflow.srcaddr, to: source.address} - - {from: aws.vpcflow.srcaddr, to: source.ip, type: ip} - - {from: aws.vpcflow.srcport, to: source.port, type: long} - - {from: aws.vpcflow.dstaddr, to: destination.address} - - {from: aws.vpcflow.dstaddr, to: destination.ip, type: ip} - - {from: aws.vpcflow.dstport, to: destination.port, type: long} - - {from: aws.vpcflow.protocol, to: network.iana_number, type: string} - - {from: aws.vpcflow.packets, to: source.packets, type: long} - - {from: aws.vpcflow.bytes, to: source.bytes, type: long} - - {from: aws.vpcflow.packets, to: network.packets, type: long} - - {from: aws.vpcflow.bytes, to: network.bytes, type: long} - - drop_fields: - fields: ["aws.vpcflow.srcaddr", "aws.vpcflow.srcport", "aws.vpcflow.dstaddr", "aws.vpcflow.dstport", "aws.vpcflow.bytes", "aws.vpcflow.packets", "aws.vpcflow.protocol"] - - community_id: ~ - # Use the aws.vpcflow.action value to set the event.outcome value to either "allow" or "deny". - - add_fields: - when.equals.aws.vpcflow.action: ACCEPT - target: event - fields: {outcome: allow} - - add_fields: - when.equals.aws.vpcflow.action: REJECT - target: event - fields: {outcome: deny} - - add_fields: - target: event - fields: {type: flow} - - add_fields: - target: event - fields: {category: network_traffic} - # Add network.type: ipv4 or ipv6 - - if: - contains.source.ip: "." - then: - - add_fields: - target: network - fields: {type: ipv4} - - if: - contains.source.ip: ":" - then: - - add_fields: - target: network - fields: {type: ipv6} - # Add network.transport: based on IANA protocol number of the traffic - # http://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml - - if: - equals.network.iana_number: "6" - then: - - add_fields: - target: network - fields: {transport: tcp} - - if: - equals.network.iana_number: "17" - then: - - add_fields: - target: network - fields: {transport: udp} - - add_fields: - target: '' - fields: - ecs.version: 1.5.0 \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpcflow/agent/stream/s3.yml.hbs b/test/packages/aws/data_stream/vpcflow/agent/stream/s3.yml.hbs index b80f186906..8241ac6fcd 100644 --- a/test/packages/aws/data_stream/vpcflow/agent/stream/s3.yml.hbs +++ b/test/packages/aws/data_stream/vpcflow/agent/stream/s3.yml.hbs @@ -26,9 +26,6 @@ session_token: {{session_token}} {{#if role_arn}} role_arn: {{role_arn}} {{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} -{{/if}} {{#if fips_enabled}} fips_enabled: {{fips_enabled}} {{/if}} diff --git a/test/packages/aws/data_stream/vpcflow/manifest.yml b/test/packages/aws/data_stream/vpcflow/manifest.yml index 919f8c553b..a27c351a81 100644 --- a/test/packages/aws/data_stream/vpcflow/manifest.yml +++ b/test/packages/aws/data_stream/vpcflow/manifest.yml @@ -1,5 +1,5 @@ title: AWS vpcflow logs -release: experimental +release: beta type: logs streams: - input: s3 diff --git a/test/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs b/test/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs index 071a862e25..a22a1d98e0 100644 --- a/test/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs +++ b/test/packages/aws/data_stream/vpn/agent/stream/stream.yml.hbs @@ -19,11 +19,14 @@ shared_credential_file: {{shared_credential_file}} role_arn: {{role_arn}} {{/if}} {{#if regions}} -regions: {{regions}} +regions: +{{#each regions as |region i|}} +- {{region}} +{{/each}} +{{/if}} +{{#if latency}} +latency: {{latency}} {{/if}} {{#if tags_filter}} tags_filter: {{tags_filter}} -{{/if}} -{{#if aws_partition}} -aws_partition: {{aws_partition}} {{/if}} \ No newline at end of file diff --git a/test/packages/aws/data_stream/vpn/fields/ecs.yml b/test/packages/aws/data_stream/vpn/fields/ecs.yml index 432ee5f4d8..a02d7269c5 100644 --- a/test/packages/aws/data_stream/vpn/fields/ecs.yml +++ b/test/packages/aws/data_stream/vpn/fields/ecs.yml @@ -45,5 +45,9 @@ ignore_above: 1024 - name: ecs.version type: keyword + description: ECS version this event conforms to. + example: 1.0.0 + ignore_above: 1024 - name: service.type type: keyword + description: Service type diff --git a/test/packages/aws/data_stream/vpn/manifest.yml b/test/packages/aws/data_stream/vpn/manifest.yml index 54c8f55ceb..7daa957da1 100644 --- a/test/packages/aws/data_stream/vpn/manifest.yml +++ b/test/packages/aws/data_stream/vpn/manifest.yml @@ -1,5 +1,5 @@ title: AWS VPN metrics -release: experimental +release: beta type: metrics streams: - input: aws/metrics diff --git a/test/packages/aws/docs/README.md b/test/packages/aws/docs/README.md index cbb773ac1e..e36b499695 100644 --- a/test/packages/aws/docs/README.md +++ b/test/packages/aws/docs/README.md @@ -14,7 +14,6 @@ AWS credentials are required for running AWS integration. * *shared_credential_file*: directory of the shared credentials file. * *endpoint*: URL of the entry point for an AWS web service. * *role_arn*: AWS IAM Role to assume. -* *aws_partition*: AWS region partition name, value is one of `aws, aws-cn, aws-us-gov`, default is `aws`. ### Credential Types There are three types of AWS credentials can be used: access keys, temporary @@ -92,2705 +91,3 @@ In order to enable AWS integration, please make sure these permissions are given * sqs:ListQueues * sts:GetCallerIdentity * iam:ListAccountAliases - -## Logs - -### cloudtrail - -The `cloudtrail` dataset collects the AWS CloudTrail logs. CloudTrail monitors -events for the account. If user creates a trail, it delivers those events as log - files to a specific Amazon S3 bucket. The `cloudtrail` dataset does not read - the CloudTrail Digest files that are delivered to the S3 bucket when Log File - Integrity is turned on, it only reads the CloudTrail logs. - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.cloudtrail.additional_eventdata | Additional data about the event that was not part of the request or response. | keyword | -| aws.cloudtrail.api_version | Identifies the API version associated with the AwsApiCall eventType value. | keyword | -| aws.cloudtrail.console_login.additional_eventdata.login_to | URL for ConsoleLogin | keyword | -| aws.cloudtrail.console_login.additional_eventdata.mfa_used | Identifies whether multi factor authentication was used during ConsoleLogin | boolean | -| aws.cloudtrail.console_login.additional_eventdata.mobile_version | Identifies whether ConsoleLogin was from mobile version | boolean | -| aws.cloudtrail.error_code | The AWS service error if the request returns an error. | keyword | -| aws.cloudtrail.error_message | If the request returns an error, the description of the error. | keyword | -| aws.cloudtrail.event_type | Identifies the type of event that generated the event record. | keyword | -| aws.cloudtrail.event_version | The CloudTrail version of the log event format. | keyword | -| aws.cloudtrail.flattened.additional_eventdata | Additional data about the event that was not part of the request or response. | flattened | -| aws.cloudtrail.flattened.request_parameters | The parameters, if any, that were sent with the request. | flattened | -| aws.cloudtrail.flattened.response_elements | The response element for actions that make changes (create, update, or delete actions). | flattened | -| aws.cloudtrail.flattened.service_event_details | Identifies the service event, including what triggered the event and the result. | flattened | -| aws.cloudtrail.management_event | A Boolean value that identifies whether the event is a management event. | keyword | -| aws.cloudtrail.read_only | Identifies whether this operation is a read-only operation. | boolean | -| aws.cloudtrail.recipient_account_id | Represents the account ID that received this event. | keyword | -| aws.cloudtrail.request_id | The value that identifies the request. The service being called generates this value. | keyword | -| aws.cloudtrail.request_parameters | The parameters, if any, that were sent with the request. | keyword | -| aws.cloudtrail.resources.account_id | Account ID of the resource owner | keyword | -| aws.cloudtrail.resources.arn | Resource ARNs | keyword | -| aws.cloudtrail.resources.type | Resource type identifier in the format: AWS::aws-service-name::data-type-name | keyword | -| aws.cloudtrail.response_elements | The response element for actions that make changes (create, update, or delete actions). | keyword | -| aws.cloudtrail.service_event_details | Identifies the service event, including what triggered the event and the result. | keyword | -| aws.cloudtrail.shared_event_id | GUID generated by CloudTrail to uniquely identify CloudTrail events from the same AWS action that is sent to different AWS accounts. | keyword | -| aws.cloudtrail.user_identity.access_key_id | The access key ID that was used to sign the request. | keyword | -| aws.cloudtrail.user_identity.arn | The Amazon Resource Name (ARN) of the principal that made the call. | keyword | -| aws.cloudtrail.user_identity.invoked_by | The name of the AWS service that made the request, such as Amazon EC2 Auto Scaling or AWS Elastic Beanstalk. | keyword | -| aws.cloudtrail.user_identity.session_context.creation_date | The date and time when the temporary security credentials were issued. | date | -| aws.cloudtrail.user_identity.session_context.mfa_authenticated | The value is true if the root user or IAM user whose credentials were used for the request also was authenticated with an MFA device; otherwise, false. | keyword | -| aws.cloudtrail.user_identity.session_context.session_issuer.account_id | The account that owns the entity that was used to get credentials. | keyword | -| aws.cloudtrail.user_identity.session_context.session_issuer.arn | The ARN of the source (account, IAM user, or role) that was used to get temporary security credentials. | keyword | -| aws.cloudtrail.user_identity.session_context.session_issuer.principal_id | The internal ID of the entity that was used to get credentials. | keyword | -| aws.cloudtrail.user_identity.session_context.session_issuer.type | The source of the temporary security credentials, such as Root, IAMUser, or Role. | keyword | -| aws.cloudtrail.user_identity.type | The type of the identity | keyword | -| aws.cloudtrail.vpc_endpoint_id | Identifies the VPC endpoint in which requests were made from a VPC to another AWS service, such as Amazon S3. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| error.message | Error message. | text | -| event.action | The action captured by the event. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. | date | -| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity. | keyword | -| event.provider | Source of the event. | keyword | -| event.type | Event severity (e.g. info, error) | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| related.user | All the user names seen on your event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | - - -### cloudwatch - -The `cloudwatch` dataset collects CloudWatch logs. Users can use Amazon -CloudWatch logs to monitor, store, and access log files from different sources. -Export logs from log groups to an Amazon S3 bucket which has SQS notification -setup already. - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.cloudwatch.message | CloudWatch log message. | text | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | - - -### ec2 - -The `ec2` dataset is specifically for EC2 logs stored in AWS CloudWatch. Export logs -from log groups to Amazon S3 bucket which has SQS notification setup already. -With this dataset, EC2 logs will be parsed into fields like `ip_address` -and `process.name`. For logs from other services, please use `cloudwatch` dataset. - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.ec2.ip_address | The internet address of the requester. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| process.name | Process name. | keyword | - - -### elb - -The `elb` dataset collects logs from AWS ELBs. Elastic Load Balancing provides -access logs that capture detailed information about requests sent to the load -balancer. Each log contains information such as the time the request was -received, the client's IP address, latencies, request paths, and server -responses. Users can use these access logs to analyze traffic patterns and to -troubleshoot issues. - -Please follow [enable access logs for classic load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html) -for sending Classic ELB access logs to S3 bucket. -For application load balancer, please follow [enable access log for application load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html#enable-access-logging). -For network load balancer, please follow [enable access log for network load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest//network/load-balancer-access-logs.html). - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.elb.action_executed | The action executed when processing the request (forward, fixed-response, authenticate...). It can contain several values. | keyword | -| aws.elb.backend.http.response.status_code | The status code from the backend (status code sent to the client from ELB is stored in `http.response.status_code` | long | -| aws.elb.backend.ip | The IP address of the backend processing this connection. | keyword | -| aws.elb.backend.port | The port in the backend processing this connection. | keyword | -| aws.elb.backend_processing_time.sec | The total time in seconds since the connection is sent to the backend till the backend starts responding. | float | -| aws.elb.chosen_cert.arn | The ARN of the chosen certificate presented to the client in TLS/SSL connections. | keyword | -| aws.elb.chosen_cert.serial | The serial number of the chosen certificate presented to the client in TLS/SSL connections. | keyword | -| aws.elb.classification | The classification for desync mitigation. | keyword | -| aws.elb.classification_reason | The classification reason code. | keyword | -| aws.elb.connection_time.ms | The total time of the connection in milliseconds, since it is opened till it is closed. | long | -| aws.elb.error.reason | The error reason if the executed action failed. | keyword | -| aws.elb.incoming_tls_alert | The integer value of TLS alerts received by the load balancer from the client, if present. | keyword | -| aws.elb.listener | The ELB listener that received the connection. | keyword | -| aws.elb.matched_rule_priority | The priority value of the rule that matched the request, if a rule matched. | keyword | -| aws.elb.name | The name of the load balancer. | keyword | -| aws.elb.protocol | The protocol of the load balancer (http or tcp). | keyword | -| aws.elb.redirect_url | The URL used if a redirection action was executed. | keyword | -| aws.elb.request_processing_time.sec | The total time in seconds since the connection or request is received until it is sent to a registered backend. | float | -| aws.elb.response_processing_time.sec | The total time in seconds since the response is received from the backend till it is sent to the client. | float | -| aws.elb.ssl_cipher | The SSL cipher used in TLS/SSL connections. | keyword | -| aws.elb.ssl_protocol | The SSL protocol used in TLS/SSL connections. | keyword | -| aws.elb.target_group.arn | The ARN of the target group handling the request. | keyword | -| aws.elb.target_port | List of IP addresses and ports for the targets that processed this request. | keyword | -| aws.elb.target_status_code | List of status codes from the responses of the targets. | keyword | -| aws.elb.tls_handshake_time.ms | The total time for the TLS handshake to complete in milliseconds once the connection has been established. | long | -| aws.elb.tls_named_group | The TLS named group. | keyword | -| aws.elb.trace_id | The contents of the `X-Amzn-Trace-Id` header. | keyword | -| aws.elb.type | The type of the load balancer for v2 Load Balancers. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | Destination domain. | keyword | -| event.category | Event category (e.g. database) | keyword | -| event.end | event.end contains the date when the event ended or when the activity was last observed. | date | -| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, sig | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | -| event.start | event.start contains the date when the event started or when the activity was first observed. | date | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.body.bytes | Size in bytes of the request body. | long | -| http.request.method | HTTP request method. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source. | ip | -| source.port | Port of the source. | keyword | -| tracing.trace.id | Unique identifier of the trace. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | - - -### s3access - -The `s3access` dataset collects server access logs from AWS S3. Server access -logging provides detailed records for the requests that are made to a bucket. -Server access logs are useful for many applications. For example, access log -information can be useful in security and access audits. It can also help users -to learn about customer base and understand Amazon S3 bill. - -Please follow [how to enable server access logging](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html#server-access-logging-overview) -for sending server access logs to S3 bucket. - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.s3access.authentication_type | The type of request authentication used, AuthHeader for authentication headers, QueryString for query string (pre-signed URL) or a - for unauthenticated requests. | keyword | -| aws.s3access.bucket | The name of the bucket that the request was processed against. | keyword | -| aws.s3access.bucket_owner | The canonical user ID of the owner of the source bucket. | keyword | -| aws.s3access.bytes_sent | The number of response bytes sent, excluding HTTP protocol overhead, or "-" if zero. | long | -| aws.s3access.cipher_suite | The Secure Sockets Layer (SSL) cipher that was negotiated for HTTPS request or a - for HTTP. | keyword | -| aws.s3access.error_code | The Amazon S3 Error Code, or "-" if no error occurred. | keyword | -| aws.s3access.host_header | The endpoint used to connect to Amazon S3. | keyword | -| aws.s3access.host_id | The x-amz-id-2 or Amazon S3 extended request ID. | keyword | -| aws.s3access.http_status | The numeric HTTP status code of the response. | long | -| aws.s3access.key | The "key" part of the request, URL encoded, or "-" if the operation does not take a key parameter. | keyword | -| aws.s3access.object_size | The total size of the object in question. | long | -| aws.s3access.operation | The operation listed here is declared as SOAP.operation, REST.HTTP_method.resource_type, WEBSITE.HTTP_method.resource_type, or BATCH.DELETE.OBJECT. | keyword | -| aws.s3access.referrer | The value of the HTTP Referrer header, if present. | keyword | -| aws.s3access.remote_ip | The apparent internet address of the requester. | ip | -| aws.s3access.request_id | A string generated by Amazon S3 to uniquely identify each request. | keyword | -| aws.s3access.request_uri | The Request-URI part of the HTTP request message. | keyword | -| aws.s3access.requester | The canonical user ID of the requester, or a - for unauthenticated requests. | keyword | -| aws.s3access.signature_version | The signature version, SigV2 or SigV4, that was used to authenticate the request or a - for unauthenticated requests. | keyword | -| aws.s3access.tls_version | The Transport Layer Security (TLS) version negotiated by the client. | keyword | -| aws.s3access.total_time | The number of milliseconds the request was in flight from the server's perspective. | long | -| aws.s3access.turn_around_time | The number of milliseconds that Amazon S3 spent processing your request. | long | -| aws.s3access.user_agent | The value of the HTTP User-Agent header. | keyword | -| aws.s3access.version_id | The version ID in the request, or "-" if the operation does not take a versionId parameter. | keyword | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | -| client.ip | IP address of the client. | ip | -| client.user.id | Unique identifiers of the user. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| event.action | The action captured by the event. | keyword | -| event.code | Identification code for this event, if one exists. | keyword | -| event.duration | Duration of the event in nanoseconds. | long | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | -| geo.city_name | City name. | keyword | -| geo.continent_name | Name of the continent. | keyword | -| geo.country_iso_code | Country ISO code. | keyword | -| geo.location | Longitude and latitude. | geo_point | -| geo.region_iso_code | Region ISO code. | keyword | -| geo.region_name | Region name. | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.status_code | HTTP response status code. | long | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names seen on your event. | keyword | -| tls.cipher | String indicating the cipher used during the current connection. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | - - -### vpcflow - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.vpcflow.account_id | The AWS account ID for the flow log. | keyword | -| aws.vpcflow.action | The action that is associated with the traffic, ACCEPT or REJECT. | keyword | -| aws.vpcflow.instance_id | The ID of the instance that's associated with network interface for which the traffic is recorded, if the instance is owned by you. | keyword | -| aws.vpcflow.interface_id | The ID of the network interface for which the traffic is recorded. | keyword | -| aws.vpcflow.log_status | The logging status of the flow log, OK, NODATA or SKIPDATA. | keyword | -| aws.vpcflow.pkt_dstaddr | The packet-level (original) destination IP address for the traffic. | ip | -| aws.vpcflow.pkt_srcaddr | The packet-level (original) source IP address of the traffic. | ip | -| aws.vpcflow.subnet_id | The ID of the subnet that contains the network interface for which the traffic is recorded. | keyword | -| aws.vpcflow.tcp_flags | The bitmask value for the following TCP flags: 2=SYN,18=SYN-ACK,1=FIN,4=RST | keyword | -| aws.vpcflow.type | The type of traffic: IPv4, IPv6, or EFA. | keyword | -| aws.vpcflow.version | The VPC Flow Logs version. If you use the default format, the version is 2. If you specify a custom format, the version is 3. | keyword | -| aws.vpcflow.vpc_id | The ID of the VPC that contains the network interface for which the traffic is recorded. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.ip | IP address of the destination. | ip | -| destination.port | Port of the destination. | long | -| event.category | Event category (e.g. database) | keyword | -| event.end | event.end contains the date when the event ended or when the activity was last observed. | date | -| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | -| event.start | event.start contains the date when the event started or when the activity was first observed. | date | -| event.type | Event severity (e.g. info, error) | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| network.bytes | Total bytes transferred in both directions. | long | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. | keyword | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.packets | Total packets transferred in both directions. | long | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.packets | Packets sent from the source to the destination. | long | -| source.port | Port of the source. | long | - - -## Metrics - -### billing - -An example event for `billing` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:17:06.212Z", - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.billing", - "module": "aws", - "duration": 1938760247 - }, - "metricset": { - "name": "billing", - "period": 43200000 - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "billing": { - "metrics": { - "EstimatedCharges": { - "max": 1625.41 - } - } - }, - "cloudwatch": { - "namespace": "AWS/Billing" - }, - "dimensions": { - "Currency": "USD" - } - }, - "service": { - "type": "aws" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.billing.metrics.EstimatedCharges.max | Maximum estimated charges for AWS acccount. | long | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.Currency | Currency name. | keyword | -| aws.dimensions.ServiceName | AWS service name. | keyword | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### cloudwatch - -An example event for `cloudwatch` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:17:02.812Z", - "event": { - "duration": 14119105951, - "dataset": "aws.cloudwatch", - "module": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "aws": { - "dimensions": { - "InstanceId": "i-0830bfecfa7173cbe" - }, - "ec2": { - "metrics": { - "DiskWriteOps": { - "avg": 0, - "max": 0 - }, - "CPUUtilization": { - "avg": 0.7661943132361363, - "max": 0.833333333333333 - } - } - }, - "cloudwatch": { - "namespace": "AWS/EC2" - } - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### dynamodb - -An example event for `dynamodb` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" - }, - "aws": { - "dimensions": { - "TableName": "TryDaxTable3" - }, - "dynamodb": { - "metrics": { - "ProvisionedWriteCapacityUnits": { - "avg": 1 - }, - "ProvisionedReadCapacityUnits": { - "avg": 1 - }, - "ConsumedWriteCapacityUnits": { - "avg": 0, - "sum": 0 - }, - "ConsumedReadCapacityUnits": { - "avg": 0, - "sum": 0 - } - } - }, - "cloudwatch": { - "namespace": "AWS/DynamoDB" - } - }, - "metricset": { - "name": "dynamodb", - "period": 300000 - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dynamodb.metrics.AccountMaxReads.max | The maximum number of read capacity units that can be used by an account. This limit does not apply to on-demand tables or global secondary indexes. | long | -| aws.dynamodb.metrics.AccountMaxTableLevelReads.max | The maximum number of read capacity units that can be used by a table or global secondary index of an account. For on-demand tables this limit caps the maximum read request units a table or a global secondary index can use. | long | -| aws.dynamodb.metrics.AccountMaxTableLevelWrites.max | The maximum number of write capacity units that can be used by a table or global secondary index of an account. For on-demand tables this limit caps the maximum write request units a table or a global secondary index can use. | long | -| aws.dynamodb.metrics.AccountMaxWrites.max | The maximum number of write capacity units that can be used by an account. This limit does not apply to on-demand tables or global secondary indexes. | long | -| aws.dynamodb.metrics.AccountProvisionedReadCapacityUtilization.avg | The average percentage of provisioned read capacity units utilized by the account. | double | -| aws.dynamodb.metrics.AccountProvisionedWriteCapacityUtilization.avg | The average percentage of provisioned write capacity units utilized by the account. | double | -| aws.dynamodb.metrics.ConditionalCheckFailedRequests.sum | The number of failed attempts to perform conditional writes. | long | -| aws.dynamodb.metrics.ConsumedReadCapacityUnits.avg | | double | -| aws.dynamodb.metrics.ConsumedReadCapacityUnits.sum | | long | -| aws.dynamodb.metrics.ConsumedWriteCapacityUnits.avg | | double | -| aws.dynamodb.metrics.ConsumedWriteCapacityUnits.sum | | long | -| aws.dynamodb.metrics.MaxProvisionedTableReadCapacityUtilization.max | The percentage of provisioned read capacity units utilized by the highest provisioned read table or global secondary index of an account. | double | -| aws.dynamodb.metrics.MaxProvisionedTableWriteCapacityUtilization.max | The percentage of provisioned write capacity utilized by the highest provisioned write table or global secondary index of an account. | double | -| aws.dynamodb.metrics.OnlineIndexPercentageProgress.avg | The percentage of completion when a new global secondary index is being added to a table. | double | -| aws.dynamodb.metrics.PendingReplicationCount.sum | The number of item updates that are written to one replica table, but that have not yet been written to another replica in the global table. | long | -| aws.dynamodb.metrics.ProvisionedReadCapacityUnits.avg | The number of provisioned read capacity units for a table or a global secondary index. | double | -| aws.dynamodb.metrics.ProvisionedWriteCapacityUnits.avg | The number of provisioned write capacity units for a table or a global secondary index. | double | -| aws.dynamodb.metrics.ReadThrottleEvents.sum | Requests to DynamoDB that exceed the provisioned read capacity units for a table or a global secondary index. | long | -| aws.dynamodb.metrics.ReplicationLatency.avg | | double | -| aws.dynamodb.metrics.ReplicationLatency.max | | double | -| aws.dynamodb.metrics.SuccessfulRequestLatency.avg | | double | -| aws.dynamodb.metrics.SuccessfulRequestLatency.max | | double | -| aws.dynamodb.metrics.SystemErrors.sum | The requests to DynamoDB or Amazon DynamoDB Streams that generate an HTTP 500 status code during the specified time period. | long | -| aws.dynamodb.metrics.ThrottledRequests.sum | Requests to DynamoDB that exceed the provisioned throughput limits on a resource (such as a table or an index). | long | -| aws.dynamodb.metrics.TransactionConflict.avg | | double | -| aws.dynamodb.metrics.TransactionConflict.sum | | long | -| aws.dynamodb.metrics.WriteThrottleEvents.sum | Requests to DynamoDB that exceed the provisioned write capacity units for a table or a global secondary index. | long | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### ebs - -An example event for `ebs` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:57:22.450Z", - "service": { - "type": "aws" - }, - "aws": { - "ebs": { - "metrics": { - "VolumeReadOps": { - "avg": 0 - }, - "VolumeQueueLength": { - "avg": 0.0000666666666666667 - }, - "VolumeWriteOps": { - "avg": 29 - }, - "VolumeTotalWriteTime": { - "sum": 0.02 - }, - "BurstBalance": { - "avg": 100 - }, - "VolumeWriteBytes": { - "avg": 14406.620689655172 - }, - "VolumeIdleTime": { - "sum": 299.98 - } - } - }, - "cloudwatch": { - "namespace": "AWS/EBS" - }, - "dimensions": { - "VolumeId": "vol-03370a204cc8b0a2f" - } - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.ebs", - "module": "aws", - "duration": 10488314037 - }, - "metricset": { - "period": 300000, - "name": "ebs" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.VolumeId | Amazon EBS volume ID | keyword | -| aws.ebs.metrics.BurstBalance.avg | Used with General Purpose SSD (gp2), Throughput Optimized HDD (st1), and Cold HDD (sc1) volumes only. Provides information about the percentage of I/O credits (for gp2) or throughput credits (for st1 and sc1) remaining in the burst bucket. | double | -| aws.ebs.metrics.VolumeConsumedReadWriteOps.avg | The total amount of read and write operations (normalized to 256K capacity units) consumed in a specified period of time. Used with Provisioned IOPS SSD volumes only. | double | -| aws.ebs.metrics.VolumeIdleTime.sum | The total number of seconds in a specified period of time when no read or write operations were submitted. | double | -| aws.ebs.metrics.VolumeQueueLength.avg | The number of read and write operation requests waiting to be completed in a specified period of time. | double | -| aws.ebs.metrics.VolumeReadBytes.avg | Average size of each read operation during the period, except on volumes attached to a Nitro-based instance, where the average represents the average over the specified period. | double | -| aws.ebs.metrics.VolumeReadOps.avg | The total number of read operations in a specified period of time. | double | -| aws.ebs.metrics.VolumeThroughputPercentage.avg | The percentage of I/O operations per second (IOPS) delivered of the total IOPS provisioned for an Amazon EBS volume. Used with Provisioned IOPS SSD volumes only. | double | -| aws.ebs.metrics.VolumeTotalReadTime.sum | The total number of seconds spent by all read operations that completed in a specified period of time. | double | -| aws.ebs.metrics.VolumeTotalWriteTime.sum | The total number of seconds spent by all write operations that completed in a specified period of time. | double | -| aws.ebs.metrics.VolumeWriteBytes.avg | Average size of each write operation during the period, except on volumes attached to a Nitro-based instance, where the average represents the average over the specified period. | double | -| aws.ebs.metrics.VolumeWriteOps.avg | The total number of write operations in a specified period of time. | double | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### ec2 - -An example event for `ec2` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:56:37.255Z", - "aws": { - "ec2": { - "network": { - "in": { - "packets": 448.4, - "bytes_per_sec": 103.10266666666666, - "packets_per_sec": 1.4946666666666666, - "bytes": 30930.8 - }, - "out": { - "packets": 233.6, - "bytes_per_sec": 51.754666666666665, - "packets_per_sec": 0.7786666666666666, - "bytes": 15526.4 - } - }, - "status": { - "check_failed": 0, - "check_failed_instance": 0, - "check_failed_system": 0 - }, - "cpu": { - "credit_usage": 0.004566, - "credit_balance": 144, - "surplus_credit_balance": 0, - "surplus_credits_charged": 0, - "total": { - "pct": 0.0999999999997574 - } - }, - "diskio": { - "read": { - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0, - "count": 0 - }, - "write": { - "count": 0, - "bytes_per_sec": 0, - "count_per_sec": 0, - "bytes": 0 - } - }, - "instance": { - "core": { - "count": 1 - }, - "threads_per_core": 1, - "public": { - "ip": "3.122.204.80", - "dns_name": "" - }, - "private": { - "ip": "10.0.0.122", - "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" - }, - "image": { - "id": "ami-0b418580298265d5c" - }, - "state": { - "name": "running", - "code": 16 - }, - "monitoring": { - "state": "disabled" - } - } - } - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - }, - "ecs": { - "version": "1.5.0" - }, - "event": { - "module": "aws", - "duration": 23217499283, - "dataset": "aws.ec2" - }, - "metricset": { - "period": 300000, - "name": "ec2" - }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "instance": { - "id": "i-04c1a32c2aace6b40" - }, - "machine": { - "type": "t2.micro" - }, - "availability_zone": "eu-central-1a" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.AutoScalingGroupName | An Auto Scaling group is a collection of instances you define if you're using Auto Scaling. | keyword | -| aws.dimensions.ImageId | This dimension filters the data you request for all instances running this Amazon EC2 Amazon Machine Image (AMI) | keyword | -| aws.dimensions.InstanceId | Amazon EC2 instance ID | keyword | -| aws.dimensions.InstanceType | This dimension filters the data you request for all instances running with this specified instance type. | keyword | -| aws.ec2.cpu.credit_balance | The number of earned CPU credits that an instance has accrued since it was launched or started. | long | -| aws.ec2.cpu.credit_usage | The number of CPU credits spent by the instance for CPU utilization. | long | -| aws.ec2.cpu.surplus_credit_balance | The number of surplus credits that have been spent by an unlimited instance when its CPUCreditBalance value is zero. | long | -| aws.ec2.cpu.surplus_credits_charged | The number of spent surplus credits that are not paid down by earned CPU credits, and which thus incur an additional charge. | long | -| aws.ec2.cpu.total.pct | The percentage of allocated EC2 compute units that are currently in use on the instance. | scaled_float | -| aws.ec2.diskio.read.bytes | Bytes read from all instance store volumes available to the instance. | long | -| aws.ec2.diskio.read.bytes_per_sec | Bytes read per second from all instance store volumes available to the instance. | long | -| aws.ec2.diskio.read.count | The number of disk IO reads | long | -| aws.ec2.diskio.read.count_per_sec | The number of disk IO reads per second | long | -| aws.ec2.diskio.read.ops | Completed read operations from all instance store volumes available to the instance in a specified period of time. | long | -| aws.ec2.diskio.read.ops_per_sec | Completed read operations per second from all instance store volumes available to the instance in a specified period of time. | long | -| aws.ec2.diskio.write.bytes | Bytes written to all instance store volumes available to the instance. | long | -| aws.ec2.diskio.write.bytes_per_sec | Bytes written per second to all instance store volumes available to the instance. | long | -| aws.ec2.diskio.write.count | The number of disk IO writes | long | -| aws.ec2.diskio.write.count_per_sec | The number of disk IO writes per second | long | -| aws.ec2.diskio.write.ops | Completed write operations to all instance store volumes available to the instance in a specified period of time. | long | -| aws.ec2.diskio.write.ops_per_sec | Completed write operations per second to all instance store volumes available to the instance in a specified period of time. | long | -| aws.ec2.instance.core.count | The number of CPU cores for the instance. | integer | -| aws.ec2.instance.image.id | The ID of the image used to launch the instance. | keyword | -| aws.ec2.instance.monitoring.state | Indicates whether detailed monitoring is enabled. | keyword | -| aws.ec2.instance.private.dns_name | The private DNS name of the network interface. | keyword | -| aws.ec2.instance.private.ip | The private IPv4 address associated with the network interface. | ip | -| aws.ec2.instance.public.dns_name | The public DNS name of the instance. | keyword | -| aws.ec2.instance.public.ip | The address of the Elastic IP address (IPv4) bound to the network interface. | ip | -| aws.ec2.instance.state.code | The state of the instance, as a 16-bit unsigned integer. | integer | -| aws.ec2.instance.state.name | The state of the instance (pending | running | shutting-down | terminated | stopping | stopped). | keyword | -| aws.ec2.instance.threads_per_core | The number of threads per CPU core. | integer | -| aws.ec2.network.in.bytes | The number of bytes received on all network interfaces by the instance. | long | -| aws.ec2.network.in.bytes_per_sec | The number of bytes per second received on all network interfaces by the instance. | long | -| aws.ec2.network.in.packets | The number of packets received on all network interfaces by the instance. | long | -| aws.ec2.network.in.packets_per_sec | The number of packets per second sent out on all network interfaces by the instance. | long | -| aws.ec2.network.out.bytes | The number of bytes sent out on all network interfaces by the instance. | long | -| aws.ec2.network.out.bytes_per_sec | The number of bytes per second sent out on all network interfaces by the instance. | long | -| aws.ec2.network.out.packets | The number of packets sent out on all network interfaces by the instance. | long | -| aws.ec2.network.out.packets_per_sec | The number of packets per second sent out on all network interfaces by the instance. | long | -| aws.ec2.status.check_failed | Reports whether the instance has passed both the instance status check and the system status check in the last minute. | long | -| aws.ec2.status.check_failed_instance | Reports whether the instance has passed the instance status check in the last minute. | long | -| aws.ec2.status.check_failed_system | Reports whether the instance has passed the system status check in the last minute. | long | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.cpu.pct | Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. | scaled_float | -| host.disk.read.bytes | The total number of bytes read successfully in a given period of time. | long | -| host.disk.write.bytes | The total number of bytes write successfully in a given period of time. | long | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.network.in.bytes | The number of bytes received on all network interfaces by the host in a given period of time. | long | -| host.network.in.packets | The number of packets received on all network interfaces by the host in a given period of time. | long | -| host.network.out.bytes | The number of bytes sent out on all network interfaces by the host in a given period of time. | long | -| host.network.out.packets | The number of packets sent out on all network interfaces by the host in a given period of time. | long | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### elb - -An example event for `elb` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:30.211Z", - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "aws": { - "elb": { - "metrics": { - "EstimatedALBNewConnectionCount": { - "avg": 32 - }, - "EstimatedALBConsumedLCUs": { - "avg": 0.00035000000000000005 - }, - "EstimatedProcessedBytes": { - "avg": 967 - }, - "EstimatedALBActiveConnectionCount": { - "avg": 5 - }, - "HealthyHostCount": { - "max": 2 - }, - "UnHealthyHostCount": { - "max": 0 - } - } - }, - "cloudwatch": { - "namespace": "AWS/ELB" - }, - "dimensions": { - "LoadBalancerName": "filebeat-aws-elb-test-elb" - } - }, - "metricset": { - "name": "elb", - "period": 60000 - }, - "event": { - "dataset": "aws.elb", - "module": "aws", - "duration": 15044430616 - }, - "service": { - "type": "aws" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.applicationelb.metrics.ActiveConnectionCount.sum | The total number of concurrent TCP connections active from clients to the load balancer and from the load balancer to targets. | long | -| aws.applicationelb.metrics.ClientTLSNegotiationErrorCount.sum | The number of TLS connections initiated by the client that did not establish a session with the load balancer due to a TLS error. | long | -| aws.applicationelb.metrics.ConsumedLCUs.avg | The number of load balancer capacity units (LCU) used by your load balancer. | double | -| aws.applicationelb.metrics.HTTPCode_ELB_3XX_Count.sum | The number of HTTP 3XX redirection codes that originate from the load balancer. | long | -| aws.applicationelb.metrics.HTTPCode_ELB_4XX_Count.sum | The number of HTTP 4XX client error codes that originate from the load balancer. | long | -| aws.applicationelb.metrics.HTTPCode_ELB_500_Count.sum | The number of HTTP 500 error codes that originate from the load balancer. | long | -| aws.applicationelb.metrics.HTTPCode_ELB_502_Count.sum | The number of HTTP 502 error codes that originate from the load balancer. | long | -| aws.applicationelb.metrics.HTTPCode_ELB_503_Count.sum | The number of HTTP 503 error codes that originate from the load balancer. | long | -| aws.applicationelb.metrics.HTTPCode_ELB_504_Count.sum | The number of HTTP 504 error codes that originate from the load balancer. | long | -| aws.applicationelb.metrics.HTTPCode_ELB_5XX_Count.sum | The number of HTTP 5XX server error codes that originate from the load balancer. | long | -| aws.applicationelb.metrics.HTTP_Fixed_Response_Count.sum | The number of fixed-response actions that were successful. | long | -| aws.applicationelb.metrics.HTTP_Redirect_Count.sum | The number of redirect actions that were successful. | long | -| aws.applicationelb.metrics.HTTP_Redirect_Url_Limit_Exceeded_Count.sum | The number of redirect actions that couldn't be completed because the URL in the response location header is larger than 8K. | long | -| aws.applicationelb.metrics.IPv6ProcessedBytes.sum | The total number of bytes processed by the load balancer over IPv6. | long | -| aws.applicationelb.metrics.IPv6RequestCount.sum | The number of IPv6 requests received by the load balancer. | long | -| aws.applicationelb.metrics.NewConnectionCount.sum | The total number of new TCP connections established from clients to the load balancer and from the load balancer to targets. | long | -| aws.applicationelb.metrics.ProcessedBytes.sum | The total number of bytes processed by the load balancer over IPv4 and IPv6. | long | -| aws.applicationelb.metrics.RejectedConnectionCount.sum | The number of connections that were rejected because the load balancer had reached its maximum number of connections. | long | -| aws.applicationelb.metrics.RequestCount.sum | The number of requests processed over IPv4 and IPv6. | long | -| aws.applicationelb.metrics.RuleEvaluations.sum | The number of rules processed by the load balancer given a request rate averaged over an hour. | long | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.AvailabilityZone | Filters the metric data by the specified Availability Zone. | keyword | -| aws.dimensions.LoadBalancer | Filters the metric data by load balancer. | keyword | -| aws.dimensions.LoadBalancerName | Filters the metric data by the specified load balancer. | keyword | -| aws.dimensions.TargetGroup | Filters the metric data by target group. | keyword | -| aws.elb.metrics.BackendConnectionErrors.sum | The number of connections that were not successfully established between the load balancer and the registered instances. | long | -| aws.elb.metrics.EstimatedALBActiveConnectionCount.avg | The estimated number of concurrent TCP connections active from clients to the load balancer and from the load balancer to targets. | double | -| aws.elb.metrics.EstimatedALBConsumedLCUs.avg | The estimated number of load balancer capacity units (LCU) used by an Application Load Balancer. | double | -| aws.elb.metrics.EstimatedALBNewConnectionCount.avg | The estimated number of new TCP connections established from clients to the load balancer and from the load balancer to targets. | double | -| aws.elb.metrics.EstimatedProcessedBytes.avg | The estimated number of bytes processed by an Application Load Balancer. | double | -| aws.elb.metrics.HTTPCode_Backend_2XX.sum | The number of HTTP 2XX response code generated by registered instances. | long | -| aws.elb.metrics.HTTPCode_Backend_3XX.sum | The number of HTTP 3XX response code generated by registered instances. | long | -| aws.elb.metrics.HTTPCode_Backend_4XX.sum | The number of HTTP 4XX response code generated by registered instances. | long | -| aws.elb.metrics.HTTPCode_Backend_5XX.sum | The number of HTTP 5XX response code generated by registered instances. | long | -| aws.elb.metrics.HTTPCode_ELB_4XX.sum | The number of HTTP 4XX client error codes generated by the load balancer. | long | -| aws.elb.metrics.HTTPCode_ELB_5XX.sum | The number of HTTP 5XX server error codes generated by the load balancer. | long | -| aws.elb.metrics.HealthyHostCount.max | The number of healthy instances registered with your load balancer. | long | -| aws.elb.metrics.Latency.avg | The total time elapsed, in seconds, from the time the load balancer sent the request to a registered instance until the instance started to send the response headers. | double | -| aws.elb.metrics.RequestCount.sum | The number of requests completed or connections made during the specified interval. | long | -| aws.elb.metrics.SpilloverCount.sum | The total number of requests that were rejected because the surge queue is full. | long | -| aws.elb.metrics.SurgeQueueLength.max | The total number of requests (HTTP listener) or connections (TCP listener) that are pending routing to a healthy instance. | long | -| aws.elb.metrics.UnHealthyHostCount.max | The number of unhealthy instances registered with your load balancer. | long | -| aws.networkelb.metrics.ActiveFlowCount.avg | The total number of concurrent flows (or connections) from clients to targets. | double | -| aws.networkelb.metrics.ActiveFlowCount_TCP.avg | The total number of concurrent TCP flows (or connections) from clients to targets. | double | -| aws.networkelb.metrics.ActiveFlowCount_TLS.avg | The total number of concurrent TLS flows (or connections) from clients to targets. | double | -| aws.networkelb.metrics.ActiveFlowCount_UDP.avg | The total number of concurrent UDP flows (or connections) from clients to targets. | double | -| aws.networkelb.metrics.ClientTLSNegotiationErrorCount.sum | The total number of TLS handshakes that failed during negotiation between a client and a TLS listener. | long | -| aws.networkelb.metrics.ConsumedLCUs.avg | The number of load balancer capacity units (LCU) used by your load balancer. | double | -| aws.networkelb.metrics.HealthyHostCount.max | The number of targets that are considered healthy. | long | -| aws.networkelb.metrics.NewFlowCount.sum | The total number of new flows (or connections) established from clients to targets in the time period. | long | -| aws.networkelb.metrics.NewFlowCount_TLS.sum | The total number of new TLS flows (or connections) established from clients to targets in the time period. | long | -| aws.networkelb.metrics.ProcessedBytes.sum | The total number of bytes processed by the load balancer, including TCP/IP headers. | long | -| aws.networkelb.metrics.ProcessedBytes_TLS.sum | The total number of bytes processed by TLS listeners. | long | -| aws.networkelb.metrics.TCP_Client_Reset_Count.sum | The total number of reset (RST) packets sent from a client to a target. | long | -| aws.networkelb.metrics.TCP_ELB_Reset_Count.sum | The total number of reset (RST) packets generated by the load balancer. | long | -| aws.networkelb.metrics.TCP_Target_Reset_Count.sum | The total number of reset (RST) packets sent from a target to a client. | long | -| aws.networkelb.metrics.TargetTLSNegotiationErrorCount.sum | The total number of TLS handshakes that failed during negotiation between a TLS listener and a target. | long | -| aws.networkelb.metrics.UnHealthyHostCount.max | The number of targets that are considered unhealthy. | long | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### lambda - -An example event for `lambda` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:17:08.666Z", - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - }, - "event": { - "dataset": "aws.dynamodb", - "module": "aws", - "duration": 10266182336 - }, - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws", - "region": "eu-central-1" - }, - "aws": { - "cloudwatch": { - "namespace": "AWS/Lambda" - }, - "dimensions": { - "FunctionName": "ec2-owner-tagger-serverless", - "Resource": "ec2-owner-tagger-serverless" - }, - "lambda": { - "metrics": { - "Duration": { - "avg": 8218.073333333334 - }, - "Errors": { - "avg": 1 - }, - "Invocations": { - "avg": 1 - }, - "Throttles": { - "avg": 0 - } - } - } - }, - "metricset": { - "name": "dynamodb", - "period": 300000 - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.ExecutedVersion | Use the ExecutedVersion dimension to compare error rates for two versions of a function that are both targets of a weighted alias. | keyword | -| aws.dimensions.FunctionName | Lambda function name. | keyword | -| aws.dimensions.Resource | Resource name. | keyword | -| aws.lambda.metrics.ConcurrentExecutions.avg | The number of function instances that are processing events. | double | -| aws.lambda.metrics.DeadLetterErrors.avg | For asynchronous invocation, the number of times Lambda attempts to send an event to a dead-letter queue but fails. | double | -| aws.lambda.metrics.DestinationDeliveryFailures.avg | For asynchronous invocation, the number of times Lambda attempts to send an event to a destination but fails. | double | -| aws.lambda.metrics.Duration.avg | The amount of time that your function code spends processing an event. | double | -| aws.lambda.metrics.Errors.avg | The number of invocations that result in a function error. | double | -| aws.lambda.metrics.Invocations.avg | The number of times your function code is executed, including successful executions and executions that result in a function error. | double | -| aws.lambda.metrics.IteratorAge.avg | For event source mappings that read from streams, the age of the last record in the event. | double | -| aws.lambda.metrics.ProvisionedConcurrencyInvocations.sum | The number of times your function code is executed on provisioned concurrency. | long | -| aws.lambda.metrics.ProvisionedConcurrencySpilloverInvocations.sum | The number of times your function code is executed on standard concurrency when all provisioned concurrency is in use. | long | -| aws.lambda.metrics.ProvisionedConcurrencyUtilization.max | For a version or alias, the value of ProvisionedConcurrentExecutions divided by the total amount of provisioned concurrency allocated. | long | -| aws.lambda.metrics.ProvisionedConcurrentExecutions.max | The number of function instances that are processing events on provisioned concurrency. | long | -| aws.lambda.metrics.Throttles.avg | The number of invocation requests that are throttled. | double | -| aws.lambda.metrics.UnreservedConcurrentExecutions.avg | For an AWS Region, the number of events that are being processed by functions that don't have reserved concurrency. | double | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### natgateway - -An example event for `natgateway` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "cloudwatch": { - "namespace": "AWS/NATGateway" - }, - "dimensions": { - "NatGatewayId": "nat-0a5cb7b9807908cc0" - }, - "natgateway": { - "metrics": { - "ActiveConnectionCount": { - "max": 0 - }, - "BytesInFromDestination": { - "sum": 0 - }, - "BytesInFromSource": { - "sum": 0 - }, - "BytesOutToDestination": { - "sum": 0 - }, - "BytesOutToSource": { - "sum": 0 - }, - "ConnectionAttemptCount": { - "sum": 0 - }, - "ConnectionEstablishedCount": { - "sum": 0 - }, - "ErrorPortAllocation": { - "sum": 0 - }, - "PacketsDropCount": { - "sum": 0 - }, - "PacketsInFromDestination": { - "sum": 0 - }, - "PacketsInFromSource": { - "sum": 0 - }, - "PacketsOutToDestination": { - "sum": 0 - }, - "PacketsOutToSource": { - "sum": 0 - } - } - } - }, - "event": { - "dataset": "aws.natgateway", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "natgateway" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.NatGatewayId | Filter the metric data by the NAT gateway ID. | keyword | -| aws.natgateway.metrics.ActiveConnectionCount.max | The total number of concurrent active TCP connections through the NAT gateway. | long | -| aws.natgateway.metrics.BytesInFromDestination.sum | The number of bytes received by the NAT gateway from the destination. | long | -| aws.natgateway.metrics.BytesInFromSource.sum | The number of bytes received by the NAT gateway from clients in your VPC. | long | -| aws.natgateway.metrics.BytesOutToDestination.sum | The number of bytes sent out through the NAT gateway to the destination. | long | -| aws.natgateway.metrics.BytesOutToSource.sum | The number of bytes sent through the NAT gateway to the clients in your VPC. | long | -| aws.natgateway.metrics.ConnectionAttemptCount.sum | The number of connection attempts made through the NAT gateway. | long | -| aws.natgateway.metrics.ConnectionEstablishedCount.sum | The number of connections established through the NAT gateway. | long | -| aws.natgateway.metrics.ErrorPortAllocation.sum | The number of times the NAT gateway could not allocate a source port. | long | -| aws.natgateway.metrics.IdleTimeoutCount.sum | The number of connections that transitioned from the active state to the idle state. | long | -| aws.natgateway.metrics.PacketsDropCount.sum | The number of packets dropped by the NAT gateway. | long | -| aws.natgateway.metrics.PacketsInFromDestination.sum | The number of packets received by the NAT gateway from the destination. | long | -| aws.natgateway.metrics.PacketsInFromSource.sum | The number of packets received by the NAT gateway from clients in your VPC. | long | -| aws.natgateway.metrics.PacketsOutToDestination.sum | The number of packets sent out through the NAT gateway to the destination. | long | -| aws.natgateway.metrics.PacketsOutToSource.sum | The number of packets sent through the NAT gateway to the clients in your VPC. | long | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### rds - -An example event for `rds` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:34.537Z", - "ecs": { - "version": "1.5.0" - }, - "service": { - "type": "aws" - }, - "aws": { - "rds": { - "latency": { - "dml": 0, - "insert": 0, - "update": 0, - "commit": 0, - "ddl": 0, - "delete": 0, - "select": 0.21927814569536422 - }, - "queries": 6.197934021992669, - "aurora_bin_log_replica_lag": 0, - "transactions": { - "blocked": 0, - "active": 0 - }, - "deadlocks": 0, - "login_failures": 0, - "throughput": { - "network": 1.399813358218904, - "insert": 0, - "ddl": 0, - "select": 2.5165408396246853, - "delete": 0, - "commit": 0, - "network_transmit": 0.699906679109452, - "update": 0, - "dml": 0, - "network_receive": 0.699906679109452 - }, - "cpu": { - "total": { - "pct": 0.03 - } - }, - "db_instance": { - "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", - "class": "db.r5.large", - "identifier": "database-1-instance-1-eu-west-1a", - "status": "available" - }, - "cache_hit_ratio.result_set": 0, - "aurora_replica.lag.ms": 19.576, - "free_local_storage.bytes": 32431271936, - "cache_hit_ratio.buffer": 100, - "disk_usage": { - "bin_log.bytes": 0 - }, - "db_instance.identifier": "database-1-instance-1-eu-west-1a", - "freeable_memory.bytes": 4436537344, - "engine_uptime.sec": 10463030, - "database_connections": 0 - } - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - }, - "availability_zone": "eu-west-1a" - }, - "event": { - "dataset": "aws.rds", - "module": "aws", - "duration": 10777919184 - }, - "metricset": { - "name": "rds", - "period": 60000 - }, - "agent": { - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.DBClusterIdentifier | This dimension filters the data that you request for a specific Amazon Aurora DB cluster. | keyword | -| aws.dimensions.DBClusterIdentifier,Role | This dimension filters the data that you request for a specific Aurora DB cluster, aggregating the metric by instance role (WRITER/READER). | keyword | -| aws.dimensions.DBInstanceIdentifier | This dimension filters the data that you request for a specific DB instance. | keyword | -| aws.dimensions.DatabaseClass | This dimension filters the data that you request for all instances in a database class. | keyword | -| aws.dimensions.DbClusterIdentifier, EngineName | This dimension filters the data that you request for a specific Aurora DB cluster, aggregating the metric by engine name. | keyword | -| aws.dimensions.EngineName | This dimension filters the data that you request for the identified engine name only. | keyword | -| aws.dimensions.SourceRegion | This dimension filters the data that you request for the specified region only. | keyword | -| aws.rds.aurora_bin_log_replica_lag | The amount of time a replica DB cluster running on Aurora with MySQL compatibility lags behind the source DB cluster. | long | -| aws.rds.aurora_global_db.data_transfer.bytes | In an Aurora Global Database, the amount of redo log data transferred from the master AWS Region to a secondary AWS Region. | long | -| aws.rds.aurora_global_db.replicated_write_io.bytes | In an Aurora Global Database, the number of write I/O operations replicated from the primary AWS Region to the cluster volume in a secondary AWS Region. | long | -| aws.rds.aurora_global_db.replication_lag.ms | For an Aurora Global Database, the amount of lag when replicating updates from the primary AWS Region, in milliseconds. | long | -| aws.rds.aurora_replica.lag.ms | For an Aurora Replica, the amount of lag when replicating updates from the primary instance, in milliseconds. | long | -| aws.rds.aurora_replica.lag_max.ms | The maximum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds. | long | -| aws.rds.aurora_replica.lag_min.ms | The minimum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds. | long | -| aws.rds.aurora_volume_left_total.bytes | The remaining available space for the cluster volume, measured in bytes. | long | -| aws.rds.backtrack_change_records.creation_rate | The number of backtrack change records created over five minutes for your DB cluster. | long | -| aws.rds.backtrack_change_records.stored | The actual number of backtrack change records used by your DB cluster. | long | -| aws.rds.backtrack_window.actual | The difference between the target backtrack window and the actual backtrack window. | long | -| aws.rds.backtrack_window.alert | The number of times that the actual backtrack window is smaller than the target backtrack window for a given period of time. | long | -| aws.rds.backup_storage_billed_total.bytes | The total amount of backup storage in bytes for which you are billed for a given Aurora DB cluster. | long | -| aws.rds.cache_hit_ratio.buffer | The percentage of requests that are served by the buffer cache. | long | -| aws.rds.cache_hit_ratio.result_set | The percentage of requests that are served by the Resultset cache. | long | -| aws.rds.cpu.credit_balance | The number of earned CPU credits that an instance has accrued since it was launched or started. | long | -| aws.rds.cpu.credit_usage | The number of CPU credits spent by the instance for CPU utilization. | long | -| aws.rds.cpu.total.pct | The percentage of CPU utilization. | scaled_float | -| aws.rds.database_connections | The number of database connections in use. | long | -| aws.rds.db_instance.arn | Amazon Resource Name(ARN) for each rds. | keyword | -| aws.rds.db_instance.class | Contains the name of the compute and memory capacity class of the DB instance. | keyword | -| aws.rds.db_instance.db_cluster_identifier | This identifier is the unique key that identifies a DB cluster specifically for Amazon Aurora DB cluster. | keyword | -| aws.rds.db_instance.engine_name | Each DB instance runs a DB engine, like MySQL, MariaDB, PostgreSQL and etc. | keyword | -| aws.rds.db_instance.identifier | Contains a user-supplied database identifier. This identifier is the unique key that identifies a DB instance. | keyword | -| aws.rds.db_instance.role | DB roles like WRITER or READER, specifically for Amazon Aurora DB cluster. | keyword | -| aws.rds.db_instance.status | Specifies the current state of this database. | keyword | -| aws.rds.deadlocks | The average number of deadlocks in the database per second. | long | -| aws.rds.disk_queue_depth | The number of outstanding IOs (read/write requests) waiting to access the disk. | float | -| aws.rds.disk_usage.bin_log.bytes | The amount of disk space occupied by binary logs on the master. Applies to MySQL read replicas. | long | -| aws.rds.disk_usage.replication_slot.mb | The disk space used by replication slot files. Applies to PostgreSQL. | long | -| aws.rds.disk_usage.transaction_logs.mb | The disk space used by transaction logs. Applies to PostgreSQL. | long | -| aws.rds.engine_uptime.sec | The amount of time that the instance has been running, in seconds. | long | -| aws.rds.failed_sql_server_agent_jobs | The number of failed SQL Server Agent jobs during the last minute. | long | -| aws.rds.free_local_storage.bytes | The amount of storage available for temporary tables and logs, in bytes. | long | -| aws.rds.free_storage.bytes | The amount of available storage space. | long | -| aws.rds.freeable_memory.bytes | The amount of available random access memory. | long | -| aws.rds.latency.commit | The amount of latency for commit operations, in milliseconds. | float | -| aws.rds.latency.ddl | The amount of latency for data definition language (DDL) requests, in milliseconds. | float | -| aws.rds.latency.delete | The amount of latency for delete queries, in milliseconds. | float | -| aws.rds.latency.dml | The amount of latency for inserts, updates, and deletes, in milliseconds. | float | -| aws.rds.latency.insert | The amount of latency for insert queries, in milliseconds. | float | -| aws.rds.latency.read | The average amount of time taken per disk I/O operation. | float | -| aws.rds.latency.select | The amount of latency for select queries, in milliseconds. | float | -| aws.rds.latency.update | The amount of latency for update queries, in milliseconds. | float | -| aws.rds.latency.write | The average amount of time taken per disk I/O operation. | float | -| aws.rds.login_failures | The average number of failed login attempts per second. | long | -| aws.rds.maximum_used_transaction_ids | The maximum transaction ID that has been used. Applies to PostgreSQL. | long | -| aws.rds.oldest_replication_slot_lag.mb | The lagging size of the replica lagging the most in terms of WAL data received. Applies to PostgreSQL. | long | -| aws.rds.queries | The average number of queries executed per second. | long | -| aws.rds.rds_to_aurora_postgresql_replica_lag.sec | The amount of lag in seconds when replicating updates from the primary RDS PostgreSQL instance to other nodes in the cluster. | long | -| aws.rds.read_io.ops_per_sec | The average number of disk read I/O operations per second. | float | -| aws.rds.replica_lag.sec | The amount of time a Read Replica DB instance lags behind the source DB instance. Applies to MySQL, MariaDB, and PostgreSQL Read Replicas. | long | -| aws.rds.storage_used.backup_retention_period.bytes | The total amount of backup storage in bytes used to support the point-in-time restore feature within the Aurora DB cluster's backup retention window. | long | -| aws.rds.storage_used.snapshot.bytes | The total amount of backup storage in bytes consumed by all Aurora snapshots for an Aurora DB cluster outside its backup retention window. | long | -| aws.rds.swap_usage.bytes | The amount of swap space used on the DB instance. This metric is not available for SQL Server. | long | -| aws.rds.throughput.commit | The average number of commit operations per second. | float | -| aws.rds.throughput.ddl | The average number of DDL requests per second. | float | -| aws.rds.throughput.delete | The average number of delete queries per second. | float | -| aws.rds.throughput.dml | The average number of inserts, updates, and deletes per second. | float | -| aws.rds.throughput.insert | The average number of insert queries per second. | float | -| aws.rds.throughput.network | The amount of network throughput both received from and transmitted to clients by each instance in the Aurora MySQL DB cluster, in bytes per second. | float | -| aws.rds.throughput.network_receive | The incoming (Receive) network traffic on the DB instance, including both customer database traffic and Amazon RDS traffic used for monitoring and replication. | float | -| aws.rds.throughput.network_transmit | The outgoing (Transmit) network traffic on the DB instance, including both customer database traffic and Amazon RDS traffic used for monitoring and replication. | float | -| aws.rds.throughput.read | The average amount of time taken per disk I/O operation. | float | -| aws.rds.throughput.select | The average number of select queries per second. | float | -| aws.rds.throughput.update | The average number of update queries per second. | float | -| aws.rds.throughput.write | The average number of bytes written to disk per second. | float | -| aws.rds.transaction_logs_generation | The disk space used by transaction logs. Applies to PostgreSQL. | long | -| aws.rds.transactions.active | The average number of current transactions executing on an Aurora database instance per second. | long | -| aws.rds.transactions.blocked | The average number of transactions in the database that are blocked per second. | long | -| aws.rds.volume.read.iops | The number of billed read I/O operations from a cluster volume, reported at 5-minute intervals. | long | -| aws.rds.volume.write.iops | The number of write disk I/O operations to the cluster volume, reported at 5-minute intervals. | long | -| aws.rds.volume_used.bytes | The amount of storage used by your Aurora DB instance, in bytes. | long | -| aws.rds.write_io.ops_per_sec | The average number of disk write I/O operations per second. | float | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### s3_daily_storage - -An example event for `s3_daily_storage` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_daily_storage": { - "bucket": { - "size": { - "bytes": 207372 - } - }, - "number_of_objects": 128 - } - }, - "event": { - "dataset": "aws.s3_daily_storage", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_daily_storage" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.BucketName | This dimension filters the data you request for the identified bucket only. | keyword | -| aws.dimensions.FilterId | This dimension filters metrics configurations that you specify for request metrics on a bucket, for example, a prefix or a tag. | keyword | -| aws.dimensions.StorageType | This dimension filters the data that you have stored in a bucket by types of storage. | keyword | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.s3_daily_storage.bucket.size.bytes | The amount of data in bytes stored in a bucket. | long | -| aws.s3_daily_storage.number_of_objects | The total number of objects stored in a bucket for all storage classes. | long | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### s3_request - -An example event for `s3_request` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "s3": { - "bucket": { - "name": "test-s3-ks-2" - } - }, - "s3_request": { - "downloaded": { - "bytes": 534 - }, - "errors": { - "4xx": 0, - "5xx": 0 - }, - "latency": { - "first_byte.ms": 214, - "total_request.ms": 533 - }, - "requests": { - "list": 2, - "put": 10, - "total": 12 - }, - "uploaded": { - "bytes": 13572 - } - } - }, - "event": { - "dataset": "aws.s3_request", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "s3_request" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.BucketName | This dimension filters the data you request for the identified bucket only. | keyword | -| aws.dimensions.FilterId | This dimension filters metrics configurations that you specify for request metrics on a bucket, for example, a prefix or a tag. | keyword | -| aws.dimensions.StorageType | This dimension filters the data that you have stored in a bucket by types of storage. | keyword | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.s3_request.downloaded.bytes | The number bytes downloaded for requests made to an Amazon S3 bucket, where the response includes a body. | long | -| aws.s3_request.errors.4xx | The number of HTTP 4xx client error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. | long | -| aws.s3_request.errors.5xx | The number of HTTP 5xx server error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. | long | -| aws.s3_request.latency.first_byte.ms | The per-request time from the complete request being received by an Amazon S3 bucket to when the response starts to be returned. | long | -| aws.s3_request.latency.total_request.ms | The elapsed per-request time from the first byte received to the last byte sent to an Amazon S3 bucket. | long | -| aws.s3_request.requests.delete | The number of HTTP DELETE requests made for objects in an Amazon S3 bucket. | long | -| aws.s3_request.requests.get | The number of HTTP GET requests made for objects in an Amazon S3 bucket. | long | -| aws.s3_request.requests.head | The number of HTTP HEAD requests made to an Amazon S3 bucket. | long | -| aws.s3_request.requests.list | The number of HTTP requests that list the contents of a bucket. | long | -| aws.s3_request.requests.post | The number of HTTP POST requests made to an Amazon S3 bucket. | long | -| aws.s3_request.requests.put | The number of HTTP PUT requests made for objects in an Amazon S3 bucket. | long | -| aws.s3_request.requests.select | The number of Amazon S3 SELECT Object Content requests made for objects in an Amazon S3 bucket. | long | -| aws.s3_request.requests.select_returned.bytes | The number of bytes of data returned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. | long | -| aws.s3_request.requests.select_scanned.bytes | The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. | long | -| aws.s3_request.requests.total | The total number of HTTP requests made to an Amazon S3 bucket, regardless of type. | long | -| aws.s3_request.uploaded.bytes | The number bytes uploaded that contain a request body, made to an Amazon S3 bucket. | long | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### sns - -An example event for `sns` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:27.154Z", - "data_stream": { - "dataset": "aws.sns", - "namespace": "default", - "type": "metrics" - }, - "aws": { - "dimensions": { - "TopicName": "test-sns-ks" - }, - "sns": { - "metrics": { - "NumberOfMessagesPublished": { - "sum": 1 - }, - "NumberOfNotificationsFailed": { - "sum": 1 - }, - "PublishSize": { - "avg": 5 - } - } - }, - "tags": { - "created-by": "ks" - } - }, - "event": { - "dataset": "aws.sns", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "sns" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.Application | Filters on application objects, which represent an app and device registered with one of the supported push notification services, such as APNs and FCM. | keyword | -| aws.dimensions.Application,Platform | Filters on application and platform objects, where the platform objects are for the supported push notification services, such as APNs and FCM. | keyword | -| aws.dimensions.Country | Filters on the destination country or region of an SMS message. | keyword | -| aws.dimensions.Platform | Filters on platform objects for the push notification services, such as APNs and FCM. | keyword | -| aws.dimensions.SMSType | Filters on the message type of SMS message. | keyword | -| aws.dimensions.TopicName | Filters on Amazon SNS topic names. | keyword | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.sns.metrics.NumberOfMessagesPublished.sum | The number of messages published to your Amazon SNS topics. | long | -| aws.sns.metrics.NumberOfNotificationsDelivered.sum | The number of messages successfully delivered from your Amazon SNS topics to subscribing endpoints. | long | -| aws.sns.metrics.NumberOfNotificationsFailed.sum | The number of messages that Amazon SNS failed to deliver. | long | -| aws.sns.metrics.NumberOfNotificationsFailedToRedriveToDlq.sum | The number of messages that couldn't be moved to a dead-letter queue. | long | -| aws.sns.metrics.NumberOfNotificationsFilteredOut-InvalidAttributes.sum | The number of messages that were rejected by subscription filter policies because the messages' attributes are invalid - for example, because the attribute JSON is incorrectly formatted. | long | -| aws.sns.metrics.NumberOfNotificationsFilteredOut-NoMessageAttributes.sum | The number of messages that were rejected by subscription filter policies because the messages have no attributes. | long | -| aws.sns.metrics.NumberOfNotificationsFilteredOut.sum | The number of messages that were rejected by subscription filter policies. | long | -| aws.sns.metrics.NumberOfNotificationsRedrivenToDlq.sum | The number of messages that have been moved to a dead-letter queue. | long | -| aws.sns.metrics.PublishSize.avg | The size of messages published. | double | -| aws.sns.metrics.SMSMonthToDateSpentUSD.sum | The charges you have accrued since the start of the current calendar month for sending SMS messages. | long | -| aws.sns.metrics.SMSSuccessRate.avg | The rate of successful SMS message deliveries. | double | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | - - -### sqs - -An example event for `sqs` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "sqs": { - "empty_receives": 0, - "messages": { - "delayed": 0, - "deleted": 0, - "not_visible": 0, - "received": 0, - "sent": 0, - "visible": 2 - }, - "oldest_message_age": { - "sec": 78494 - }, - "queue": { - "name": "test-s3-notification" - }, - "sent_message_size": {} - } - }, - "event": { - "dataset": "aws.sqs", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "sqs" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.QueueName | SQS queue name | keyword | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.sqs.empty_receives | The number of ReceiveMessage API calls that did not return a message. | long | -| aws.sqs.messages.delayed | TThe number of messages in the queue that are delayed and not available for reading immediately. | long | -| aws.sqs.messages.deleted | The number of messages deleted from the queue. | long | -| aws.sqs.messages.not_visible | The number of messages that are in flight. | long | -| aws.sqs.messages.received | The number of messages returned by calls to the ReceiveMessage action. | long | -| aws.sqs.messages.sent | The number of messages added to a queue. | long | -| aws.sqs.messages.visible | The number of messages available for retrieval from the queue. | long | -| aws.sqs.oldest_message_age.sec | The approximate age of the oldest non-deleted message in the queue. | long | -| aws.sqs.queue.name | SQS queue name | keyword | -| aws.sqs.sent_message_size.bytes | The size of messages added to a queue. | long | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### transitgateway - -An example event for `transitgateway` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T20:10:20.953Z", - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "aws": { - "transitgateway": { - "metrics": { - "PacketsIn": { - "sum": 0 - }, - "BytesIn": { - "sum": 0 - }, - "BytesOut": { - "sum": 0 - }, - "PacketsOut": { - "sum": 0 - }, - "PacketDropCountBlackhole": { - "sum": 0 - }, - "PacketDropCountNoRoute": { - "sum": 0 - } - } - }, - "cloudwatch": { - "namespace": "AWS/TransitGateway" - }, - "dimensions": { - "TransitGateway": "tgw-0630672a32f12808a" - } - }, - "ecs": { - "version": "1.5.0" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" - }, - "event": { - "dataset": "aws.transitgateway", - "module": "aws", - "duration": 12762825681 - }, - "metricset": { - "period": 60000, - "name": "transitgateway" - }, - "service": { - "type": "aws" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.TransitGateway | Filters the metric data by transit gateway. | keyword | -| aws.dimensions.TransitGatewayAttachment | Filters the metric data by transit gateway attachment. | keyword | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| aws.transitgateway.metrics.BytesIn.sum | The number of bytes received by the transit gateway. | long | -| aws.transitgateway.metrics.BytesOut.sum | The number of bytes sent from the transit gateway. | long | -| aws.transitgateway.metrics.PacketDropCountBlackhole.sum | The number of packets dropped because they matched a blackhole route. | long | -| aws.transitgateway.metrics.PacketDropCountNoRoute.sum | The number of packets dropped because they did not match a route. | long | -| aws.transitgateway.metrics.PacketsIn.sum | The number of packets received by the transit gateway. | long | -| aws.transitgateway.metrics.PacketsOut.sum | The number of packets sent by the transit gateway. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### usage - -An example event for `usage` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:30.929Z", - "aws": { - "usage": { - "metrics": { - "CallCount": { - "sum": 1 - } - } - }, - "cloudwatch": { - "namespace": "AWS/Usage" - }, - "dimensions": { - "Type": "API", - "Resource": "GetMetricData", - "Service": "CloudWatch", - "Class": "None" - } - }, - "event": { - "duration": 1191329839, - "dataset": "aws.usage", - "module": "aws" - }, - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "cloud": { - "provider": "aws", - "region": "eu-north-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "metricset": { - "name": "usage", - "period": 60000 - }, - "agent": { - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.Class | The class of resource being tracked. | keyword | -| aws.dimensions.Resource | The name of the API operation. | keyword | -| aws.dimensions.Service | The name of the AWS service containing the resource. | keyword | -| aws.dimensions.Type | The type of resource being tracked. | keyword | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| aws.usage.metrics.CallCount.sum | The number of specified API operations performed in your account. | long | -| aws.usage.metrics.ResourceCount.sum | The number of the specified resources running in your account. The resources are defined by the dimensions associated with the metric. | long | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - - -### vpn - -An example event for `vpn` looks as following: - -```$json -{ - "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" - }, - "aws": { - "vpn": { - "metrics": { - "TunnelState": { - "avg": 0 - }, - "TunnelDataIn": { - "sum": 0 - }, - "TunnelDataOut": { - "sum": 0 - } - } - }, - "cloudwatch": { - "namespace": "AWS/VPN" - } - }, - "event": { - "dataset": "aws.vpn", - "module": "aws", - "duration": 10418157072 - }, - "metricset": { - "period": 60000, - "name": "vpn" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" - } -} -``` - -**Exported fields** - -| Field | Description | Type | -|---|---|---| -| @timestamp | Event timestamp. | date | -| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | -| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | -| aws.dimensions.* | Metric dimensions. | object | -| aws.dimensions.TunnelIpAddress | Filters the metric data by the IP address of the tunnel for the virtual private gateway. | keyword | -| aws.dimensions.VpnId | Filters the metric data by the Site-to-Site VPN connection ID. | keyword | -| aws.s3.bucket.name | Name of a S3 bucket. | keyword | -| aws.tags.* | Tag key value pairs from aws resources. | object | -| aws.vpn.metrics.TunnelDataIn.sum | The bytes received through the VPN tunnel. | double | -| aws.vpn.metrics.TunnelDataOut.sum | The bytes sent through the VPN tunnel. | double | -| aws.vpn.metrics.TunnelState.avg | The state of the tunnel. For static VPNs, 0 indicates DOWN and 1 indicates UP. For BGP VPNs, 1 indicates ESTABLISHED and 0 is used for all other states. | double | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host is running. | keyword | -| cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | Name of the project in Google Cloud. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host is running. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| data_stream.dataset | Data stream dataset. | constant_keyword | -| data_stream.namespace | Data stream namespace. | constant_keyword | -| data_stream.type | Data stream type. | constant_keyword | -| ecs.version | | keyword | -| host.architecture | Operating system architecture. | keyword | -| host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host mac addresses. | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | -| host.os.build | OS build information. | keyword | -| host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| service.type | | keyword | - diff --git a/test/packages/aws/docs/billing.md b/test/packages/aws/docs/billing.md new file mode 100644 index 0000000000..fef6f8d65b --- /dev/null +++ b/test/packages/aws/docs/billing.md @@ -0,0 +1,121 @@ +# billing + +## Metrics + +An example event for `billing` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:17:06.212Z", + "cloud": { + "provider": "aws", + "region": "us-east-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "event": { + "dataset": "aws.billing", + "module": "aws", + "duration": 1938760247 + }, + "metricset": { + "name": "billing", + "period": 43200000 + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "billing": { + "metrics": { + "EstimatedCharges": { + "max": 1625.41 + } + } + }, + "cloudwatch": { + "namespace": "AWS/Billing" + }, + "dimensions": { + "Currency": "USD" + } + }, + "service": { + "type": "aws" + }, + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.billing.AmortizedCost.amount | Amortized cost amount. | double | +| aws.billing.AmortizedCost.unit | Amortized cost unit. | keyword | +| aws.billing.BlendedCost.amount | Blended cost amount. | double | +| aws.billing.BlendedCost.unit | Blended cost unit. | keyword | +| aws.billing.Currency | Currency name. | keyword | +| aws.billing.EstimatedCharges.max | Maximum estimated charges for AWS acccount. | long | +| aws.billing.NormalizedUsageAmount.amount | Normalized usage amount. | double | +| aws.billing.NormalizedUsageAmount.unit | Normalized usage amount unit. | keyword | +| aws.billing.ServiceName | AWS service name. | keyword | +| aws.billing.UnblendedCost.amount | Unblended cost amount. | double | +| aws.billing.UnblendedCost.unit | Unblended cost unit. | keyword | +| aws.billing.UsageQuantity.amount | Usage quantity amount. | double | +| aws.billing.UsageQuantity.unit | Usage quantity unit. | keyword | +| aws.billing.end_date | End date for retrieving AWS costs. | keyword | +| aws.billing.group_by | Cost explorer group by key values. | object | +| aws.billing.group_definition.key | The string that represents a key for a specified group. | keyword | +| aws.billing.group_definition.type | The string that represents the type of group. | keyword | +| aws.billing.start_date | Start date for retrieving AWS costs. | keyword | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | + diff --git a/test/packages/aws/docs/cloudtrail.md b/test/packages/aws/docs/cloudtrail.md new file mode 100644 index 0000000000..7edd7c85a8 --- /dev/null +++ b/test/packages/aws/docs/cloudtrail.md @@ -0,0 +1,125 @@ +# cloudtrail + +## Logs + +The `cloudtrail` dataset collects the AWS CloudTrail logs. CloudTrail monitors +events for the account. If user creates a trail, it delivers those events as log + files to a specific Amazon S3 bucket. The `cloudtrail` dataset does not read + the CloudTrail Digest files that are delivered to the S3 bucket when Log File + Integrity is turned on, it only reads the CloudTrail logs. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.cloudtrail.additional_eventdata | Additional data about the event that was not part of the request or response. | keyword | +| aws.cloudtrail.api_version | Identifies the API version associated with the AwsApiCall eventType value. | keyword | +| aws.cloudtrail.console_login.additional_eventdata.login_to | URL for ConsoleLogin | keyword | +| aws.cloudtrail.console_login.additional_eventdata.mfa_used | Identifies whether multi factor authentication was used during ConsoleLogin | boolean | +| aws.cloudtrail.console_login.additional_eventdata.mobile_version | Identifies whether ConsoleLogin was from mobile version | boolean | +| aws.cloudtrail.error_code | The AWS service error if the request returns an error. | keyword | +| aws.cloudtrail.error_message | If the request returns an error, the description of the error. | keyword | +| aws.cloudtrail.event_category | The CloudTrail event category. | keyword | +| aws.cloudtrail.event_type | Identifies the type of event that generated the event record. | keyword | +| aws.cloudtrail.event_version | The CloudTrail version of the log event format. | keyword | +| aws.cloudtrail.flattened.additional_eventdata | Additional data about the event that was not part of the request or response. | flattened | +| aws.cloudtrail.flattened.digest | Additional digest information. | flattened | +| aws.cloudtrail.flattened.insight_details | Additional insight details. | flattened | +| aws.cloudtrail.flattened.request_parameters | The parameters, if any, that were sent with the request. | flattened | +| aws.cloudtrail.flattened.response_elements | The response element for actions that make changes (create, update, or delete actions). | flattened | +| aws.cloudtrail.flattened.service_event_details | Identifies the service event, including what triggered the event and the result. | flattened | +| aws.cloudtrail.management_event | A Boolean value that identifies whether the event is a management event. | keyword | +| aws.cloudtrail.read_only | Identifies whether this operation is a read-only operation. | boolean | +| aws.cloudtrail.recipient_account_id | Represents the account ID that received this event. | keyword | +| aws.cloudtrail.request_id | The value that identifies the request. The service being called generates this value. | keyword | +| aws.cloudtrail.request_parameters | The parameters, if any, that were sent with the request. | keyword | +| aws.cloudtrail.resources.account_id | Account ID of the resource owner | keyword | +| aws.cloudtrail.resources.arn | Resource ARNs | keyword | +| aws.cloudtrail.resources.type | Resource type identifier in the format: AWS::aws-service-name::data-type-name | keyword | +| aws.cloudtrail.response_elements | The response element for actions that make changes (create, update, or delete actions). | keyword | +| aws.cloudtrail.service_event_details | Identifies the service event, including what triggered the event and the result. | keyword | +| aws.cloudtrail.shared_event_id | GUID generated by CloudTrail to uniquely identify CloudTrail events from the same AWS action that is sent to different AWS accounts. | keyword | +| aws.cloudtrail.user_identity.access_key_id | The access key ID that was used to sign the request. | keyword | +| aws.cloudtrail.user_identity.arn | The Amazon Resource Name (ARN) of the principal that made the call. | keyword | +| aws.cloudtrail.user_identity.invoked_by | The name of the AWS service that made the request, such as Amazon EC2 Auto Scaling or AWS Elastic Beanstalk. | keyword | +| aws.cloudtrail.user_identity.session_context.creation_date | The date and time when the temporary security credentials were issued. | date | +| aws.cloudtrail.user_identity.session_context.mfa_authenticated | The value is true if the root user or IAM user whose credentials were used for the request also was authenticated with an MFA device; otherwise, false. | keyword | +| aws.cloudtrail.user_identity.session_context.session_issuer.account_id | The account that owns the entity that was used to get credentials. | keyword | +| aws.cloudtrail.user_identity.session_context.session_issuer.arn | The ARN of the source (account, IAM user, or role) that was used to get temporary security credentials. | keyword | +| aws.cloudtrail.user_identity.session_context.session_issuer.principal_id | The internal ID of the entity that was used to get credentials. | keyword | +| aws.cloudtrail.user_identity.session_context.session_issuer.type | The source of the temporary security credentials, such as Root, IAMUser, or Role. | keyword | +| aws.cloudtrail.user_identity.type | The type of the identity | keyword | +| aws.cloudtrail.vpc_endpoint_id | Identifies the VPC endpoint in which requests were made from a VPC to another AWS service, such as Amazon S3. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| error.message | Error message. | text | +| event.action | The action captured by the event. | keyword | +| event.ingested | Timestamp when an event arrived in the central data store. | date | +| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity. | keyword | +| event.provider | Source of the event. | keyword | +| event.type | Event severity (e.g. info, error) | keyword | +| file.hash.md5 | MD5 hash. | keyword | +| file.hash.sha1 | SHA1 hash. | keyword | +| file.hash.sha256 | SHA256 hash. | keyword | +| file.hash.sha512 | SHA512 hash. | keyword | +| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | +| group.id | Unique identifier for the group on the system/platform. | keyword | +| group.name | Name of the group. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| related.hash | All the hashes seen on your event. | keyword | +| related.user | All the user names seen on your event. | keyword | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.country_name | Country name. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| user.changes.name | Short name or login of the user. | keyword | +| user.id | Unique identifier of the user. | keyword | +| user.name | Short name or login of the user. | keyword | +| user.target.id | Unique identifier of the user. | keyword | +| user.target.name | Short name or login of the user. | keyword | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | +| user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.os.full | Operating system name, including the version or code name. | keyword | +| user_agent.os.name | Operating system name, without the version. | keyword | +| user_agent.os.version | Operating system version as a raw string. | keyword | +| user_agent.version | Version of the user agent. | keyword | + diff --git a/test/packages/aws/docs/cloudwatch.md b/test/packages/aws/docs/cloudwatch.md new file mode 100644 index 0000000000..06685928cc --- /dev/null +++ b/test/packages/aws/docs/cloudwatch.md @@ -0,0 +1,154 @@ +# cloudwatch + +## Logs + +The `cloudwatch` dataset collects CloudWatch logs. Users can use Amazon +CloudWatch logs to monitor, store, and access log files from different sources. +Export logs from log groups to an Amazon S3 bucket which has SQS notification +setup already. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.cloudwatch.message | CloudWatch log message. | text | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | + + +## Metrics + +An example event for `cloudwatch` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:17:02.812Z", + "event": { + "duration": 14119105951, + "dataset": "aws.cloudwatch", + "module": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "dimensions": { + "InstanceId": "i-0830bfecfa7173cbe" + }, + "ec2": { + "metrics": { + "DiskWriteOps": { + "avg": 0, + "max": 0 + }, + "CPUUtilization": { + "avg": 0.7661943132361363, + "max": 0.833333333333333 + } + } + }, + "cloudwatch": { + "namespace": "AWS/EC2" + } + }, + "metricset": { + "period": 300000, + "name": "cloudwatch" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | diff --git a/test/packages/aws/docs/dynamodb.md b/test/packages/aws/docs/dynamodb.md new file mode 100644 index 0000000000..72f60c8c2a --- /dev/null +++ b/test/packages/aws/docs/dynamodb.md @@ -0,0 +1,140 @@ +# dynamodb + +## Metrics + +An example event for `dynamodb` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "dimensions": { + "TableName": "TryDaxTable3" + }, + "dynamodb": { + "metrics": { + "ProvisionedWriteCapacityUnits": { + "avg": 1 + }, + "ProvisionedReadCapacityUnits": { + "avg": 1 + }, + "ConsumedWriteCapacityUnits": { + "avg": 0, + "sum": 0 + }, + "ConsumedReadCapacityUnits": { + "avg": 0, + "sum": 0 + } + } + }, + "cloudwatch": { + "namespace": "AWS/DynamoDB" + } + }, + "metricset": { + "name": "dynamodb", + "period": 300000 + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dynamodb.metrics.AccountMaxReads.max | The maximum number of read capacity units that can be used by an account. This limit does not apply to on-demand tables or global secondary indexes. | long | +| aws.dynamodb.metrics.AccountMaxTableLevelReads.max | The maximum number of read capacity units that can be used by a table or global secondary index of an account. For on-demand tables this limit caps the maximum read request units a table or a global secondary index can use. | long | +| aws.dynamodb.metrics.AccountMaxTableLevelWrites.max | The maximum number of write capacity units that can be used by a table or global secondary index of an account. For on-demand tables this limit caps the maximum write request units a table or a global secondary index can use. | long | +| aws.dynamodb.metrics.AccountMaxWrites.max | The maximum number of write capacity units that can be used by an account. This limit does not apply to on-demand tables or global secondary indexes. | long | +| aws.dynamodb.metrics.AccountProvisionedReadCapacityUtilization.avg | The average percentage of provisioned read capacity units utilized by the account. | double | +| aws.dynamodb.metrics.AccountProvisionedWriteCapacityUtilization.avg | The average percentage of provisioned write capacity units utilized by the account. | double | +| aws.dynamodb.metrics.ConditionalCheckFailedRequests.sum | The number of failed attempts to perform conditional writes. | long | +| aws.dynamodb.metrics.ConsumedReadCapacityUnits.avg | | double | +| aws.dynamodb.metrics.ConsumedReadCapacityUnits.sum | | long | +| aws.dynamodb.metrics.ConsumedWriteCapacityUnits.avg | | double | +| aws.dynamodb.metrics.ConsumedWriteCapacityUnits.sum | | long | +| aws.dynamodb.metrics.MaxProvisionedTableReadCapacityUtilization.max | The percentage of provisioned read capacity units utilized by the highest provisioned read table or global secondary index of an account. | double | +| aws.dynamodb.metrics.MaxProvisionedTableWriteCapacityUtilization.max | The percentage of provisioned write capacity utilized by the highest provisioned write table or global secondary index of an account. | double | +| aws.dynamodb.metrics.OnlineIndexPercentageProgress.avg | The percentage of completion when a new global secondary index is being added to a table. | double | +| aws.dynamodb.metrics.PendingReplicationCount.sum | The number of item updates that are written to one replica table, but that have not yet been written to another replica in the global table. | long | +| aws.dynamodb.metrics.ProvisionedReadCapacityUnits.avg | The number of provisioned read capacity units for a table or a global secondary index. | double | +| aws.dynamodb.metrics.ProvisionedWriteCapacityUnits.avg | The number of provisioned write capacity units for a table or a global secondary index. | double | +| aws.dynamodb.metrics.ReadThrottleEvents.sum | Requests to DynamoDB that exceed the provisioned read capacity units for a table or a global secondary index. | long | +| aws.dynamodb.metrics.ReplicationLatency.avg | | double | +| aws.dynamodb.metrics.ReplicationLatency.max | | double | +| aws.dynamodb.metrics.SuccessfulRequestLatency.avg | | double | +| aws.dynamodb.metrics.SuccessfulRequestLatency.max | | double | +| aws.dynamodb.metrics.SystemErrors.sum | The requests to DynamoDB or Amazon DynamoDB Streams that generate an HTTP 500 status code during the specified time period. | long | +| aws.dynamodb.metrics.ThrottledRequests.sum | Requests to DynamoDB that exceed the provisioned throughput limits on a resource (such as a table or an index). | long | +| aws.dynamodb.metrics.TransactionConflict.avg | | double | +| aws.dynamodb.metrics.TransactionConflict.sum | | long | +| aws.dynamodb.metrics.WriteThrottleEvents.sum | Requests to DynamoDB that exceed the provisioned write capacity units for a table or a global secondary index. | long | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | diff --git a/test/packages/aws/docs/ebs.md b/test/packages/aws/docs/ebs.md new file mode 100644 index 0000000000..9164abd149 --- /dev/null +++ b/test/packages/aws/docs/ebs.md @@ -0,0 +1,132 @@ +# ebs + +## Metrics + +An example event for `ebs` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:57:22.450Z", + "service": { + "type": "aws" + }, + "aws": { + "ebs": { + "metrics": { + "VolumeReadOps": { + "avg": 0 + }, + "VolumeQueueLength": { + "avg": 0.0000666666666666667 + }, + "VolumeWriteOps": { + "avg": 29 + }, + "VolumeTotalWriteTime": { + "sum": 0.02 + }, + "BurstBalance": { + "avg": 100 + }, + "VolumeWriteBytes": { + "avg": 14406.620689655172 + }, + "VolumeIdleTime": { + "sum": 299.98 + } + } + }, + "cloudwatch": { + "namespace": "AWS/EBS" + }, + "dimensions": { + "VolumeId": "vol-03370a204cc8b0a2f" + } + }, + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "event": { + "dataset": "aws.ebs", + "module": "aws", + "duration": 10488314037 + }, + "metricset": { + "period": 300000, + "name": "ebs" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.VolumeId | Amazon EBS volume ID | keyword | +| aws.ebs.metrics.BurstBalance.avg | Used with General Purpose SSD (gp2), Throughput Optimized HDD (st1), and Cold HDD (sc1) volumes only. Provides information about the percentage of I/O credits (for gp2) or throughput credits (for st1 and sc1) remaining in the burst bucket. | double | +| aws.ebs.metrics.VolumeConsumedReadWriteOps.avg | The total amount of read and write operations (normalized to 256K capacity units) consumed in a specified period of time. Used with Provisioned IOPS SSD volumes only. | double | +| aws.ebs.metrics.VolumeIdleTime.sum | The total number of seconds in a specified period of time when no read or write operations were submitted. | double | +| aws.ebs.metrics.VolumeQueueLength.avg | The number of read and write operation requests waiting to be completed in a specified period of time. | double | +| aws.ebs.metrics.VolumeReadBytes.avg | Average size of each read operation during the period, except on volumes attached to a Nitro-based instance, where the average represents the average over the specified period. | double | +| aws.ebs.metrics.VolumeReadOps.avg | The total number of read operations in a specified period of time. | double | +| aws.ebs.metrics.VolumeThroughputPercentage.avg | The percentage of I/O operations per second (IOPS) delivered of the total IOPS provisioned for an Amazon EBS volume. Used with Provisioned IOPS SSD volumes only. | double | +| aws.ebs.metrics.VolumeTotalReadTime.sum | The total number of seconds spent by all read operations that completed in a specified period of time. | double | +| aws.ebs.metrics.VolumeTotalWriteTime.sum | The total number of seconds spent by all write operations that completed in a specified period of time. | double | +| aws.ebs.metrics.VolumeWriteBytes.avg | Average size of each write operation during the period, except on volumes attached to a Nitro-based instance, where the average represents the average over the specified period. | double | +| aws.ebs.metrics.VolumeWriteOps.avg | The total number of write operations in a specified period of time. | double | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | diff --git a/test/packages/aws/docs/ec2.md b/test/packages/aws/docs/ec2.md new file mode 100644 index 0000000000..b0e5741fc7 --- /dev/null +++ b/test/packages/aws/docs/ec2.md @@ -0,0 +1,256 @@ +# ec2 + +## Logs + +The `ec2` dataset is specifically for EC2 logs stored in AWS CloudWatch. Export logs +from log groups to Amazon S3 bucket which has SQS notification setup already. +With this dataset, EC2 logs will be parsed into fields like `ip_address` +and `process.name`. For logs from other services, please use `cloudwatch` dataset. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.ec2.ip_address | The internet address of the requester. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| process.name | Process name. | keyword | + + +## Metrics + +An example event for `ec2` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:56:37.255Z", + "aws": { + "ec2": { + "network": { + "in": { + "packets": 448.4, + "bytes_per_sec": 103.10266666666666, + "packets_per_sec": 1.4946666666666666, + "bytes": 30930.8 + }, + "out": { + "packets": 233.6, + "bytes_per_sec": 51.754666666666665, + "packets_per_sec": 0.7786666666666666, + "bytes": 15526.4 + } + }, + "status": { + "check_failed": 0, + "check_failed_instance": 0, + "check_failed_system": 0 + }, + "cpu": { + "credit_usage": 0.004566, + "credit_balance": 144, + "surplus_credit_balance": 0, + "surplus_credits_charged": 0, + "total": { + "pct": 0.0999999999997574 + } + }, + "diskio": { + "read": { + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0, + "count": 0 + }, + "write": { + "count": 0, + "bytes_per_sec": 0, + "count_per_sec": 0, + "bytes": 0 + } + }, + "instance": { + "core": { + "count": 1 + }, + "threads_per_core": 1, + "public": { + "ip": "3.122.204.80", + "dns_name": "" + }, + "private": { + "ip": "10.0.0.122", + "dns_name": "ip-10-0-0-122.eu-central-1.compute.internal" + }, + "image": { + "id": "ami-0b418580298265d5c" + }, + "state": { + "name": "running", + "code": 16 + }, + "monitoring": { + "state": "disabled" + } + } + } + }, + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + }, + "ecs": { + "version": "1.5.0" + }, + "event": { + "module": "aws", + "duration": 23217499283, + "dataset": "aws.ec2" + }, + "metricset": { + "period": 300000, + "name": "ec2" + }, + "service": { + "type": "aws" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "instance": { + "id": "i-04c1a32c2aace6b40" + }, + "machine": { + "type": "t2.micro" + }, + "availability_zone": "eu-central-1a" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.AutoScalingGroupName | An Auto Scaling group is a collection of instances you define if you're using Auto Scaling. | keyword | +| aws.dimensions.ImageId | This dimension filters the data you request for all instances running this Amazon EC2 Amazon Machine Image (AMI) | keyword | +| aws.dimensions.InstanceId | Amazon EC2 instance ID | keyword | +| aws.dimensions.InstanceType | This dimension filters the data you request for all instances running with this specified instance type. | keyword | +| aws.ec2.cpu.credit_balance | The number of earned CPU credits that an instance has accrued since it was launched or started. | long | +| aws.ec2.cpu.credit_usage | The number of CPU credits spent by the instance for CPU utilization. | long | +| aws.ec2.cpu.surplus_credit_balance | The number of surplus credits that have been spent by an unlimited instance when its CPUCreditBalance value is zero. | long | +| aws.ec2.cpu.surplus_credits_charged | The number of spent surplus credits that are not paid down by earned CPU credits, and which thus incur an additional charge. | long | +| aws.ec2.cpu.total.pct | The percentage of allocated EC2 compute units that are currently in use on the instance. | scaled_float | +| aws.ec2.diskio.read.bytes | Bytes read from all instance store volumes available to the instance. | long | +| aws.ec2.diskio.read.bytes_per_sec | Bytes read per second from all instance store volumes available to the instance. | long | +| aws.ec2.diskio.read.count | Completed read operations from all instance store volumes available to the instance in a specified period of time. | long | +| aws.ec2.diskio.read.count_per_sec | Completed read operations per second from all instance store volumes available to the instance in a specified period of time. | long | +| aws.ec2.diskio.write.bytes | Bytes written to all instance store volumes available to the instance. | long | +| aws.ec2.diskio.write.bytes_per_sec | Bytes written per second to all instance store volumes available to the instance. | long | +| aws.ec2.diskio.write.count | Completed write operations to all instance store volumes available to the instance in a specified period of time. | long | +| aws.ec2.diskio.write.count_per_sec | Completed write operations per second to all instance store volumes available to the instance in a specified period of time. | long | +| aws.ec2.instance.core.count | The number of CPU cores for the instance. | integer | +| aws.ec2.instance.image.id | The ID of the image used to launch the instance. | keyword | +| aws.ec2.instance.monitoring.state | Indicates whether detailed monitoring is enabled. | keyword | +| aws.ec2.instance.private.dns_name | The private DNS name of the network interface. | keyword | +| aws.ec2.instance.private.ip | The private IPv4 address associated with the network interface. | ip | +| aws.ec2.instance.public.dns_name | The public DNS name of the instance. | keyword | +| aws.ec2.instance.public.ip | The address of the Elastic IP address (IPv4) bound to the network interface. | ip | +| aws.ec2.instance.state.code | The state of the instance, as a 16-bit unsigned integer. | integer | +| aws.ec2.instance.state.name | The state of the instance (pending | running | shutting-down | terminated | stopping | stopped). | keyword | +| aws.ec2.instance.threads_per_core | The number of threads per CPU core. | integer | +| aws.ec2.network.in.bytes | The number of bytes received on all network interfaces by the instance. | long | +| aws.ec2.network.in.bytes_per_sec | The number of bytes per second received on all network interfaces by the instance. | long | +| aws.ec2.network.in.packets | The number of packets received on all network interfaces by the instance. | long | +| aws.ec2.network.in.packets_per_sec | The number of packets per second sent out on all network interfaces by the instance. | long | +| aws.ec2.network.out.bytes | The number of bytes sent out on all network interfaces by the instance. | long | +| aws.ec2.network.out.bytes_per_sec | The number of bytes per second sent out on all network interfaces by the instance. | long | +| aws.ec2.network.out.packets | The number of packets sent out on all network interfaces by the instance. | long | +| aws.ec2.network.out.packets_per_sec | The number of packets per second sent out on all network interfaces by the instance. | long | +| aws.ec2.status.check_failed | Reports whether the instance has passed both the instance status check and the system status check in the last minute. | long | +| aws.ec2.status.check_failed_instance | Reports whether the instance has passed the instance status check in the last minute. | long | +| aws.ec2.status.check_failed_system | Reports whether the instance has passed the system status check in the last minute. | long | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.cpu.pct | Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 1. | scaled_float | +| host.disk.read.bytes | The total number of bytes read successfully in a given period of time. | long | +| host.disk.write.bytes | The total number of bytes write successfully in a given period of time. | long | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.network.in.bytes | The number of bytes received on all network interfaces by the host in a given period of time. | long | +| host.network.in.packets | The number of packets received on all network interfaces by the host in a given period of time. | long | +| host.network.out.bytes | The number of bytes sent out on all network interfaces by the host in a given period of time. | long | +| host.network.out.packets | The number of packets sent out on all network interfaces by the host in a given period of time. | long | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | diff --git a/test/packages/aws/docs/elb.md b/test/packages/aws/docs/elb.md new file mode 100644 index 0000000000..f7cf15fe4b --- /dev/null +++ b/test/packages/aws/docs/elb.md @@ -0,0 +1,281 @@ +# elb + +## Logs + +The `elb` dataset collects logs from AWS ELBs. Elastic Load Balancing provides +access logs that capture detailed information about requests sent to the load +balancer. Each log contains information such as the time the request was +received, the client's IP address, latencies, request paths, and server +responses. Users can use these access logs to analyze traffic patterns and to +troubleshoot issues. + +Please follow [enable access logs for classic load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-access-logs.html) +for sending Classic ELB access logs to S3 bucket. +For application load balancer, please follow [enable access log for application load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html#enable-access-logging). +For network load balancer, please follow [enable access log for network load balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest//network/load-balancer-access-logs.html). + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.elb.action_executed | The action executed when processing the request (forward, fixed-response, authenticate...). It can contain several values. | keyword | +| aws.elb.backend.http.response.status_code | The status code from the backend (status code sent to the client from ELB is stored in `http.response.status_code` | long | +| aws.elb.backend.ip | The IP address of the backend processing this connection. | keyword | +| aws.elb.backend.port | The port in the backend processing this connection. | keyword | +| aws.elb.backend_processing_time.sec | The total time in seconds since the connection is sent to the backend till the backend starts responding. | float | +| aws.elb.chosen_cert.arn | The ARN of the chosen certificate presented to the client in TLS/SSL connections. | keyword | +| aws.elb.chosen_cert.serial | The serial number of the chosen certificate presented to the client in TLS/SSL connections. | keyword | +| aws.elb.classification | The classification for desync mitigation. | keyword | +| aws.elb.classification_reason | The classification reason code. | keyword | +| aws.elb.connection_time.ms | The total time of the connection in milliseconds, since it is opened till it is closed. | long | +| aws.elb.error.reason | The error reason if the executed action failed. | keyword | +| aws.elb.incoming_tls_alert | The integer value of TLS alerts received by the load balancer from the client, if present. | keyword | +| aws.elb.listener | The ELB listener that received the connection. | keyword | +| aws.elb.matched_rule_priority | The priority value of the rule that matched the request, if a rule matched. | keyword | +| aws.elb.name | The name of the load balancer. | keyword | +| aws.elb.protocol | The protocol of the load balancer (http or tcp). | keyword | +| aws.elb.redirect_url | The URL used if a redirection action was executed. | keyword | +| aws.elb.request_processing_time.sec | The total time in seconds since the connection or request is received until it is sent to a registered backend. | float | +| aws.elb.response_processing_time.sec | The total time in seconds since the response is received from the backend till it is sent to the client. | float | +| aws.elb.ssl_cipher | The SSL cipher used in TLS/SSL connections. | keyword | +| aws.elb.ssl_protocol | The SSL protocol used in TLS/SSL connections. | keyword | +| aws.elb.target_group.arn | The ARN of the target group handling the request. | keyword | +| aws.elb.target_port | List of IP addresses and ports for the targets that processed this request. | keyword | +| aws.elb.target_status_code | List of status codes from the responses of the targets. | keyword | +| aws.elb.tls_handshake_time.ms | The total time for the TLS handshake to complete in milliseconds once the connection has been established. | long | +| aws.elb.tls_named_group | The TLS named group. | keyword | +| aws.elb.trace_id | The contents of the `X-Amzn-Trace-Id` header. | keyword | +| aws.elb.type | The type of the load balancer for v2 Load Balancers. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.bytes | Bytes sent from the destination to the source. | long | +| destination.domain | Destination domain. | keyword | +| event.category | Event category (e.g. database) | keyword | +| event.end | event.end contains the date when the event ended or when the activity was last observed. | date | +| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, sig | keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | +| event.start | event.start contains the date when the event started or when the activity was first observed. | date | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.body.bytes | Size in bytes of the request body. | long | +| http.request.method | HTTP request method. | keyword | +| http.request.referrer | Referrer for this HTTP request. | keyword | +| http.response.body.bytes | Size in bytes of the response body. | long | +| http.response.status_code | HTTP response status code. | long | +| http.version | HTTP version. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source. | ip | +| source.port | Port of the source. | keyword | +| tracing.trace.id | Unique identifier of the trace. | keyword | +| user_agent.original | Unparsed user_agent string. | keyword | + + +## Metrics + +An example event for `elb` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:30.211Z", + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-central-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + } + }, + "aws": { + "elb": { + "metrics": { + "EstimatedALBNewConnectionCount": { + "avg": 32 + }, + "EstimatedALBConsumedLCUs": { + "avg": 0.00035000000000000005 + }, + "EstimatedProcessedBytes": { + "avg": 967 + }, + "EstimatedALBActiveConnectionCount": { + "avg": 5 + }, + "HealthyHostCount": { + "max": 2 + }, + "UnHealthyHostCount": { + "max": 0 + } + } + }, + "cloudwatch": { + "namespace": "AWS/ELB" + }, + "dimensions": { + "LoadBalancerName": "filebeat-aws-elb-test-elb" + } + }, + "metricset": { + "name": "elb", + "period": 60000 + }, + "event": { + "dataset": "aws.elb", + "module": "aws", + "duration": 15044430616 + }, + "service": { + "type": "aws" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.applicationelb.metrics.ActiveConnectionCount.sum | The total number of concurrent TCP connections active from clients to the load balancer and from the load balancer to targets. | long | +| aws.applicationelb.metrics.ClientTLSNegotiationErrorCount.sum | The number of TLS connections initiated by the client that did not establish a session with the load balancer due to a TLS error. | long | +| aws.applicationelb.metrics.ConsumedLCUs.avg | The number of load balancer capacity units (LCU) used by your load balancer. | double | +| aws.applicationelb.metrics.HTTPCode_ELB_3XX_Count.sum | The number of HTTP 3XX redirection codes that originate from the load balancer. | long | +| aws.applicationelb.metrics.HTTPCode_ELB_4XX_Count.sum | The number of HTTP 4XX client error codes that originate from the load balancer. | long | +| aws.applicationelb.metrics.HTTPCode_ELB_500_Count.sum | The number of HTTP 500 error codes that originate from the load balancer. | long | +| aws.applicationelb.metrics.HTTPCode_ELB_502_Count.sum | The number of HTTP 502 error codes that originate from the load balancer. | long | +| aws.applicationelb.metrics.HTTPCode_ELB_503_Count.sum | The number of HTTP 503 error codes that originate from the load balancer. | long | +| aws.applicationelb.metrics.HTTPCode_ELB_504_Count.sum | The number of HTTP 504 error codes that originate from the load balancer. | long | +| aws.applicationelb.metrics.HTTPCode_ELB_5XX_Count.sum | The number of HTTP 5XX server error codes that originate from the load balancer. | long | +| aws.applicationelb.metrics.HTTP_Fixed_Response_Count.sum | The number of fixed-response actions that were successful. | long | +| aws.applicationelb.metrics.HTTP_Redirect_Count.sum | The number of redirect actions that were successful. | long | +| aws.applicationelb.metrics.HTTP_Redirect_Url_Limit_Exceeded_Count.sum | The number of redirect actions that couldn't be completed because the URL in the response location header is larger than 8K. | long | +| aws.applicationelb.metrics.IPv6ProcessedBytes.sum | The total number of bytes processed by the load balancer over IPv6. | long | +| aws.applicationelb.metrics.IPv6RequestCount.sum | The number of IPv6 requests received by the load balancer. | long | +| aws.applicationelb.metrics.NewConnectionCount.sum | The total number of new TCP connections established from clients to the load balancer and from the load balancer to targets. | long | +| aws.applicationelb.metrics.ProcessedBytes.sum | The total number of bytes processed by the load balancer over IPv4 and IPv6. | long | +| aws.applicationelb.metrics.RejectedConnectionCount.sum | The number of connections that were rejected because the load balancer had reached its maximum number of connections. | long | +| aws.applicationelb.metrics.RequestCount.sum | The number of requests processed over IPv4 and IPv6. | long | +| aws.applicationelb.metrics.RuleEvaluations.sum | The number of rules processed by the load balancer given a request rate averaged over an hour. | long | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.AvailabilityZone | Filters the metric data by the specified Availability Zone. | keyword | +| aws.dimensions.LoadBalancer | Filters the metric data by load balancer. | keyword | +| aws.dimensions.LoadBalancerName | Filters the metric data by the specified load balancer. | keyword | +| aws.dimensions.TargetGroup | Filters the metric data by target group. | keyword | +| aws.elb.metrics.BackendConnectionErrors.sum | The number of connections that were not successfully established between the load balancer and the registered instances. | long | +| aws.elb.metrics.EstimatedALBActiveConnectionCount.avg | The estimated number of concurrent TCP connections active from clients to the load balancer and from the load balancer to targets. | double | +| aws.elb.metrics.EstimatedALBConsumedLCUs.avg | The estimated number of load balancer capacity units (LCU) used by an Application Load Balancer. | double | +| aws.elb.metrics.EstimatedALBNewConnectionCount.avg | The estimated number of new TCP connections established from clients to the load balancer and from the load balancer to targets. | double | +| aws.elb.metrics.EstimatedProcessedBytes.avg | The estimated number of bytes processed by an Application Load Balancer. | double | +| aws.elb.metrics.HTTPCode_Backend_2XX.sum | The number of HTTP 2XX response code generated by registered instances. | long | +| aws.elb.metrics.HTTPCode_Backend_3XX.sum | The number of HTTP 3XX response code generated by registered instances. | long | +| aws.elb.metrics.HTTPCode_Backend_4XX.sum | The number of HTTP 4XX response code generated by registered instances. | long | +| aws.elb.metrics.HTTPCode_Backend_5XX.sum | The number of HTTP 5XX response code generated by registered instances. | long | +| aws.elb.metrics.HTTPCode_ELB_4XX.sum | The number of HTTP 4XX client error codes generated by the load balancer. | long | +| aws.elb.metrics.HTTPCode_ELB_5XX.sum | The number of HTTP 5XX server error codes generated by the load balancer. | long | +| aws.elb.metrics.HealthyHostCount.max | The number of healthy instances registered with your load balancer. | long | +| aws.elb.metrics.Latency.avg | The total time elapsed, in seconds, from the time the load balancer sent the request to a registered instance until the instance started to send the response headers. | double | +| aws.elb.metrics.RequestCount.sum | The number of requests completed or connections made during the specified interval. | long | +| aws.elb.metrics.SpilloverCount.sum | The total number of requests that were rejected because the surge queue is full. | long | +| aws.elb.metrics.SurgeQueueLength.max | The total number of requests (HTTP listener) or connections (TCP listener) that are pending routing to a healthy instance. | long | +| aws.elb.metrics.UnHealthyHostCount.max | The number of unhealthy instances registered with your load balancer. | long | +| aws.networkelb.metrics.ActiveFlowCount.avg | The total number of concurrent flows (or connections) from clients to targets. | double | +| aws.networkelb.metrics.ActiveFlowCount_TCP.avg | The total number of concurrent TCP flows (or connections) from clients to targets. | double | +| aws.networkelb.metrics.ActiveFlowCount_TLS.avg | The total number of concurrent TLS flows (or connections) from clients to targets. | double | +| aws.networkelb.metrics.ActiveFlowCount_UDP.avg | The total number of concurrent UDP flows (or connections) from clients to targets. | double | +| aws.networkelb.metrics.ClientTLSNegotiationErrorCount.sum | The total number of TLS handshakes that failed during negotiation between a client and a TLS listener. | long | +| aws.networkelb.metrics.ConsumedLCUs.avg | The number of load balancer capacity units (LCU) used by your load balancer. | double | +| aws.networkelb.metrics.HealthyHostCount.max | The number of targets that are considered healthy. | long | +| aws.networkelb.metrics.NewFlowCount.sum | The total number of new flows (or connections) established from clients to targets in the time period. | long | +| aws.networkelb.metrics.NewFlowCount_TLS.sum | The total number of new TLS flows (or connections) established from clients to targets in the time period. | long | +| aws.networkelb.metrics.ProcessedBytes.sum | The total number of bytes processed by the load balancer, including TCP/IP headers. | long | +| aws.networkelb.metrics.ProcessedBytes_TLS.sum | The total number of bytes processed by TLS listeners. | long | +| aws.networkelb.metrics.TCP_Client_Reset_Count.sum | The total number of reset (RST) packets sent from a client to a target. | long | +| aws.networkelb.metrics.TCP_ELB_Reset_Count.sum | The total number of reset (RST) packets generated by the load balancer. | long | +| aws.networkelb.metrics.TCP_Target_Reset_Count.sum | The total number of reset (RST) packets sent from a target to a client. | long | +| aws.networkelb.metrics.TargetTLSNegotiationErrorCount.sum | The total number of TLS handshakes that failed during negotiation between a TLS listener and a target. | long | +| aws.networkelb.metrics.UnHealthyHostCount.max | The number of targets that are considered unhealthy. | long | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | + diff --git a/test/packages/aws/docs/lambda.md b/test/packages/aws/docs/lambda.md new file mode 100644 index 0000000000..c5cadae76e --- /dev/null +++ b/test/packages/aws/docs/lambda.md @@ -0,0 +1,129 @@ +# lambda + +## Metrics + +An example event for `lambda` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:17:08.666Z", + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + }, + "event": { + "dataset": "aws.dynamodb", + "module": "aws", + "duration": 10266182336 + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/Lambda" + }, + "dimensions": { + "FunctionName": "ec2-owner-tagger-serverless", + "Resource": "ec2-owner-tagger-serverless" + }, + "lambda": { + "metrics": { + "Duration": { + "avg": 8218.073333333334 + }, + "Errors": { + "avg": 1 + }, + "Invocations": { + "avg": 1 + }, + "Throttles": { + "avg": 0 + } + } + } + }, + "metricset": { + "name": "dynamodb", + "period": 300000 + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.ExecutedVersion | Use the ExecutedVersion dimension to compare error rates for two versions of a function that are both targets of a weighted alias. | keyword | +| aws.dimensions.FunctionName | Lambda function name. | keyword | +| aws.dimensions.Resource | Resource name. | keyword | +| aws.lambda.metrics.ConcurrentExecutions.avg | The number of function instances that are processing events. | double | +| aws.lambda.metrics.DeadLetterErrors.avg | For asynchronous invocation, the number of times Lambda attempts to send an event to a dead-letter queue but fails. | double | +| aws.lambda.metrics.DestinationDeliveryFailures.avg | For asynchronous invocation, the number of times Lambda attempts to send an event to a destination but fails. | double | +| aws.lambda.metrics.Duration.avg | The amount of time that your function code spends processing an event. | double | +| aws.lambda.metrics.Errors.avg | The number of invocations that result in a function error. | double | +| aws.lambda.metrics.Invocations.avg | The number of times your function code is executed, including successful executions and executions that result in a function error. | double | +| aws.lambda.metrics.IteratorAge.avg | For event source mappings that read from streams, the age of the last record in the event. | double | +| aws.lambda.metrics.ProvisionedConcurrencyInvocations.sum | The number of times your function code is executed on provisioned concurrency. | long | +| aws.lambda.metrics.ProvisionedConcurrencySpilloverInvocations.sum | The number of times your function code is executed on standard concurrency when all provisioned concurrency is in use. | long | +| aws.lambda.metrics.ProvisionedConcurrencyUtilization.max | For a version or alias, the value of ProvisionedConcurrentExecutions divided by the total amount of provisioned concurrency allocated. | long | +| aws.lambda.metrics.ProvisionedConcurrentExecutions.max | The number of function instances that are processing events on provisioned concurrency. | long | +| aws.lambda.metrics.Throttles.avg | The number of invocation requests that are throttled. | double | +| aws.lambda.metrics.UnreservedConcurrentExecutions.avg | For an AWS Region, the number of events that are being processed by functions that don't have reserved concurrency. | double | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | + diff --git a/test/packages/aws/docs/natgateway.md b/test/packages/aws/docs/natgateway.md new file mode 100644 index 0000000000..b9ef0d503d --- /dev/null +++ b/test/packages/aws/docs/natgateway.md @@ -0,0 +1,153 @@ +# natgateway + +## Metrics + +An example event for `natgateway` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/NATGateway" + }, + "dimensions": { + "NatGatewayId": "nat-0a5cb7b9807908cc0" + }, + "natgateway": { + "metrics": { + "ActiveConnectionCount": { + "max": 0 + }, + "BytesInFromDestination": { + "sum": 0 + }, + "BytesInFromSource": { + "sum": 0 + }, + "BytesOutToDestination": { + "sum": 0 + }, + "BytesOutToSource": { + "sum": 0 + }, + "ConnectionAttemptCount": { + "sum": 0 + }, + "ConnectionEstablishedCount": { + "sum": 0 + }, + "ErrorPortAllocation": { + "sum": 0 + }, + "PacketsDropCount": { + "sum": 0 + }, + "PacketsInFromDestination": { + "sum": 0 + }, + "PacketsInFromSource": { + "sum": 0 + }, + "PacketsOutToDestination": { + "sum": 0 + }, + "PacketsOutToSource": { + "sum": 0 + } + } + } + }, + "event": { + "dataset": "aws.natgateway", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "natgateway" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.NatGatewayId | Filter the metric data by the NAT gateway ID. | keyword | +| aws.natgateway.metrics.ActiveConnectionCount.max | The total number of concurrent active TCP connections through the NAT gateway. | long | +| aws.natgateway.metrics.BytesInFromDestination.sum | The number of bytes received by the NAT gateway from the destination. | long | +| aws.natgateway.metrics.BytesInFromSource.sum | The number of bytes received by the NAT gateway from clients in your VPC. | long | +| aws.natgateway.metrics.BytesOutToDestination.sum | The number of bytes sent out through the NAT gateway to the destination. | long | +| aws.natgateway.metrics.BytesOutToSource.sum | The number of bytes sent through the NAT gateway to the clients in your VPC. | long | +| aws.natgateway.metrics.ConnectionAttemptCount.sum | The number of connection attempts made through the NAT gateway. | long | +| aws.natgateway.metrics.ConnectionEstablishedCount.sum | The number of connections established through the NAT gateway. | long | +| aws.natgateway.metrics.ErrorPortAllocation.sum | The number of times the NAT gateway could not allocate a source port. | long | +| aws.natgateway.metrics.IdleTimeoutCount.sum | The number of connections that transitioned from the active state to the idle state. | long | +| aws.natgateway.metrics.PacketsDropCount.sum | The number of packets dropped by the NAT gateway. | long | +| aws.natgateway.metrics.PacketsInFromDestination.sum | The number of packets received by the NAT gateway from the destination. | long | +| aws.natgateway.metrics.PacketsInFromSource.sum | The number of packets received by the NAT gateway from clients in your VPC. | long | +| aws.natgateway.metrics.PacketsOutToDestination.sum | The number of packets sent out through the NAT gateway to the destination. | long | +| aws.natgateway.metrics.PacketsOutToSource.sum | The number of packets sent through the NAT gateway to the clients in your VPC. | long | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | diff --git a/test/packages/aws/docs/rds.md b/test/packages/aws/docs/rds.md new file mode 100644 index 0000000000..dc86925809 --- /dev/null +++ b/test/packages/aws/docs/rds.md @@ -0,0 +1,223 @@ +# rds + +## Metrics + +An example event for `rds` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:34.537Z", + "ecs": { + "version": "1.5.0" + }, + "service": { + "type": "aws" + }, + "aws": { + "rds": { + "latency": { + "dml": 0, + "insert": 0, + "update": 0, + "commit": 0, + "ddl": 0, + "delete": 0, + "select": 0.21927814569536422 + }, + "queries": 6.197934021992669, + "aurora_bin_log_replica_lag": 0, + "transactions": { + "blocked": 0, + "active": 0 + }, + "deadlocks": 0, + "login_failures": 0, + "throughput": { + "network": 1.399813358218904, + "insert": 0, + "ddl": 0, + "select": 2.5165408396246853, + "delete": 0, + "commit": 0, + "network_transmit": 0.699906679109452, + "update": 0, + "dml": 0, + "network_receive": 0.699906679109452 + }, + "cpu": { + "total": { + "pct": 0.03 + } + }, + "db_instance": { + "arn": "arn:aws:rds:eu-west-1:428152502467:db:database-1-instance-1-eu-west-1a", + "class": "db.r5.large", + "identifier": "database-1-instance-1-eu-west-1a", + "status": "available" + }, + "cache_hit_ratio.result_set": 0, + "aurora_replica.lag.ms": 19.576, + "free_local_storage.bytes": 32431271936, + "cache_hit_ratio.buffer": 100, + "disk_usage": { + "bin_log.bytes": 0 + }, + "db_instance.identifier": "database-1-instance-1-eu-west-1a", + "freeable_memory.bytes": 4436537344, + "engine_uptime.sec": 10463030, + "database_connections": 0 + } + }, + "cloud": { + "provider": "aws", + "region": "eu-west-1", + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "availability_zone": "eu-west-1a" + }, + "event": { + "dataset": "aws.rds", + "module": "aws", + "duration": 10777919184 + }, + "metricset": { + "name": "rds", + "period": 60000 + }, + "agent": { + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.DBClusterIdentifier | This dimension filters the data that you request for a specific Amazon Aurora DB cluster. | keyword | +| aws.dimensions.DBClusterIdentifier,Role | This dimension filters the data that you request for a specific Aurora DB cluster, aggregating the metric by instance role (WRITER/READER). | keyword | +| aws.dimensions.DBInstanceIdentifier | This dimension filters the data that you request for a specific DB instance. | keyword | +| aws.dimensions.DatabaseClass | This dimension filters the data that you request for all instances in a database class. | keyword | +| aws.dimensions.DbClusterIdentifier, EngineName | This dimension filters the data that you request for a specific Aurora DB cluster, aggregating the metric by engine name. | keyword | +| aws.dimensions.EngineName | This dimension filters the data that you request for the identified engine name only. | keyword | +| aws.dimensions.SourceRegion | This dimension filters the data that you request for the specified region only. | keyword | +| aws.rds.aurora_bin_log_replica_lag | The amount of time a replica DB cluster running on Aurora with MySQL compatibility lags behind the source DB cluster. | long | +| aws.rds.aurora_global_db.data_transfer.bytes | In an Aurora Global Database, the amount of redo log data transferred from the master AWS Region to a secondary AWS Region. | long | +| aws.rds.aurora_global_db.replicated_write_io.bytes | In an Aurora Global Database, the number of write I/O operations replicated from the primary AWS Region to the cluster volume in a secondary AWS Region. | long | +| aws.rds.aurora_global_db.replication_lag.ms | For an Aurora Global Database, the amount of lag when replicating updates from the primary AWS Region, in milliseconds. | long | +| aws.rds.aurora_replica.lag.ms | For an Aurora Replica, the amount of lag when replicating updates from the primary instance, in milliseconds. | long | +| aws.rds.aurora_replica.lag_max.ms | The maximum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds. | long | +| aws.rds.aurora_replica.lag_min.ms | The minimum amount of lag between the primary instance and each Aurora DB instance in the DB cluster, in milliseconds. | long | +| aws.rds.aurora_volume_left_total.bytes | The remaining available space for the cluster volume, measured in bytes. | long | +| aws.rds.backtrack_change_records.creation_rate | The number of backtrack change records created over five minutes for your DB cluster. | long | +| aws.rds.backtrack_change_records.stored | The actual number of backtrack change records used by your DB cluster. | long | +| aws.rds.backtrack_window.actual | The difference between the target backtrack window and the actual backtrack window. | long | +| aws.rds.backtrack_window.alert | The number of times that the actual backtrack window is smaller than the target backtrack window for a given period of time. | long | +| aws.rds.backup_storage_billed_total.bytes | The total amount of backup storage in bytes for which you are billed for a given Aurora DB cluster. | long | +| aws.rds.cache_hit_ratio.buffer | The percentage of requests that are served by the buffer cache. | long | +| aws.rds.cache_hit_ratio.result_set | The percentage of requests that are served by the Resultset cache. | long | +| aws.rds.cpu.credit_balance | The number of earned CPU credits that an instance has accrued since it was launched or started. | long | +| aws.rds.cpu.credit_usage | The number of CPU credits spent by the instance for CPU utilization. | long | +| aws.rds.cpu.total.pct | The percentage of CPU utilization. | scaled_float | +| aws.rds.database_connections | The number of database connections in use. | long | +| aws.rds.db_instance.arn | Amazon Resource Name(ARN) for each rds. | keyword | +| aws.rds.db_instance.class | Contains the name of the compute and memory capacity class of the DB instance. | keyword | +| aws.rds.db_instance.db_cluster_identifier | This identifier is the unique key that identifies a DB cluster specifically for Amazon Aurora DB cluster. | keyword | +| aws.rds.db_instance.engine_name | Each DB instance runs a DB engine, like MySQL, MariaDB, PostgreSQL and etc. | keyword | +| aws.rds.db_instance.identifier | Contains a user-supplied database identifier. This identifier is the unique key that identifies a DB instance. | keyword | +| aws.rds.db_instance.role | DB roles like WRITER or READER, specifically for Amazon Aurora DB cluster. | keyword | +| aws.rds.db_instance.status | Specifies the current state of this database. | keyword | +| aws.rds.deadlocks | The average number of deadlocks in the database per second. | long | +| aws.rds.disk_queue_depth | The number of outstanding IOs (read/write requests) waiting to access the disk. | float | +| aws.rds.disk_usage.bin_log.bytes | The amount of disk space occupied by binary logs on the master. Applies to MySQL read replicas. | long | +| aws.rds.disk_usage.replication_slot.mb | The disk space used by replication slot files. Applies to PostgreSQL. | long | +| aws.rds.disk_usage.transaction_logs.mb | The disk space used by transaction logs. Applies to PostgreSQL. | long | +| aws.rds.engine_uptime.sec | The amount of time that the instance has been running, in seconds. | long | +| aws.rds.failed_sql_server_agent_jobs | The number of failed SQL Server Agent jobs during the last minute. | long | +| aws.rds.free_local_storage.bytes | The amount of storage available for temporary tables and logs, in bytes. | long | +| aws.rds.free_storage.bytes | The amount of available storage space. | long | +| aws.rds.freeable_memory.bytes | The amount of available random access memory. | long | +| aws.rds.latency.commit | The amount of latency for commit operations, in milliseconds. | float | +| aws.rds.latency.ddl | The amount of latency for data definition language (DDL) requests, in milliseconds. | float | +| aws.rds.latency.delete | The amount of latency for delete queries, in milliseconds. | float | +| aws.rds.latency.dml | The amount of latency for inserts, updates, and deletes, in milliseconds. | float | +| aws.rds.latency.insert | The amount of latency for insert queries, in milliseconds. | float | +| aws.rds.latency.read | The average amount of time taken per disk I/O operation. | float | +| aws.rds.latency.select | The amount of latency for select queries, in milliseconds. | float | +| aws.rds.latency.update | The amount of latency for update queries, in milliseconds. | float | +| aws.rds.latency.write | The average amount of time taken per disk I/O operation. | float | +| aws.rds.login_failures | The average number of failed login attempts per second. | long | +| aws.rds.maximum_used_transaction_ids | The maximum transaction ID that has been used. Applies to PostgreSQL. | long | +| aws.rds.oldest_replication_slot_lag.mb | The lagging size of the replica lagging the most in terms of WAL data received. Applies to PostgreSQL. | long | +| aws.rds.queries | The average number of queries executed per second. | long | +| aws.rds.rds_to_aurora_postgresql_replica_lag.sec | The amount of lag in seconds when replicating updates from the primary RDS PostgreSQL instance to other nodes in the cluster. | long | +| aws.rds.read_io.ops_per_sec | The average number of disk read I/O operations per second. | float | +| aws.rds.replica_lag.sec | The amount of time a Read Replica DB instance lags behind the source DB instance. Applies to MySQL, MariaDB, and PostgreSQL Read Replicas. | long | +| aws.rds.storage_used.backup_retention_period.bytes | The total amount of backup storage in bytes used to support the point-in-time restore feature within the Aurora DB cluster's backup retention window. | long | +| aws.rds.storage_used.snapshot.bytes | The total amount of backup storage in bytes consumed by all Aurora snapshots for an Aurora DB cluster outside its backup retention window. | long | +| aws.rds.swap_usage.bytes | The amount of swap space used on the DB instance. This metric is not available for SQL Server. | long | +| aws.rds.throughput.commit | The average number of commit operations per second. | float | +| aws.rds.throughput.ddl | The average number of DDL requests per second. | float | +| aws.rds.throughput.delete | The average number of delete queries per second. | float | +| aws.rds.throughput.dml | The average number of inserts, updates, and deletes per second. | float | +| aws.rds.throughput.insert | The average number of insert queries per second. | float | +| aws.rds.throughput.network | The amount of network throughput both received from and transmitted to clients by each instance in the Aurora MySQL DB cluster, in bytes per second. | float | +| aws.rds.throughput.network_receive | The incoming (Receive) network traffic on the DB instance, including both customer database traffic and Amazon RDS traffic used for monitoring and replication. | float | +| aws.rds.throughput.network_transmit | The outgoing (Transmit) network traffic on the DB instance, including both customer database traffic and Amazon RDS traffic used for monitoring and replication. | float | +| aws.rds.throughput.read | The average amount of time taken per disk I/O operation. | float | +| aws.rds.throughput.select | The average number of select queries per second. | float | +| aws.rds.throughput.update | The average number of update queries per second. | float | +| aws.rds.throughput.write | The average number of bytes written to disk per second. | float | +| aws.rds.transaction_logs_generation | The disk space used by transaction logs. Applies to PostgreSQL. | long | +| aws.rds.transactions.active | The average number of current transactions executing on an Aurora database instance per second. | long | +| aws.rds.transactions.blocked | The average number of transactions in the database that are blocked per second. | long | +| aws.rds.volume.read.iops | The number of billed read I/O operations from a cluster volume, reported at 5-minute intervals. | long | +| aws.rds.volume.write.iops | The number of write disk I/O operations to the cluster volume, reported at 5-minute intervals. | long | +| aws.rds.volume_used.bytes | The amount of storage used by your Aurora DB instance, in bytes. | long | +| aws.rds.write_io.ops_per_sec | The average number of disk write I/O operations per second. | float | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | diff --git a/test/packages/aws/docs/s3.md b/test/packages/aws/docs/s3.md new file mode 100644 index 0000000000..fe90ac0310 --- /dev/null +++ b/test/packages/aws/docs/s3.md @@ -0,0 +1,350 @@ +# S3 + +## Logs +The `s3access` dataset collects server access logs from AWS S3. Server access +logging provides detailed records for the requests that are made to a bucket. +Server access logs are useful for many applications. For example, access log +information can be useful in security and access audits. It can also help users +to learn about customer base and understand Amazon S3 bill. + +Please follow [how to enable server access logging](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html#server-access-logging-overview) +for sending server access logs to S3 bucket. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.s3access.authentication_type | The type of request authentication used, AuthHeader for authentication headers, QueryString for query string (pre-signed URL) or a - for unauthenticated requests. | keyword | +| aws.s3access.bucket | The name of the bucket that the request was processed against. | keyword | +| aws.s3access.bucket_owner | The canonical user ID of the owner of the source bucket. | keyword | +| aws.s3access.bytes_sent | The number of response bytes sent, excluding HTTP protocol overhead, or "-" if zero. | long | +| aws.s3access.cipher_suite | The Secure Sockets Layer (SSL) cipher that was negotiated for HTTPS request or a - for HTTP. | keyword | +| aws.s3access.error_code | The Amazon S3 Error Code, or "-" if no error occurred. | keyword | +| aws.s3access.host_header | The endpoint used to connect to Amazon S3. | keyword | +| aws.s3access.host_id | The x-amz-id-2 or Amazon S3 extended request ID. | keyword | +| aws.s3access.http_status | The numeric HTTP status code of the response. | long | +| aws.s3access.key | The "key" part of the request, URL encoded, or "-" if the operation does not take a key parameter. | keyword | +| aws.s3access.object_size | The total size of the object in question. | long | +| aws.s3access.operation | The operation listed here is declared as SOAP.operation, REST.HTTP_method.resource_type, WEBSITE.HTTP_method.resource_type, or BATCH.DELETE.OBJECT. | keyword | +| aws.s3access.referrer | The value of the HTTP Referrer header, if present. | keyword | +| aws.s3access.remote_ip | The apparent internet address of the requester. | ip | +| aws.s3access.request_id | A string generated by Amazon S3 to uniquely identify each request. | keyword | +| aws.s3access.request_uri | The Request-URI part of the HTTP request message. | keyword | +| aws.s3access.requester | The canonical user ID of the requester, or a - for unauthenticated requests. | keyword | +| aws.s3access.signature_version | The signature version, SigV2 or SigV4, that was used to authenticate the request or a - for unauthenticated requests. | keyword | +| aws.s3access.tls_version | The Transport Layer Security (TLS) version negotiated by the client. | keyword | +| aws.s3access.total_time | The number of milliseconds the request was in flight from the server's perspective. | long | +| aws.s3access.turn_around_time | The number of milliseconds that Amazon S3 spent processing your request. | long | +| aws.s3access.user_agent | The value of the HTTP User-Agent header. | keyword | +| aws.s3access.version_id | The version ID in the request, or "-" if the operation does not take a versionId parameter. | keyword | +| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | +| client.ip | IP address of the client. | ip | +| client.user.id | Unique identifiers of the user. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| event.action | The action captured by the event. | keyword | +| event.code | Identification code for this event, if one exists. | keyword | +| event.duration | Duration of the event in nanoseconds. | long | +| event.id | Unique ID to describe the event. | keyword | +| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | +| geo.city_name | City name. | keyword | +| geo.continent_name | Name of the continent. | keyword | +| geo.country_iso_code | Country ISO code. | keyword | +| geo.country_name | Country name. | keyword | +| geo.location | Longitude and latitude. | geo_point | +| geo.region_iso_code | Region ISO code. | keyword | +| geo.region_name | Region name. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| http.request.method | HTTP request method. | keyword | +| http.request.referrer | Referrer for this HTTP request. | keyword | +| http.response.body.bytes | Size in bytes of the response body. | long | +| http.response.status_code | HTTP response status code. | long | +| http.version | HTTP version. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| related.user | All the user names seen on your event. | keyword | +| tls.cipher | String indicating the cipher used during the current connection. | keyword | +| tls.version | Numeric part of the version parsed from the original string. | keyword | +| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | +| url.original | Unmodified original url as seen in the event source. | keyword | +| url.path | Path of the request, such as "/search". | keyword | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". | keyword | +| user_agent.device.name | Name of the device. | keyword | +| user_agent.name | Name of the user agent. | keyword | +| user_agent.original | Unparsed user_agent string. | keyword | +| user_agent.os.full | Operating system name, including the version or code name. | keyword | +| user_agent.os.name | Operating system name, without the version. | keyword | +| user_agent.os.version | Operating system version as a raw string. | keyword | +| user_agent.version | Version of the user agent. | keyword | + + +## Metrics + +### s3_daily_storage + +An example event for `s3_daily_storage` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" + } + }, + "s3_daily_storage": { + "bucket": { + "size": { + "bytes": 207372 + } + }, + "number_of_objects": 128 + } + }, + "event": { + "dataset": "aws.s3_daily_storage", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "s3_daily_storage" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.BucketName | This dimension filters the data you request for the identified bucket only. | keyword | +| aws.dimensions.FilterId | This dimension filters metrics configurations that you specify for request metrics on a bucket, for example, a prefix or a tag. | keyword | +| aws.dimensions.StorageType | This dimension filters the data that you have stored in a bucket by types of storage. | keyword | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.s3_daily_storage.bucket.size.bytes | The amount of data in bytes stored in a bucket. | long | +| aws.s3_daily_storage.number_of_objects | The total number of objects stored in a bucket for all storage classes. | long | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | + + +### s3_request + +An example event for `s3_request` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "s3": { + "bucket": { + "name": "test-s3-ks-2" + } + }, + "s3_request": { + "downloaded": { + "bytes": 534 + }, + "errors": { + "4xx": 0, + "5xx": 0 + }, + "latency": { + "first_byte.ms": 214, + "total_request.ms": 533 + }, + "requests": { + "list": 2, + "put": 10, + "total": 12 + }, + "uploaded": { + "bytes": 13572 + } + } + }, + "event": { + "dataset": "aws.s3_request", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "s3_request" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.BucketName | This dimension filters the data you request for the identified bucket only. | keyword | +| aws.dimensions.FilterId | This dimension filters metrics configurations that you specify for request metrics on a bucket, for example, a prefix or a tag. | keyword | +| aws.dimensions.StorageType | This dimension filters the data that you have stored in a bucket by types of storage. | keyword | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.s3_request.downloaded.bytes | The number bytes downloaded for requests made to an Amazon S3 bucket, where the response includes a body. | long | +| aws.s3_request.errors.4xx | The number of HTTP 4xx client error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. | long | +| aws.s3_request.errors.5xx | The number of HTTP 5xx server error status code requests made to an Amazon S3 bucket with a value of either 0 or 1. | long | +| aws.s3_request.latency.first_byte.ms | The per-request time from the complete request being received by an Amazon S3 bucket to when the response starts to be returned. | long | +| aws.s3_request.latency.total_request.ms | The elapsed per-request time from the first byte received to the last byte sent to an Amazon S3 bucket. | long | +| aws.s3_request.requests.delete | The number of HTTP DELETE requests made for objects in an Amazon S3 bucket. | long | +| aws.s3_request.requests.get | The number of HTTP GET requests made for objects in an Amazon S3 bucket. | long | +| aws.s3_request.requests.head | The number of HTTP HEAD requests made to an Amazon S3 bucket. | long | +| aws.s3_request.requests.list | The number of HTTP requests that list the contents of a bucket. | long | +| aws.s3_request.requests.post | The number of HTTP POST requests made to an Amazon S3 bucket. | long | +| aws.s3_request.requests.put | The number of HTTP PUT requests made for objects in an Amazon S3 bucket. | long | +| aws.s3_request.requests.select | The number of Amazon S3 SELECT Object Content requests made for objects in an Amazon S3 bucket. | long | +| aws.s3_request.requests.select_returned.bytes | The number of bytes of data returned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. | long | +| aws.s3_request.requests.select_scanned.bytes | The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. | long | +| aws.s3_request.requests.total | The total number of HTTP requests made to an Amazon S3 bucket, regardless of type. | long | +| aws.s3_request.uploaded.bytes | The number bytes uploaded that contain a request body, made to an Amazon S3 bucket. | long | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | + diff --git a/test/packages/aws/docs/sns.md b/test/packages/aws/docs/sns.md new file mode 100644 index 0000000000..ad5dfa0bf9 --- /dev/null +++ b/test/packages/aws/docs/sns.md @@ -0,0 +1,128 @@ +# sns + +## Metrics + +An example event for `sns` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "cloudwatch": { + "namespace": "AWS/SNS" + }, + "dimensions": { + "TopicName": "test-sns-ks" + }, + "sns": { + "metrics": { + "NumberOfMessagesPublished": { + "sum": 1 + }, + "NumberOfNotificationsFailed": { + "sum": 1 + }, + "PublishSize": { + "avg": 5 + } + } + }, + "tags": { + "created-by": "ks" + } + }, + "event": { + "dataset": "aws.sns", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "sns" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.Application | Filters on application objects, which represent an app and device registered with one of the supported push notification services, such as APNs and FCM. | keyword | +| aws.dimensions.Application,Platform | Filters on application and platform objects, where the platform objects are for the supported push notification services, such as APNs and FCM. | keyword | +| aws.dimensions.Country | Filters on the destination country or region of an SMS message. | keyword | +| aws.dimensions.Platform | Filters on platform objects for the push notification services, such as APNs and FCM. | keyword | +| aws.dimensions.SMSType | Filters on the message type of SMS message. | keyword | +| aws.dimensions.TopicName | Filters on Amazon SNS topic names. | keyword | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.sns.metrics.NumberOfMessagesPublished.sum | The number of messages published to your Amazon SNS topics. | long | +| aws.sns.metrics.NumberOfNotificationsDelivered.sum | The number of messages successfully delivered from your Amazon SNS topics to subscribing endpoints. | long | +| aws.sns.metrics.NumberOfNotificationsFailed.sum | The number of messages that Amazon SNS failed to deliver. | long | +| aws.sns.metrics.NumberOfNotificationsFailedToRedriveToDlq.sum | The number of messages that couldn't be moved to a dead-letter queue. | long | +| aws.sns.metrics.NumberOfNotificationsFilteredOut-InvalidAttributes.sum | The number of messages that were rejected by subscription filter policies because the messages' attributes are invalid - for example, because the attribute JSON is incorrectly formatted. | long | +| aws.sns.metrics.NumberOfNotificationsFilteredOut-NoMessageAttributes.sum | The number of messages that were rejected by subscription filter policies because the messages have no attributes. | long | +| aws.sns.metrics.NumberOfNotificationsFilteredOut.sum | The number of messages that were rejected by subscription filter policies. | long | +| aws.sns.metrics.NumberOfNotificationsRedrivenToDlq.sum | The number of messages that have been moved to a dead-letter queue. | long | +| aws.sns.metrics.PublishSize.avg | The size of messages published. | double | +| aws.sns.metrics.SMSMonthToDateSpentUSD.sum | The charges you have accrued since the start of the current calendar month for sending SMS messages. | long | +| aws.sns.metrics.SMSSuccessRate.avg | The rate of successful SMS message deliveries. | double | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | diff --git a/test/packages/aws/docs/sqs.md b/test/packages/aws/docs/sqs.md new file mode 100644 index 0000000000..aaed8e9ce8 --- /dev/null +++ b/test/packages/aws/docs/sqs.md @@ -0,0 +1,117 @@ +# sqs + +## Metrics + +An example event for `sqs` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "sqs": { + "empty_receives": 0, + "messages": { + "delayed": 0, + "deleted": 0, + "not_visible": 0, + "received": 0, + "sent": 0, + "visible": 2 + }, + "oldest_message_age": { + "sec": 78494 + }, + "queue": { + "name": "test-s3-notification" + }, + "sent_message_size": {} + } + }, + "event": { + "dataset": "aws.sqs", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "sqs" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.QueueName | SQS queue name | keyword | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.sqs.empty_receives | The number of ReceiveMessage API calls that did not return a message. | long | +| aws.sqs.messages.delayed | TThe number of messages in the queue that are delayed and not available for reading immediately. | long | +| aws.sqs.messages.deleted | The number of messages deleted from the queue. | long | +| aws.sqs.messages.not_visible | The number of messages that are in flight. | long | +| aws.sqs.messages.received | The number of messages returned by calls to the ReceiveMessage action. | long | +| aws.sqs.messages.sent | The number of messages added to a queue. | long | +| aws.sqs.messages.visible | The number of messages available for retrieval from the queue. | long | +| aws.sqs.oldest_message_age.sec | The approximate age of the oldest non-deleted message in the queue. | long | +| aws.sqs.queue.name | SQS queue name | keyword | +| aws.sqs.sent_message_size.bytes | The size of messages added to a queue. | long | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | diff --git a/test/packages/aws/docs/transitgateway.md b/test/packages/aws/docs/transitgateway.md new file mode 100644 index 0000000000..2250173d04 --- /dev/null +++ b/test/packages/aws/docs/transitgateway.md @@ -0,0 +1,126 @@ +# transitgateway + +## Metrics + +An example event for `transitgateway` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T20:10:20.953Z", + "cloud": { + "provider": "aws", + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "aws": { + "transitgateway": { + "metrics": { + "PacketsIn": { + "sum": 0 + }, + "BytesIn": { + "sum": 0 + }, + "BytesOut": { + "sum": 0 + }, + "PacketsOut": { + "sum": 0 + }, + "PacketDropCountBlackhole": { + "sum": 0 + }, + "PacketDropCountNoRoute": { + "sum": 0 + } + } + }, + "cloudwatch": { + "namespace": "AWS/TransitGateway" + }, + "dimensions": { + "TransitGateway": "tgw-0630672a32f12808a" + } + }, + "ecs": { + "version": "1.5.0" + }, + "agent": { + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" + }, + "event": { + "dataset": "aws.transitgateway", + "module": "aws", + "duration": 12762825681 + }, + "metricset": { + "period": 60000, + "name": "transitgateway" + }, + "service": { + "type": "aws" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.TransitGateway | Filters the metric data by transit gateway. | keyword | +| aws.dimensions.TransitGatewayAttachment | Filters the metric data by transit gateway attachment. | keyword | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| aws.transitgateway.metrics.BytesIn.sum | The number of bytes received by the transit gateway. | long | +| aws.transitgateway.metrics.BytesOut.sum | The number of bytes sent from the transit gateway. | long | +| aws.transitgateway.metrics.PacketDropCountBlackhole.sum | The number of packets dropped because they matched a blackhole route. | long | +| aws.transitgateway.metrics.PacketDropCountNoRoute.sum | The number of packets dropped because they did not match a route. | long | +| aws.transitgateway.metrics.PacketsIn.sum | The number of packets received by the transit gateway. | long | +| aws.transitgateway.metrics.PacketsOut.sum | The number of packets sent by the transit gateway. | long | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | + diff --git a/test/packages/aws/docs/usage.md b/test/packages/aws/docs/usage.md new file mode 100644 index 0000000000..d378189c74 --- /dev/null +++ b/test/packages/aws/docs/usage.md @@ -0,0 +1,112 @@ +# usage + +## Metrics + +An example event for `usage` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:30.929Z", + "aws": { + "usage": { + "metrics": { + "CallCount": { + "sum": 1 + } + } + }, + "cloudwatch": { + "namespace": "AWS/Usage" + }, + "dimensions": { + "Type": "API", + "Resource": "GetMetricData", + "Service": "CloudWatch", + "Class": "None" + } + }, + "event": { + "duration": 1191329839, + "dataset": "aws.usage", + "module": "aws" + }, + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "cloud": { + "provider": "aws", + "region": "eu-north-1", + "account": { + "name": "elastic-beats", + "id": "428152502467" + } + }, + "metricset": { + "name": "usage", + "period": 60000 + }, + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.Class | The class of resource being tracked. | keyword | +| aws.dimensions.Resource | The name of the API operation. | keyword | +| aws.dimensions.Service | The name of the AWS service containing the resource. | keyword | +| aws.dimensions.Type | The type of resource being tracked. | keyword | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| aws.usage.metrics.CallCount.sum | The number of specified API operations performed in your account. | long | +| aws.usage.metrics.ResourceCount.sum | The number of the specified resources running in your account. The resources are defined by the dimensions associated with the metric. | long | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | + diff --git a/test/packages/aws/docs/vpcflow.md b/test/packages/aws/docs/vpcflow.md new file mode 100644 index 0000000000..c2eadee598 --- /dev/null +++ b/test/packages/aws/docs/vpcflow.md @@ -0,0 +1,89 @@ +# vpcflow + +## Logs + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.vpcflow.account_id | The AWS account ID for the flow log. | keyword | +| aws.vpcflow.action | The action that is associated with the traffic, ACCEPT or REJECT. | keyword | +| aws.vpcflow.instance_id | The ID of the instance that's associated with network interface for which the traffic is recorded, if the instance is owned by you. | keyword | +| aws.vpcflow.interface_id | The ID of the network interface for which the traffic is recorded. | keyword | +| aws.vpcflow.log_status | The logging status of the flow log, OK, NODATA or SKIPDATA. | keyword | +| aws.vpcflow.pkt_dstaddr | The packet-level (original) destination IP address for the traffic. | ip | +| aws.vpcflow.pkt_srcaddr | The packet-level (original) source IP address of the traffic. | ip | +| aws.vpcflow.subnet_id | The ID of the subnet that contains the network interface for which the traffic is recorded. | keyword | +| aws.vpcflow.tcp_flags | The bitmask value for the following TCP flags: 2=SYN,18=SYN-ACK,1=FIN,4=RST | keyword | +| aws.vpcflow.type | The type of traffic: IPv4, IPv6, or EFA. | keyword | +| aws.vpcflow.version | The VPC Flow Logs version. If you use the default format, the version is 2. If you specify a custom format, the version is 3. | keyword | +| aws.vpcflow.vpc_id | The ID of the VPC that contains the network interface for which the traffic is recorded. | keyword | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | +| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| destination.as.organization.name | Organization name. | keyword | +| destination.geo.continent_name | Name of the continent. | keyword | +| destination.geo.country_iso_code | Country ISO code. | keyword | +| destination.geo.location | Longitude and latitude. | geo_point | +| destination.ip | IP address of the destination. | ip | +| destination.port | Port of the destination. | long | +| event.category | Event category (e.g. database) | keyword | +| event.end | event.end contains the date when the event ended or when the activity was last observed. | date | +| event.kind | Event kind (e.g. event, alert, metric, state, pipeline_error, signal) | keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity. | keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. | keyword | +| event.start | event.start contains the date when the event started or when the activity was first observed. | date | +| event.type | Event severity (e.g. info, error) | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| network.bytes | Total bytes transferred in both directions. | long | +| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. | keyword | +| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | +| network.packets | Total packets transferred in both directions. | long | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the .address field. | keyword | +| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| source.as.organization.name | Organization name. | keyword | +| source.bytes | Bytes sent from the source to the destination. | long | +| source.geo.city_name | City name. | keyword | +| source.geo.continent_name | Name of the continent. | keyword | +| source.geo.country_iso_code | Country ISO code. | keyword | +| source.geo.location | Longitude and latitude. | geo_point | +| source.geo.region_iso_code | Region ISO code. | keyword | +| source.geo.region_name | Region name. | keyword | +| source.ip | IP address of the source (IPv4 or IPv6). | ip | +| source.packets | Packets sent from the source to the destination. | long | +| source.port | Port of the source. | long | + diff --git a/test/packages/aws/docs/vpn.md b/test/packages/aws/docs/vpn.md new file mode 100644 index 0000000000..3942dbd812 --- /dev/null +++ b/test/packages/aws/docs/vpn.md @@ -0,0 +1,111 @@ +# vpn + +## Metrics + +An example event for `vpn` looks as following: + +```$json +{ + "@timestamp": "2020-05-28T17:58:27.154Z", + "service": { + "type": "aws" + }, + "ecs": { + "version": "1.5.0" + }, + "aws": { + "vpn": { + "metrics": { + "TunnelState": { + "avg": 0 + }, + "TunnelDataIn": { + "sum": 0 + }, + "TunnelDataOut": { + "sum": 0 + } + } + }, + "cloudwatch": { + "namespace": "AWS/VPN" + } + }, + "event": { + "dataset": "aws.vpn", + "module": "aws", + "duration": 10418157072 + }, + "metricset": { + "period": 60000, + "name": "vpn" + }, + "cloud": { + "region": "us-west-2", + "account": { + "name": "elastic-beats", + "id": "428152502467" + }, + "provider": "aws" + }, + "agent": { + "version": "8.0.0", + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| aws.*.metrics.*.* | Metrics that returned from Cloudwatch API query. | object | +| aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword | +| aws.dimensions.* | Metric dimensions. | object | +| aws.dimensions.TunnelIpAddress | Filters the metric data by the IP address of the tunnel for the virtual private gateway. | keyword | +| aws.dimensions.VpnId | Filters the metric data by the Site-to-Site VPN connection ID. | keyword | +| aws.s3.bucket.name | Name of a S3 bucket. | keyword | +| aws.tags.* | Tag key value pairs from aws resources. | object | +| aws.vpn.metrics.TunnelDataIn.sum | The bytes received through the VPN tunnel. | double | +| aws.vpn.metrics.TunnelDataOut.sum | The bytes sent through the VPN tunnel. | double | +| aws.vpn.metrics.TunnelState.avg | The state of the tunnel. For static VPNs, 0 indicates DOWN and 1 indicates UP. For BGP VPNs, 1 indicates ESTABLISHED and 0 is used for all other states. | double | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| service.type | Service type | keyword | + diff --git a/test/packages/aws/img/logo_billing.svg b/test/packages/aws/img/logo_billing.svg new file mode 100644 index 0000000000..673d58ca61 --- /dev/null +++ b/test/packages/aws/img/logo_billing.svg @@ -0,0 +1 @@ +AWS-Cost-Explorer \ No newline at end of file diff --git a/test/packages/aws/img/logo_cloudtrail.svg b/test/packages/aws/img/logo_cloudtrail.svg new file mode 100644 index 0000000000..df6bd21c02 --- /dev/null +++ b/test/packages/aws/img/logo_cloudtrail.svg @@ -0,0 +1 @@ +AWS-CloudTrail \ No newline at end of file diff --git a/test/packages/aws/img/logo_cloudwatch.svg b/test/packages/aws/img/logo_cloudwatch.svg new file mode 100644 index 0000000000..46e787df15 --- /dev/null +++ b/test/packages/aws/img/logo_cloudwatch.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/aws/img/logo_dynamodb.svg b/test/packages/aws/img/logo_dynamodb.svg new file mode 100644 index 0000000000..b8f0d359e7 --- /dev/null +++ b/test/packages/aws/img/logo_dynamodb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/aws/img/logo_ebs.svg b/test/packages/aws/img/logo_ebs.svg new file mode 100644 index 0000000000..e0211057aa --- /dev/null +++ b/test/packages/aws/img/logo_ebs.svg @@ -0,0 +1 @@ +Amazon-Elastic-Block-Store-EBS \ No newline at end of file diff --git a/test/packages/aws/img/logo_ec2.svg b/test/packages/aws/img/logo_ec2.svg new file mode 100644 index 0000000000..2bbf9402b3 --- /dev/null +++ b/test/packages/aws/img/logo_ec2.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/aws/img/logo_elb.svg b/test/packages/aws/img/logo_elb.svg new file mode 100644 index 0000000000..6a313d453e --- /dev/null +++ b/test/packages/aws/img/logo_elb.svg @@ -0,0 +1 @@ +Elastic-Load-Balancing_light-bg \ No newline at end of file diff --git a/test/packages/aws/img/logo_lambda.svg b/test/packages/aws/img/logo_lambda.svg new file mode 100644 index 0000000000..0975f058ff --- /dev/null +++ b/test/packages/aws/img/logo_lambda.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/aws/img/logo_natgateway.svg b/test/packages/aws/img/logo_natgateway.svg new file mode 100644 index 0000000000..5ccd131261 --- /dev/null +++ b/test/packages/aws/img/logo_natgateway.svg @@ -0,0 +1 @@ +Amazon-VPC_NAT-Gateway_light-bg \ No newline at end of file diff --git a/test/packages/aws/img/logo_rds.svg b/test/packages/aws/img/logo_rds.svg new file mode 100644 index 0000000000..952292b532 --- /dev/null +++ b/test/packages/aws/img/logo_rds.svg @@ -0,0 +1 @@ +Amazon-RDS \ No newline at end of file diff --git a/test/packages/aws/img/logo_s3.svg b/test/packages/aws/img/logo_s3.svg new file mode 100644 index 0000000000..34ad3db7c4 --- /dev/null +++ b/test/packages/aws/img/logo_s3.svg @@ -0,0 +1 @@ +Amazon-Simple-Storage-Service-S3 \ No newline at end of file diff --git a/test/packages/aws/img/logo_sns.svg b/test/packages/aws/img/logo_sns.svg new file mode 100644 index 0000000000..468f745437 --- /dev/null +++ b/test/packages/aws/img/logo_sns.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/aws/img/logo_sqs.svg b/test/packages/aws/img/logo_sqs.svg new file mode 100644 index 0000000000..c586307c55 --- /dev/null +++ b/test/packages/aws/img/logo_sqs.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/aws/img/logo_transitgateway.svg b/test/packages/aws/img/logo_transitgateway.svg new file mode 100644 index 0000000000..22e262b849 --- /dev/null +++ b/test/packages/aws/img/logo_transitgateway.svg @@ -0,0 +1 @@ +AWS-Transit-Gateway \ No newline at end of file diff --git a/test/packages/aws/img/logo_vpcflow.svg b/test/packages/aws/img/logo_vpcflow.svg new file mode 100644 index 0000000000..4393e385f5 --- /dev/null +++ b/test/packages/aws/img/logo_vpcflow.svg @@ -0,0 +1 @@ +Amazon-VPC_Flow-Logs_light-bg \ No newline at end of file diff --git a/test/packages/aws/img/logo_vpn.svg b/test/packages/aws/img/logo_vpn.svg new file mode 100644 index 0000000000..e304dc767b --- /dev/null +++ b/test/packages/aws/img/logo_vpn.svg @@ -0,0 +1 @@ +AWS-Client-VPN \ No newline at end of file diff --git a/test/packages/aws/manifest.yml b/test/packages/aws/manifest.yml index b40c65ed32..3350f8a718 100644 --- a/test/packages/aws/manifest.yml +++ b/test/packages/aws/manifest.yml @@ -13,127 +13,96 @@ categories: - cloud - network - security -release: experimental +release: beta conditions: kibana.version: '^7.9.0' screenshots: - - src: /img/filebeat-aws-cloudtrail.png - title: filebeat aws cloudtrail - size: 1702x1063 - type: image/png - - src: /img/filebeat-aws-elb-overview.png - title: filebeat aws elb overview - size: 5120x2704 - type: image/png - - src: /img/filebeat-aws-s3access-overview.png - title: filebeat aws s3access overview - size: 1684x897 - type: image/png - - src: /img/filebeat-aws-vpcflow-overview.png - title: filebeat aws vpcflow overview - size: 5111x2609 - type: image/png - src: /img/metricbeat-aws-overview.png title: metricbeat aws overview size: 3848x2440 type: image/png - - src: /img/metricbeat-aws-billing-overview.png - title: metricbeat aws billing overview - size: 2176x1826 - type: image/png - - src: /img/metricbeat-aws-dynamodb-overview.png - title: metricbeat aws dynamodb overview - size: 1873x846 - type: image/png - - src: /img/metricbeat-aws-ebs-overview.png - title: metricbeat aws ebs overview - size: 3372x2104 - type: image/png - - src: /img/metricbeat-aws-ec2-overview.png - title: metricbeat aws ec2 overview - size: 2640x2240 - type: image/png - - src: /img/metricbeat-aws-elb-overview.png - title: metricbeat aws elb overview - size: 2676x2384 - type: image/png - - src: /img/metricbeat-aws-lambda-overview.png - title: metricbeat aws lambda overview - size: 2582x2206 - type: image/png - - src: /img/metricbeat-aws-rds-overview.png - title: metricbeat aws rds overview - size: 3468x2290 - type: image/png - - src: /img/metricbeat-aws-s3-overview.png - title: metricbeat aws s3 overview - size: 2048x1504 - type: image/png - - src: /img/metricbeat-aws-sqs-overview.png - title: metricbeat aws sqs overview - size: 2560x1440 - type: image/png - - src: /img/metricbeat-aws-usage-overview.png - title: metricbeat aws usage overview - size: 2238x2438 - type: image/png - - src: /img/metricbeat-aws-billing-overview.png - title: metricbeat aws billing overview - size: 2176x1826 - type: image/png - - src: /img/metricbeat-aws-ebs-overview.png - title: metricbeat aws ebs overview - size: 3372x2104 - type: image/png - - src: /img/metricbeat-aws-ec2-overview.png - title: metricbeat aws ec2 overview - size: 2640x2240 - type: image/png - - src: /img/metricbeat-aws-elb-overview.png - title: metricbeat aws elb overview - size: 2676x2384 - type: image/png - - src: /img/metricbeat-aws-lambda-overview.png - title: metricbeat aws lambda overview - size: 2582x2206 - type: image/png - - src: /img/metricbeat-aws-rds-overview.png - title: metricbeat aws rds overview - size: 3468x2290 - type: image/png - - src: /img/metricbeat-aws-s3-overview.png - title: metricbeat aws s3 overview - size: 2048x1504 - type: image/png - - src: /img/metricbeat-aws-s3-overview.png - title: metricbeat aws s3 overview - size: 2048x1504 - type: image/png - - src: /img/metricbeat-aws-sns-overview.png - title: metricbeat aws sns overview - size: 3840x2676 - type: image/png - - src: /img/metricbeat-aws-sqs-overview.png - title: metricbeat aws sqs overview - size: 2560x1440 - type: image/png - - src: /img/metricbeat-aws-usage-overview.png - title: metricbeat aws usage overview - size: 2238x2438 - type: image/png icons: - src: /img/logo_aws.svg title: logo aws size: 32x32 type: image/svg+xml +vars: + - name: shared_credential_file + type: text + title: Shared Credential File + multi: false + required: false + show_user: false + description: Directory of the shared credentials file. + - name: credential_profile_name + type: text + title: Credential Profile Name + multi: false + required: false + show_user: true + - name: access_key_id + type: text + title: Access Key ID + multi: false + required: false + show_user: false + - name: secret_access_key + type: text + title: Secret Access Key + multi: false + required: false + show_user: false + - name: session_token + type: text + title: Session Token + multi: false + required: false + show_user: false + - name: role_arn + type: text + title: Role ARN + multi: false + required: false + show_user: false + - name: endpoint + type: text + title: Endpoint + multi: false + required: false + show_user: false + default: "amazonaws.com" + description: URL of the entry point for an AWS web service. policy_templates: - - name: aws - title: AWS logs and metrics - description: Collect logs and metrics from AWS services + - name: billing + title: AWS Billing + description: Collect AWS billing metrics + data_streams: + - billing + inputs: + - type: aws/metrics + title: Collect billing metrics + description: Collect billing metrics + input_group: metrics + icons: + - src: /img/logo_billing.svg + title: AWS Billing logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-billing-overview.png + title: metricbeat aws billing overview + size: 2640x2240 + type: image/png + - name: cloudtrail + title: AWS Cloudtrail + description: Collect logs from AWS Cloudtrail + data_streams: + - cloudtrail inputs: - type: s3 - title: Collect logs from AWS services - description: Collecting AWS CloudTrail, CloudWatch, EC2, ELB, S3 access logs and VPC flow logs logs + title: Collect logs from Cloudtrail service + description: Collecting Cloudtrail logs using S3 input + input_group: logs vars: - name: visibility_timeout type: text @@ -149,114 +118,411 @@ policy_templates: required: false show_user: false description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - - name: shared_credential_file - type: text - title: Shared Credential File - multi: false - required: false - show_user: false - description: Directory of the shared credentials file. - - name: credential_profile_name - type: text - title: Credential Profile Name - multi: false - required: false - show_user: true - - name: access_key_id - type: text - title: Access Key ID - multi: false - required: false - show_user: false - - name: secret_access_key + icons: + - src: /img/logo_cloudtrail.svg + title: AWS Cloudtrail logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/filebeat-aws-cloudtrail.png + title: filebeat aws cloudtrail + size: 1702x1063 + type: image/png + - name: cloudwatch + title: AWS CloudWatch + description: Collect logs and metrics from CloudWatch + data_streams: + - cloudwatch_logs + - cloudwatch_metrics + inputs: + - type: s3 + title: Collect logs from CloudWatch + description: Collecting logs from CloudWatch using S3 input + input_group: logs + vars: + - name: visibility_timeout type: text - title: Secret Access Key + title: Visibility Timeout multi: false required: false show_user: false - - name: session_token + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout type: text - title: Session Token + title: API Timeout multi: false required: false show_user: false - - name: role_arn - type: text - title: Role ARN + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. + - type: aws/metrics + title: Collect metrics from CloudWatch + description: Collecting metrics from AWS CloudWatch + input_group: metrics + vars: + - name: metrics + type: yaml + title: Metrics multi: false - required: false - show_user: false - - name: endpoint + required: true + show_user: true + default: | + - namespace: AWS/EC2 + resource_type: ec2:instance + name: + - CPUUtilization + - DiskWriteOps + statistic: + - Average + - Maximum + # dimensions: + # - name: InstanceId + # value: i-123456 + # tags: + # - key: created-by + # value: foo + icons: + - src: /img/logo_cloudwatch.svg + title: AWS CloudWatch logo + size: 32x32 + type: image/svg+xml + - name: dynamodb + title: AWS DynamoDB + description: Collect AWS DynamoDB metrics + data_streams: + - dynamodb + inputs: + - type: aws/metrics + title: Collect dynamodb metrics + description: Collect dynamodb metrics + input_group: metrics + icons: + - src: /img/logo_dynamodb.svg + title: AWS DynamoDB logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-dynamodb-overview.png + title: metricbeat aws dynamodb overview + size: 2640x2240 + type: image/png + - name: ebs + title: AWS EBS + description: Collect AWS EBS metrics + data_streams: + - ebs + inputs: + - type: aws/metrics + title: Collect EBS metrics + description: Collect EBS metrics + input_group: metrics + icons: + - src: /img/logo_ebs.svg + title: AWS EBS logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-ebs-overview.png + title: metricbeat aws ebs overview + size: 2640x2240 + type: image/png + - name: ec2 + title: AWS EC2 + description: Collect logs and metrics from EC2 service + data_streams: + - ec2_logs + - ec2_metrics + inputs: + - type: s3 + title: Collect logs from EC2 service + description: Collecting EC2 logs using S3 input + input_group: logs + vars: + - name: visibility_timeout type: text - title: Endpoint + title: Visibility Timeout multi: false required: false show_user: false - default: "amazonaws.com" - description: URL of the entry point for an AWS web service. - - name: aws_partition + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout type: text - title: AWS Partition + title: API Timeout multi: false required: false show_user: false - default: "aws" - description: AWS region partition name, value is one of aws, aws-cn, aws-us-gov. + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. - type: aws/metrics - title: Collect metrics from AWS services - description: Collecting AWS billing, cloudwatch, dynamodb, ebs, ec2, elb, lambda, natgateway, rds, s3_daily_storage, s3_request, sns, sqs, transitgateway, usage and vpn metrics + title: Collect metrics from EC2 service + description: Collecting EC2 metrics using AWS CloudWatch + input_group: metrics + icons: + - src: /img/logo_ec2.svg + title: AWS EC2 logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-ec2-overview.png + title: metricbeat aws ec2 overview + size: 2640x2240 + type: image/png + - name: elb + title: AWS ELB + description: Collect logs and metrics from ELB service + data_streams: + - elb_logs + - elb_metrics + inputs: + - type: s3 + title: Collect logs from ELB service + description: Collecting ELB logs using S3 input + input_group: logs vars: - - name: access_key_id - type: text - title: Access Key ID - multi: false - required: false - show_user: false - - name: secret_access_key + - name: visibility_timeout type: text - title: Secret Access Key + title: Visibility Timeout multi: false required: false show_user: false - - name: session_token + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout type: text - title: Session Token + title: API Timeout multi: false required: false show_user: false - - name: shared_credential_file + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. + - type: aws/metrics + title: Collect metrics from ELB service + description: Collecting ELB metrics using AWS CloudWatch + input_group: metrics + icons: + - src: /img/logo_elb.svg + title: AWS ELB logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-elb-overview.png + title: metricbeat aws elb overview + size: 2640x2240 + type: image/png + - src: /img/filebeat-aws-elb-overview.png + title: filebeat aws elb overview + size: 1684x897 + type: image/png + - name: lambda + title: AWS Lambda + description: Collect AWS Lambda metrics + data_streams: + - lambda + inputs: + - type: aws/metrics + title: Collect Lambda metrics + description: Collect Lambda metrics + input_group: metrics + icons: + - src: /img/logo_lambda.svg + title: AWS Lambda logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-lambda-overview.png + title: metricbeat aws lambda overview + size: 2640x2240 + type: image/png + - name: natgateway + title: AWS NATGateway + description: Collect AWS NATGateway metrics + data_streams: + - natgateway + inputs: + - type: aws/metrics + title: Collect NATGateway metrics + description: Collect NATGateway metrics + input_group: metrics + icons: + - src: /img/logo_natgateway.svg + title: AWS NATGateway logo + size: 32x32 + type: image/svg+xml + - name: rds + title: AWS RDS + description: Collect AWS RDS metrics + data_streams: + - rds + inputs: + - type: aws/metrics + title: Collect RDS metrics + description: Collect RDS metrics + input_group: metrics + icons: + - src: /img/logo_rds.svg + title: AWS RDS logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-rds-overview.png + title: metricbeat aws rds overview + size: 2640x2240 + type: image/png + - name: s3 + title: AWS S3 + description: Collect AWS S3 metrics + data_streams: + - s3_daily_storage + - s3_request + - s3access + inputs: + - type: s3 + title: Collect S3 access logs + description: Collecting S3 access logs using S3 input + input_group: logs + vars: + - name: visibility_timeout type: text - title: Shared Credential File + title: Visibility Timeout multi: false required: false show_user: false - - name: credential_profile_name - type: text - title: Credential Profile Name - multi: false - required: false - show_user: true - - name: role_arn + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout type: text - title: Role ARN + title: API Timeout multi: false required: false show_user: false - - name: endpoint + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. + - type: aws/metrics + title: Collect metrics from S3 + description: Collecting S3 metrics using AWS CloudWatch + input_group: metrics + icons: + - src: /img/logo_s3.svg + title: AWS S3 logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-s3-overview.png + title: metricbeat aws s3 overview + size: 2640x2240 + type: image/png + - src: /img/filebeat-aws-s3access-overview.png + title: filebeat aws s3access overview + size: 1684x897 + type: image/png + - name: sns + title: AWS SNS + description: Collect AWS SNS metrics + data_streams: + - sns + inputs: + - type: aws/metrics + title: Collect SNS metrics + description: Collect SNS metrics + input_group: metrics + icons: + - src: /img/logo_sns.svg + title: AWS SNS logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-sns-overview.png + title: metricbeat aws sns overview + size: 2640x2240 + type: image/png + - name: sqs + title: AWS SQS + description: Collect AWS SQS metrics + data_streams: + - sqs + inputs: + - type: aws/metrics + title: Collect SQS metrics + description: Collect SQS metrics + input_group: metrics + icons: + - src: /img/logo_sqs.svg + title: AWS SQS logo + size: 32x32 + type: image/svg+xml + screenshots: + - src: /img/metricbeat-aws-sqs-overview.png + title: metricbeat aws sqs overview + size: 2640x2240 + type: image/png + - name: transitgateway + title: AWS Transit Gateway + description: Collect AWS Transit Gateway metrics + data_streams: + - transitgateway + inputs: + - type: aws/metrics + title: Collect Transit Gateway metrics + description: Collect Transit Gateway metrics + input_group: metrics + icons: + - src: /img/logo_transitgateway.svg + title: AWS Transit Gateway logo + size: 32x32 + type: image/svg+xml + - name: usage + title: AWS Usage + description: Collect AWS Usage metrics + data_streams: + - usage + inputs: + - type: aws/metrics + title: Collect Usage metrics + description: Collect Usage metrics + input_group: metrics + screenshots: + - src: /img/metricbeat-aws-usage-overview.png + title: metricbeat aws sns overview + size: 2640x2240 + type: image/png + - name: vpcflow + title: AWS VPC Flow + description: Collect AWS vpcflow logs + data_streams: + - vpcflow + inputs: + - type: s3 + title: Collect VPC Flow logs + description: Collecting VPC Flow logs using S3 input + input_group: logs + vars: + - name: visibility_timeout type: text - title: Endpoint + title: Visibility Timeout multi: false required: false show_user: false - default: "amazonaws.com" - description: URL of the entry point for an AWS web service. - - name: aws_partition + description: The duration that the received messages are hidden from subsequent retrieve requests after being retrieved by a ReceiveMessage request. The maximum is 12 hours. + - name: api_timeout type: text - title: AWS Partition + title: API Timeout multi: false required: false show_user: false - default: "aws" - description: AWS region partition name, value is one of aws, aws-cn, aws-us-gov. + description: The maximum duration of AWS API can take. The maximum is half of the visibility timeout value. + icons: + - src: /img/logo_vpcflow.svg + title: AWS VPC logo + size: 32x32 + type: image/svg+xml + - name: vpn + title: AWS VPN + description: Collect AWS VPN metrics + data_streams: + - vpn + inputs: + - type: aws/metrics + title: Collect VPN metrics + description: Collect VPN metrics + input_group: metrics + icons: + - src: /img/logo_vpn.svg + title: AWS VPN logo + size: 32x32 + type: image/svg+xml owner: github: elastic/integrations-platforms