diff --git a/code/go/ecs/base.go b/code/go/ecs/base.go index 8960635b75..80c6d36b09 100644 --- a/code/go/ecs/base.go +++ b/code/go/ecs/base.go @@ -35,7 +35,7 @@ type Base struct { Timestamp time.Time `ecs:"@timestamp"` // List of keywords used to tag each event. - Tags string `ecs:"tags"` + Tags []string `ecs:"tags"` // Custom key/value pairs. // Can be used to add meta information to events. Should not contain nested diff --git a/code/go/ecs/container.go b/code/go/ecs/container.go index 34c5698ba5..066fe1668f 100644 --- a/code/go/ecs/container.go +++ b/code/go/ecs/container.go @@ -33,7 +33,7 @@ type Container struct { ImageName string `ecs:"image.name"` // Container image tags. - ImageTag string `ecs:"image.tag"` + ImageTag []string `ecs:"image.tag"` // Container name. Name string `ecs:"name"` diff --git a/code/go/ecs/dns.go b/code/go/ecs/dns.go index 08f37b81cf..dfc4e85af4 100644 --- a/code/go/ecs/dns.go +++ b/code/go/ecs/dns.go @@ -45,7 +45,7 @@ type Dns struct { // Array of 2 letter DNS header flags. // Expected values are: AA, TC, RD, RA, AD, CD, DO. - HeaderFlags string `ecs:"header_flags"` + HeaderFlags []string `ecs:"header_flags"` // The DNS response code. ResponseCode string `ecs:"response_code"` @@ -96,7 +96,7 @@ type Dns struct { // answer objects must contain the `data` key. If more information is // available, map as much of it to ECS as possible, and add any additional // fields to the answer objects as custom fields. - Answers map[string]interface{} `ecs:"answers"` + Answers []map[string]interface{} `ecs:"answers"` // The domain name to which this resource record pertains. // If a chain of CNAME is being resolved, each answer's `name` should be @@ -125,5 +125,5 @@ type Dns struct { // data formats it can contain. Extracting all IP addresses seen in there // to `dns.resolved_ip` makes it possible to index them as IP addresses, // and makes them easier to visualize and query for. - ResolvedIP string `ecs:"resolved_ip"` + ResolvedIP []string `ecs:"resolved_ip"` } diff --git a/code/go/ecs/event.go b/code/go/ecs/event.go index 410d49a0c9..aed848aee5 100644 --- a/code/go/ecs/event.go +++ b/code/go/ecs/event.go @@ -64,7 +64,7 @@ type Event struct { // `event.type`, which is used as a subcategory. // This field is an array. This will allow proper categorization of some // events that fall in multiple categories. - Category string `ecs:"category"` + Category []string `ecs:"category"` // The action captured by the event. // This describes the information in the event. It is more specific than @@ -96,7 +96,7 @@ type Event struct { // down to a level appropriate for single visualization. // This field is an array. This will allow proper categorization of some // events that fall in multiple event types. - Type string `ecs:"type"` + Type []string `ecs:"type"` // Name of the module this data is coming from. // If your monitoring agent supports the concept of modules or plugins to diff --git a/code/go/ecs/file.go b/code/go/ecs/file.go index 1dc53d28b0..a4c99b5eac 100644 --- a/code/go/ecs/file.go +++ b/code/go/ecs/file.go @@ -37,7 +37,7 @@ type File struct { // Attributes names will vary by platform. Here's a non-exhaustive list of // values that are expected in this field: archive, compressed, directory, // encrypted, execute, hidden, read, readonly, system, write. - Attributes string `ecs:"attributes"` + Attributes []string `ecs:"attributes"` // Directory where the file is located. It should include the drive letter, // when appropriate. diff --git a/code/go/ecs/host.go b/code/go/ecs/host.go index 1d66d78832..d2895e7ede 100644 --- a/code/go/ecs/host.go +++ b/code/go/ecs/host.go @@ -42,10 +42,10 @@ type Host struct { ID string `ecs:"id"` // Host ip addresses. - IP string `ecs:"ip"` + IP []string `ecs:"ip"` // Host mac addresses. - MAC string `ecs:"mac"` + MAC []string `ecs:"mac"` // Type of host. // For Cloud providers this can be the machine type like `t2.medium`. If diff --git a/code/go/ecs/observer.go b/code/go/ecs/observer.go index a7459aa11a..bebb521a19 100644 --- a/code/go/ecs/observer.go +++ b/code/go/ecs/observer.go @@ -33,10 +33,10 @@ package ecs // observers in ECS. type Observer struct { // MAC addresses of the observer - MAC string `ecs:"mac"` + MAC []string `ecs:"mac"` // IP addresses of the observer. - IP string `ecs:"ip"` + IP []string `ecs:"ip"` // Hostname of the observer. Hostname string `ecs:"hostname"` diff --git a/code/go/ecs/registry.go b/code/go/ecs/registry.go index 54a01777cf..13eba471bb 100644 --- a/code/go/ecs/registry.go +++ b/code/go/ecs/registry.go @@ -43,7 +43,7 @@ type Registry struct { // array will be variable length. For numeric data, such as REG_DWORD and // REG_QWORD, this should be populated with the decimal representation (e.g // `"1"`). - DataStrings string `ecs:"data.strings"` + DataStrings []string `ecs:"data.strings"` // Original bytes written with base64 encoding. // For Windows registry operations, such as SetValueEx and RegQueryValueEx, diff --git a/code/go/ecs/related.go b/code/go/ecs/related.go index 8facf9bcec..68a788b666 100644 --- a/code/go/ecs/related.go +++ b/code/go/ecs/related.go @@ -29,13 +29,13 @@ package ecs // matter where it appeared, by querying `related.ip:192.0.2.15`. type Related struct { // All of the IPs seen on your event. - IP string `ecs:"ip"` + IP []string `ecs:"ip"` // All the user names seen on your event. - User string `ecs:"user"` + User []string `ecs:"user"` // All the hashes seen on your event. Populating this field, then using it // to search for hashes can help in situations where you're unsure what the // hash algorithm is (and therefore which key name to search). - Hash string `ecs:"hash"` + Hash []string `ecs:"hash"` } diff --git a/code/go/ecs/rule.go b/code/go/ecs/rule.go index ae07c808f5..476eb71dd7 100644 --- a/code/go/ecs/rule.go +++ b/code/go/ecs/rule.go @@ -60,7 +60,7 @@ type Rule struct { // Name, organization, or pseudonym of the author or authors who created // the rule used to generate this event. - Author string `ecs:"author"` + Author []string `ecs:"author"` // Name of the license under which the rule used to generate this event is // made available. diff --git a/code/go/ecs/threat.go b/code/go/ecs/threat.go index a77aa888e1..91d86a19b7 100644 --- a/code/go/ecs/threat.go +++ b/code/go/ecs/threat.go @@ -37,27 +37,27 @@ type Threat struct { // Name of the type of tactic used by this threat. You can use a MITRE // ATT&CK® tactic, for example. (ex. // https://attack.mitre.org/tactics/TA0040/) - TacticName string `ecs:"tactic.name"` + TacticName []string `ecs:"tactic.name"` // The id of tactic used by this threat. You can use a MITRE ATT&CK® // tactic, for example. (ex. https://attack.mitre.org/tactics/TA0040/ ) - TacticID string `ecs:"tactic.id"` + TacticID []string `ecs:"tactic.id"` // The reference url of tactic used by this threat. You can use a MITRE // ATT&CK® tactic, for example. (ex. // https://attack.mitre.org/tactics/TA0040/ ) - TacticReference string `ecs:"tactic.reference"` + TacticReference []string `ecs:"tactic.reference"` // The name of technique used by this threat. You can use a MITRE ATT&CK® // technique, for example. (ex. https://attack.mitre.org/techniques/T1499/) - TechniqueName string `ecs:"technique.name"` + TechniqueName []string `ecs:"technique.name"` // The id of technique used by this threat. You can use a MITRE ATT&CK® // technique, for example. (ex. https://attack.mitre.org/techniques/T1499/) - TechniqueID string `ecs:"technique.id"` + TechniqueID []string `ecs:"technique.id"` // The reference url of technique used by this threat. You can use a MITRE // ATT&CK® technique, for example. (ex. // https://attack.mitre.org/techniques/T1499/ ) - TechniqueReference string `ecs:"technique.reference"` + TechniqueReference []string `ecs:"technique.reference"` } diff --git a/code/go/ecs/tls.go b/code/go/ecs/tls.go index 424a12e71a..cca537c403 100644 --- a/code/go/ecs/tls.go +++ b/code/go/ecs/tls.go @@ -63,7 +63,7 @@ type Tls struct { ClientServerName string `ecs:"client.server_name"` // Array of ciphers offered by the client during the client hello. - ClientSupportedCiphers string `ecs:"client.supported_ciphers"` + ClientSupportedCiphers []string `ecs:"client.supported_ciphers"` // Distinguished name of subject of the x.509 certificate presented by the // client. @@ -84,7 +84,7 @@ type Tls struct { // offered by the client. This is usually mutually-exclusive of // `client.certificate` since that value should be the first certificate in // the chain. - ClientCertificateChain string `ecs:"client.certificate_chain"` + ClientCertificateChain []string `ecs:"client.certificate_chain"` // PEM-encoded stand-alone certificate offered by the client. This is // usually mutually-exclusive of `client.certificate_chain` since this @@ -127,7 +127,7 @@ type Tls struct { // offered by the server. This is usually mutually-exclusive of // `server.certificate` since that value should be the first certificate in // the chain. - ServerCertificateChain string `ecs:"server.certificate_chain"` + ServerCertificateChain []string `ecs:"server.certificate_chain"` // PEM-encoded stand-alone certificate offered by the server. This is // usually mutually-exclusive of `server.certificate_chain` since this diff --git a/code/go/ecs/vulnerability.go b/code/go/ecs/vulnerability.go index bda83eae5c..39b4c7d8cc 100644 --- a/code/go/ecs/vulnerability.go +++ b/code/go/ecs/vulnerability.go @@ -69,7 +69,7 @@ type Vulnerability struct { // (https://qualysguard.qualys.com/qwebhelp/fo_portal/knowledgebase/vulnerability_categories.htm[Qualys // vulnerability categories]) // This field must be an array. - Category string `ecs:"category"` + Category []string `ecs:"category"` // The description of the vulnerability that provides additional context of // the vulnerability. For example diff --git a/code/go/ecs/x509.go b/code/go/ecs/x509.go index 5aba49e350..3f2c36876a 100644 --- a/code/go/ecs/x509.go +++ b/code/go/ecs/x509.go @@ -45,22 +45,22 @@ type X509 struct { IssuerDistinguishedName string `ecs:"issuer.distinguished_name"` // List of common name (CN) of issuing certificate authority. - IssuerCommonName string `ecs:"issuer.common_name"` + IssuerCommonName []string `ecs:"issuer.common_name"` // List of organizational units (OU) of issuing certificate authority. - IssuerOrganizationalUnit string `ecs:"issuer.organizational_unit"` + IssuerOrganizationalUnit []string `ecs:"issuer.organizational_unit"` // List of organizations (O) of issuing certificate authority. - IssuerOrganization string `ecs:"issuer.organization"` + IssuerOrganization []string `ecs:"issuer.organization"` // List of locality names (L) - IssuerLocality string `ecs:"issuer.locality"` + IssuerLocality []string `ecs:"issuer.locality"` // List of state or province names (ST, S, or P) - IssuerStateOrProvince string `ecs:"issuer.state_or_province"` + IssuerStateOrProvince []string `ecs:"issuer.state_or_province"` // List of country (C) codes - IssuerCountry string `ecs:"issuer.country"` + IssuerCountry []string `ecs:"issuer.country"` // Identifier for certificate signature algorithm. Recommend using names // found in Go Lang Crypto library (See @@ -77,22 +77,22 @@ type X509 struct { SubjectDistinguishedName string `ecs:"subject.distinguished_name"` // List of common names (CN) of subject. - SubjectCommonName string `ecs:"subject.common_name"` + SubjectCommonName []string `ecs:"subject.common_name"` // List of organizational units (OU) of subject. - SubjectOrganizationalUnit string `ecs:"subject.organizational_unit"` + SubjectOrganizationalUnit []string `ecs:"subject.organizational_unit"` // List of organizations (O) of subject. - SubjectOrganization string `ecs:"subject.organization"` + SubjectOrganization []string `ecs:"subject.organization"` // List of locality names (L) - SubjectLocality string `ecs:"subject.locality"` + SubjectLocality []string `ecs:"subject.locality"` // List of state or province names (ST, S, or P) - SubjectStateOrProvince string `ecs:"subject.state_or_province"` + SubjectStateOrProvince []string `ecs:"subject.state_or_province"` // List of country (C) code - SubjectCountry string `ecs:"subject.country"` + SubjectCountry []string `ecs:"subject.country"` // Algorithm used to generate the public key. PublicKeyAlgorithm string `ecs:"public_key_algorithm"` @@ -110,5 +110,5 @@ type X509 struct { // List of subject alternative names (SAN). Name types vary by certificate // authority and certificate type but commonly contain IP addresses, DNS // names (and wildcards), and email addresses. - AlternativeNames string `ecs:"alternative_names"` + AlternativeNames []string `ecs:"alternative_names"` } diff --git a/scripts/cmd/gocodegen/gocodegen.go b/scripts/cmd/gocodegen/gocodegen.go index c202691ce0..87f8463582 100644 --- a/scripts/cmd/gocodegen/gocodegen.go +++ b/scripts/cmd/gocodegen/gocodegen.go @@ -31,7 +31,6 @@ import ( wordwrap "github.com/mitchellh/go-wordwrap" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/go-ucfg/yaml" ) @@ -111,6 +110,16 @@ type Field struct { JSONKey string } +type YamlField struct { + Name string `config:"name"` + Type string `config:"type"` + Description string `config:"description"` + Format string `config:"format"` + Fields []YamlField `config:"fields"` + MultiFields []YamlField `config:"multi_fields"` + Normalize []string `config:"normalize"` +} + // Flags var ( schemaDir string @@ -138,9 +147,9 @@ func main() { } // Load schema files. - fields := common.Fields{} + fields := []YamlField{} for _, path := range paths { - f := common.Fields{} + f := []YamlField{} cfg, err := yaml.NewConfigWithFile(path) if err != nil { @@ -170,7 +179,7 @@ func main() { } for _, field := range group.Fields { - dataType := goDataType(field.Name, field.Type) + dataType := goDataType(field) if strings.HasPrefix(dataType, "time.") { t.ImportTime = true } @@ -215,6 +224,15 @@ func main() { } } +func isArrayField(field YamlField) bool { + for _, normalizations := range field.Normalize { + if normalizations == "array" { + return true + } + } + return false +} + // isSeparate returns true if the character is a field name separator. This is // used to detect the separators in fields like ephemeral_id or instance.name. func isSeparator(c rune) bool { @@ -264,33 +282,46 @@ func trimTrailingWhitespace(text string) string { } // goDataType returns the Go type to use for Elasticsearch mapping data type. -func goDataType(fieldName, elasticsearchDataType string) string { +func goDataType(field YamlField) string { + dataType, special := nonNormalizedGoDataType(field) + if !special && isArrayField(field) { + return "[]" + dataType + } + return dataType +} + +// nonNormalizedGoDataType returns the Go type without consideration of normalizations +// it also returns whether or not this was a "special" case that avoids additional normalizations +func nonNormalizedGoDataType(field YamlField) (string, bool) { + fieldName := field.Name + elasticsearchDataType := field.Type + // Special cases. switch { case fieldName == "duration" && elasticsearchDataType == "long": - return "time.Duration" + return "time.Duration", true case fieldName == "args" && elasticsearchDataType == "keyword": - return "[]string" + return "[]string", true } switch elasticsearchDataType { case "keyword", "text", "ip", "geo_point": - return "string" + return "string", false case "long": - return "int64" + return "int64", false case "integer": - return "int32" + return "int32", false case "float": - return "float64" + return "float64", false case "date": - return "time.Time" + return "time.Time", false case "boolean": - return "bool" + return "bool", false case "object": - return "map[string]interface{}" + return "map[string]interface{}", false default: log.Fatalf("no translation for %v (field %s)", elasticsearchDataType, fieldName) - return "" + return "", false } }