diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index 7551d2173..c15daac68 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -77,6 +77,14 @@ func TestValidateFile(t *testing.T) { "field \"example.agent.call_duration\" of type histogram can't be a dimension, allowed types for dimensions: constant_keyword, keyword, long, integer, short, byte, double, float, half_float, scaled_float, unsigned_long, ip", }, }, + "bad_field_typos": { + "data_stream/example/fields/fields.yml", + []string{ + "field 0: Additional property descriiption is not allowed", + "field 0: Additional property dimensions is not allowed", + "field 1: Additional property default_fields is not allowed", + }, + }, } for pkgName, test := range tests { diff --git a/test/packages/bad_field_typos/changelog.yml b/test/packages/bad_field_typos/changelog.yml new file mode 100644 index 000000000..7b5f7fd8f --- /dev/null +++ b/test/packages/bad_field_typos/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/0 # FIXME Replace with the real PR link diff --git a/test/packages/bad_field_typos/data_stream/example/agent/stream/stream.yml.hbs b/test/packages/bad_field_typos/data_stream/example/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..3e1cf2259 --- /dev/null +++ b/test/packages/bad_field_typos/data_stream/example/agent/stream/stream.yml.hbs @@ -0,0 +1,6 @@ +metricsets: ["sample_metricset"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} diff --git a/test/packages/bad_field_typos/data_stream/example/fields/base-fields.yml b/test/packages/bad_field_typos/data_stream/example/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/bad_field_typos/data_stream/example/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/bad_field_typos/data_stream/example/fields/ecs.yml b/test/packages/bad_field_typos/data_stream/example/fields/ecs.yml new file mode 100644 index 000000000..1706e98b6 --- /dev/null +++ b/test/packages/bad_field_typos/data_stream/example/fields/ecs.yml @@ -0,0 +1,3 @@ +- name: container.id + external: ecs + dimension: true diff --git a/test/packages/bad_field_typos/data_stream/example/fields/fields.yml b/test/packages/bad_field_typos/data_stream/example/fields/fields.yml new file mode 100644 index 000000000..eec99deb7 --- /dev/null +++ b/test/packages/bad_field_typos/data_stream/example/fields/fields.yml @@ -0,0 +1,11 @@ +- name: example.agent.id + descriiption: Agent ID + type: keyword + dimensions: true +- name: example.agent.call_count + type: long + metric_type: counter + default_fields: false +- name: example.agent.current_count + type: long + metric_type: gauge diff --git a/test/packages/bad_field_typos/data_stream/example/manifest.yml b/test/packages/bad_field_typos/data_stream/example/manifest.yml new file mode 100644 index 000000000..ad6224a65 --- /dev/null +++ b/test/packages/bad_field_typos/data_stream/example/manifest.yml @@ -0,0 +1,17 @@ +title: "Example data stream for time series" +type: metrics +streams: + - input: sample/metrics + title: Sample metrics + description: Collect sample metrics + vars: + - name: period + type: text + title: Period + default: 10s + +elasticsearch: + index_template: + settings: + # Defaults to 16 + index.mapping.dimension_fields.limit: 32 diff --git a/test/packages/bad_field_typos/docs/README.md b/test/packages/bad_field_typos/docs/README.md new file mode 100644 index 000000000..76705cd41 --- /dev/null +++ b/test/packages/bad_field_typos/docs/README.md @@ -0,0 +1,5 @@ +# Package with time series + +This is a new integration created using the [elastic-package](https://github.com/elastic/elastic-package) tool. + +Consider using the README template file `_dev/build/docs/README.md`to generate a list of exported fields or include a sample event. \ No newline at end of file diff --git a/test/packages/bad_field_typos/img/sample-logo.svg b/test/packages/bad_field_typos/img/sample-logo.svg new file mode 100644 index 000000000..6268dd88f --- /dev/null +++ b/test/packages/bad_field_typos/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/bad_field_typos/img/sample-screenshot.png b/test/packages/bad_field_typos/img/sample-screenshot.png new file mode 100644 index 000000000..d7a56a3ec Binary files /dev/null and b/test/packages/bad_field_typos/img/sample-screenshot.png differ diff --git a/test/packages/bad_field_typos/manifest.yml b/test/packages/bad_field_typos/manifest.yml new file mode 100644 index 000000000..181ce0805 --- /dev/null +++ b/test/packages/bad_field_typos/manifest.yml @@ -0,0 +1,33 @@ +format_version: 1.2.0 +name: time_series +title: "Package with time series" +version: 0.0.1 +license: basic +description: "This is a package with time series data" +type: integration +categories: + - custom + - monitoring +release: beta +conditions: + kibana.version: "^7.16.0" +screenshots: + - src: /img/sample-screenshot.png + title: Sample screenshot + size: 600x600 + type: image/png +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations-developer-experience diff --git a/test/packages/good/data_stream/foo/fields/some_fields.yml b/test/packages/good/data_stream/foo/fields/some_fields.yml index 1fcd4f743..9e693e46f 100644 --- a/test/packages/good/data_stream/foo/fields/some_fields.yml +++ b/test/packages/good/data_stream/foo/fields/some_fields.yml @@ -1,5 +1,6 @@ - name: source title: Source + footnote: "Some detailed information about these fields" group: 2 type: group fields: @@ -26,9 +27,16 @@ type: text description: A field with a pattern defined pattern: '^[a-zA-Z]$' + example: foobartext - name: aaa type: integer metric_type: gauge - name: vehicle_type type: constant_keyword value: truck +- name: magic_number + type: integer + # TODO: Fix this: example: 42 +- name: location + type: alias + path: source.geo.location diff --git a/test/packages/good/data_stream/k8s_data_stream/fields/base-fields.yml b/test/packages/good/data_stream/k8s_data_stream/fields/base-fields.yml index e6e1d439f..b6586ea17 100644 --- a/test/packages/good/data_stream/k8s_data_stream/fields/base-fields.yml +++ b/test/packages/good/data_stream/k8s_data_stream/fields/base-fields.yml @@ -21,4 +21,24 @@ level: core type: keyword description: Region name. - ignore_above: 1024 \ No newline at end of file + ignore_above: 1024 +- name: host + title: Host + group: 2 + type: group + fields: + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + - name: labels + level: extended + type: object + object_type: keyword + object_type_mapping_type: "*" + description: Image labels. diff --git a/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml b/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml index bde93f380..5c0c5687a 100644 --- a/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml +++ b/test/packages/good/data_stream/skipped_tests/fields/some_fields.yml @@ -25,4 +25,4 @@ - name: foobar type: text description: A field with a pattern defined - pattern: '^[a-zA-Z]$' \ No newline at end of file + pattern: '^[a-zA-Z]$' diff --git a/test/packages/time_series/data_stream/example/fields/fields.yml b/test/packages/time_series/data_stream/example/fields/fields.yml index 1dd40c43a..cd773967a 100644 --- a/test/packages/time_series/data_stream/example/fields/fields.yml +++ b/test/packages/time_series/data_stream/example/fields/fields.yml @@ -7,3 +7,8 @@ - name: example.agent.current_count type: long metric_type: gauge +- name: example.agent.received.bytes + type: long + metric_type: gauge + unit: byte + format: bytes diff --git a/versions/1/changelog.yml b/versions/1/changelog.yml index 0faf318c1..e279526c4 100644 --- a/versions/1/changelog.yml +++ b/versions/1/changelog.yml @@ -10,6 +10,9 @@ - description: Add kibana/csp-rule-template asset type: enhancement link: https://github.com/elastic/package-spec/pull/276 + - description: Disallow unknown properties on fields definitions + type: breaking-change + link: https://github.com/elastic/package-spec/pull/281 - version: 1.4.1 changes: - description: ML model file name now matches the id of the model. diff --git a/versions/1/data_stream/fields/fields.spec.yml b/versions/1/data_stream/fields/fields.spec.yml index aacdaad8d..951f43e0b 100644 --- a/versions/1/data_stream/fields/fields.spec.yml +++ b/versions/1/data_stream/fields/fields.spec.yml @@ -6,7 +6,7 @@ spec: type: array items: type: object - additionalProperties: true + additionalProperties: false properties: name: description: Name of field @@ -51,12 +51,65 @@ spec: - wildcard - version - unsigned_long + object_type: + description: Datatype of elements in object field + type: string + enum: + - histogram + - text + - keyword + - long + - integer + - short + - byte + - double + - float + - half_float + - scaled_float + - date + - date_nanos + - boolean + - binary + - integer_range + - float_range + - long_range + - double_range + - date_range + - ip_range + - geo_point + - ip + - array + - flattened + - wildcard + - version + - unsigned_long + object_type_mapping_type: + description: Types in the source document that should be mapped to the type defined in object_type, it can be "*" to indicate all types. + type: string + examples: + - "double" + - "*" + group: + description: Group the field belongs to. + type: integer + level: + description: ECS level the field belongs to. + type: string + examples: + - core description: - description: Short description of field + description: Short description of field. + type: string + footnote: + description: Additional detains on the field. type: string value: description: The value to associate with a constant_keyword field. type: string + default_field: + description: Set to false if the field should not be installed as a default field in Kibana index patterns. + type: boolean + default: true metric_type: description: Metric type type: string @@ -76,20 +129,43 @@ spec: - ms - micros - nanos + format: + description: Formatting used by Kibana to display this field + type: string + enum: + - percent + - bytes + - duration + path: + description: Target of alias field. + type: string # TODO: Add sematic validation to check that the referenced field exists. dimension: description: Declare a field as dimension of time series type: boolean default: false + example: + description: Examples for this field. + type: string pattern: description: Regular expression pattern of the field value type: string examples: - '^[a-zA-Z]$' + ignore_above: + description: Index only the first number of characters indicated by this option. + type: integer external: description: External source reference type: string enum: - ecs + norms: + description: Set to false to disable normalization factors used for scoring calculation. + type: boolean + default: true + multi_fields: + description: Additional mappings for this field. + $ref: "#" fields: description: Sub-fields, when type is group $ref: "#" # JSON-schema syntax for pointing to the root of the schema