diff --git a/code/go/internal/validator/semantic/validate_integration_input_qualifier.go b/code/go/internal/validator/semantic/validate_integration_input_qualifier.go new file mode 100644 index 000000000..ad1bef56e --- /dev/null +++ b/code/go/internal/validator/semantic/validate_integration_input_qualifier.go @@ -0,0 +1,79 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "fmt" + "io/fs" + + "gopkg.in/yaml.v3" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" +) + +type integrationInputQualifier struct { + Name string `yaml:"name"` + Type string `yaml:"type"` +} + +type integrationPolicyTemplateQualifier struct { + Name string `yaml:"name"` + Inputs []integrationInputQualifier `yaml:"inputs"` +} + +type integrationPackageManifestQualifier struct { + Type string `yaml:"type"` + PolicyTemplates []integrationPolicyTemplateQualifier `yaml:"policy_templates"` +} + +// ValidateIntegrationInputQualifier checks that if a policy template contains +// multiple inputs of the same type, all of them must have a name set. Without +// names, Fleet cannot distinguish them, leading to the ambiguity this field +// aims to solve. +func ValidateIntegrationInputQualifier(fsys fspath.FS) specerrors.ValidationErrors { + manifestPath := "manifest.yml" + data, err := fs.ReadFile(fsys, manifestPath) + if err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to read manifest: %w", fsys.Path(manifestPath), err)} + } + + var manifest integrationPackageManifestQualifier + if err := yaml.Unmarshal(data, &manifest); err != nil { + return specerrors.ValidationErrors{ + specerrors.NewStructuredErrorf("file \"%s\" is invalid: failed to parse manifest: %w", fsys.Path(manifestPath), err)} + } + + if manifest.Type != integrationPackageType { + return nil + } + + return validateInputQualifiers(fsys, manifest, manifestPath) +} + +func validateInputQualifiers(fsys fspath.FS, manifest integrationPackageManifestQualifier, manifestPath string) specerrors.ValidationErrors { + var errs specerrors.ValidationErrors + + for _, policyTemplate := range manifest.PolicyTemplates { + typeCounts := make(map[string]int) + for _, input := range policyTemplate.Inputs { + typeCounts[input.Type]++ + } + + reported := make(map[string]bool) + for _, input := range policyTemplate.Inputs { + if typeCounts[input.Type] > 1 && input.Name == "" && !reported[input.Type] { + reported[input.Type] = true + errs = append(errs, specerrors.NewStructuredError( + fmt.Errorf("file \"%s\" is invalid: policy template \"%s\": input with type \"%s\" must have a name when multiple inputs of the same type are present", + fsys.Path(manifestPath), policyTemplate.Name, input.Type), + specerrors.CodeIntegrationInputQualifierRequired)) + } + } + } + + return errs +} diff --git a/code/go/internal/validator/semantic/validate_integration_input_qualifier_test.go b/code/go/internal/validator/semantic/validate_integration_input_qualifier_test.go new file mode 100644 index 000000000..73ab16d9e --- /dev/null +++ b/code/go/internal/validator/semantic/validate_integration_input_qualifier_test.go @@ -0,0 +1,134 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package semantic + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/package-spec/v3/code/go/internal/fspath" + "github.com/elastic/package-spec/v3/code/go/pkg/specerrors" +) + +func TestValidateIntegrationInputQualifier(t *testing.T) { + tests := map[string]struct { + manifest string + expectedErrs []string + expectedCodes []string + }{ + "single_input_per_type_no_name": { + manifest: ` +type: integration +policy_templates: + - name: nginx + inputs: + - type: logfile + title: Logs + description: Collect logs + - type: httpjson + title: Metrics + description: Collect metrics +`, + }, + "multiple_inputs_same_type_with_names": { + manifest: ` +type: integration +policy_templates: + - name: nginx + inputs: + - name: filelog_otel + type: otelcol + title: Logs + description: Collect logs + - name: nginx_otel + type: otelcol + title: Metrics + description: Collect metrics +`, + }, + "multiple_inputs_same_type_no_names": { + manifest: ` +type: integration +policy_templates: + - name: nginx + inputs: + - type: otelcol + title: Logs + description: Collect logs + - type: otelcol + title: Metrics + description: Collect metrics +`, + expectedErrs: []string{ + `policy template "nginx": input with type "otelcol" must have a name when multiple inputs of the same type are present`, + }, + expectedCodes: []string{ + specerrors.CodeIntegrationInputQualifierRequired, + }, + }, + "multiple_inputs_same_type_partial_names": { + manifest: ` +type: integration +policy_templates: + - name: nginx + inputs: + - name: filelog_otel + type: otelcol + title: Logs + description: Collect logs + - type: otelcol + title: Metrics + description: Collect metrics +`, + expectedErrs: []string{ + `policy template "nginx": input with type "otelcol" must have a name when multiple inputs of the same type are present`, + }, + expectedCodes: []string{ + specerrors.CodeIntegrationInputQualifierRequired, + }, + }, + "non_integration_package": { + manifest: ` +type: input +policy_templates: + - name: sample + inputs: + - type: logfile + title: Logs + description: Collect logs + - type: logfile + title: More logs + description: Collect more logs +`, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + d := t.TempDir() + err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(tc.manifest), 0o644) + require.NoError(t, err) + + errs := ValidateIntegrationInputQualifier(fspath.DirFS(d)) + + if len(tc.expectedErrs) == 0 { + require.Empty(t, errs) + return + } + + require.Len(t, errs, len(tc.expectedErrs)) + for i, expected := range tc.expectedErrs { + assert.True(t, strings.Contains(errs[i].Error(), expected), + "error %q does not contain %q", errs[i].Error(), expected) + assert.Equal(t, tc.expectedCodes[i], errs[i].Code()) + } + }) + } +} diff --git a/code/go/internal/validator/spec.go b/code/go/internal/validator/spec.go index 3247e42eb..44b7995f1 100644 --- a/code/go/internal/validator/spec.go +++ b/code/go/internal/validator/spec.go @@ -237,6 +237,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules {fn: semantic.ValidateKibanaTagDuplicates}, {fn: semantic.ValidatePipelineOnFailure, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, {fn: semantic.ValidateIntegrationInputsDeprecation, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, + {fn: semantic.ValidateIntegrationInputQualifier, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, {fn: semantic.ValidateDeprecatedReplacedBy, since: semver.MustParse("3.6.0")}, {fn: semantic.ValidatePackageReferences, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, {fn: semantic.ValidateTestPackageRequirements, types: []string{"integration"}, since: semver.MustParse("3.6.0")}, diff --git a/code/go/pkg/specerrors/constants.go b/code/go/pkg/specerrors/constants.go index 10ce295f3..eab3b35a4 100644 --- a/code/go/pkg/specerrors/constants.go +++ b/code/go/pkg/specerrors/constants.go @@ -25,4 +25,5 @@ const ( CodeKibanaTagDuplicates = "SVR00007" CodePipelineOnFailureEventKind = "SVR00008" CodePipelineOnFailureMessage = "SVR00009" + CodeIntegrationInputQualifierRequired = "SVR00010" ) diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index 7512e1477..68e41b6d4 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -36,6 +36,7 @@ func TestValidateFile(t *testing.T) { "good_var_groups_input": {}, "good_input": {}, "good_input_otel": {}, + "good_input_qualifier": {}, "good_input_dynamic_signal_type": {}, "good_input_profiles": {}, "good_input_template_paths": {}, @@ -421,6 +422,19 @@ func TestValidateFile(t *testing.T) { "field policy_templates.0.inputs.0.type: Must not be present", }, }, + "bad_input_qualifier_ambiguous": { + "manifest.yml", + []string{ + `policy template "nginx": input with type "otelcol" must have a name when multiple inputs of the same type are present (SVR00010)`, + }, + }, + "bad_input_qualifier_old_version": { + "manifest.yml", + []string{ + "field policy_templates.0.inputs.0.type: Must not be present", + "field policy_templates.0.inputs.0: Additional property name is not allowed", + }, + }, "bad_input_dynamic_signal_types_old_version": { "manifest.yml", []string{ diff --git a/spec/changelog.yml b/spec/changelog.yml index 5209fc912..b02417985 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -13,6 +13,10 @@ - description: Add var_groups support to policy template and input levels in integration packages, and to policy template and package root levels in input packages. type: enhancement link: https://github.com/elastic/package-spec/pull/1120 + # Pending on https://github.com/elastic/kibana/pull/262138 + - description: Add support for named inputs in policy templates to disambiguate multiple inputs of the same type. + type: enhancement + link: https://github.com/elastic/package-spec/pull/1135 - version: 3.6.0 changes: - description: Add pipeline tag validations. diff --git a/spec/integration/manifest.spec.yml b/spec/integration/manifest.spec.yml index 5b4589a89..279d596ea 100644 --- a/spec/integration/manifest.spec.yml +++ b/spec/integration/manifest.spec.yml @@ -779,6 +779,12 @@ spec: type: object additionalProperties: false properties: + name: + description: > + Unique name for this input within the policy template. When set, data streams + reference this input by name instead of type, allowing multiple inputs of the + same type to coexist in the same policy template. + type: string type: description: Type of input. type: string @@ -933,6 +939,9 @@ spec: versions: - before: 3.6.0 patch: + # Input qualifier (named inputs). + - op: remove + path: "/properties/policy_templates/items/properties/inputs/items/properties/name" # Support for otelcol and dynamic_signal_types in integration packages. - op: add path: "/properties/policy_templates/items/properties/inputs/items/properties/type" diff --git a/test/packages/bad_input_qualifier_ambiguous/changelog.yml b/test/packages/bad_input_qualifier_ambiguous/changelog.yml new file mode 100644 index 000000000..5a976949e --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1234 diff --git a/test/packages/bad_input_qualifier_ambiguous/data_stream/access/agent/stream/stream.yml.hbs b/test/packages/bad_input_qualifier_ambiguous/data_stream/access/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..d0a3b3825 --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/data_stream/access/agent/stream/stream.yml.hbs @@ -0,0 +1,15 @@ +receivers: + filelog: + include: +{{#each paths}} + - {{this}} +{{/each}} +exporters: + {{#each exporters}} + {{this}} + {{/each}} +service: + pipelines: + {{#each pipelines}} + {{this}} + {{/each}} diff --git a/test/packages/bad_input_qualifier_ambiguous/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/test/packages/bad_input_qualifier_ambiguous/data_stream/access/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..f55f76887 --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,18 @@ +--- +description: Pipeline for processing Nginx access logs +processors: + - set: + tag: set_data_stream_type + field: data_stream.type + value: logs +on_failure: + - set: + field: event.kind + value: pipeline_error + - set: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + with tag '{{{ _ingest.on_failure_processor_tag }}}' + in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/test/packages/bad_input_qualifier_ambiguous/data_stream/access/fields/fields.yml b/test/packages/bad_input_qualifier_ambiguous/data_stream/access/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/data_stream/access/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/bad_input_qualifier_ambiguous/data_stream/access/manifest.yml b/test/packages/bad_input_qualifier_ambiguous/data_stream/access/manifest.yml new file mode 100644 index 000000000..f95887a26 --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/data_stream/access/manifest.yml @@ -0,0 +1,15 @@ +title: Nginx Access Logs +type: logs +streams: + - input: filelog_otel + title: Collect Nginx access logs via filelog OTel receiver + description: Collecting Nginx access logs via filelog OpenTelemetry receiver. + vars: + - name: paths + type: text + title: Log file paths + multi: true + required: true + show_user: true + default: + - /var/log/nginx/access.log diff --git a/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/agent/stream/stream.yml.hbs b/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5a44c963b --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +receivers: + nginx: + endpoint: http://127.0.0.1/nginx_status + collection_interval: {{period}} +exporters: + {{#each exporters}} + {{this}} + {{/each}} +service: + pipelines: + {{#each pipelines}} + {{this}} + {{/each}} diff --git a/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/elasticsearch/ingest_pipeline/default.yml b/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..aea0906a0 --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,18 @@ +--- +description: Pipeline for processing Nginx stub status metrics +processors: + - set: + tag: set_data_stream_type + field: data_stream.type + value: metrics +on_failure: + - set: + field: event.kind + value: pipeline_error + - set: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + with tag '{{{ _ingest.on_failure_processor_tag }}}' + in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/fields/fields.yml b/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/manifest.yml b/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/manifest.yml new file mode 100644 index 000000000..3e01137a6 --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/data_stream/stubstatus/manifest.yml @@ -0,0 +1,14 @@ +title: Nginx Stub Status Metrics +type: metrics +streams: + - input: nginx_otel + title: Collect Nginx stub status metrics via OTel receiver + description: Collecting Nginx stub_status metrics via OpenTelemetry receiver. + vars: + - name: period + type: text + title: Collection period + multi: false + required: true + show_user: true + default: 10s diff --git a/test/packages/bad_input_qualifier_ambiguous/docs/README.md b/test/packages/bad_input_qualifier_ambiguous/docs/README.md new file mode 100644 index 000000000..2dc5571b4 --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/docs/README.md @@ -0,0 +1,3 @@ +# Good Input Qualifier + +Integration package with multiple inputs of the same type using named qualifiers. diff --git a/test/packages/bad_input_qualifier_ambiguous/manifest.yml b/test/packages/bad_input_qualifier_ambiguous/manifest.yml new file mode 100644 index 000000000..6457b75ed --- /dev/null +++ b/test/packages/bad_input_qualifier_ambiguous/manifest.yml @@ -0,0 +1,29 @@ +format_version: 3.6.0 +name: bad_input_qualifier_ambiguous +title: "Bad package with ambiguous inputs" +version: 0.0.1 +source: + license: "Apache-2.0" +description: "Integration package with multiple inputs of the same type without names." +type: integration +categories: + - observability +conditions: + kibana: + version: "^9.4.0" + elastic: + subscription: "basic" +policy_templates: + - name: nginx + title: Nginx logs and metrics + description: Collect logs and metrics from Nginx instances + inputs: + - type: otelcol + title: Collect Nginx access logs via filelog OTel receiver + description: Use the filelog OTel receiver to tail Nginx access log files. + - type: otelcol + title: Collect Nginx stub status metrics via OTel receiver + description: Use the nginx OTel receiver to scrape Nginx stub_status metrics. +owner: + github: elastic/foobar + type: elastic diff --git a/test/packages/bad_input_qualifier_old_version/LICENSE.txt b/test/packages/bad_input_qualifier_old_version/LICENSE.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/test/packages/bad_input_qualifier_old_version/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/test/packages/bad_input_qualifier_old_version/changelog.yml b/test/packages/bad_input_qualifier_old_version/changelog.yml new file mode 100644 index 000000000..bb0320a52 --- /dev/null +++ b/test/packages/bad_input_qualifier_old_version/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/bad_input_qualifier_old_version/docs/README.md b/test/packages/bad_input_qualifier_old_version/docs/README.md new file mode 100644 index 000000000..d28db2ce3 --- /dev/null +++ b/test/packages/bad_input_qualifier_old_version/docs/README.md @@ -0,0 +1 @@ +# Bad integration because of use of otel before 3.6.0 diff --git a/test/packages/bad_input_qualifier_old_version/img/sample-logo.svg b/test/packages/bad_input_qualifier_old_version/img/sample-logo.svg new file mode 100644 index 000000000..6268dd88f --- /dev/null +++ b/test/packages/bad_input_qualifier_old_version/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/bad_input_qualifier_old_version/img/sample-screenshot.png b/test/packages/bad_input_qualifier_old_version/img/sample-screenshot.png new file mode 100644 index 000000000..d7a56a3ec Binary files /dev/null and b/test/packages/bad_input_qualifier_old_version/img/sample-screenshot.png differ diff --git a/test/packages/bad_input_qualifier_old_version/manifest.yml b/test/packages/bad_input_qualifier_old_version/manifest.yml new file mode 100644 index 000000000..12c90ecdc --- /dev/null +++ b/test/packages/bad_input_qualifier_old_version/manifest.yml @@ -0,0 +1,32 @@ +format_version: 3.5.7 +name: bad_input_qualifier_old_version +title: "Integration package with named inputs and old version" +version: 0.0.1 +source: + license: "Apache-2.0" +description: "Test package for integration with named inputs and old format_version" +type: integration +categories: + - custom +conditions: + kibana: + version: "^9.4.0" + elastic: + subscription: "basic" +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: nginx + title: Nginx logs and metrics + description: Collect logs and metrics from Nginx instances + inputs: + - name: filelog_otel + type: otelcol + title: Collect Nginx access logs via filelog OTel receiver + description: Use the filelog OTel receiver to tail Nginx access log files. +owner: + github: elastic/ecosystem + type: elastic diff --git a/test/packages/bad_input_qualifier_old_version/sample_event.json b/test/packages/bad_input_qualifier_old_version/sample_event.json new file mode 100644 index 000000000..e4a442fcb --- /dev/null +++ b/test/packages/bad_input_qualifier_old_version/sample_event.json @@ -0,0 +1,3 @@ +{ + "description": "This is an example sample-event for Integration package with otelcol input and old version. Replace it with a real sample event. Hint: If system tests exist, running `elastic-package test system --generate` will generate this file." +} diff --git a/test/packages/good_input_qualifier/changelog.yml b/test/packages/good_input_qualifier/changelog.yml new file mode 100644 index 000000000..5a976949e --- /dev/null +++ b/test/packages/good_input_qualifier/changelog.yml @@ -0,0 +1,5 @@ +- version: 0.0.1 + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/package-spec/pull/1234 diff --git a/test/packages/good_input_qualifier/data_stream/access/agent/stream/stream.yml.hbs b/test/packages/good_input_qualifier/data_stream/access/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..d0a3b3825 --- /dev/null +++ b/test/packages/good_input_qualifier/data_stream/access/agent/stream/stream.yml.hbs @@ -0,0 +1,15 @@ +receivers: + filelog: + include: +{{#each paths}} + - {{this}} +{{/each}} +exporters: + {{#each exporters}} + {{this}} + {{/each}} +service: + pipelines: + {{#each pipelines}} + {{this}} + {{/each}} diff --git a/test/packages/good_input_qualifier/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/test/packages/good_input_qualifier/data_stream/access/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..f55f76887 --- /dev/null +++ b/test/packages/good_input_qualifier/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,18 @@ +--- +description: Pipeline for processing Nginx access logs +processors: + - set: + tag: set_data_stream_type + field: data_stream.type + value: logs +on_failure: + - set: + field: event.kind + value: pipeline_error + - set: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + with tag '{{{ _ingest.on_failure_processor_tag }}}' + in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/test/packages/good_input_qualifier/data_stream/access/fields/fields.yml b/test/packages/good_input_qualifier/data_stream/access/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_input_qualifier/data_stream/access/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_input_qualifier/data_stream/access/manifest.yml b/test/packages/good_input_qualifier/data_stream/access/manifest.yml new file mode 100644 index 000000000..f95887a26 --- /dev/null +++ b/test/packages/good_input_qualifier/data_stream/access/manifest.yml @@ -0,0 +1,15 @@ +title: Nginx Access Logs +type: logs +streams: + - input: filelog_otel + title: Collect Nginx access logs via filelog OTel receiver + description: Collecting Nginx access logs via filelog OpenTelemetry receiver. + vars: + - name: paths + type: text + title: Log file paths + multi: true + required: true + show_user: true + default: + - /var/log/nginx/access.log diff --git a/test/packages/good_input_qualifier/data_stream/stubstatus/agent/stream/stream.yml.hbs b/test/packages/good_input_qualifier/data_stream/stubstatus/agent/stream/stream.yml.hbs new file mode 100644 index 000000000..5a44c963b --- /dev/null +++ b/test/packages/good_input_qualifier/data_stream/stubstatus/agent/stream/stream.yml.hbs @@ -0,0 +1,13 @@ +receivers: + nginx: + endpoint: http://127.0.0.1/nginx_status + collection_interval: {{period}} +exporters: + {{#each exporters}} + {{this}} + {{/each}} +service: + pipelines: + {{#each pipelines}} + {{this}} + {{/each}} diff --git a/test/packages/good_input_qualifier/data_stream/stubstatus/elasticsearch/ingest_pipeline/default.yml b/test/packages/good_input_qualifier/data_stream/stubstatus/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..aea0906a0 --- /dev/null +++ b/test/packages/good_input_qualifier/data_stream/stubstatus/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,18 @@ +--- +description: Pipeline for processing Nginx stub status metrics +processors: + - set: + tag: set_data_stream_type + field: data_stream.type + value: metrics +on_failure: + - set: + field: event.kind + value: pipeline_error + - set: + field: error.message + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + with tag '{{{ _ingest.on_failure_processor_tag }}}' + in pipeline '{{{ _ingest.pipeline }}}' + failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/test/packages/good_input_qualifier/data_stream/stubstatus/fields/fields.yml b/test/packages/good_input_qualifier/data_stream/stubstatus/fields/fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_input_qualifier/data_stream/stubstatus/fields/fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_input_qualifier/data_stream/stubstatus/manifest.yml b/test/packages/good_input_qualifier/data_stream/stubstatus/manifest.yml new file mode 100644 index 000000000..3e01137a6 --- /dev/null +++ b/test/packages/good_input_qualifier/data_stream/stubstatus/manifest.yml @@ -0,0 +1,14 @@ +title: Nginx Stub Status Metrics +type: metrics +streams: + - input: nginx_otel + title: Collect Nginx stub status metrics via OTel receiver + description: Collecting Nginx stub_status metrics via OpenTelemetry receiver. + vars: + - name: period + type: text + title: Collection period + multi: false + required: true + show_user: true + default: 10s diff --git a/test/packages/good_input_qualifier/docs/README.md b/test/packages/good_input_qualifier/docs/README.md new file mode 100644 index 000000000..2dc5571b4 --- /dev/null +++ b/test/packages/good_input_qualifier/docs/README.md @@ -0,0 +1,3 @@ +# Good Input Qualifier + +Integration package with multiple inputs of the same type using named qualifiers. diff --git a/test/packages/good_input_qualifier/manifest.yml b/test/packages/good_input_qualifier/manifest.yml new file mode 100644 index 000000000..61feb6433 --- /dev/null +++ b/test/packages/good_input_qualifier/manifest.yml @@ -0,0 +1,31 @@ +format_version: 3.6.0 +name: good_input_qualifier +title: "Good package with named inputs" +version: 0.0.1 +source: + license: "Apache-2.0" +description: "Integration package with multiple inputs of the same type using named qualifiers." +type: integration +categories: + - observability +conditions: + kibana: + version: "^9.4.0" + elastic: + subscription: "basic" +policy_templates: + - name: nginx + title: Nginx logs and metrics + description: Collect logs and metrics from Nginx instances + inputs: + - name: filelog_otel + type: otelcol + title: Collect Nginx access logs via filelog OTel receiver + description: Use the filelog OTel receiver to tail Nginx access log files. + - name: nginx_otel + type: otelcol + title: Collect Nginx stub status metrics via OTel receiver + description: Use the nginx OTel receiver to scrape Nginx stub_status metrics. +owner: + github: elastic/foobar + type: elastic