Initial definition of content pack[age]s

code/go/internal/loader/spec.go

@@ -5,7 +5,10 @@
 package loader
 
 import (
+	"errors"
+	"fmt"
 	"io/fs"
+	"os"
 
 	"github.com/Masterminds/semver/v3"
 
@@ -18,5 +21,12 @@ import (
 func LoadSpec(fsys fs.FS, version semver.Version, pkgType string) (spectypes.ItemSpec, error) {
 	fileSpecLoader := yamlschema.NewFileSchemaLoader()
 	loader := specschema.NewFolderSpecLoader(fsys, fileSpecLoader, version)
-	return loader.Load(pkgType)
+	spec, err := loader.Load(pkgType)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, fmt.Errorf("package type %q not supported (%w)", pkgType, err)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return spec, nil
 }

code/go/internal/spec_test.go

@@ -19,6 +19,7 @@ func TestLoadAllBundledVersions(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, version := range versions {
+		testForVersionType(t, version, "content")
 		testForVersionType(t, version, "input")
 		testForVersionType(t, version, "integration")
 	}

code/go/pkg/specerrors/filter.go

@@ -23,25 +23,33 @@ type Filter struct {
 type FilterResult struct {
 	Processed error
 	Removed   error
+
+	UnusedProcessors []Processor
 }
 
 // Run runs all the processors over all the validation errors and return the filtered ones
 func (r *Filter) Run(allErrors ValidationErrors) (FilterResult, error) {
 	newErrors := allErrors
 	var allFiltered ValidationErrors
 
+	var unused []Processor
 	for _, p := range r.processors {
 		result, err := p.Process(newErrors)
 		if err != nil {
 			return FilterResult{Processed: allErrors, Removed: nil}, err
 		}
 		newErrors = result.Processed
 		allFiltered.Append(result.Removed)
+
+		if len(result.Removed) == 0 {
+			unused = append(unused, p)
+		}
 	}
 
 	return FilterResult{
-		Processed: nilOrValidationErrors(newErrors),
-		Removed:   nilOrValidationErrors(allFiltered),
+		Processed:        nilOrValidationErrors(newErrors),
+		Removed:          nilOrValidationErrors(allFiltered),
+		UnusedProcessors: unused,
 	}, nil
 }
 

code/go/pkg/validator/validator_test.go

@@ -33,6 +33,7 @@ func TestValidateFile(t *testing.T) {
 		"good_v2":                            {},
 		"good_v3":                            {},
 		"good_input":                         {},
+		"good_content":                       {},
 		"deploy_custom_agent":                {},
 		"deploy_custom_agent_multi_services": {},
 		"deploy_docker":                      {},
@@ -232,6 +233,7 @@ func TestValidateFile(t *testing.T) {
 					filter := specerrors.NewFilter(filterConfig)
 					result, err := filter.Run(verrs)
 					require.NoError(t, err)
+					assert.Empty(t, result.UnusedProcessors, "There are unused exclusion checks in the validation.yml file")
 					errs = result.Processed
 				}
 			}

spec/changelog.yml

@@ -2,11 +2,24 @@
 ## This file documents changes in the package specification. It is NOT a package specification file.
 ## Newer entries go at the bottom of each in-development version.
 ##
+- version: 3.4.0-next
+  changes:
+  - description: Add support for content packages.
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/777
 - version: 3.3.0-next
   changes:
   - description: Add support for `slo` assets.
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/767
+  - description: Add spec for img directory.
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/777
+- version: 3.2.2-next
+  changes:
+  - description: Improved error message for unsupported package types.
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/777
 - version: 3.2.1
   changes:
   - description: Improve error information for missing dataset field in manifest.

spec/content/kibana/spec.yml

@@ -0,0 +1,31 @@
+spec:
+  additionalContents: false
+  contents:
+  - description: Folder containing Kibana dashboard assets
+    type: folder
+    name: dashboard
+    required: false
+    contents:
+    - description: A dashboard asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: '^{PACKAGE_NAME}-.+\.json$'
+      forbiddenPatterns:
+        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden
+  - description: File containing saved object tag definitions for assets
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "tags.yml"
+    required: false
+    $ref: "../../integration/kibana/tags.spec.yml"
+  - description: Folder containing Kibana SLO assets
+    type: folder
+    name: slo
+    required: false
+    contents:
+    - description: An SLO asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: '^{PACKAGE_NAME}-.+\.json$'
+      forbiddenPatterns:
+        - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden

spec/content/manifest.spec.yml

@@ -0,0 +1,70 @@
+##
+## Describes the specification for the content package's main manifest.yml file
+##
+spec:
+  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+  type: object
+  additionalProperties: false
+  definitions:
+    discovery:
+      description: >
+        Description of the data this package can be used with. It can be used to discover
+        the package from elements in the existing data.
+      type: object
+      additionalProperties: false
+      properties:
+        fields:
+          description: Description of the fields this package can be used with.
+          type: array
+          items:
+            type: object
+            properties:
+              name:
+                description: Name of the field.
+                type: string
+  properties:
+    format_version:
+      description: The version of the package specification format used by this package.
+      $ref: "../integration/manifest.spec.yml#/definitions/version"
+    name:
+      description: The name of the package.
+      type: string
+      pattern: '^[a-z0-9_]+$'
+      examples:
+      - apache
+    title:
+      $ref: "../integration/manifest.spec.yml#/definitions/title"
+    description:
+      $ref: "../integration/manifest.spec.yml#/definitions/description"
+    version:
+      description: The version of the package.
+      $ref: "../integration/manifest.spec.yml#/definitions/version"
+    source:
+      $ref: "../integration/manifest.spec.yml#/definitions/source"
+    type:
+      description: The type of package.
+      type: string
+      enum:
+      - content
+      examples:
+      - content
+    categories:
+      $ref: "../integration/manifest.spec.yml#/definitions/categories"
+    conditions:
+      $ref: "../integration/manifest.spec.yml#/definitions/conditions"
+    discovery:
+      $ref: "#/definitions/discovery"
+    icons:
+      $ref: "../integration/manifest.spec.yml#/definitions/icons"
+    screenshots:
+      $ref: "../integration/manifest.spec.yml#/definitions/screenshots"
+    owner:
+      $ref: "../integration/manifest.spec.yml#/definitions/owner"
+  required:
+  - format_version
+  - name
+  - title
+  - description
+  - version
+  - type
+  - owner

spec/content/spec.yml

@@ -0,0 +1,61 @@
+##
+## Entrypoint of "content packages" specification.
+## The specification is considered "beta" at the moment, so it may change until we release it as GA.
+##
+## Describes the folders and files that make up a package.
+##
+spec:
+  additionalContents: false
+  totalContentsLimit: 65535
+  totalSizeLimit: 50MB
+  sizeLimit: 30MB
+  configurationSizeLimit: 5MB
+  relativePathSizeLimit: 3MB
+  fieldsPerDataStreamLimit: 2048
+  release: beta
+  contents:
+  - description: The main package manifest file
+    type: file
+    contentMediaType: "application/x-yaml"
+    sizeLimit: 5MB
+    name: "manifest.yml"
+    required: true
+    $ref: "./manifest.spec.yml"
+  - description: The package's CHANGELOG file
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "changelog.yml"
+    required: true
+    $ref: "../integration/changelog.spec.yml"
+  - description: The package's license file
+    type: file
+    contentMediaType: "text/plain"
+    name: "LICENSE.txt"
+    required: false
+  - description: Folder containing documentation for the package
+    type: folder
+    name: docs
+    required: true
+    $ref: "../integration/docs/spec.yml"
+  - description: Folder containing images for the package
+    type: folder
+    name: img
+    required: false
+    $ref: "../integration/img/spec.yml"
+  - description: Folder containing Kibana assets provided by the package
+    type: folder
+    name: kibana
+    required: false
+    $ref: "./kibana/spec.yml"
+  - description: Configuration file to process the results returned from the package validation. This file is just for package validation and it should be ignored when installing or using the package.
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "validation.yml"
+    required: false
+    $ref: "../integration/validation.spec.yml"
+
+versions:
+  - before: 3.4.0
+    patch:
+      - op: remove
+        path: "/contents" # Package type not available before this version.

spec/input/spec.yml

@@ -1,6 +1,5 @@
 ##
 ## Entrypoint of "input packages" specification.
-## The specification is considered "beta" at the moment, so it may change until we release it as GA.
 ##
 ## Describes the folders and files that make up a package.
 ##
@@ -69,8 +68,17 @@ spec:
     name: "validation.yml"
     required: false
     $ref: "../integration/validation.spec.yml"
+  - description: Folder containing images for the package
+    type: folder
+    name: img
+    required: false
+    $ref: "../integration/img/spec.yml"
 
 versions:
+  - before: 3.3.0
+    patch:
+      - op: remove
+        path: "/contents/10" # Definition for img folder.
   - before: 2.10.0
     patch:
       - op: remove

spec/integration/docs/spec.yml

@@ -1,13 +1,13 @@
-  spec:
-    additionalContents: false
-    contents:
-    - description: Main README file
-      type: file
-      contentMediaType: "text/markdown"
-      name: "README.md"
-      required: true
-    - description: Other README files (can be used by policy templates)
-      type: file
-      contentMediaType: "text/markdown"
-      pattern: '^.+.md'
-      required: false
+spec:
+  additionalContents: false
+  contents:
+  - description: Main README file
+    type: file
+    contentMediaType: "text/markdown"
+    name: "README.md"
+    required: true
+  - description: Other README files (can be used by policy templates)
+    type: file
+    contentMediaType: "text/markdown"
+    pattern: '^.+.md'
+    required: false

spec/integration/img/spec.yml

@@ -0,0 +1,2 @@
+spec:
+  additionalContents: true # TODO: Add definition for this dir?

spec/integration/kibana/tags.spec.yml

@@ -1,5 +1,5 @@
 ##
-## Describes the specification for a routing rules yml file
+## Describes the specification for the additional kibana tags provided by the package.
 ##
 spec:
   # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability

spec/integration/spec.yml

@@ -72,3 +72,14 @@ spec:
     name: "validation.yml"
     required: false
     $ref: "./validation.spec.yml"
+  - description: Folder containing images for the package
+    type: folder
+    name: img
+    required: false
+    $ref: "./img/spec.yml"
+
+versions:
+  - before: 3.3.0
+    patch:
+      - op: remove
+        path: "/contents/11" # Definition for img folder.

test/packages/good_content/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.1.0
+  changes:
+    - description: Initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/777

test/packages/good_content/docs/README.md

@@ -0,0 +1 @@
+# Reference package of content type

test/packages/good_content/manifest.yml

@@ -0,0 +1,32 @@
+format_version: 3.4.0
+name: good_content
+title: Good content package
+description: >
+  This package is a dummy example for packages with the content type.
+  These packages contain resources that are useful with data ingested by other integrations.
+  They are not used to configure data sources.
+version: 0.1.0
+type: content
+source:
+  license: "Apache-2.0"
+conditions:
+  kibana:
+    version: '^8.16.0' #TBD
+  elastic:
+    subscription: 'basic'
+discovery:
+  fields:
+    - name: process.pid
+screenshots:
+  - src: /img/kibana-system.png
+    title: kibana system
+    size: 1220x852
+    type: image/png
+icons:
+  - src: /img/system.svg
+    title: system
+    size: 1000x1000
+    type: image/svg+xml
+owner:
+  github: elastic/ecosystem
+  type: elastic

test/packages/good_content/validation.yml

@@ -0,0 +1,3 @@
+errors:
+  exclude_checks:
+    - PSR00002 # Allow to use non-GA features.