Adding initial spec files

README.md

@@ -7,3 +7,20 @@ In the future it may also contain code for validating said specifications.
 # Purpose
 
 Please use this repository to discuss any changes to the specification, either my making issues or PRs to the specification.
+
+# Specification Format 
+
+An Elastic Package specification describes:
+1. the folder structure of packages and expected files within these folders; and
+2. the structure of the expected files' contents.
+
+There may be multiple versions of specifications. At the root of this repository is a `versions` folder. In this folder you will find sub-folders for each active version of the specification.
+
+Within each version folder, there must be a `spec.yml` file. This file is the entry point for the specification for a package's contents. It describes the the folder structure of packages and expected files within these folders (this is point 1. above). The specification is expressed using a schema similar to [JSON Schema](https://json-schema.org/), but with a couple of differences:
+- The `type` field can be either `folder` or `file`,
+- A new field field, `specRef` is introduced to (recursively) reference other specification files, and
+- The specification is written as YAML for readability.
+
+Expected package files, e.g. `manifest.yml` themselves have a structure to their contents. This structure is described in specification files using JSON schema (this is point 2. above). These specification files are also written as YAML for readability.
+
+Note that the specification files primarily defined the structure (syntax) of a package's contents. To a limited extent they may also define some semantics, e.g. enumeration values for certain fields. Richer semantics, however, will need to be expressed as validation code.

versions/1.0.0/changelog.spec.yml

@@ -0,0 +1,46 @@
+##
+## Describes the specification for the package's CHANGELOG file
+##
+spec:
+  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+  type: array
+  items:
+    type: object
+    properties:
+      version:
+        description: Package version.
+        type: string
+        pattern: "^[0-9]+\.[0-9]+\.[0-9]+$"
+        examples:
+        - "1.0.4"
+      changes:
+        description: List of changes in package version.
+        type: array
+        items:
+          type: object
+          properties:
+            description:
+              description: Description of change.
+              type: string
+              examples:
+              - "Fix broken template"
+            type:
+              description: Type of change.
+              type: string
+              enum:
+              - "breaking-change"
+              - "bugfix"
+              - "enhancement"
+              - "known-issue"
+            link:
+              description: Link to issue or PR describing change in detail.
+              type: string
+              examples:
+              - "https://github.com/elastic/beats/issues/13507"
+          required:
+          - description
+          - type
+          - link
+    required:
+    - version
+    - changes

versions/1.0.0/dataset/fields/fields.spec.yml

@@ -0,0 +1,41 @@
+##
+## Describes the specification for a dataset's various field definition files
+##
+spec:
+  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+  type: object
+  properties:
+    name:
+      description: Name of field
+      type: string
+    title:
+      description: Title of field
+      type: string
+    type: 
+      description: Datatype of field
+      type: string
+      enum:
+      - constant_keyword
+      - date
+      - group
+    description:
+      description: Short description of field
+      type: string
+    group:
+      description: TODO
+      type: int
+    level:
+      description: TODO
+      type: string
+      enum:
+      - custom
+    default_field:
+      description: "Is this field a default? TODO: better description"
+      type: boolean
+      default: true
+    fields:
+      description: Sub-fields, when type is group
+      $ref: "#"     # JSON-schema syntax for pointing to the root of the schema
+  required:
+  - name
+  - type

versions/1.0.0/dataset/fields/spec.yml

@@ -0,0 +1,27 @@
+spec:
+- name: base fields
+  description: Base fields definitions
+  type: file
+  name: "base-fields.yml"
+  required: true
+  contentMediaType: "application/x-yaml"
+  specRef: "fields.spec.yml"
+- name: dataset fields
+  description: Dataset-specific fields definitions
+  type: file
+  name: "fields.yml"
+  required: true
+  contentMediaType: "application/x-yaml"
+  specRef: "fields.spec.yml"
+- name: package fields
+  description: Package-specific fields definitions
+  type: file
+  name: "package-fields.yml"
+  contentMediaType: "application/x-yaml"
+  specRef: "fields.spec.yml"
+- name: ECS fields
+  description: ECS fields definitions
+  type: file
+  name: "ecs.yml"
+  contentMediaType: "application/x-yaml"
+  specRef: "fields.spec.yml"

versions/1.0.0/dataset/manifest.spec.yml

@@ -0,0 +1,104 @@
+##
+## Describes the specification for a dataset's manifest.yml file
+##
+spec:
+  # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability
+  type: object
+  properties:
+    title:
+      description: Title of dataset.
+      type: string
+      examples:
+      - AWS billing metrics
+    release:
+      description: Stability of dataset.
+      type: string
+      enum:
+      - experimental
+      - beta
+      examples:
+      - beta
+    type:
+      description: Type of dataset
+      type: string
+      enum:
+      - metrics
+      - logs
+      examples:
+      - metrics
+    streams:
+      description: Streams offered by dataset.
+      type: array
+      items:
+        type: object
+        properties:
+          title:
+            type: string
+            examples:
+            - AWS Billing metrics
+          description:
+            type: string
+            examples:
+            - Collect AWS billing metrics
+          input:
+            type: string
+            examples:
+            - aws/metrics
+            - s3
+            - file
+          vars:
+            description: Input variables.
+            type: array
+            items:
+              type: object
+              properties:
+                name:
+                  description: Variable name.
+                  type: string
+                  examples:
+                  - hosts
+                type:
+                  description: Data type of variable.
+                  type: string
+                  enum:
+                  - text
+                  examples:
+                  - text
+                title:
+                  description: Title of variable
+                  type: string
+                  examples:
+                  - Hosts
+                multi:
+                  description: TODO
+                  type: boolean
+                  default: false
+                  examples:
+                  - true
+                required:
+                  description: Is variable required?
+                  type: boolean
+                  default: false
+                  examples:
+                  - true
+                show_user:
+                  description: TODO
+                  type: boolean
+                  default: true
+                  examples:
+                  - false
+                default:
+                  description: Default value(s) for variable
+                  type: array
+                  items:
+                    type: string
+                    examples:
+                    - "http://127.0.0.1"
+              required:
+              - name
+              - type                  
+        required:
+        - title
+        - description
+  required:
+  - title

versions/1.0.0/dataset/spec.yml

@@ -0,0 +1,45 @@
+spec:
+- description: Folder containing a single dataset definition
+  type: folder
+  pattern: "[a-z0-9][a-z0-9_]+[a-z0-9]"
+  minOccurences: 1
+  spec:
+  - description: A dataset's manifest file
+    type: file
+    contentMediaType: "application/x-yaml"
+    name: "manifest.yml"
+    required: true
+    specRef: "manifest.spec.yml"
+  - description: Folder containing field definitions
+    type: folder
+    name: fields
+    required: true
+    specRef: "fields/spec.yml"
+  - description: Folder containing agent-related definitions
+    type: folder
+    name: agent
+    required: true
+    spec:
+    - description: Folder containing agent stream definitions
+      type: folder
+      name: stream
+      required: true
+      spec:
+      - description: Agent stream definition
+        type: file
+        name: "stream.yml.hbs"
+        contentMediaType: "application/x-hbs"
+        required: true
+  - description: Folder containing Elasticsearch assets
+    type: folder
+    name: elasticsearch
+    spec:
+    - description: Folder containing Elasticsearch Ingest Node pipeline definitions
+      type: folder
+      name: ingest_pipeline
+      spec:
+      - description: Default ingest pipeline definition
+        type: file
+        name: "default.yml"
+        contentMediaType: "application/x-yaml"
+        required: true

versions/1.0.0/kibana/spec.yml

@@ -0,0 +1,37 @@
+  spec:
+  - description: Folder containing Kibana dashboard assets
+    type: folder
+    name: dashboard
+    required: false
+    spec:
+    - description: A dashboard asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: ".+\.json"
+  - description: Folder containing Kibana visualization assets
+    type: folder
+    name: visualization
+    required: false
+    spec:
+    - description: A visualization asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: ".+\.json"
+  - description: Folder containing Kibana saved search assets
+    type: folder
+    name: search
+    required: false
+    spec:
+    - description: A saved search asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: ".+\.json"
+  - description: Folder containing Kibana map assets
+    type: folder
+    name: map
+    required: false
+    spec:
+    - description: A map asset file
+      type: file
+      contentMediaType: "application/json"
+      pattern: ".+\.json"