Merge remote-tracking branch 'upstream/master' into template-timeline

Author: angorayc

.eslintrc.js

@@ -562,7 +562,7 @@ module.exports = {
      */
     {
       // front end typescript and javascript files only
-      files: ['x-pack/legacy/plugins/siem/public/**/*.{js,ts,tsx}'],
+      files: ['x-pack/plugins/siem/public/**/*.{js,ts,tsx}'],
       rules: {
         'import/no-nodejs-modules': 'error',
         'no-restricted-imports': [
@@ -611,7 +611,7 @@ module.exports = {
     // {
     //   // will introduced after the other warns are fixed
     //   // typescript and javascript for front end react performance
-    //   files: ['x-pack/legacy/plugins/siem/public/**/!(*.test).{js,ts,tsx}'],
+    //   files: ['x-pack/plugins/siem/public/**/!(*.test).{js,ts,tsx}'],
     //   plugins: ['react-perf'],
     //   rules: {
     //     // 'react-perf/jsx-no-new-object-as-prop': 'error',

Jenkinsfile

@@ -4,54 +4,56 @@ library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
 kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
-  githubPr.withDefaultPrComments {
-    catchError {
-      retryable.enable()
-      parallel([
-        'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-        'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-        'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-          'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
-          'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-          'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-          'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-          'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-          'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-          'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-          'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-          'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-          'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-          'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-          'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-          'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-          'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
-          // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
-        ]),
-        'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-          'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
-          'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-          'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-          'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-          'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-          'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-          'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-          'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
-          'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
-          'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
-          'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
-          'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
-          'xpack-siemCypress': { processNumber ->
-            whenChanged(['x-pack/plugins/siem/', 'x-pack/legacy/plugins/siem/', 'x-pack/test/siem_cypress/']) {
-              kibanaPipeline.functionalTestProcess('xpack-siemCypress', './test/scripts/jenkins_siem_cypress.sh')(processNumber)
-            }
-          },
+  ciStats.trackBuild {
+    githubPr.withDefaultPrComments {
+      catchError {
+        retryable.enable()
+        parallel([
+          'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
+          'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
+          'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+            'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
+            'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
+            'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
+            'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
+            'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
+            'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
+            'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
+            'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
+            'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
+            'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
+            'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
+            'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
+            'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
+            'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
+            // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
+          ]),
+          'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
+            'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
+            'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
+            'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
+            'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
+            'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
+            'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
+            'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
+            'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
+            'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
+            'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
+            'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
+            'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
+            'xpack-siemCypress': { processNumber ->
+              whenChanged(['x-pack/plugins/siem/', 'x-pack/test/siem_cypress/']) {
+                kibanaPipeline.functionalTestProcess('xpack-siemCypress', './test/scripts/jenkins_siem_cypress.sh')(processNumber)
+              }
+            },
 
-          // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
-        ]),
-      ])
+            // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
+          ]),
+        ])
+      }
     }
-  }
 
-  retryable.printFlakyFailures()
-  kibanaPipeline.sendMail()
+    retryable.printFlakyFailures()
+    kibanaPipeline.sendMail()
+  }
 }

docs/developer/core-development.asciidoc

@@ -7,6 +7,7 @@
 * <<development-elasticsearch>>
 * <<development-unit-tests>>
 * <<development-functional-tests>>
+* <<development-es-snapshots>>
 
 include::core/development-basepath.asciidoc[]
 
@@ -19,3 +20,5 @@ include::core/development-elasticsearch.asciidoc[]
 include::core/development-unit-tests.asciidoc[]
 
 include::core/development-functional-tests.asciidoc[]
+
+include::core/development-es-snapshots.asciidoc[]

docs/developer/core/development-es-snapshots.asciidoc

@@ -0,0 +1,113 @@
+[[development-es-snapshots]]
+=== Daily Elasticsearch Snapshots
+
+For local development and CI, Kibana, by default, uses Elasticsearch snapshots that are built daily when running tasks that require Elasticsearch (e.g. functional tests).
+
+A snapshot is just a group of tarballs, one for each supported distribution/architecture/os of Elasticsearch, and a JSON-based manifest file containing metadata about the distributions.
+
+https://ci.kibana.dev/es-snapshots[A dashboard] is available that shows the current status and compatibility of the latest Elasticsearch snapshots.
+
+==== Process Overview
+
+1. Elasticsearch snapshots are built for each current tracked branch of Kibana.
+2. Each snapshot is uploaded to a public Google Cloud Storage bucket, `kibana-ci-es-snapshots-daily`.
+** At this point, the snapshot is not automatically used in CI or local development. It needs to be tested/verified first.
+3. Each snapshot is tested with the latest commit of the corresponding Kibana branch, using the full CI suite.
+4. After CI
+** If the snapshot passes, it is promoted and automatically used in CI and local development.
+** If the snapshot fails, the issue must be investigated and resolved. A new incompatibility may exist between Elasticsearch and Kibana.
+
+==== Using the latest snapshot
+
+When developing locally, you may wish to use the most recent Elasticsearch snapshot, even if it's failing CI. To do so, prefix your commands with the follow environment variable:
+
+["source","bash"]
+-----------
+KBN_ES_SNAPSHOT_USE_UNVERIFIED=true
+-----------
+
+You can use this flag with any command that downloads and runs Elasticsearch snapshots, such as `scripts/es` or the FTR.
+
+For example, to run functional tests with the latest snapshot:
+
+["source","bash"]
+-----------
+KBN_ES_SNAPSHOT_USE_UNVERIFIED=true node scripts/functional_tests_server
+-----------
+
+===== For Pull Requests
+
+Currently, there is not a way to run your pull request with the latest unverified snapshot without a code change. You can, however, do it with a small code change.
+
+1. Edit `Jenkinsfile` in the root of the Kibana repo
+2. Add `env.KBN_ES_SNAPSHOT_USE_UNVERIFIED = 'true'` at the top of the file.
+3. Commit the change
+
+Your pull request should then use the latest snapshot the next time that it runs. Just don't merge the change to `Jenkinsfile`!
+
+==== Google Cloud Storage buckets
+
+===== kibana-ci-es-snapshots-daily
+
+This bucket stores snapshots that are created on a daily basis, and is the primary location used by `kbn-es` to download snapshots.
+
+Snapshots are automatically deleted after 10 days.
+
+The file structure for this bucket looks like this:
+
+* `<version>/manifest-latest.json`
+* `<version>/manifest-latest-verified.json`
+* `<version>/archives/<unique id>/*.tar.gz`
+* `<version>/archives/<unique id>/*.tar.gz.sha512`
+* `<version>/archives/<unique id>/manifest.json`
+
+===== kibana-ci-es-snapshots-permanent
+
+This bucket stores only the most recently promoted snapshot for each version. Old snapshots are only deleted when new ones are uploaded.
+
+This bucket serves as permanent snapshot storage for old branches/versions that are no longer being built. `kbn-es` checks the daily bucket first, followed by this one if no snapshots were found.
+
+The file structure for this bucket looks like this:
+
+* `<version>/*.tar.gz`
+* `<version>/*.tar.gz.sha512`
+* `<version>/manifest.json`
+
+==== How snapshots are built, tested, and promoted
+
+Each day, a https://kibana-ci.elastic.co/job/elasticsearch+snapshots+trigger/[Jenkins job] runs that triggers Elasticsearch builds for each currently tracked branch/version. This job is automatically updated with the correct branches whenever we release new versions of Kibana.
+
+===== Build
+
+https://kibana-ci.elastic.co/job/elasticsearch+snapshots+build/[This Jenkins job] builds the Elasticsearch snapshots and uploads them to GCS.
+
+The Jenkins job pipeline definition is https://github.com/elastic/kibana/blob/master/.ci/es-snapshots/Jenkinsfile_build_es[in the kibana repo].
+
+1. Checkout Elasticsearch repo for the given branch/version.
+2. Run `./gradlew -p distribution/archives assemble --parallel` to create all of the Elasticsearch distributions.
+3. Create a tarball for each distribution.
+4. Create a manifest JSON file containing info about the distribution, as well as its download URL.
+5. Upload the tarballs and manifest to a unique location in the GCS bucket `kibana-ci-es-snapshots-daily`.
+** e.g. `<version>/archives/<unique id>`
+6. Replace `<version>/manifest-latest.json` in GCS with this newest manifest.
+** This allows the `KBN_ES_SNAPSHOT_USE_UNVERIFIED` flag to work.
+7. Trigger the verification job, to run the full Kibana CI test suite with this snapshot.
+
+===== Verification and Promotion
+
+https://kibana-ci.elastic.co/job/elasticsearch+snapshots+verify/[This Jenkins job] tests the latest Elasticsearch snapshot with the full Kibana CI pipeline, and promotes if it there are no test failures.
+
+The Jenkins job pipeline definition is https://github.com/elastic/kibana/blob/master/.ci/es-snapshots/Jenkinsfile_verify_es[in the kibana repo].
+
+1. Checkout Kibana and set up CI environment as normal.
+2. Set the `ES_SNAPSHOT_MANIFEST` env var to point to the latest snapshot manifest.
+3. Run CI (functional tests, integration tests, etc).
+4. After CI
+** If there was a test failure or other build error, send out an e-mail notification and stop.
+** If there were no errors, promote the snapshot.
+
+Promotion is done as part of the same pipeline:
+
+1. Replace the manifest at `kibana-ci-es-snapshots-daily/<version>/manifest-latest-verified.json` with the manifest from the tested snapshot.
+** At this point, the snapshot has been promoted and will automatically be used in CI and in local development.
+2. Replace the snapshot at `kibana-ci-es-snapshots-permanent/<version>/` with the tested snapshot by copying all of the tarballs and the manifest file.

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.field.__spec.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [Field](./kibana-plugin-plugins-data-public.field.md) &gt; [$$spec](./kibana-plugin-plugins-data-public.field.__spec.md)
+
+## Field.$$spec property
+
+<b>Signature:</b>
+
+```typescript
+$$spec: FieldSpec;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.indexpatternfield._constructor_.md renamed to docs/development/plugins/data/public/kibana-plugin-plugins-data-public.field._constructor_.md

@@ -1,8 +1,8 @@
 <!-- Do not edit this file. It is automatically generated by API Documenter. -->
 
-[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [IndexPatternField](./kibana-plugin-plugins-data-public.indexpatternfield.md) &gt; [(constructor)](./kibana-plugin-plugins-data-public.indexpatternfield._constructor_.md)
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [Field](./kibana-plugin-plugins-data-public.field.md) &gt; [(constructor)](./kibana-plugin-plugins-data-public.field._constructor_.md)
 
-## IndexPatternField.(constructor)
+## Field.(constructor)
 
 Constructs a new instance of the `Field` class
 

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.field.aggregatable.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [Field](./kibana-plugin-plugins-data-public.field.md) &gt; [aggregatable](./kibana-plugin-plugins-data-public.field.aggregatable.md)
+
+## Field.aggregatable property
+
+<b>Signature:</b>
+
+```typescript
+aggregatable?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.field.conflictdescriptions.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [Field](./kibana-plugin-plugins-data-public.field.md) &gt; [conflictDescriptions](./kibana-plugin-plugins-data-public.field.conflictdescriptions.md)
+
+## Field.conflictDescriptions property
+
+<b>Signature:</b>
+
+```typescript
+conflictDescriptions?: Record<string, string[]>;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.field.count.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [Field](./kibana-plugin-plugins-data-public.field.md) &gt; [count](./kibana-plugin-plugins-data-public.field.count.md)
+
+## Field.count property
+
+<b>Signature:</b>
+
+```typescript
+count?: number;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.field.displayname.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [Field](./kibana-plugin-plugins-data-public.field.md) &gt; [displayName](./kibana-plugin-plugins-data-public.field.displayname.md)
+
+## Field.displayName property
+
+<b>Signature:</b>
+
+```typescript
+displayName?: string;
+```