Merge branch 'drilldowns' into drilldown-demo-2

Author: elasticmachine

.eslintignore

@@ -10,7 +10,6 @@ bower_components
 /html_docs
 /src/plugins/data/common/es_query/kuery/ast/_generated_/**
 /src/plugins/vis_type_timelion/public/_generated_/**
-src/legacy/core_plugins/vis_type_vislib/public/vislib/__tests__/lib/fixtures/mock_data
 /src/legacy/ui/public/flot-charts
 /test/fixtures/scenarios
 /src/legacy/core_plugins/console/public/webpackShims

.eslintrc.js

@@ -217,6 +217,9 @@ module.exports = {
                   'examples/**/*',
                   '!(src|x-pack)/**/*.test.*',
                   '!(x-pack/)?test/**/*',
+                  // next folder contains legacy browser tests which can't be migrated to jest
+                  // which import np files
+                  '!src/legacy/core_plugins/kibana/public/__tests__/**/*',
                 ],
                 from: [
                   '(src|x-pack)/plugins/**/(public|server)/**/*',
@@ -562,7 +565,7 @@ module.exports = {
      */
     {
       // front end typescript and javascript files only
-      files: ['x-pack/legacy/plugins/siem/public/**/*.{js,ts,tsx}'],
+      files: ['x-pack/plugins/siem/public/**/*.{js,ts,tsx}'],
       rules: {
         'import/no-nodejs-modules': 'error',
         'no-restricted-imports': [
@@ -611,7 +614,7 @@ module.exports = {
     // {
     //   // will introduced after the other warns are fixed
     //   // typescript and javascript for front end react performance
-    //   files: ['x-pack/legacy/plugins/siem/public/**/!(*.test).{js,ts,tsx}'],
+    //   files: ['x-pack/plugins/siem/public/**/!(*.test).{js,ts,tsx}'],
     //   plugins: ['react-perf'],
     //   rules: {
     //     // 'react-perf/jsx-no-new-object-as-prop': 'error',

.github/CODEOWNERS

@@ -12,7 +12,7 @@
 /src/legacy/core_plugins/kibana/public/discover/ @elastic/kibana-app
 /src/legacy/core_plugins/kibana/public/local_application_service/ @elastic/kibana-app
 /src/legacy/core_plugins/kibana/public/dev_tools/ @elastic/kibana-app
-/src/legacy/core_plugins/vis_type_vislib/ @elastic/kibana-app
+/src/plugins/vis_type_vislib/ @elastic/kibana-app
 /src/plugins/vis_type_xy/ @elastic/kibana-app
 /src/plugins/vis_type_table/ @elastic/kibana-app
 /src/plugins/kibana_legacy/ @elastic/kibana-app

.i18nrc.json

@@ -51,10 +51,10 @@
     "visTypeMetric": "src/plugins/vis_type_metric",
     "visTypeTable": "src/plugins/vis_type_table",
     "visTypeTagCloud": "src/plugins/vis_type_tagcloud",
-    "visTypeTimeseries": ["src/legacy/core_plugins/vis_type_timeseries", "src/plugins/vis_type_timeseries"],
+    "visTypeTimeseries": "src/plugins/vis_type_timeseries",
     "visTypeVega": "src/plugins/vis_type_vega",
-    "visTypeVislib": "src/legacy/core_plugins/vis_type_vislib",
-    "visTypeXy": "src/legacy/core_plugins/vis_type_xy",
+    "visTypeVislib": "src/plugins/vis_type_vislib",
+    "visTypeXy": "src/plugins/vis_type_xy",
     "visualizations": "src/plugins/visualizations",
     "visualize": "src/plugins/visualize"
   },

.sass-lint.yml

@@ -2,8 +2,8 @@ files:
   include:
     - 'src/legacy/core_plugins/metrics/**/*.s+(a|c)ss'
     - 'src/legacy/core_plugins/timelion/**/*.s+(a|c)ss'
-    - 'src/legacy/core_plugins/vis_type_vislib/**/*.s+(a|c)ss'
-    - 'src/legacy/core_plugins/vis_type_xy/**/*.s+(a|c)ss'
+    - 'src/plugins/vis_type_vislib/**/*.s+(a|c)ss'
+    - 'src/plugins/vis_type_xy/**/*.s+(a|c)ss'
     - 'x-pack/legacy/plugins/security/**/*.s+(a|c)ss'
     - 'x-pack/legacy/plugins/canvas/**/*.s+(a|c)ss'
     - 'x-pack/plugins/triggers_actions_ui/**/*.s+(a|c)ss'

Jenkinsfile

@@ -4,54 +4,56 @@ library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
 kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
-  githubPr.withDefaultPrComments {
-    catchError {
-      retryable.enable()
-      parallel([
-        'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-        'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-        'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-          'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
-          'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-          'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-          'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-          'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-          'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-          'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-          'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-          'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-          'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-          'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-          'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-          'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-          'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
-          // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
-        ]),
-        'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-          'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
-          'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-          'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-          'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-          'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-          'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-          'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-          'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
-          'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
-          'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
-          'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
-          'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
-          'xpack-siemCypress': { processNumber ->
-            whenChanged(['x-pack/plugins/siem/', 'x-pack/legacy/plugins/siem/', 'x-pack/test/siem_cypress/']) {
-              kibanaPipeline.functionalTestProcess('xpack-siemCypress', './test/scripts/jenkins_siem_cypress.sh')(processNumber)
-            }
-          },
+  ciStats.trackBuild {
+    githubPr.withDefaultPrComments {
+      catchError {
+        retryable.enable()
+        parallel([
+          'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
+          'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
+          'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+            'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
+            'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
+            'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
+            'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
+            'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
+            'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
+            'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
+            'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
+            'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
+            'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
+            'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
+            'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
+            'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
+            'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
+            // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
+          ]),
+          'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
+            'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
+            'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
+            'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
+            'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
+            'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
+            'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
+            'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
+            'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
+            'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
+            'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
+            'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
+            'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
+            'xpack-siemCypress': { processNumber ->
+              whenChanged(['x-pack/plugins/siem/', 'x-pack/test/siem_cypress/']) {
+                kibanaPipeline.functionalTestProcess('xpack-siemCypress', './test/scripts/jenkins_siem_cypress.sh')(processNumber)
+              }
+            },
 
-          // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
-        ]),
-      ])
+            // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
+          ]),
+        ])
+      }
     }
-  }
 
-  retryable.printFlakyFailures()
-  kibanaPipeline.sendMail()
+    retryable.printFlakyFailures()
+    kibanaPipeline.sendMail()
+  }
 }

docs/developer/core-development.asciidoc

@@ -7,6 +7,7 @@
 * <<development-elasticsearch>>
 * <<development-unit-tests>>
 * <<development-functional-tests>>
+* <<development-es-snapshots>>
 
 include::core/development-basepath.asciidoc[]
 
@@ -19,3 +20,5 @@ include::core/development-elasticsearch.asciidoc[]
 include::core/development-unit-tests.asciidoc[]
 
 include::core/development-functional-tests.asciidoc[]
+
+include::core/development-es-snapshots.asciidoc[]

docs/developer/core/development-es-snapshots.asciidoc

@@ -0,0 +1,113 @@
+[[development-es-snapshots]]
+=== Daily Elasticsearch Snapshots
+
+For local development and CI, Kibana, by default, uses Elasticsearch snapshots that are built daily when running tasks that require Elasticsearch (e.g. functional tests).
+
+A snapshot is just a group of tarballs, one for each supported distribution/architecture/os of Elasticsearch, and a JSON-based manifest file containing metadata about the distributions.
+
+https://ci.kibana.dev/es-snapshots[A dashboard] is available that shows the current status and compatibility of the latest Elasticsearch snapshots.
+
+==== Process Overview
+
+1. Elasticsearch snapshots are built for each current tracked branch of Kibana.
+2. Each snapshot is uploaded to a public Google Cloud Storage bucket, `kibana-ci-es-snapshots-daily`.
+** At this point, the snapshot is not automatically used in CI or local development. It needs to be tested/verified first.
+3. Each snapshot is tested with the latest commit of the corresponding Kibana branch, using the full CI suite.
+4. After CI
+** If the snapshot passes, it is promoted and automatically used in CI and local development.
+** If the snapshot fails, the issue must be investigated and resolved. A new incompatibility may exist between Elasticsearch and Kibana.
+
+==== Using the latest snapshot
+
+When developing locally, you may wish to use the most recent Elasticsearch snapshot, even if it's failing CI. To do so, prefix your commands with the follow environment variable:
+
+["source","bash"]
+-----------
+KBN_ES_SNAPSHOT_USE_UNVERIFIED=true
+-----------
+
+You can use this flag with any command that downloads and runs Elasticsearch snapshots, such as `scripts/es` or the FTR.
+
+For example, to run functional tests with the latest snapshot:
+
+["source","bash"]
+-----------
+KBN_ES_SNAPSHOT_USE_UNVERIFIED=true node scripts/functional_tests_server
+-----------
+
+===== For Pull Requests
+
+Currently, there is not a way to run your pull request with the latest unverified snapshot without a code change. You can, however, do it with a small code change.
+
+1. Edit `Jenkinsfile` in the root of the Kibana repo
+2. Add `env.KBN_ES_SNAPSHOT_USE_UNVERIFIED = 'true'` at the top of the file.
+3. Commit the change
+
+Your pull request should then use the latest snapshot the next time that it runs. Just don't merge the change to `Jenkinsfile`!
+
+==== Google Cloud Storage buckets
+
+===== kibana-ci-es-snapshots-daily
+
+This bucket stores snapshots that are created on a daily basis, and is the primary location used by `kbn-es` to download snapshots.
+
+Snapshots are automatically deleted after 10 days.
+
+The file structure for this bucket looks like this:
+
+* `<version>/manifest-latest.json`
+* `<version>/manifest-latest-verified.json`
+* `<version>/archives/<unique id>/*.tar.gz`
+* `<version>/archives/<unique id>/*.tar.gz.sha512`
+* `<version>/archives/<unique id>/manifest.json`
+
+===== kibana-ci-es-snapshots-permanent
+
+This bucket stores only the most recently promoted snapshot for each version. Old snapshots are only deleted when new ones are uploaded.
+
+This bucket serves as permanent snapshot storage for old branches/versions that are no longer being built. `kbn-es` checks the daily bucket first, followed by this one if no snapshots were found.
+
+The file structure for this bucket looks like this:
+
+* `<version>/*.tar.gz`
+* `<version>/*.tar.gz.sha512`
+* `<version>/manifest.json`
+
+==== How snapshots are built, tested, and promoted
+
+Each day, a https://kibana-ci.elastic.co/job/elasticsearch+snapshots+trigger/[Jenkins job] runs that triggers Elasticsearch builds for each currently tracked branch/version. This job is automatically updated with the correct branches whenever we release new versions of Kibana.
+
+===== Build
+
+https://kibana-ci.elastic.co/job/elasticsearch+snapshots+build/[This Jenkins job] builds the Elasticsearch snapshots and uploads them to GCS.
+
+The Jenkins job pipeline definition is https://github.com/elastic/kibana/blob/master/.ci/es-snapshots/Jenkinsfile_build_es[in the kibana repo].
+
+1. Checkout Elasticsearch repo for the given branch/version.
+2. Run `./gradlew -p distribution/archives assemble --parallel` to create all of the Elasticsearch distributions.
+3. Create a tarball for each distribution.
+4. Create a manifest JSON file containing info about the distribution, as well as its download URL.
+5. Upload the tarballs and manifest to a unique location in the GCS bucket `kibana-ci-es-snapshots-daily`.
+** e.g. `<version>/archives/<unique id>`
+6. Replace `<version>/manifest-latest.json` in GCS with this newest manifest.
+** This allows the `KBN_ES_SNAPSHOT_USE_UNVERIFIED` flag to work.
+7. Trigger the verification job, to run the full Kibana CI test suite with this snapshot.
+
+===== Verification and Promotion
+
+https://kibana-ci.elastic.co/job/elasticsearch+snapshots+verify/[This Jenkins job] tests the latest Elasticsearch snapshot with the full Kibana CI pipeline, and promotes if it there are no test failures.
+
+The Jenkins job pipeline definition is https://github.com/elastic/kibana/blob/master/.ci/es-snapshots/Jenkinsfile_verify_es[in the kibana repo].
+
+1. Checkout Kibana and set up CI environment as normal.
+2. Set the `ES_SNAPSHOT_MANIFEST` env var to point to the latest snapshot manifest.
+3. Run CI (functional tests, integration tests, etc).
+4. After CI
+** If there was a test failure or other build error, send out an e-mail notification and stop.
+** If there were no errors, promote the snapshot.
+
+Promotion is done as part of the same pipeline:
+
+1. Replace the manifest at `kibana-ci-es-snapshots-daily/<version>/manifest-latest-verified.json` with the manifest from the tested snapshot.
+** At this point, the snapshot has been promoted and will automatically be used in CI and in local development.
+2. Replace the snapshot at `kibana-ci-es-snapshots-permanent/<version>/` with the tested snapshot by copying all of the tarballs and the manifest file.