Merge branch 'main' into bug/watcher-stuck-firing-state

Author: CJ Cenizal

.buildkite/agents.json

@@ -26,10 +26,22 @@ export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
 KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_VERSION
 
-export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
-export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
-export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
-export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress"
+BUILDKITE_AGENT_GCP_REGION=""
+if [[ "$(curl -is metadata.google.internal || true)" ]]; then
+  # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1
+  BUILDKITE_AGENT_GCP_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev)
+fi
+export BUILDKITE_AGENT_GCP_REGION
+
+CI_PROXY_CACHE_SUFFIX=""
+if [[ "$BUILDKITE_AGENT_GCP_REGION" ]]; then
+  CI_PROXY_CACHE_SUFFIX="/region/$BUILDKITE_AGENT_GCP_REGION"
+fi
+
+export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
+export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
+export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
+export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX/cypress"
 
 export NODE_OPTIONS="--max-old-space-size=4096"
 

.buildkite/scripts/common/env.sh

@@ -16,12 +16,7 @@ BAZEL_CACHE_MODE=${BAZEL_CACHE_MODE:-gcs}
 if [[ "$BAZEL_CACHE_MODE" == "gcs" ]]; then
   echo "[bazel] enabling caching with GCS buckets"
 
-  BAZEL_REGION="us-central1"
-  if [[ "$(curl -is metadata.google.internal || true)" ]]; then
-    # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1
-    BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev)
-  fi
-
+  BAZEL_REGION="${BUILDKITE_AGENT_GCP_REGION:-us-central1}"
   BAZEL_BUCKET="kibana-ci-bazel_$BAZEL_REGION"
 
   echo "[bazel] using GCS bucket: $BAZEL_BUCKET"

.buildkite/scripts/common/setup_bazel.sh

@@ -122,7 +122,11 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ux_plugin_e2e.yml'));
     }
 
-    if (GITHUB_PR_LABELS.includes('ci:deploy-cloud')) {
+    if (
+      GITHUB_PR_LABELS.includes('ci:deploy-cloud') ||
+      GITHUB_PR_LABELS.includes('ci:cloud-deploy') ||
+      GITHUB_PR_LABELS.includes('ci:cloud-redeploy')
+    ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
     }
 

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -9,6 +9,7 @@ source .buildkite/scripts/common/util.sh
 export KBN_NP_PLUGINS_BUILT=true
 
 VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
+ECCTL_LOGS=$(mktemp --suffix ".json")
 
 echo "--- Download Kibana Distribution"
 
@@ -46,9 +47,19 @@ fi
 
 docker logout docker.elastic.co
 
+if is_pr_with_label "ci:cloud-redeploy"; then
+  echo "--- Shutdown Previous Deployment"
+  CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
+  if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
+    echo "No deployment to remove"
+  else
+    echo "Shutting down previous deployment..."
+    ecctl deployment shutdown "$CLOUD_DEPLOYMENT_ID" --force --track --output json > "$ECCTL_LOGS"
+  fi
+fi
+
 echo "--- Create Deployment"
 CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
-JSON_FILE=$(mktemp --suffix ".json")
 if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
   jq '
     .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
@@ -60,45 +71,49 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
     .resources.integrations_server[0].plan.integrations_server.version = "'$VERSION'"
     ' .buildkite/scripts/steps/cloud/deploy.json > /tmp/deploy.json
 
-  echo -n "Creating deployment..."
-  ecctl deployment create --track --output json --file /tmp/deploy.json > "$JSON_FILE"
-  echo "done"
+  echo "Creating deployment..."
+  ecctl deployment create --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 
-  CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$JSON_FILE")
-  CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$JSON_FILE")
-  CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$JSON_FILE")
-  CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$JSON_FILE")
+  CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$ECCTL_LOGS")
+  CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$ECCTL_LOGS")
+  CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$ECCTL_LOGS")
+  CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$ECCTL_LOGS")
 
-  echo -n "Writing to vault..."
+  echo "Writing to vault..."
   VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
   VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
   VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
   retry 5 30 vault login -no-print "$VAULT_TOKEN"
   retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
 
-  echo -n "Enabling Stack Monitoring..."
+  echo "Enabling Stack Monitoring..."
   jq '
     .settings.observability.metrics.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" |
     .settings.observability.logging.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'"
     ' .buildkite/scripts/steps/cloud/stack_monitoring.json > /tmp/stack_monitoring.json
-  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json > "$JSON_FILE"
-  echo "done"
 
-  echo -n "Enabling verbose logging..."
+  # After a deployment is created, a new Kibana plan is automatically added to update settings
+  # and restart Kibana.  Polling for a plan state isn't especially reliable because it flips
+  # between empty and queued, and then empty again depending on how quickly setup is run in cloud.
+  # We want to enable monitoring after the automatic plan has run, if not we get an error:
+  # * deployments.resource_plan_state_error: Kibana resource [main-kibana] has a plan still pending, cancel that or wait for it to complete (settings.observability.plan)
+  # This adds a sleep and retry to see if we can make this step more reliable
+  sleep 120
+  retry 5 60 ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json > "$ECCTL_LOGS"
+
+  echo "Enabling verbose logging..."
   ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
     .resources.kibana[0].plan.kibana.user_settings_yaml = "logging.root.level: all"
     ' > /tmp/verbose_logging.json
-  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/verbose_logging.json > "$JSON_FILE"
-  echo "done"
+  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/verbose_logging.json > "$ECCTL_LOGS"
 else
-ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
-  .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
-  (.. | select(.version? != null).version) = "'$VERSION'"
-  ' > /tmp/deploy.json
-
-  echo -n "Updating deployment..."
-  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$JSON_FILE"
-  echo "done"
+  ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
+    .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
+    (.. | select(.version? != null).version) = "'$VERSION'"
+    ' > /tmp/deploy.json
+
+  echo "Updating deployment..."
+  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 fi
 
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -31,9 +31,13 @@ for (const deployment of prDeployments) {
     if (pullRequest.state !== 'OPEN') {
       console.log(`Pull Request #${prNumber} is no longer open, will delete associated deployment`);
       deploymentsToPurge.push(deployment);
-    } else if (!pullRequest.labels.filter((label: any) => label.name === 'ci:deploy-cloud')) {
+    } else if (
+      !pullRequest.labels.filter((label: any) =>
+        /^ci:(deploy-cloud|cloud-deploy|cloud-redeploy)$/.test(label.name)
+      )
+    ) {
       console.log(
-        `Pull Request #${prNumber} no longer has the ci:deploy-cloud label, will delete associated deployment`
+        `Pull Request #${prNumber} no longer has the a cloud deployment label, will delete associated deployment`
       );
       deploymentsToPurge.push(deployment);
     } else if (lastCommitTimestamp < NOW - 60 * 60 * 24 * 7) {

.buildkite/scripts/steps/cloud/purge_deployments.ts

@@ -178,6 +178,7 @@ const DEV_PATTERNS = [
   'src/dev/**/*',
   'x-pack/{dev-tools,tasks,scripts,test,build_chromium}/**/*',
   'x-pack/plugins/*/server/scripts/**/*',
+  'x-pack/plugins/fleet/cypress',
 ];
 
 /** Restricted imports with suggested alternatives */
@@ -1341,7 +1342,7 @@ module.exports = {
      * Discover overrides
      */
     {
-      files: ['src/plugins/discover/**/*.{ts,tsx}'],
+      files: ['src/plugins/discover/**/*.{ts,tsx}', 'src/plugins/saved_search/**/*.{ts,tsx}'],
       rules: {
         '@typescript-eslint/no-explicit-any': 'error',
         '@typescript-eslint/ban-ts-comment': [

.eslintrc.js

@@ -10,6 +10,7 @@
 
 # Data Discovery
 /src/plugins/discover/ @elastic/kibana-data-discovery
+/src/plugins/saved_search/ @elastic/kibana-data-discovery
 /x-pack/plugins/discover_enhanced/ @elastic/kibana-data-discovery
 /test/functional/apps/discover/ @elastic/kibana-data-discovery
 /x-pack/plugins/graph/ @elastic/kibana-data-discovery

.github/CODEOWNERS

@@ -18,6 +18,7 @@
     "dataViews": "src/plugins/data_views",
     "devTools": "src/plugins/dev_tools",
     "discover": "src/plugins/discover",
+    "savedSearch": "src/plugins/saved_search",
     "embeddableApi": "src/plugins/embeddable",
     "embeddableExamples": "examples/embeddable_examples",
     "esQuery": "packages/kbn-es-query/src",

.i18nrc.json

@@ -1622,7 +1622,7 @@
         "label": "ActionParamsType",
         "description": [],
         "signature": [
-          "{ readonly group?: string | undefined; readonly source?: string | undefined; readonly component?: string | undefined; readonly summary?: string | undefined; readonly timestamp?: string | undefined; readonly eventAction?: \"resolve\" | \"trigger\" | \"acknowledge\" | undefined; readonly dedupKey?: string | undefined; readonly severity?: \"error\" | \"warning\" | \"info\" | \"critical\" | undefined; readonly class?: string | undefined; }"
+          "{ readonly group?: string | undefined; readonly source?: string | undefined; readonly summary?: string | undefined; readonly component?: string | undefined; readonly timestamp?: string | undefined; readonly eventAction?: \"resolve\" | \"trigger\" | \"acknowledge\" | undefined; readonly dedupKey?: string | undefined; readonly severity?: \"error\" | \"warning\" | \"info\" | \"critical\" | undefined; readonly class?: string | undefined; }"
         ],
         "path": "x-pack/plugins/actions/server/builtin_action_types/pagerduty.ts",
         "deprecated": false,
@@ -1814,7 +1814,9 @@
           "ActionsConfigurationUtilities",
           ") => Promise<{ accessToken: string | null; }>; enqueueExecution: (options: ",
           "ExecuteOptions",
-          ") => Promise<void>; ephemeralEnqueuedExecution: (options: ",
+          ") => Promise<void>; bulkEnqueueExecution: (options: ",
+          "ExecuteOptions",
+          "[]) => Promise<void>; ephemeralEnqueuedExecution: (options: ",
           "ExecuteOptions",
           ") => Promise<",
           {