Merge branch 'master' of github.com:elastic/kibana into implement/report-doc-changes-url-in-prs

Author: spalger

vars/kibanaPipeline.groovy

@@ -447,13 +447,27 @@ def withTasks(Map params = [worker: [:]], Closure closure) {
 }
 
 def allCiTasks() {
-  withTasks {
-    tasks.check()
-    tasks.lint()
-    tasks.test()
-    tasks.functionalOss()
-    tasks.functionalXpack()
-  }
+  parallel([
+    general: {
+      withTasks {
+        tasks.check()
+        tasks.lint()
+        tasks.test()
+        tasks.functionalOss()
+        tasks.functionalXpack()
+      }
+    },
+    jest: {
+      workers.ci(name: 'jest', size: 'c2-8', ramDisk: true) {
+        scriptTask('Jest Unit Tests', 'test/scripts/test/jest_unit.sh')()
+      }
+    },
+    xpackJest: {
+      workers.ci(name: 'xpack-jest', size: 'c2-8', ramDisk: true) {
+        scriptTask('X-Pack Jest Unit Tests', 'test/scripts/test/xpack_jest_unit.sh')()
+      }
+    },
+  ])
 }
 
 def pipelineLibraryTests() {

vars/tasks.groovy

@@ -30,12 +30,9 @@ def lint() {
 
 def test() {
   tasks([
-    // These 2 tasks require isolation because of hard-coded, conflicting ports and such, so let's use Docker here
+    // This task requires isolation because of hard-coded, conflicting ports and such, so let's use Docker here
     kibanaPipeline.scriptTaskDocker('Jest Integration Tests', 'test/scripts/test/jest_integration.sh'),
-
-    kibanaPipeline.scriptTask('Jest Unit Tests', 'test/scripts/test/jest_unit.sh'),
     kibanaPipeline.scriptTask('API Integration Tests', 'test/scripts/test/api_integration.sh'),
-    kibanaPipeline.scriptTask('X-Pack Jest Unit Tests', 'test/scripts/test/xpack_jest_unit.sh'),
   ])
 }
 

vars/workers.groovy

@@ -19,6 +19,8 @@ def label(size) {
       return 'docker && tests-xl-highmem'
     case 'xxl':
       return 'docker && tests-xxl && gobld/machineType:custom-64-270336'
+    case 'c2-8':
+      return 'docker && linux && immutable && gobld/machineType:c2-standard-8'
   }
 
   error "unknown size '${size}'"

x-pack/build_chromium/README.md

@@ -6,50 +6,65 @@ to accept a commit hash from the Chromium repository, and initialize the build
 environments and run the build on Mac, Windows, and Linux.
 
 ## Before you begin
+
 If you wish to use a remote VM to build, you'll need access to our GCP account,
 which is where we have two machines provisioned for the Linux and Windows
 builds. Mac builds can be achieved locally, and are a great place to start to
 gain familiarity.
 
+**NOTE:** Linux builds should be done in Ubuntu on x86 architecture. ARM builds
+are created in x86. CentOS is not supported for building Chromium.
+
 1. Login to our GCP instance [here using your okta credentials](https://console.cloud.google.com/).
 2. Click the "Compute Engine" tab.
-3. Ensure that `chromium-build-linux` and `chromium-build-windows-12-beefy` are there.
-4. If #3 fails, you'll have to spin up new instances. Generally, these need `n1-standard-8` types or 8 vCPUs/30 GB memory.
-5. Ensure that there's enough room left on the disk: 100GB is required. `ncdu` is a good linux util to verify what's claming space.
-
-## Usage
+3. Find `chromium-build-linux` or `chromium-build-windows-12-beefy` and start the instance.
+4. Install [Google Cloud SDK](https://cloud.google.com/sdk) locally to ssh into the GCP instance
+5. System dependencies:
+    - 8 CPU
+    - 30GB memory
+    - 80GB free space on disk (Try `ncdu /home` to see where space is used.)
+    - git
+    - python2 (`python` must link to `python2`)
+    - lsb_release
+    - tmux is recommended in case your ssh session is interrupted
+6. Copy the entire `build_chromium` directory into a GCP storage bucket, so you can copy the scripts into the instance and run them.
+
+## Build Script Usage
 
 ```
+# Allow our scripts to use depot_tools commands
 export PATH=$HOME/chromium/depot_tools:$PATH
+
 # Create a dedicated working directory for this directory of Python scripts.
 mkdir ~/chromium && cd ~/chromium
+
 # Copy the scripts from the Kibana repo to use them conveniently in the working directory
-cp -r ~/path/to/kibana/x-pack/build_chromium .
-# Install the OS packages, configure the environment, download the chromium source
+gsutil cp -r gs://my-bucket/build_chromium .
+
+# Install the OS packages, configure the environment, download the chromium source (25GB)
 python ./build_chromium/init.sh [arch_name]
 
 # Run the build script with the path to the chromium src directory, the git commit id
-python ./build_chromium/build.py <commit_id>
+python ./build_chromium/build.py <commit_id> x86
 
-# You can add an architecture flag for ARM
+# OR You can build for ARM
 python ./build_chromium/build.py <commit_id> arm64
 ```
 
+**NOTE:** The `init.py` script updates git config to make it more possible for
+the Chromium repo to be cloned successfully. If checking out the Chromium fails
+with "early EOF" errors, the instance could be low on memory or disk space.
+
 ## Getting the Commit ID
-Getting `<commit_id>` can be tricky. The best technique seems to be:
+The `build.py` script requires a commit ID of the Chromium repo. Getting `<commit_id>` can be tricky. The best technique seems to be:
 1. Create a temporary working directory and intialize yarn
 2. `yarn add puppeteer # install latest puppeter`
-3. Look through puppeteer's node module files to find the "chromium revision" (a custom versioning convention for Chromium).
+3. Look through Puppeteer documentation and Changelogs to find information
+about where the "chromium revision" is located in the Puppeteer code. The code
+containing it might not be distributed in the node module.
+    - Example: https://github.com/puppeteer/puppeteer/blob/b549256/src/revisions.ts
 4. Use `https://crrev.com` and look up the revision and find the git commit info.
-
-The official Chromium build process is poorly documented, and seems to have
-breaking changes fairly regularly. The build pre-requisites, and the build
-flags change over time, so it is likely that the scripts in this directory will
-be out of date by the time we have to do another Chromium build.
-
-This document is an attempt to note all of the gotchas we've come across while
-building, so that the next time we have to tinker here, we'll have a good
-starting point.
+    - Example: http://crrev.com/818858 leads to the git commit e62cb7e3fc7c40548cef66cdf19d270535d9350b
 
 ## Build args
 
@@ -115,8 +130,8 @@ The more cores the better, as the build makes effective use of each. For Linux,
 
 - Linux:
   - SSH in using [gcloud](https://cloud.google.com/sdk/)
-  - Get the ssh command in the [GCP console](https://console.cloud.google.com/) -> VM instances -> your-vm-name -> SSH -> gcloud
-  - Their in-browser UI is kinda sluggish, so use the commandline tool
+  - Get the ssh command in the [GCP console](https://console.cloud.google.com/) -> VM instances -> your-vm-name -> SSH -> "View gcloud command"
+  - Their in-browser UI is kinda sluggish, so use the commandline tool (Google Cloud SDK is required)
 
 - Windows:
   - Install Microsoft's Remote Desktop tools

x-pack/build_chromium/build.py

@@ -33,10 +33,10 @@
 base_version = source_version[:7].strip('.')
 
 # Set to "arm" to build for ARM on Linux
-arch_name = sys.argv[2] if len(sys.argv) >= 3 else 'x64'
+arch_name = sys.argv[2] if len(sys.argv) >= 3 else 'unknown'
 
 if arch_name != 'x64' and arch_name != 'arm64':
-  raise Exception('Unexpected architecture: ' + arch_name)
+  raise Exception('Unexpected architecture: ' + arch_name + '. `x64` and `arm64` are supported.')
 
 print('Building Chromium ' + source_version + ' for ' + arch_name + ' from ' + src_path)
 print('src path: ' + src_path)

x-pack/build_chromium/init.py

@@ -8,18 +8,19 @@
 # call this once the platform-specific initialization has completed.
 
 # Set to "arm" to build for ARM on Linux
-arch_name = sys.argv[1] if len(sys.argv) >= 2 else 'x64'
+arch_name = sys.argv[1] if len(sys.argv) >= 2 else 'undefined'
 build_path = path.abspath(os.curdir)
 src_path = path.abspath(path.join(build_path, 'chromium', 'src'))
 
 if arch_name != 'x64' and arch_name != 'arm64':
-  raise Exception('Unexpected architecture: ' + arch_name)
+  raise Exception('Unexpected architecture: ' + arch_name + '. `x64` and `arm64` are supported.')
 
 # Configure git
 print('Configuring git globals...')
 runcmd('git config --global core.autocrlf false')
 runcmd('git config --global core.filemode false')
 runcmd('git config --global branch.autosetuprebase always')
+runcmd('git config --global core.compression 0')
 
 # Grab Chromium's custom build tools, if they aren't already installed
 # (On Windows, they are installed before this Python script is run)
@@ -35,13 +36,14 @@
   runcmd('git pull origin master')
   os.chdir(original_dir)
 
-configure_environment(arch_name, build_path, src_path)
-
 # Fetch the Chromium source code
 chromium_dir = path.join(build_path, 'chromium')
 if not path.isdir(chromium_dir):
   mkdir(chromium_dir)
   os.chdir(chromium_dir)
-  runcmd('fetch chromium')
+  runcmd('fetch chromium --nohooks=1 --no-history=1')
 else:
   print('Directory exists: ' + chromium_dir + '. Skipping chromium fetch.')
+
+# This depends on having the chromium/src directory with the complete checkout
+configure_environment(arch_name, build_path, src_path)

x-pack/plugins/enterprise_search/public/applications/workplace_search/app_logic.ts

@@ -21,6 +21,7 @@ interface AppValues extends WorkplaceSearchInitialData {
 interface AppActions {
   initializeAppData(props: InitialAppData): InitialAppData;
   setContext(isOrganization: boolean): boolean;
+  setSourceRestriction(canCreatePersonalSources: boolean): boolean;
 }
 
 const emptyOrg = {} as Organization;
@@ -34,6 +35,7 @@ export const AppLogic = kea<MakeLogicType<AppValues, AppActions>>({
       isFederatedAuth,
     }),
     setContext: (isOrganization) => isOrganization,
+    setSourceRestriction: (canCreatePersonalSources: boolean) => canCreatePersonalSources,
   },
   reducers: {
     hasInitialized: [
@@ -64,6 +66,10 @@ export const AppLogic = kea<MakeLogicType<AppValues, AppActions>>({
       emptyAccount,
       {
         initializeAppData: (_, { workplaceSearch }) => workplaceSearch?.account || emptyAccount,
+        setSourceRestriction: (state, canCreatePersonalSources) => ({
+          ...state,
+          canCreatePersonalSources,
+        }),
       },
     ],
   },

x-pack/plugins/enterprise_search/public/applications/workplace_search/components/layout/nav.tsx

@@ -45,9 +45,7 @@ export const WorkplaceSearchNav: React.FC<Props> = ({
     <SideNavLink isExternal to={getWorkplaceSearchUrl(`#${ROLE_MAPPINGS_PATH}`)}>
       {NAV.ROLE_MAPPINGS}
     </SideNavLink>
-    <SideNavLink isExternal to={getWorkplaceSearchUrl(`#${SECURITY_PATH}`)}>
-      {NAV.SECURITY}
-    </SideNavLink>
+    <SideNavLink to={SECURITY_PATH}>{NAV.SECURITY}</SideNavLink>
     <SideNavLink subNav={settingsSubNav} to={ORG_SETTINGS_PATH}>
       {NAV.SETTINGS}
     </SideNavLink>

x-pack/plugins/enterprise_search/public/applications/workplace_search/constants.ts

@@ -289,6 +289,87 @@ export const DOCUMENTATION_LINK_TITLE = i18n.translate(
   }
 );
 
+export const PRIVATE_SOURCES_DESCRIPTION = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.privateSources.description',
+  {
+    defaultMessage:
+      'Private sources are connected by users in your organization to create a personalized search experience.',
+  }
+);
+
+export const PRIVATE_SOURCES_TOGGLE_DESCRIPTION = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.privateSourcesToggle.description',
+  {
+    defaultMessage: 'Enable private sources for your organization',
+  }
+);
+
+export const REMOTE_SOURCES_TOGGLE_TEXT = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.remoteSourcesToggle.text',
+  {
+    defaultMessage: 'Enable remote private sources',
+  }
+);
+
+export const REMOTE_SOURCES_TABLE_DESCRIPTION = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.remoteSourcesTable.description',
+  {
+    defaultMessage:
+      'Remote sources synchronize and store a limited amount of data on disk, with a low impact on storage resources.',
+  }
+);
+
+export const REMOTE_SOURCES_EMPTY_TABLE_TITLE = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.remoteSourcesEmptyTable.title',
+  {
+    defaultMessage: 'No remote private sources configured yet',
+  }
+);
+
+export const STANDARD_SOURCES_TOGGLE_TEXT = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.standardSourcesToggle.text',
+  {
+    defaultMessage: 'Enable standard private sources',
+  }
+);
+
+export const STANDARD_SOURCES_TABLE_DESCRIPTION = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.standardSourcesTable.description',
+  {
+    defaultMessage:
+      'Standard sources synchronize and store all searchable data on disk, with a directly correlated impact on storage resources.',
+  }
+);
+
+export const STANDARD_SOURCES_EMPTY_TABLE_TITLE = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.standardSourcesEmptyTable.title',
+  {
+    defaultMessage: 'No standard private sources configured yet',
+  }
+);
+
+export const SECURITY_UNSAVED_CHANGES_MESSAGE = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.unsavedChanges.message',
+  {
+    defaultMessage:
+      'Your private sources settings have not been saved. Are you sure you want to leave?',
+  }
+);
+
+export const PRIVATE_SOURCES_UPDATE_CONFIRMATION_TEXT = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.privateSourcesUpdateConfirmation.text',
+  {
+    defaultMessage: 'Updates to private source configuration will take effect immediately.',
+  }
+);
+
+export const SOURCE_RESTRICTIONS_SUCCESS_MESSAGE = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.security.sourceRestrictionsSuccess.message',
+  {
+    defaultMessage: 'Successfully updated source restrictions.',
+  }
+);
+
 export const PUBLIC_KEY_LABEL = i18n.translate(
   'xpack.enterpriseSearch.workplaceSearch.publicKey.label',
   {
@@ -382,6 +463,20 @@ export const SAVE_CHANGES_BUTTON = i18n.translate(
   }
 );
 
+export const SAVE_SETTINGS_BUTTON = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.saveSettings.button',
+  {
+    defaultMessage: 'Save settings',
+  }
+);
+
+export const KEEP_EDITING_BUTTON = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.keepEditing.button',
+  {
+    defaultMessage: 'Keep editing',
+  }
+);
+
 export const NAME_LABEL = i18n.translate('xpack.enterpriseSearch.workplaceSearch.name.label', {
   defaultMessage: 'Name',
 });
@@ -493,6 +588,10 @@ export const UPDATE_BUTTON = i18n.translate(
   }
 );
 
+export const RESET_BUTTON = i18n.translate('xpack.enterpriseSearch.workplaceSearch.reset.button', {
+  defaultMessage: 'Reset',
+});
+
 export const CONFIGURE_BUTTON = i18n.translate(
   'xpack.enterpriseSearch.workplaceSearch.configure.button',
   {
@@ -522,13 +621,31 @@ export const PRIVATE_PLATINUM_LICENSE_CALLOUT = i18n.translate(
   }
 );
 
+export const SOURCE = i18n.translate('xpack.enterpriseSearch.workplaceSearch.source.text', {
+  defaultMessage: 'Source',
+});
+
 export const PRIVATE_SOURCE = i18n.translate(
   'xpack.enterpriseSearch.workplaceSearch.privateSource.text',
   {
     defaultMessage: 'Private Source',
   }
 );
 
+export const PRIVATE_SOURCES = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.privateSources.text',
+  {
+    defaultMessage: 'Private Sources',
+  }
+);
+
+export const CONFIRM_CHANGES_TEXT = i18n.translate(
+  'xpack.enterpriseSearch.workplaceSearch.confirmChanges.text',
+  {
+    defaultMessage: 'Confirm changes',
+  }
+);
+
 export const CONNECTORS_HEADER_TITLE = i18n.translate(
   'xpack.enterpriseSearch.workplaceSearch.connectors.header.title',
   {