[7.17][ES 8.1] Support for superuser not having write access

.buildkite/scripts/common/env.sh

@@ -95,12 +95,3 @@ fi
 
 export BUILD_TS_REFS_DISABLE=true
 export DISABLE_BOOTSTRAP_VALIDATION=true
-
-export TEST_KIBANA_HOST=localhost
-export TEST_KIBANA_PORT=6101
-export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101"
-export TEST_ES_URL="http://elastic:changeme@localhost:6102"
-export TEST_ES_TRANSPORT_PORT=6301-6309
-export TEST_CORS_SERVER_PORT=6106
-export ALERTING_PROXY_PORT=6105
-export TEST_PROXY_SERVER_PORT=6107

.buildkite/scripts/steps/functional/common.sh

@@ -4,6 +4,9 @@ set -euo pipefail
 
 # Note, changes here might also need to be made in other scripts, e.g. uptime.sh
 
+# TEMP: DO NOT MERGE
+export ES_SNAPSHOT_MANIFEST="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/8.0.2/archives/20220301-190149_d95c69ce/manifest.json"
+
 source .buildkite/scripts/common/util.sh
 
 .buildkite/scripts/bootstrap.sh

packages/kbn-es/src/index.ts

@@ -10,3 +10,4 @@
 export { run } from './cli';
 // @ts-expect-error not typed yet
 export { Cluster } from './cluster';
+export { SYSTEM_INDICES_SUPERUSER } from './utils';

packages/kbn-es/src/integration_tests/__fixtures__/es_bin.js

@@ -69,6 +69,16 @@ const { ES_KEY_PATH, ES_CERT_PATH } = require('@kbn/dev-utils');
         });
       }
 
+      if (url.pathname === '/_cluster/health') {
+        return send(
+          200,
+          {
+            status: 'green',
+          },
+          { 'x-elastic-product': 'Elasticsearch' }
+        );
+      }
+
       return send(404, {
         error: {
           reason: 'not found',

packages/kbn-es/src/utils/index.ts

@@ -14,6 +14,6 @@ export { findMostRecentlyChanged } from './find_most_recently_changed';
 // @ts-expect-error not typed yet
 export { extractConfigFiles } from './extract_config_files';
 // @ts-expect-error not typed yet
-export { NativeRealm } from './native_realm';
+export { NativeRealm, SYSTEM_INDICES_SUPERUSER } from './native_realm';
 export { buildSnapshot } from './build_snapshot';
 export { archiveForPlatform } from './build_snapshot';

packages/kbn-es/src/utils/native_realm.js

@@ -11,6 +11,9 @@ const chalk = require('chalk');
 
 const { log: defaultLog } = require('./log');
 
+export const SYSTEM_INDICES_SUPERUSER =
+  process.env.TEST_ES_SYSTEM_INDICES_USER || 'system_indices_superuser';
+
 exports.NativeRealm = class NativeRealm {
   constructor({ elasticPassword, port, log = defaultLog, ssl = false, caCert }) {
     this._client = new Client({
@@ -53,18 +56,33 @@ exports.NativeRealm = class NativeRealm {
     });
   }
 
+  async clusterReady() {
+    return await this._autoRetry({ maxAttempts: 10 }, async () => {
+      const {
+        body: { status: status },
+      } = await this._client.cluster.health({ wait_for_status: 'yellow' });
+
+      if (status === 'red') {
+        throw new Error(`not ready, cluster health is ${status}`);
+      }
+    });
+  }
+
   async setPasswords(options) {
+    await this.clusterReady();
+
     if (!(await this.isSecurityEnabled())) {
       this._log.info('security is not enabled, unable to set native realm passwords');
       return;
     }
 
     const reservedUsers = await this.getReservedUsers();
-    await Promise.all(
-      reservedUsers.map(async (user) => {
+    await Promise.all([
+      ...reservedUsers.map(async (user) => {
         await this.setPassword(user, options[`password.${user}`]);
-      })
-    );
+      }),
+      this._createSystemIndicesUser(),
+    ]);
   }
 
   async getReservedUsers(retryOpts = {}) {
@@ -100,7 +118,7 @@ exports.NativeRealm = class NativeRealm {
   }
 
   async _autoRetry(opts, fn) {
-    const { attempt = 1, maxAttempts = 3 } = opts;
+    const { attempt = 1, maxAttempts = 3, sleep = 1000 } = opts;
 
     try {
       return await fn(attempt);
@@ -111,7 +129,7 @@ exports.NativeRealm = class NativeRealm {
 
       const sec = 1.5 * attempt;
       this._log.warning(`assuming ES isn't initialized completely, trying again in ${sec} seconds`);
-      await new Promise((resolve) => setTimeout(resolve, sec * 1000));
+      await new Promise((resolve) => setTimeout(resolve, sleep));
 
       const nextOpts = {
         ...opts,
@@ -120,4 +138,43 @@ exports.NativeRealm = class NativeRealm {
       return await this._autoRetry(nextOpts, fn);
     }
   }
+
+  async _createSystemIndicesUser() {
+    if (!(await this.isSecurityEnabled())) {
+      this._log.info('security is not enabled, unable to create role and user');
+      return;
+    }
+
+    await this._client.security.putRole({
+      name: SYSTEM_INDICES_SUPERUSER,
+      refresh: 'wait_for',
+      body: {
+        cluster: ['all'],
+        indices: [
+          {
+            names: ['*'],
+            privileges: ['all'],
+            allow_restricted_indices: true,
+          },
+        ],
+        applications: [
+          {
+            application: '*',
+            privileges: ['*'],
+            resources: ['*'],
+          },
+        ],
+        run_as: ['*'],
+      },
+    });
+
+    await this._client.security.putUser({
+      username: SYSTEM_INDICES_SUPERUSER,
+      refresh: 'wait_for',
+      body: {
+        password: this._elasticPassword,
+        roles: [SYSTEM_INDICES_SUPERUSER],
+      },
+    });
+  }
 };

packages/kbn-es/src/utils/native_realm.test.js

@@ -18,9 +18,14 @@ const mockClient = {
   xpack: {
     info: jest.fn(),
   },
+  cluster: {
+    health: jest.fn(),
+  },
   security: {
     changePassword: jest.fn(),
     getUser: jest.fn(),
+    putRole: jest.fn(),
+    putUser: jest.fn(),
   },
 };
 Client.mockImplementation(() => mockClient);
@@ -49,6 +54,12 @@ function mockXPackInfo(available, enabled) {
   }));
 }
 
+function mockClusterStatus(status) {
+  mockClient.cluster.health.mockImplementation(() => {
+    return { body: status };
+  });
+}
+
 describe('isSecurityEnabled', () => {
   test('returns true if enabled and available', async () => {
     mockXPackInfo(true, true);
@@ -95,6 +106,7 @@ describe('isSecurityEnabled', () => {
 describe('setPasswords', () => {
   it('uses provided passwords', async () => {
     mockXPackInfo(true, true);
+    mockClusterStatus('green');
 
     mockClient.security.getUser.mockImplementation(() => ({
       body: {
@@ -127,49 +139,51 @@ describe('setPasswords', () => {
     }));
 
     await nativeRealm.setPasswords({
-      'password.kibana_system': 'bar',
+      body: {
+        'password.kibana_system': 'bar',
+      },
     });
 
     expect(mockClient.security.changePassword.mock.calls).toMatchInlineSnapshot(`
-Array [
-  Array [
-    Object {
-      "body": Object {
-        "password": "bar",
-      },
-      "refresh": "wait_for",
-      "username": "kibana_system",
-    },
-  ],
-  Array [
-    Object {
-      "body": Object {
-        "password": "changeme",
-      },
-      "refresh": "wait_for",
-      "username": "logstash_system",
-    },
-  ],
-  Array [
-    Object {
-      "body": Object {
-        "password": "changeme",
-      },
-      "refresh": "wait_for",
-      "username": "elastic",
-    },
-  ],
-  Array [
-    Object {
-      "body": Object {
-        "password": "changeme",
-      },
-      "refresh": "wait_for",
-      "username": "beats_system",
-    },
-  ],
-]
-`);
+      Array [
+        Array [
+          Object {
+            "body": Object {
+              "password": "changeme",
+            },
+            "refresh": "wait_for",
+            "username": "kibana_system",
+          },
+        ],
+        Array [
+          Object {
+            "body": Object {
+              "password": "changeme",
+            },
+            "refresh": "wait_for",
+            "username": "logstash_system",
+          },
+        ],
+        Array [
+          Object {
+            "body": Object {
+              "password": "changeme",
+            },
+            "refresh": "wait_for",
+            "username": "elastic",
+          },
+        ],
+        Array [
+          Object {
+            "body": Object {
+              "password": "changeme",
+            },
+            "refresh": "wait_for",
+            "username": "beats_system",
+          },
+        ],
+      ]
+    `);
   });
 });
 

packages/kbn-test/src/es/es_test_config.ts

@@ -8,7 +8,7 @@
 
 import { kibanaPackageJson as pkg } from '@kbn/dev-utils';
 import Url from 'url';
-import { adminTestUser } from '../kbn';
+import { systemIndicesSuperuser } from '../kbn';
 
 class EsTestConfig {
   getVersion() {
@@ -51,8 +51,8 @@ class EsTestConfig {
       };
     }
 
-    const username = process.env.TEST_ES_USERNAME || adminTestUser.username;
-    const password = process.env.TEST_ES_PASSWORD || adminTestUser.password;
+    const username = process.env.TEST_ES_USERNAME || systemIndicesSuperuser.username;
+    const password = process.env.TEST_ES_PASSWORD || systemIndicesSuperuser.password;
 
     const port = process.env.TEST_ES_PORT ? parseInt(process.env.TEST_ES_PORT, 10) : 9220;
 

packages/kbn-test/src/index.ts

@@ -28,7 +28,13 @@ export { KIBANA_ROOT } from './functional_tests/lib/paths';
 export type { CreateTestEsClusterOptions, EsTestCluster, ICluster } from './es';
 export { esTestConfig, createTestEsCluster } from './es';
 
-export { kbnTestConfig, kibanaServerTestUser, kibanaTestUser, adminTestUser } from './kbn';
+export {
+  kbnTestConfig,
+  kibanaServerTestUser,
+  kibanaTestUser,
+  adminTestUser,
+  systemIndicesSuperuser,
+} from './kbn';
 
 export { readConfigFile } from './functional_test_runner/lib/config/read_config_file';
 

packages/kbn-test/src/kbn/index.ts

@@ -7,4 +7,9 @@
  */
 
 export { kbnTestConfig } from './kbn_test_config';
-export { kibanaTestUser, kibanaServerTestUser, adminTestUser } from './users';
+export {
+  kibanaTestUser,
+  kibanaServerTestUser,
+  adminTestUser,
+  systemIndicesSuperuser,
+} from './users';

packages/kbn-test/src/kbn/users.ts

@@ -6,6 +6,9 @@
  * Side Public License, v 1.
  */
 
+// @ts-expect-error no types
+import { SYSTEM_INDICES_SUPERUSER } from '@kbn/es';
+
 const env = process.env;
 
 export const kibanaTestUser = {
@@ -22,3 +25,11 @@ export const adminTestUser = {
   username: env.TEST_ES_USER || 'elastic',
   password: env.TEST_ES_PASS || 'changeme',
 };
+
+/**
+ * User with higher privileges than regular superuser role for writing to system indices
+ */
+export const systemIndicesSuperuser = {
+  username: SYSTEM_INDICES_SUPERUSER,
+  password: env.TEST_ES_PASS || 'changeme',
+};

src/core/server/saved_objects/migrationsv2/integration_tests/batch_size_bytes.test.ts

@@ -27,7 +27,7 @@ async function removeLogFile() {
   await fs.unlink(logFilePath).catch(() => void 0);
 }
 
-describe('migration v2', () => {
+describe('migration v2', function () {
   let esServer: kbnTestServer.TestElasticsearchUtils;
   let root: Root;
   let startES: () => Promise<kbnTestServer.TestElasticsearchUtils>;

src/core/test_helpers/kbn_server.ts

@@ -12,7 +12,7 @@ import {
   CreateTestEsClusterOptions,
   esTestConfig,
   kibanaServerTestUser,
-  kibanaTestUser,
+  systemIndicesSuperuser,
 } from '@kbn/test';
 import { defaultsDeep } from 'lodash';
 import { resolve } from 'path';
@@ -73,7 +73,9 @@ export function createRootWithSettings(
  * @param path
  */
 export function getSupertest(root: Root, method: HttpMethod, path: string) {
-  const testUserCredentials = Buffer.from(`${kibanaTestUser.username}:${kibanaTestUser.password}`);
+  const testUserCredentials = Buffer.from(
+    `${systemIndicesSuperuser.username}:${systemIndicesSuperuser.password}`
+  );
   return supertest((root as any).server.http.httpServer.server.listener)
     [method](path)
     .set('Authorization', `Basic ${testUserCredentials.toString('base64')}`);