From 3ef0399484b7834815459e01472811385c5e4006 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 14 Apr 2026 16:28:16 +0200 Subject: [PATCH 1/5] [StorageIndexAdapter] Set auto_expand_replicas to fix yellow health on single-node ES clusters StorageIndexAdapter did not include index settings in its template, causing all managed indices to default to number_of_replicas: 1. On single-node Elasticsearch clusters, the replica shard cannot be allocated, leaving cluster health yellow indefinitely. This adds auto_expand_replicas: '0-1' and number_of_shards: 1 to the index template and updates existing indices on write if their settings differ. Fixes #263048 --- .../src/index_adapter/index.test.ts | 73 +++++++++++++++++++ .../src/index_adapter/index.ts | 42 ++++++++++- .../integration_tests/index.test.ts | 55 ++++++++++++++ 3 files changed, 166 insertions(+), 4 deletions(-) diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts index cfab00f92e280..91f16880c51a7 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts @@ -76,6 +76,16 @@ const createMockEsClient = () => { template: { mappings: {} }, }), putMapping: jest.fn().mockResolvedValue({}), + getSettings: jest.fn().mockResolvedValue({ + 'test_index-000001': { + settings: { + index: { + auto_expand_replicas: '0-1', + }, + }, + }, + }), + putSettings: jest.fn().mockResolvedValue({}), }, } as unknown as jest.Mocked; return client; @@ -162,6 +172,69 @@ describe('StorageIndexAdapter - transport options forwarding', () => { ); }); + it('includes settings in the index template', async () => { + const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings); + const client = adapter.getClient(); + + await client.index({ id: 'doc1', document: { foo: 'bar' } }); + + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + template: expect.objectContaining({ + settings: expect.objectContaining({ + auto_expand_replicas: '0-1', + number_of_shards: 1, + }), + }), + }) + ); + }); + + it('updates settings on an existing write index when auto_expand_replicas differs', async () => { + const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings); + const client = adapter.getClient(); + + (esClient.indices.getSettings as jest.Mock).mockResolvedValue({ + 'test_index-000001': { + settings: { + index: { + auto_expand_replicas: '1-1', + }, + }, + }, + }); + + await client.index({ id: 'doc1', document: { foo: 'bar' } }); + + expect(esClient.indices.putSettings).toHaveBeenCalledWith( + expect.objectContaining({ + index: 'test_index-000001', + settings: { + auto_expand_replicas: '0-1', + }, + }) + ); + }); + + it('does not update settings when auto_expand_replicas is already 0-1', async () => { + const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings); + const client = adapter.getClient(); + + (esClient.indices.getSettings as jest.Mock).mockResolvedValue({ + 'test_index-000001': { + settings: { + index: { + auto_expand_replicas: '0-1', + }, + }, + }, + }); + + await client.index({ id: 'doc1', document: { foo: 'bar' } }); + + expect(esClient.indices.putSettings).not.toHaveBeenCalled(); + }); + it('works without transport options (backward compatible)', async () => { const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings); const client = adapter.getClient(); diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts index 287c07b75b58a..e4ca3d1e6d0ce 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts @@ -122,7 +122,7 @@ export interface StorageIndexAdapterOptions { */ export class StorageIndexAdapter< TStorageSettings extends IndexStorageSettings, - TApplicationType extends Partial> + TApplicationType extends Partial>, > { private readonly logger: Logger; constructor( @@ -146,6 +146,10 @@ export class StorageIndexAdapter< const version = getSchemaVersion(this.storage); const template: IndicesPutIndexTemplateIndexTemplateMapping = { + settings: { + auto_expand_replicas: '0-1', + number_of_shards: 1, + }, mappings: { _meta: { version, @@ -254,6 +258,30 @@ export class StorageIndexAdapter< ).catch(catchConflictError); } + private async updateSettingsOfExistingIndex({ name }: { name: string }) { + const currentIndexSettings = await wrapEsCall( + this.esClient.indices.getSettings({ + index: name, + flat_settings: true, + }) + ); + + const indexSettings = currentIndexSettings[name]?.settings?.index; + const currentAutoExpandReplicas = indexSettings?.auto_expand_replicas; + + if (currentAutoExpandReplicas !== '0-1') { + this.logger.debug(`Updating settings of existing index to set auto_expand_replicas=0-1`); + await wrapEsCall( + this.esClient.indices.putSettings({ + index: name, + settings: { + auto_expand_replicas: '0-1', + }, + }) + ); + } + } + private async updateMappingsOfExistingIndex({ name }: { name: string }) { const simulateIndexTemplateResponse = await this.esClient.indices.simulateIndexTemplate({ name: getIndexName(this.storage.name, 999999), @@ -282,11 +310,17 @@ export class StorageIndexAdapter< if (!writeIndex) { this.logger.debug(`Creating index`); await this.createIndex(); - } else if (writeIndex?.state.mappings?._meta?.version !== expectedSchemaVersion) { - this.logger.debug(`Updating mappings of existing index due to schema version mismatch`); - await this.updateMappingsOfExistingIndex({ + } else { + await this.updateSettingsOfExistingIndex({ name: writeIndex.name, }); + + if (writeIndex.state.mappings?._meta?.version !== expectedSchemaVersion) { + this.logger.debug(`Updating mappings of existing index due to schema version mismatch`); + await this.updateMappingsOfExistingIndex({ + name: writeIndex.name, + }); + } } return await cb(); diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts index 4ace63dd5092c..8bf331ba26195 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts @@ -315,6 +315,57 @@ describe('StorageIndexAdapter', () => { }); }); + describe('when writing/bootstrapping with an existing index missing auto_expand_replicas', () => { + const LEGACY_INDEX_NAME = 'test_legacy_settings_index'; + + afterAll(async () => { + await esClient.indices.delete({ index: `${LEGACY_INDEX_NAME}*` }).catch(() => {}); + await esClient.indices.deleteIndexTemplate({ name: LEGACY_INDEX_NAME }).catch(() => {}); + }); + + it('updates auto_expand_replicas on the existing index', async () => { + const legacySettings = { + name: LEGACY_INDEX_NAME, + schema: { + properties: { + foo: { + type: 'keyword', + }, + }, + }, + } satisfies StorageSettings; + + const legacyAdapter = createStorageIndexAdapter(legacySettings); + const legacyClient = legacyAdapter.getClient(); + + await legacyClient.index({ id: 'doc1', document: { foo: 'bar' } }); + + const writeIndexName = `${LEGACY_INDEX_NAME}-000001`; + + const beforeUpdate = await esClient.indices.get({ index: LEGACY_INDEX_NAME }); + expect(beforeUpdate[writeIndexName].settings?.index?.auto_expand_replicas).toEqual('0-1'); + + await esClient.indices.putSettings({ + index: writeIndexName, + settings: { + auto_expand_replicas: '1-1', + }, + }); + + const afterManualChange = await esClient.indices.get({ index: LEGACY_INDEX_NAME }); + expect(afterManualChange[writeIndexName].settings?.index?.auto_expand_replicas).toEqual( + '1-1' + ); + + await legacyClient.index({ id: 'doc2', document: { foo: 'baz' } }); + + const afterSecondWrite = await esClient.indices.get({ index: LEGACY_INDEX_NAME }); + expect(afterSecondWrite[writeIndexName].settings?.index?.auto_expand_replicas).toEqual('0-1'); + + await legacyClient.clean(); + }); + }); + describe('when writing/bootstrapping with an legacy index', () => { beforeAll(async () => { await client.index({ id: 'foo', document: { foo: 'bar' } }); @@ -576,6 +627,10 @@ describe('StorageIndexAdapter', () => { is_write_index: true, }, }); + + expect(getIndexResponse[writeIndexName].settings?.index?.auto_expand_replicas).toEqual('0-1'); + + expect(getIndexResponse[writeIndexName].settings?.index?.number_of_shards).toEqual('1'); } async function verifyClean() { From 2229578e01c3968174c3eeb6240290ac42165238 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 14 Apr 2026 16:55:01 +0200 Subject: [PATCH 2/5] Remove flat_settings from getSettings call in updateSettingsOfExistingIndex With flat_settings: true, Elasticsearch returns dot-notation keys like 'index.auto_expand_replicas' instead of nested objects. This caused currentAutoExpandReplicas to always be undefined, making putSettings run on every write even when the setting was already correct. --- .../shared/kbn-storage-adapter/src/index_adapter/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts index e4ca3d1e6d0ce..ae20f29c2d992 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts @@ -262,7 +262,6 @@ export class StorageIndexAdapter< const currentIndexSettings = await wrapEsCall( this.esClient.indices.getSettings({ index: name, - flat_settings: true, }) ); From e4c70855448be2dc5ca847b2210c4da4541d32c2 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 14 Apr 2026 15:31:31 +0000 Subject: [PATCH 3/5] Changes from node scripts/eslint_all_files --no-cache --fix --- .../shared/kbn-storage-adapter/src/index_adapter/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts index ae20f29c2d992..03eb0da236a38 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts @@ -122,7 +122,7 @@ export interface StorageIndexAdapterOptions { */ export class StorageIndexAdapter< TStorageSettings extends IndexStorageSettings, - TApplicationType extends Partial>, + TApplicationType extends Partial> > { private readonly logger: Logger; constructor( From fd65d54e1f96c7e74eb973fdf7ec2665ea96c3ed Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 17 Apr 2026 13:06:58 +0000 Subject: [PATCH 4/5] Ralph: simplify the implementation so the auto_expand_replicas as rud... --- .../src/index_adapter/index.test.ts | 54 ------------------- .../src/index_adapter/index.ts | 29 +--------- .../integration_tests/index.test.ts | 51 ------------------ 3 files changed, 1 insertion(+), 133 deletions(-) diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts index 91f16880c51a7..5c165c46b3a32 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts @@ -76,15 +76,6 @@ const createMockEsClient = () => { template: { mappings: {} }, }), putMapping: jest.fn().mockResolvedValue({}), - getSettings: jest.fn().mockResolvedValue({ - 'test_index-000001': { - settings: { - index: { - auto_expand_replicas: '0-1', - }, - }, - }, - }), putSettings: jest.fn().mockResolvedValue({}), }, } as unknown as jest.Mocked; @@ -190,51 +181,6 @@ describe('StorageIndexAdapter - transport options forwarding', () => { ); }); - it('updates settings on an existing write index when auto_expand_replicas differs', async () => { - const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings); - const client = adapter.getClient(); - - (esClient.indices.getSettings as jest.Mock).mockResolvedValue({ - 'test_index-000001': { - settings: { - index: { - auto_expand_replicas: '1-1', - }, - }, - }, - }); - - await client.index({ id: 'doc1', document: { foo: 'bar' } }); - - expect(esClient.indices.putSettings).toHaveBeenCalledWith( - expect.objectContaining({ - index: 'test_index-000001', - settings: { - auto_expand_replicas: '0-1', - }, - }) - ); - }); - - it('does not update settings when auto_expand_replicas is already 0-1', async () => { - const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings); - const client = adapter.getClient(); - - (esClient.indices.getSettings as jest.Mock).mockResolvedValue({ - 'test_index-000001': { - settings: { - index: { - auto_expand_replicas: '0-1', - }, - }, - }, - }); - - await client.index({ id: 'doc1', document: { foo: 'bar' } }); - - expect(esClient.indices.putSettings).not.toHaveBeenCalled(); - }); - it('works without transport options (backward compatible)', async () => { const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings); const client = adapter.getClient(); diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts index 03eb0da236a38..ca3e3e55f9971 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts @@ -122,7 +122,7 @@ export interface StorageIndexAdapterOptions { */ export class StorageIndexAdapter< TStorageSettings extends IndexStorageSettings, - TApplicationType extends Partial> + TApplicationType extends Partial>, > { private readonly logger: Logger; constructor( @@ -258,29 +258,6 @@ export class StorageIndexAdapter< ).catch(catchConflictError); } - private async updateSettingsOfExistingIndex({ name }: { name: string }) { - const currentIndexSettings = await wrapEsCall( - this.esClient.indices.getSettings({ - index: name, - }) - ); - - const indexSettings = currentIndexSettings[name]?.settings?.index; - const currentAutoExpandReplicas = indexSettings?.auto_expand_replicas; - - if (currentAutoExpandReplicas !== '0-1') { - this.logger.debug(`Updating settings of existing index to set auto_expand_replicas=0-1`); - await wrapEsCall( - this.esClient.indices.putSettings({ - index: name, - settings: { - auto_expand_replicas: '0-1', - }, - }) - ); - } - } - private async updateMappingsOfExistingIndex({ name }: { name: string }) { const simulateIndexTemplateResponse = await this.esClient.indices.simulateIndexTemplate({ name: getIndexName(this.storage.name, 999999), @@ -310,10 +287,6 @@ export class StorageIndexAdapter< this.logger.debug(`Creating index`); await this.createIndex(); } else { - await this.updateSettingsOfExistingIndex({ - name: writeIndex.name, - }); - if (writeIndex.state.mappings?._meta?.version !== expectedSchemaVersion) { this.logger.debug(`Updating mappings of existing index due to schema version mismatch`); await this.updateMappingsOfExistingIndex({ diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts index 8bf331ba26195..4a2ea79b93b63 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/integration_tests/index.test.ts @@ -315,57 +315,6 @@ describe('StorageIndexAdapter', () => { }); }); - describe('when writing/bootstrapping with an existing index missing auto_expand_replicas', () => { - const LEGACY_INDEX_NAME = 'test_legacy_settings_index'; - - afterAll(async () => { - await esClient.indices.delete({ index: `${LEGACY_INDEX_NAME}*` }).catch(() => {}); - await esClient.indices.deleteIndexTemplate({ name: LEGACY_INDEX_NAME }).catch(() => {}); - }); - - it('updates auto_expand_replicas on the existing index', async () => { - const legacySettings = { - name: LEGACY_INDEX_NAME, - schema: { - properties: { - foo: { - type: 'keyword', - }, - }, - }, - } satisfies StorageSettings; - - const legacyAdapter = createStorageIndexAdapter(legacySettings); - const legacyClient = legacyAdapter.getClient(); - - await legacyClient.index({ id: 'doc1', document: { foo: 'bar' } }); - - const writeIndexName = `${LEGACY_INDEX_NAME}-000001`; - - const beforeUpdate = await esClient.indices.get({ index: LEGACY_INDEX_NAME }); - expect(beforeUpdate[writeIndexName].settings?.index?.auto_expand_replicas).toEqual('0-1'); - - await esClient.indices.putSettings({ - index: writeIndexName, - settings: { - auto_expand_replicas: '1-1', - }, - }); - - const afterManualChange = await esClient.indices.get({ index: LEGACY_INDEX_NAME }); - expect(afterManualChange[writeIndexName].settings?.index?.auto_expand_replicas).toEqual( - '1-1' - ); - - await legacyClient.index({ id: 'doc2', document: { foo: 'baz' } }); - - const afterSecondWrite = await esClient.indices.get({ index: LEGACY_INDEX_NAME }); - expect(afterSecondWrite[writeIndexName].settings?.index?.auto_expand_replicas).toEqual('0-1'); - - await legacyClient.clean(); - }); - }); - describe('when writing/bootstrapping with an legacy index', () => { beforeAll(async () => { await client.index({ id: 'foo', document: { foo: 'bar' } }); From 8ba7f52a23feda7d408f5dbc1174e3701f5ce9e5 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:45:58 +0000 Subject: [PATCH 5/5] Changes from node scripts/eslint_all_files --no-cache --fix --- .../shared/kbn-storage-adapter/src/index_adapter/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts index fff0714c0adf8..dba0aed9b0874 100644 --- a/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts +++ b/src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts @@ -134,7 +134,7 @@ export interface StorageIndexAdapterOptions { */ export class StorageIndexAdapter< TStorageSettings extends IndexStorageSettings, - TApplicationType extends Partial>, + TApplicationType extends Partial> > { private readonly logger: Logger; private updateMappingsPromise: Promise | undefined;