[Storage Index Adapter] Fix index template creation on Serverless

config/serverless.oblt.yml

@@ -109,3 +109,6 @@ xpack.ml.compatibleModuleType: 'observability'
 
 # Disable the embedded Dev Console
 console.ui.embeddedEnabled: false
+
+# TODO: Remove this before merging
+xpack.evals.enabled: true

src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.test.ts

@@ -8,6 +8,8 @@
  */
 
 import type { ElasticsearchClient, Logger } from '@kbn/core/server';
+import type { TransportResult } from '@elastic/elasticsearch';
+import { errors } from '@elastic/elasticsearch';
 import type { StorageTransportOptions } from '../..';
 import { StorageIndexAdapter, type StorageSettings } from '../..';
 
@@ -181,6 +183,75 @@ describe('StorageIndexAdapter - transport options forwarding', () => {
     );
   });
 
+  it('retries index template without settings on serverless', async () => {
+    const serverlessError = new errors.ResponseError({
+      statusCode: 400,
+      headers: {},
+      warnings: [],
+      meta: {} as any,
+      body: {
+        error: {
+          type: 'illegal_argument_exception',
+          reason:
+            'Settings [index.auto_expand_replicas,index.number_of_shards] are not available when running in serverless mode',
+        },
+      },
+    } as TransportResult);
+    (esClient.indices.putIndexTemplate as jest.Mock).mockRejectedValueOnce(serverlessError);
+
+    const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings);
+    const client = adapter.getClient();
+
+    await client.index({ id: 'doc1', document: { foo: 'bar' } });
+
+    expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(2);
+    expect(esClient.indices.putIndexTemplate).toHaveBeenNthCalledWith(
+      1,
+      expect.objectContaining({
+        template: expect.objectContaining({
+          settings: expect.objectContaining({ auto_expand_replicas: '0-1' }),
+        }),
+      })
+    );
+    expect(esClient.indices.putIndexTemplate).toHaveBeenNthCalledWith(
+      2,
+      expect.objectContaining({
+        template: expect.not.objectContaining({ settings: expect.anything() }),
+      })
+    );
+  });
+
+  it('skips settings on subsequent writes after serverless detection', async () => {
+    const serverlessError = new errors.ResponseError({
+      statusCode: 400,
+      headers: {},
+      warnings: [],
+      meta: {} as any,
+      body: {
+        error: {
+          type: 'illegal_argument_exception',
+          reason:
+            'Settings [index.auto_expand_replicas,index.number_of_shards] are not available when running in serverless mode',
+        },
+      },
+    } as TransportResult);
+    (esClient.indices.putIndexTemplate as jest.Mock).mockRejectedValueOnce(serverlessError);
+
+    const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings);
+    const client = adapter.getClient();
+
+    await client.index({ id: 'doc1', document: { foo: 'bar' } });
+    await client.index({ id: 'doc2', document: { foo: 'baz' } });
+
+    expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3);
+    expect(esClient.indices.putIndexTemplate).toHaveBeenNthCalledWith(
+      3,
+      expect.objectContaining({
+        template: expect.not.objectContaining({ settings: expect.anything() }),
+      })
+    );
+  });
+
   it('works without transport options (backward compatible)', async () => {
     const adapter = new StorageIndexAdapter(esClient, loggerMock, storageSettings);
     const client = adapter.getClient();

src/platform/packages/shared/kbn-storage-adapter/src/index_adapter/index.ts

@@ -84,6 +84,14 @@ function isNotFoundError(error: Error): error is errors.ResponseError & { status
   return isResponseError(error) && error.statusCode === 404;
 }
 
+function isServerlessSettingsError(error: unknown): boolean {
+  if (!isResponseError(error as Error)) {
+    return false;
+  }
+  const reason: string = (error as errors.ResponseError).body?.error?.reason ?? '';
+  return reason.includes('not available when running in serverless mode');
+}
+
 /*
  * When calling into Elasticsearch, the stack trace is lost.
  * If we create an error before calling, and append it to
@@ -136,8 +144,14 @@ export class StorageIndexAdapter<
   TStorageSettings extends IndexStorageSettings,
   TApplicationType extends Partial<StorageDocumentOf<TStorageSettings>>
 > {
+  private static readonly INDEX_SETTINGS = {
+    auto_expand_replicas: '0-1',
+    number_of_shards: 1,
+  } as const;
+
   private readonly logger: Logger;
   private updateMappingsPromise: Promise<void> | undefined;
+  private isServerless: boolean | undefined;
 
   constructor(
     private readonly esClient: ElasticsearchClient,
@@ -159,39 +173,59 @@ export class StorageIndexAdapter<
   private async createOrUpdateIndexTemplate(): Promise<void> {
     const version = getSchemaVersion(this.storage);
 
-    const template: IndicesPutIndexTemplateIndexTemplateMapping = {
-      settings: {
-        auto_expand_replicas: '0-1',
-        number_of_shards: 1,
-      },
-      mappings: {
-        _meta: {
-          version,
-        },
-        dynamic: 'strict',
-        properties: {
-          ...mapValues(this.storage.schema.properties, toElasticsearchMappingProperty),
-        },
+    const mappings: IndicesPutIndexTemplateIndexTemplateMapping['mappings'] = {
+      _meta: { version },
+      dynamic: 'strict',
+      properties: {
+        ...mapValues(this.storage.schema.properties, toElasticsearchMappingProperty),
       },
-      aliases: {
-        [getAliasName(this.storage.name)]: {
-          is_write_index: true,
-        },
+    };
+
+    const aliases: IndicesPutIndexTemplateIndexTemplateMapping['aliases'] = {
+      [getAliasName(this.storage.name)]: {
+        is_write_index: true,
       },
     };
 
-    await wrapEsCall(
-      this.esClient.indices.putIndexTemplate({
-        name: getIndexTemplateName(this.storage.name),
-        create: false,
-        allow_auto_create: false,
-        index_patterns: getIndexPattern(this.storage.name),
-        _meta: {
-          version,
-        },
-        template,
-      })
-    ).catch(catchConflictError);
+    const baseRequest = {
+      name: getIndexTemplateName(this.storage.name),
+      create: false as const,
+      allow_auto_create: false as const,
+      index_patterns: getIndexPattern(this.storage.name),
+      _meta: { version },
+    };
+
+    const putTemplate = (includeSettings: boolean) =>
+      wrapEsCall(
+        this.esClient.indices.putIndexTemplate({
+          ...baseRequest,
+          template: {
+            ...(includeSettings ? { settings: StorageIndexAdapter.INDEX_SETTINGS } : {}),
+            mappings,
+            aliases,
+          },
+        })
+      ).catch(catchConflictError);
+
+    if (this.isServerless) {
+      await putTemplate(false);
+      return;
+    }
+
+    try {
+      await putTemplate(true);
+      this.isServerless = false;
+    } catch (error) {
+      if (isServerlessSettingsError(error)) {
+        this.isServerless = true;
+        this.logger.debug(
+          'Index settings are unavailable (serverless ES); retrying template without settings'
+        );
+        await putTemplate(false);
+      } else {
+        throw error;
+      }
+    }
   }
 
   private async getExistingIndexTemplate(): Promise<IndicesIndexTemplate | undefined> {