[8.x] [Feature Flags] Retry provider setup (#214200)

src/core/packages/feature-flags/server-internal/src/feature_flags_service.test.ts

@@ -36,7 +36,9 @@ describe('FeatureFlagsService Server', () => {
   });
 
   afterEach(async () => {
+    jest.useRealTimers();
     await featureFlagsService.stop();
+    jest.spyOn(OpenFeature, 'setProviderAndWait').mockRestore(); // Make sure that we clean up any previous mocked implementations
     jest.clearAllMocks();
     await OpenFeature.clearProviders();
   });
@@ -45,7 +47,7 @@ describe('FeatureFlagsService Server', () => {
     test('appends a provider (no async operation)', () => {
       expect.assertions(1);
       const { setProvider } = featureFlagsService.setup();
-      const spy = jest.spyOn(OpenFeature, 'setProvider');
+      const spy = jest.spyOn(OpenFeature, 'setProviderAndWait');
       const fakeProvider = { metadata: { name: 'fake provider' } } as Provider;
       setProvider(fakeProvider);
       expect(spy).toHaveBeenCalledWith(fakeProvider);

src/core/packages/feature-flags/server-internal/src/feature_flags_service.ts

@@ -24,6 +24,7 @@ import {
 } from '@openfeature/server-sdk';
 import deepMerge from 'deepmerge';
 import { filter, switchMap, startWith, Subject } from 'rxjs';
+import { setProviderWithRetries } from './set_provider_with_retries';
 import { type FeatureFlagsConfig, featureFlagsConfig } from './feature_flags_config';
 
 /**
@@ -76,7 +77,7 @@ export class FeatureFlagsService {
         if (OpenFeature.providerMetadata !== NOOP_PROVIDER.metadata) {
           throw new Error('A provider has already been set. This API cannot be called twice.');
         }
-        OpenFeature.setProvider(provider);
+        setProviderWithRetries(provider, this.logger);
       },
       appendContext: (contextToAppend) => this.appendContext(contextToAppend),
     };

src/core/packages/feature-flags/server-internal/src/set_provider_with_retries.test.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { OpenFeature, type Provider } from '@openfeature/server-sdk';
+import { setProviderWithRetries } from './set_provider_with_retries';
+import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
+
+describe('setProviderWithRetries', () => {
+  const fakeProvider = { metadata: { name: 'fake provider' } } as Provider;
+  let logger: MockedLogger;
+
+  beforeEach(() => {
+    jest.useFakeTimers();
+    logger = loggerMock.create();
+  });
+
+  afterEach(() => {
+    jest.clearAllTimers();
+    jest.clearAllMocks();
+    jest.useRealTimers();
+  });
+
+  test('sets the provider and logs the success', async () => {
+    expect.assertions(3);
+    const spy = jest.spyOn(OpenFeature, 'setProviderAndWait');
+
+    setProviderWithRetries(fakeProvider, logger);
+
+    expect(spy).toHaveBeenCalledWith(fakeProvider);
+    expect(spy).toHaveBeenCalledTimes(1);
+
+    await jest.runAllTimersAsync();
+
+    expect(logger.info.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Feature flags provider successfully set up.",
+        ],
+      ]
+    `);
+  });
+
+  test('should retry up to 5 times (and does not throw/reject)', async () => {
+    expect.assertions(15);
+    const spy = jest
+      .spyOn(OpenFeature, 'setProviderAndWait')
+      .mockRejectedValue(new Error('Something went terribly wrong!'));
+
+    setProviderWithRetries(fakeProvider, logger);
+
+    expect(spy).toHaveBeenCalledWith(fakeProvider);
+
+    // Initial attempt
+    expect(spy).toHaveBeenCalledTimes(1);
+
+    // 5 retries
+    for (let i = 0; i < 5; i++) {
+      await jest.advanceTimersByTimeAsync(1000 * Math.pow(2, i)); // exponential backoff of factor 2
+      expect(spy).toHaveBeenCalledTimes(i + 2);
+      expect(logger.warn).toHaveBeenCalledTimes(i + 2);
+    }
+
+    // Given up retrying
+    await jest.advanceTimersByTimeAsync(32000);
+    expect(spy).toHaveBeenCalledTimes(6);
+
+    expect(logger.warn.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 5 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 4 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 3 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 2 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 1 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 0 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+      ]
+    `);
+    expect(logger.error.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+      ]
+    `);
+  });
+});

src/core/packages/feature-flags/server-internal/src/set_provider_with_retries.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { Logger } from '@kbn/logging';
+import { type Provider, OpenFeature } from '@openfeature/server-sdk';
+import pRetry from 'p-retry';
+
+/**
+ * Handles the setting of the Feature Flags provider and any retries that may be required.
+ * This method is intentionally synchronous (no async/await) to avoid holding Kibana's startup on the feature flags setup.
+ * @param provider The OpenFeature provider to set up.
+ * @param logger You know, for logging.
+ */
+export function setProviderWithRetries(provider: Provider, logger: Logger): void {
+  pRetry(() => OpenFeature.setProviderAndWait(provider), {
+    retries: 5,
+    onFailedAttempt: (error) => {
+      logger.warn(
+        `Failed to set up the feature flags provider: ${error.message}. Retrying ${error.retriesLeft} times more...`,
+        { error }
+      );
+    },
+  })
+    .then(() => {
+      logger.info('Feature flags provider successfully set up.');
+    })
+    .catch((error) => {
+      logger.error(`Failed to set up the feature flags provider: ${error.message}`, {
+        error,
+      });
+    });
+}

src/core/packages/feature-flags/server-internal/tsconfig.json

@@ -20,5 +20,6 @@
     "@kbn/core-base-server-mocks",
     "@kbn/config-schema",
     "@kbn/config-mocks",
+    "@kbn/logging-mocks",
   ]
 }