diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/server/routes/knowledge_base/route.ts index a04b088c55775..6647e88d48d1f 100644 --- a/x-pack/platform/plugins/shared/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/platform/plugins/shared/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -103,11 +103,11 @@ const reIndexKnowledgeBase = createObservabilityAIAssistantServerRoute({ requiredPrivileges: ['ai_assistant'], }, }, - handler: async (resources): Promise<{ result: boolean }> => { + handler: async (resources): Promise<{ success: boolean }> => { const client = await resources.service.getClient({ request: resources.request }); const { inference_id: inferenceId } = resources.params.query; - const result = await client.reIndexKnowledgeBaseWithLock(inferenceId); - return { result }; + await client.reIndexKnowledgeBaseWithLock(inferenceId); + return { success: true }; }, }); diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts index 9b08e32ddf86c..694e86871a8e9 100644 --- a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -33,7 +33,6 @@ import { hasKbWriteIndex } from './has_kb_index'; import { getInferenceIdFromWriteIndex } from './get_inference_id_from_write_index'; import { reIndexKnowledgeBaseWithLock } from './reindex_knowledge_base'; import { isSemanticTextUnsupportedError } from '../startup_migrations/run_startup_migrations'; -import { isKnowledgeBaseIndexWriteBlocked } from './index_write_block_utils'; interface Dependencies { core: CoreSetup; @@ -461,12 +460,6 @@ export class KnowledgeBaseService { ); } - if (isKnowledgeBaseIndexWriteBlocked(error)) { - throw new Error( - `Writes to the knowledge base are currently blocked due to an Elasticsearch write index block. This is most likely due to an ongoing re-indexing operation. Please try again later. Error: ${error.message}` - ); - } - throw error; } }; diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index_write_block_utils.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index_write_block_utils.ts deleted file mode 100644 index 053240191e34d..0000000000000 --- a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/index_write_block_utils.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { errors } from '@elastic/elasticsearch'; -import { ElasticsearchClient, Logger } from '@kbn/core/server'; -import pRetry from 'p-retry'; -import { resourceNames } from '..'; - -export async function addIndexWriteBlock({ - esClient, - index, -}: { - esClient: { asInternalUser: ElasticsearchClient }; - index: string; -}) { - await esClient.asInternalUser.indices.addBlock({ index, block: 'write' }); -} - -export function removeIndexWriteBlock({ - esClient, - index, -}: { - esClient: { asInternalUser: ElasticsearchClient }; - index: string; -}) { - return esClient.asInternalUser.indices.putSettings({ - index, - body: { 'index.blocks.write': false }, - }); -} - -export async function hasIndexWriteBlock({ - esClient, - index, -}: { - esClient: { asInternalUser: ElasticsearchClient }; - index: string; -}) { - const response = await esClient.asInternalUser.indices.getSettings({ index }); - const writeBlockSetting = Object.values(response)[0]?.settings?.index?.blocks?.write; - return writeBlockSetting === 'true' || writeBlockSetting === true; -} - -export async function waitForWriteBlockToBeRemoved({ - esClient, - logger, - index, -}: { - esClient: { asInternalUser: ElasticsearchClient }; - logger: Logger; - index: string; -}) { - return pRetry( - async () => { - const isBlocked = await hasIndexWriteBlock({ esClient, index }); - if (isBlocked) { - logger.debug(`Waiting for the write block to be removed from "${index}"...`); - throw new Error( - 'Waiting for the re-index operation to complete and the write block to be removed...' - ); - } - }, - { forever: true, maxTimeout: 10000 } - ); -} - -export function isKnowledgeBaseIndexWriteBlocked(error: any) { - return ( - error instanceof errors.ResponseError && - error.message.includes(`cluster_block_exception`) && - error.message.includes(resourceNames.writeIndexAlias.kb) - ); -} diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/reindex_knowledge_base.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/reindex_knowledge_base.ts index 4f11d8cc0ad5f..931c45833cd8b 100644 --- a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/reindex_knowledge_base.ts +++ b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/reindex_knowledge_base.ts @@ -13,11 +13,6 @@ import { CoreSetup } from '@kbn/core/server'; import { LockManagerService } from '@kbn/lock-manager'; import { resourceNames } from '..'; import { ObservabilityAIAssistantPluginStartDependencies } from '../../types'; -import { - addIndexWriteBlock, - hasIndexWriteBlock, - removeIndexWriteBlock, -} from './index_write_block_utils'; import { createKnowledgeBaseIndex } from './create_knowledge_base_index'; import { updateKnowledgeBaseWriteIndexAlias } from './update_knowledge_base_index_alias'; @@ -34,47 +29,13 @@ export async function reIndexKnowledgeBaseWithLock({ asInternalUser: ElasticsearchClient; }; inferenceId: string; -}): Promise { +}): Promise { const lmService = new LockManagerService(core, logger); return lmService.withLock(KB_REINDEXING_LOCK_ID, () => - reIndexKnowledgeBaseWithWriteIndexBlock({ - logger: logger.get('kb-reindex'), - esClient, - inferenceId, - }) + reIndexKnowledgeBase({ logger, esClient, inferenceId }) ); } -async function reIndexKnowledgeBaseWithWriteIndexBlock({ - logger, - esClient, - inferenceId, -}: { - logger: Logger; - esClient: { asInternalUser: ElasticsearchClient }; - inferenceId: string; -}): Promise { - logger.debug('Initializing re-indexing of knowledge base...'); - if (await hasIndexWriteBlock({ esClient, index: resourceNames.writeIndexAlias.kb })) { - throw new Error( - `Write block is already set on the knowledge base index: ${resourceNames.writeIndexAlias.kb}` - ); - } - - try { - await addIndexWriteBlock({ esClient, index: resourceNames.writeIndexAlias.kb }); - await reIndexKnowledgeBase({ logger, esClient, inferenceId }); - logger.info('Re-indexing knowledge base completed successfully.'); - } catch (error) { - logger.error(`Re-indexing knowledge base failed: ${error.message}`); - throw error; - } finally { - await removeIndexWriteBlock({ esClient, index: resourceNames.writeIndexAlias.kb }); - } - - return true; -} - async function reIndexKnowledgeBase({ logger, esClient, diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/index.ts index 06bb2af4d583a..675248c90b09a 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/index.ts @@ -32,7 +32,7 @@ export default function aiAssistantApiIntegrationTests({ loadTestFile(require.resolve('./knowledge_base/knowledge_base_setup.spec.ts')); loadTestFile(require.resolve('./knowledge_base/knowledge_base_status.spec.ts')); loadTestFile(require.resolve('./knowledge_base/knowledge_base_user_instructions.spec.ts')); - loadTestFile(require.resolve('./knowledge_base/knowledge_base.spec.ts')); + loadTestFile(require.resolve('./knowledge_base/knowledge_base_basic_operations.spec.ts')); loadTestFile( require.resolve('./knowledge_base/knowledge_base_change_model_from_elser_to_e5.spec.ts') ); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.10_upgrade_test.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.10_upgrade_test.spec.ts index 534633fa40b13..c833f84d302da 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.10_upgrade_test.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.10_upgrade_test.spec.ts @@ -33,7 +33,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon // Sparse vector field was introduced in Elasticsearch 8.11 // The semantic text field was added to the knowledge base index in 8.17 // Indices created in 8.10 do not support semantic text field and need to be reindexed - describe('when upgrading from 8.10 to 8.18', function () { + describe('Knowledge base: when upgrading from 8.10 to 8.18', function () { // Intentionally skipped in all serverless environnments (local and MKI) // because the migration scenario being tested is not relevant to MKI and Serverless. this.tags(['skipServerless']); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.16_upgrade_test.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.16_upgrade_test.spec.ts index 498fe75f6d435..55c9cc88406cd 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.16_upgrade_test.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.16_upgrade_test.spec.ts @@ -34,7 +34,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon // In 8.16 and earlier embeddings were stored in the `ml.tokens` field // In 8.17 `ml.tokens` is replaced with `semantic_text` field and the custom ELSER inference endpoint "obs_ai_assistant_kb_inference" is introduced // When upgrading we must ensure that the semantic_text field is populated - describe('when upgrading from 8.16 to 8.17', function () { + describe('Knowledge base: when upgrading from 8.16 to 8.17', function () { // Intentionally skipped in all serverless environnments (local and MKI) // because the migration scenario being tested is not relevant to MKI and Serverless. this.tags(['skipServerless']); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.18_upgrade_test.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.18_upgrade_test.spec.ts index 3ba050327b659..65ef4a2bb0bcf 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.18_upgrade_test.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_8.18_upgrade_test.spec.ts @@ -35,7 +35,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon // In 8.19 / 9.1 the custom inference endpoint ("obs_ai_assistant_kb_inference") is replaced with the preconfigured endpoint ".elser-2-elasticsearch" // We need to make sure that the custom inference endpoint continues to work after the migration - describe('when upgrading from 8.18 to 8.19', function () { + describe('Knowledge base: when upgrading from 8.18 to 8.19', function () { this.tags(['skipServerless']); before(async () => { diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_basic_operations.spec.ts similarity index 99% rename from x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base.spec.ts rename to x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_basic_operations.spec.ts index 18c548971b8fd..67b1dc71cea28 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_basic_operations.spec.ts @@ -44,7 +44,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon return res.body.entries; } - describe('Knowledge base', function () { + describe('Knowledge base: Basic operations', function () { before(async () => { await deployTinyElserAndSetupKb(getService); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_change_model_from_elser_to_e5.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_change_model_from_elser_to_e5.spec.ts index e852a7a79fef1..74a4d94f25f69 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_change_model_from_elser_to_e5.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_change_model_from_elser_to_e5.spec.ts @@ -42,7 +42,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon type KnowledgeBaseEsEntry = Awaited>[0]; - describe('when changing from ELSER to E5-like model', function () { + describe('Knowledge base: when changing from ELSER to E5-like model', function () { let elserEntriesFromApi: KnowledgeBaseEntry[]; let elserEntriesFromEs: KnowledgeBaseEsEntry[]; let elserInferenceId: string; diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_reindex_concurrency.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_reindex_concurrency.spec.ts index fb9f5ed62cf49..a7f268dc26b77 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_reindex_concurrency.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_reindex_concurrency.spec.ts @@ -26,7 +26,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi'); const es = getService('es'); - describe('POST /internal/observability_ai_assistant/kb/reindex', function () { + describe('Knowledge base: POST /internal/observability_ai_assistant/kb/reindex', function () { // Intentionally skipped in all serverless environnments (local and MKI) // because the migration scenario being tested is not relevant to MKI and Serverless. this.tags(['skipServerless']); @@ -82,7 +82,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon const iterations = 5; describe(`when running ${iterations} re-index operations in sequence`, () => { - let results: Array<{ status: number; result: boolean; errorMessage: string | undefined }>; + let results: Array<{ status: number; success: boolean; errorMessage: string | undefined }>; let initialIndexSequenceNumber: number; before(async () => { @@ -103,7 +103,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon it('every re-index operation succeeds', async () => { const successResults = results.filter((result) => result.status === 200); expect(successResults).to.have.length(iterations); - expect(successResults.every((r) => r.result === true)).to.be(true); + expect(successResults.every((r) => r.success === true)).to.be(true); }); it('no requests should fail', async () => { @@ -126,7 +126,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon return { status: res.status, - result: res.body.result, + success: res.body.success, errorMessage: 'message' in res.body ? (res.body.message as string) : undefined, }; } diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_setup.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_setup.spec.ts index fc39b3b4d2164..3500826d9ead0 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_setup.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_setup.spec.ts @@ -32,7 +32,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon const ml = getService('ml'); const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi'); - describe('/internal/observability_ai_assistant/kb/setup', function () { + describe('Knowledge base: POST /internal/observability_ai_assistant/kb/setup', function () { before(async () => { await teardownTinyElserModelAndInferenceEndpoint(getService); await restoreIndexAssets(observabilityAIAssistantAPIClient, es); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_status.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_status.spec.ts index 07edcbab12704..0fb48f361ffc4 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_status.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_status.spec.ts @@ -21,10 +21,7 @@ import { export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) { const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi'); - describe('/internal/observability_ai_assistant/kb/status', function () { - // see details: https://github.com/elastic/kibana/issues/219217 - this.tags(['failsOnMKI']); - + describe('Knowledge base: GET /internal/observability_ai_assistant/kb/status', function () { it('returns correct status before knowledge base is setup', async () => { const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/status', diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_user_instructions.spec.ts index 96055d962fbc4..badc0919922cd 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -31,7 +31,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon const log = getService('log'); const retry = getService('retry'); - describe('Knowledge base user instructions', function () { + describe('Knowledge base: user instructions', function () { before(async () => { await deployTinyElserAndSetupKb(getService); }); diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_warmup_model.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_warmup_model.spec.ts index 07df98671cfab..1c9ce539e59f1 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_warmup_model.spec.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_warmup_model.spec.ts @@ -38,7 +38,7 @@ export default function WarmupModelApiTest({ getService }: DeploymentAgnosticFtr }); } - describe('/internal/observability_ai_assistant/kb/warmup_model', function () { + describe('Knowledge base: POST /internal/observability_ai_assistant/kb/warmup_model', function () { const inferenceId = TINY_ELSER_INFERENCE_ID; before(async () => { diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/knowledge_base.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/knowledge_base.ts index 0b6667b27e932..5cf451471ca39 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/knowledge_base.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/knowledge_base.ts @@ -162,12 +162,6 @@ export async function getConcreteWriteIndexFromAlias(es: Client) { return writeIndex; } -export async function hasIndexWriteBlock(es: Client, index: string) { - const response = await es.indices.getSettings({ index }); - const writeBlockSetting = Object.values(response)[0]?.settings?.index?.blocks?.write; - return writeBlockSetting === 'true' || writeBlockSetting === true; -} - export async function getKbIndexCreatedVersion(es: Client) { const indexSettings = await es.indices.getSettings({ index: resourceNames.writeIndexAlias.kb,