diff --git a/x-pack/platform/plugins/shared/cases/common/constants/index.ts b/x-pack/platform/plugins/shared/cases/common/constants/index.ts index 03fb62b1b5f5e..1452f227f7e50 100644 --- a/x-pack/platform/plugins/shared/cases/common/constants/index.ts +++ b/x-pack/platform/plugins/shared/cases/common/constants/index.ts @@ -176,6 +176,8 @@ export const DEFAULT_FEATURES: CasesFeaturesAllRequired = Object.freeze({ */ export const CASES_TELEMETRY_TASK_NAME = 'cases-telemetry-task'; +export const ANALYTICS_BACKFILL_TASK_TYPE = 'cai:cases_analytics_index_backfill'; +export const ANALYTICS_SYNCHRONIZATION_TASK_TYPE = 'cai:cases_analytics_index_synchronization'; /** * Telemetry diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/analytics_index.test.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/analytics_index.test.ts index 6117fbf6b4157..accf93c950f64 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/analytics_index.test.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/analytics_index.test.ts @@ -118,6 +118,7 @@ describe('AnalyticsIndex', () => { }, settings: { index: { + hidden: true, auto_expand_replicas: '0-1', mode: 'lookup', number_of_shards: 1, diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/analytics_index.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/analytics_index.ts index 325b2a4ce5420..01c87b1e678f0 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/analytics_index.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/analytics_index.ts @@ -93,6 +93,7 @@ export class AnalyticsIndex { this.sourceIndex = sourceIndex; this.sourceQuery = sourceQuery; this.indexSettings = { + hidden: true, // settings are not supported on serverless ES ...(isServerless ? {} diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/constants.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/constants.ts index 8e0bcc0a20fff..8206a81e0124a 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/constants.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/constants.ts @@ -5,5 +5,4 @@ * 2.0. */ -export const TASK_TYPE = 'cai:cases_analytics_index_backfill'; export const BACKFILL_RUN_AT = 60 * 1000; // milliseconds diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/index.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/index.ts index ae0b54dd0b2cc..72f4ec66e765a 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/index.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/backfill_task/index.ts @@ -13,9 +13,10 @@ import type { } from '@kbn/task-manager-plugin/server'; import type { CoreSetup, ElasticsearchClient } from '@kbn/core/server'; import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { ANALYTICS_BACKFILL_TASK_TYPE } from '../../../../common/constants'; import type { CasesServerStartDependencies } from '../../../types'; import { CaseAnalyticsIndexBackfillTaskFactory } from './backfill_task_factory'; -import { TASK_TYPE, BACKFILL_RUN_AT } from './constants'; +import { BACKFILL_RUN_AT } from './constants'; export function registerCAIBackfillTask({ taskManager, @@ -32,7 +33,7 @@ export function registerCAIBackfillTask({ }; taskManager.registerTaskDefinitions({ - [TASK_TYPE]: { + [ANALYTICS_BACKFILL_TASK_TYPE]: { title: 'Backfill cases analytics indexes.', maxAttempts: 3, createTaskRunner: (context: RunContext) => { @@ -60,7 +61,7 @@ export async function scheduleCAIBackfillTask({ try { await taskManager.ensureScheduled({ id: taskId, - taskType: TASK_TYPE, + taskType: ANALYTICS_BACKFILL_TASK_TYPE, params: { sourceIndex, destIndex, sourceQuery }, runAt: new Date(Date.now() + BACKFILL_RUN_AT), // todo, value is short for testing but should run after 5 minutes state: {}, diff --git a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/synchronization_task/index.ts b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/synchronization_task/index.ts index 1e4c8e73f135d..a7743f980255b 100644 --- a/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/synchronization_task/index.ts +++ b/x-pack/platform/plugins/shared/cases/server/cases_analytics/tasks/synchronization_task/index.ts @@ -13,10 +13,10 @@ import type { TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; import type { CoreSetup, ElasticsearchClient } from '@kbn/core/server'; +import { ANALYTICS_SYNCHRONIZATION_TASK_TYPE } from '../../../../common/constants'; import type { CasesServerStartDependencies } from '../../../types'; import { AnalyticsIndexSynchronizationTaskFactory } from './synchronization_task_factory'; -const TASK_TYPE = 'cai:cases_analytics_index_synchronization'; const SCHEDULE: IntervalSchedule = { interval: '5m' }; export function registerCAISynchronizationTask({ @@ -34,7 +34,7 @@ export function registerCAISynchronizationTask({ }; taskManager.registerTaskDefinitions({ - [TASK_TYPE]: { + [ANALYTICS_SYNCHRONIZATION_TASK_TYPE]: { title: 'Synchronization for the cases analytics index', createTaskRunner: (context: RunContext) => { return new AnalyticsIndexSynchronizationTaskFactory({ getESClient, logger }).create( @@ -64,7 +64,7 @@ export async function scheduleCAISynchronizationTask({ try { await taskManager.ensureScheduled({ id: taskId, - taskType: TASK_TYPE, + taskType: ANALYTICS_SYNCHRONIZATION_TASK_TYPE, params: { sourceIndex, destIndex }, schedule: SCHEDULE, // every 5 minutes state: {}, diff --git a/x-pack/platform/plugins/shared/cases/server/config.test.ts b/x-pack/platform/plugins/shared/cases/server/config.test.ts index c82c43ba2e00b..080ae28cceeff 100644 --- a/x-pack/platform/plugins/shared/cases/server/config.test.ts +++ b/x-pack/platform/plugins/shared/cases/server/config.test.ts @@ -12,7 +12,11 @@ describe('config validation', () => { it('sets the defaults correctly', () => { expect(ConfigSchema.validate({})).toMatchInlineSnapshot(` Object { - "analytics": Object {}, + "analytics": Object { + "index": Object { + "enabled": true, + }, + }, "enabled": true, "files": Object { "allowedMimeTypes": Array [ diff --git a/x-pack/platform/plugins/shared/cases/server/config.ts b/x-pack/platform/plugins/shared/cases/server/config.ts index c8cedcd042d54..21f6b4f88428e 100644 --- a/x-pack/platform/plugins/shared/cases/server/config.ts +++ b/x-pack/platform/plugins/shared/cases/server/config.ts @@ -44,11 +44,9 @@ export const ConfigSchema = schema.object({ }), }), analytics: schema.object({ - index: schema.maybe( - schema.object({ - enabled: schema.boolean({ defaultValue: true }), - }) - ), + index: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + }), }), enabled: schema.boolean({ defaultValue: true }), }); diff --git a/x-pack/platform/plugins/shared/cases/server/plugin.test.ts b/x-pack/platform/plugins/shared/cases/server/plugin.test.ts index 89b389e7f1892..78be0a0bf4b21 100644 --- a/x-pack/platform/plugins/shared/cases/server/plugin.test.ts +++ b/x-pack/platform/plugins/shared/cases/server/plugin.test.ts @@ -30,7 +30,7 @@ function getConfig(overrides: Partial = {}): ConfigType { files: { maxSize: 1, allowedMimeTypes: ALLOWED_MIME_TYPES }, stack: { enabled: true }, incrementalId: { enabled: true, taskIntervalMinutes: 10, taskStartDelayMinutes: 10 }, - analytics: {}, + analytics: { index: { enabled: true } }, ...overrides, }; } diff --git a/x-pack/platform/plugins/shared/cases/server/plugin.ts b/x-pack/platform/plugins/shared/cases/server/plugin.ts index 524af9599650c..a1cb71549ac86 100644 --- a/x-pack/platform/plugins/shared/cases/server/plugin.ts +++ b/x-pack/platform/plugins/shared/cases/server/plugin.ts @@ -224,6 +224,7 @@ export class CasePlugin if (this.caseConfig.incrementalId.enabled) { void this.incrementalIdTaskManager?.setupIncrementIdTask(plugins.taskManager, core); } + if (this.caseConfig.analytics.index?.enabled) { scheduleCasesAnalyticsSyncTasks({ taskManager: plugins.taskManager, logger: this.logger }); createCasesAnalyticsIndexes({ diff --git a/x-pack/platform/test/api_integration/apis/search/search.ts b/x-pack/platform/test/api_integration/apis/search/search.ts index b55f43715c1df..70675244e1d82 100644 --- a/x-pack/platform/test/api_integration/apis/search/search.ts +++ b/x-pack/platform/test/api_integration/apis/search/search.ts @@ -63,6 +63,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -89,6 +90,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -116,6 +118,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -159,6 +162,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -196,6 +200,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -258,6 +263,7 @@ export default function ({ getService }: FtrProviderContext) { .set(ELASTIC_HTTP_VERSION_HEADER, '1') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -277,6 +283,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -298,6 +305,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -478,6 +486,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -519,6 +528,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, diff --git a/x-pack/platform/test/cases_api_integration/common/lib/api/analytics.ts b/x-pack/platform/test/cases_api_integration/common/lib/api/analytics.ts new file mode 100644 index 0000000000000..e1521d7e19b7d --- /dev/null +++ b/x-pack/platform/test/cases_api_integration/common/lib/api/analytics.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type SuperTest from 'supertest'; + +import { + CAI_ACTIVITY_BACKFILL_TASK_ID, + CAI_ACTIVITY_SYNCHRONIZATION_TASK_ID, + CAI_ACTIVITY_SOURCE_INDEX, + CAI_ACTIVITY_INDEX_NAME, + CAI_ACTIVITY_SOURCE_QUERY, +} from '@kbn/cases-plugin/server/cases_analytics/activity_index/constants'; +import { + CAI_ATTACHMENTS_BACKFILL_TASK_ID, + CAI_ATTACHMENTS_SYNCHRONIZATION_TASK_ID, + CAI_ATTACHMENTS_SOURCE_INDEX, + CAI_ATTACHMENTS_INDEX_NAME, + CAI_ATTACHMENTS_SOURCE_QUERY, +} from '@kbn/cases-plugin/server/cases_analytics/attachments_index/constants'; +import { + CAI_CASES_BACKFILL_TASK_ID, + CAI_CASES_SYNCHRONIZATION_TASK_ID, + CAI_CASES_SOURCE_INDEX, + CAI_CASES_INDEX_NAME, + CAI_CASES_SOURCE_QUERY, +} from '@kbn/cases-plugin/server/cases_analytics/cases_index/constants'; +import { + CAI_COMMENTS_BACKFILL_TASK_ID, + CAI_COMMENTS_SYNCHRONIZATION_TASK_ID, + CAI_COMMENTS_SOURCE_INDEX, + CAI_COMMENTS_INDEX_NAME, + CAI_COMMENTS_SOURCE_QUERY, +} from '@kbn/cases-plugin/server/cases_analytics/comments_index/constants'; + +export const runCasesBackfillTask = async (supertest: SuperTest.Agent) => { + await supertest + .post('/api/analytics_index/backfill/run_soon') + .set('kbn-xsrf', 'xxx') + .send({ + taskId: CAI_CASES_BACKFILL_TASK_ID, + sourceIndex: CAI_CASES_SOURCE_INDEX, + destIndex: CAI_CASES_INDEX_NAME, + sourceQuery: JSON.stringify(CAI_CASES_SOURCE_QUERY), + }) + .expect(200); +}; + +export const runCasesSynchronizationTask = async (supertest: SuperTest.Agent) => { + await supertest + .post('/api/analytics_index/synchronization/run_soon') + .set('kbn-xsrf', 'xxx') + .send({ taskId: CAI_CASES_SYNCHRONIZATION_TASK_ID }) + .expect(200); +}; + +export const runAttachmentsBackfillTask = async (supertest: SuperTest.Agent) => { + await supertest + .post('/api/analytics_index/backfill/run_soon') + .set('kbn-xsrf', 'xxx') + .send({ + taskId: CAI_ATTACHMENTS_BACKFILL_TASK_ID, + sourceIndex: CAI_ATTACHMENTS_SOURCE_INDEX, + destIndex: CAI_ATTACHMENTS_INDEX_NAME, + sourceQuery: JSON.stringify(CAI_ATTACHMENTS_SOURCE_QUERY), + }) + .expect(200); +}; + +export const runAttachmentsSynchronizationTask = async (supertest: SuperTest.Agent) => { + await supertest + .post('/api/analytics_index/synchronization/run_soon') + .set('kbn-xsrf', 'xxx') + .send({ taskId: CAI_ATTACHMENTS_SYNCHRONIZATION_TASK_ID }) + .expect(200); +}; + +export const runCommentsBackfillTask = async (supertest: SuperTest.Agent) => { + await supertest + .post('/api/analytics_index/backfill/run_soon') + .set('kbn-xsrf', 'xxx') + .send({ + taskId: CAI_COMMENTS_BACKFILL_TASK_ID, + sourceIndex: CAI_COMMENTS_SOURCE_INDEX, + destIndex: CAI_COMMENTS_INDEX_NAME, + sourceQuery: JSON.stringify(CAI_COMMENTS_SOURCE_QUERY), + }) + .expect(200); +}; + +export const runCommentsSynchronizationTask = async (supertest: SuperTest.Agent) => { + await supertest + .post('/api/analytics_index/synchronization/run_soon') + .set('kbn-xsrf', 'xxx') + .send({ taskId: CAI_COMMENTS_SYNCHRONIZATION_TASK_ID }) + .expect(200); +}; + +export const runActivityBackfillTask = async (supertest: SuperTest.Agent) => { + await supertest + .post('/api/analytics_index/backfill/run_soon') + .set('kbn-xsrf', 'xxx') + .send({ + taskId: CAI_ACTIVITY_BACKFILL_TASK_ID, + sourceIndex: CAI_ACTIVITY_SOURCE_INDEX, + destIndex: CAI_ACTIVITY_INDEX_NAME, + sourceQuery: JSON.stringify(CAI_ACTIVITY_SOURCE_QUERY), + }) + .expect(200); +}; + +export const runActivitySynchronizationTask = async (supertest: SuperTest.Agent) => { + await supertest + .post('/api/analytics_index/synchronization/run_soon') + .set('kbn-xsrf', 'xxx') + .send({ taskId: CAI_ACTIVITY_SYNCHRONIZATION_TASK_ID }) + .expect(200); +}; diff --git a/x-pack/platform/test/cases_api_integration/common/lib/api/index.ts b/x-pack/platform/test/cases_api_integration/common/lib/api/index.ts index ccb63fc2c49b5..0fb0eea9b5c5d 100644 --- a/x-pack/platform/test/cases_api_integration/common/lib/api/index.ts +++ b/x-pack/platform/test/cases_api_integration/common/lib/api/index.ts @@ -904,3 +904,52 @@ export const findInternalCaseUserActions = async ({ return userActions; }; + +export const deleteAllCaseAnalyticsItems = async (es: Client) => { + await Promise.all([ + deleteCasesAnalytics(es), + deleteAttachmentsAnalytics(es), + deleteCommentsAnalytics(es), + deleteActivityAnalytics(es), + ]); +}; + +export const deleteCasesAnalytics = async (es: Client): Promise => { + await es.deleteByQuery({ + index: '.internal.cases', + query: { match_all: {} }, + wait_for_completion: true, + refresh: true, + conflicts: 'proceed', + }); +}; + +export const deleteAttachmentsAnalytics = async (es: Client): Promise => { + await es.deleteByQuery({ + index: '.internal.cases-attachments', + query: { match_all: {} }, + wait_for_completion: true, + refresh: true, + conflicts: 'proceed', + }); +}; + +export const deleteCommentsAnalytics = async (es: Client): Promise => { + await es.deleteByQuery({ + index: '.internal.cases-comments', + query: { match_all: {} }, + wait_for_completion: true, + refresh: true, + conflicts: 'proceed', + }); +}; + +export const deleteActivityAnalytics = async (es: Client): Promise => { + await es.deleteByQuery({ + index: '.internal.cases-activity', + query: { match_all: {} }, + wait_for_completion: true, + refresh: true, + conflicts: 'proceed', + }); +}; diff --git a/x-pack/platform/test/cases_api_integration/common/plugins/cases/server/plugin.ts b/x-pack/platform/test/cases_api_integration/common/plugins/cases/server/plugin.ts index b4bd6c11344ce..db93842ad01d4 100644 --- a/x-pack/platform/test/cases_api_integration/common/plugins/cases/server/plugin.ts +++ b/x-pack/platform/test/cases_api_integration/common/plugins/cases/server/plugin.ts @@ -5,13 +5,7 @@ * 2.0. */ -import type { - Plugin, - CoreSetup, - CoreStart, - PluginInitializerContext, - Logger, -} from '@kbn/core/server'; +import type { Plugin, CoreSetup, PluginInitializerContext, Logger } from '@kbn/core/server'; import type { FeaturesPluginSetup } from '@kbn/features-plugin/server'; import type { SpacesPluginStart } from '@kbn/spaces-plugin/server'; import type { SecurityPluginStart } from '@kbn/security-plugin/server'; @@ -93,6 +87,7 @@ export class FixturePlugin implements Plugin { @@ -288,4 +291,68 @@ export const registerRoutes = (core: CoreSetup, logger: Logger } } ); + + router.post( + { + path: '/api/analytics_index/backfill/run_soon', + security: { + authz: { + enabled: false, + reason: 'This route is opted out from authorization', + }, + }, + validate: { + body: schema.object({ + taskId: schema.string(), + sourceIndex: schema.string(), + destIndex: schema.string(), + sourceQuery: schema.string(), + }), + }, + }, + async (context, req, res) => { + const { taskId, sourceIndex, destIndex, sourceQuery } = req.body; + try { + const [_, { taskManager }] = await core.getStartServices(); + + return res.ok({ + body: await taskManager.ensureScheduled({ + id: taskId, + taskType: ANALYTICS_BACKFILL_TASK_TYPE, + params: { sourceIndex, destIndex, sourceQuery: JSON.parse(sourceQuery) }, + runAt: new Date(), + state: {}, + }), + }); + } catch (err) { + return res.ok({ body: { id: taskId, error: `${err}` } }); + } + } + ); + + router.post( + { + path: '/api/analytics_index/synchronization/run_soon', + security: { + authz: { + enabled: false, + reason: 'This route is opted out from authorization', + }, + }, + validate: { + body: schema.object({ + taskId: schema.string(), + }), + }, + }, + async (context, req, res) => { + const { taskId } = req.body; + try { + const [_, { taskManager }] = await core.getStartServices(); + return res.ok({ body: await taskManager.runSoon(taskId) }); + } catch (err) { + return res.ok({ body: { id: taskId, error: `${err}` } }); + } + } + ); }; diff --git a/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/backfill.ts b/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/backfill.ts new file mode 100644 index 0000000000000..2e923dbd02efc --- /dev/null +++ b/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/backfill.ts @@ -0,0 +1,306 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { + CaseSeverity, + CaseStatuses, + CustomFieldTypes, +} from '@kbn/cases-plugin/common/types/domain'; +import { SECURITY_SOLUTION_OWNER } from '@kbn/cases-plugin/common/constants'; +import { + runActivityBackfillTask, + runAttachmentsBackfillTask, + runCasesBackfillTask, + runCommentsBackfillTask, +} from '../../../../../common/lib/api/analytics'; +import { + createCase, + createConfiguration, + createComment, + createFileAttachment, + deleteAllCaseItems, + deleteAllFiles, + getAuthWithSuperUser, + getConfigurationRequest, + updateCase, + deleteCases, + deleteAllCaseAnalyticsItems, +} from '../../../../../common/lib/api'; +import { + getPostCaseRequest, + postCaseReq, + postFileReq, + postCommentAlertReq, + postCommentUserReq, +} from '../../../../../common/lib/mock'; +import type { FtrProviderContext } from '../../../../../../common/ftr_provider_context'; + +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + const esClient = getService('es'); + const retry = getService('retry'); + const authSpace1 = getAuthWithSuperUser(); + + describe('analytics indexes backfill task', () => { + beforeEach(async () => { + await deleteAllCaseAnalyticsItems(esClient); + await deleteAllCaseItems(esClient); + await deleteAllFiles({ + supertest, + auth: authSpace1, + }); + }); + + after(async () => { + await deleteAllCaseItems(esClient); + }); + + it('should backfill the cases index', async () => { + await createConfiguration( + supertest, + getConfigurationRequest({ + overrides: { + customFields: [ + { + key: 'test_custom_field', + label: 'text', + type: CustomFieldTypes.TEXT, + required: false, + }, + ], + }, + }) + ); + + const postCaseRequest = getPostCaseRequest({ + category: 'foobar', + customFields: [ + { + key: 'test_custom_field', + type: CustomFieldTypes.TEXT, + value: 'value', + }, + ], + }); + + const caseToBackfill = await createCase(supertest, postCaseRequest, 200); + + await runCasesBackfillTask(supertest); + + await retry.try(async () => { + const caseAnalytics = await esClient.get({ + index: '.internal.cases', + id: `cases:${caseToBackfill.id}`, + }); + + expect(caseAnalytics.found).to.be(true); + + const { + '@timestamp': timestamp, + created_at: createdAt, + created_at_ms: createdAtMs, + ...analyticsFields + } = caseAnalytics._source as any; + + expect(timestamp).not.to.be(null); + expect(timestamp).not.to.be(undefined); + expect(createdAt).not.to.be(null); + expect(createdAt).not.to.be(undefined); + expect(createdAtMs).not.to.be(null); + expect(createdAtMs).not.to.be(undefined); + + expect(analyticsFields).to.eql({ + assignees: [], + category: 'foobar', + created_by: { + email: null, + full_name: null, + profile_uid: null, + username: 'elastic', + }, + custom_fields: [ + { + key: 'test_custom_field', + type: 'text', + value: 'value', + }, + ], + description: 'This is a brand new case of a bad meanie defacing data', + observables: [], + owner: 'securitySolutionFixture', + severity: 'low', + severity_sort: 0, + space_ids: ['default'], + status: 'open', + status_sort: 0, + tags: ['defacement'], + title: 'Super Bad Security Issue', + total_alerts: 0, + total_assignees: 0, + total_comments: 0, + }); + }); + }); + + // This test passes locally but fails in the flaky test runner. + // Increasing the timeout did not work. + it.skip('should backfill the cases attachments index', async () => { + const postedCase = await createCase( + supertest, + { ...postCaseReq, owner: SECURITY_SOLUTION_OWNER }, + 200, + authSpace1 + ); + + await createFileAttachment({ + supertest, + caseId: postedCase.id, + params: postFileReq, + auth: authSpace1, + }); + + const postedCaseWithAttachments = await createComment({ + supertest, + caseId: postedCase.id, + params: { + ...postCommentAlertReq, + alertId: 'test-id-2', + index: 'test-index-2', + owner: SECURITY_SOLUTION_OWNER, + }, + auth: authSpace1, + }); + + await runAttachmentsBackfillTask(supertest); + + await retry.tryForTime(300000, async () => { + const firstAttachmentAnalytics = await esClient.get({ + index: '.internal.cases-attachments', + id: `cases-comments:${postedCaseWithAttachments.comments![0].id}`, + }); + + expect(firstAttachmentAnalytics.found).to.be(true); + }); + + const secondAttachmentAnalytics = await esClient.get({ + index: '.internal.cases-attachments', + id: `cases-comments:${postedCaseWithAttachments.comments![1].id}`, + }); + + expect(secondAttachmentAnalytics.found).to.be(true); + }); + + it('should backfill the cases comments index', async () => { + const postedCase = await createCase(supertest, postCaseReq, 200); + const patchedCase = await createComment({ + supertest, + caseId: postedCase.id, + params: postCommentUserReq, + }); + + await runCommentsBackfillTask(supertest); + + await retry.try(async () => { + const commentAnalytics = await esClient.get({ + index: '.internal.cases-comments', + id: `cases-comments:${patchedCase.comments![0].id}`, + }); + + expect(commentAnalytics.found).to.be(true); + + const { + '@timestamp': timestamp, + created_at: createdAt, + case_id: caseId, + ...analyticsFields + } = commentAnalytics._source as any; + + expect(caseId).to.be(postedCase.id); + + expect(timestamp).not.to.be(null); + expect(timestamp).not.to.be(undefined); + expect(createdAt).not.to.be(null); + expect(createdAt).not.to.be(undefined); + + expect(analyticsFields).to.eql({ + comment: 'This is a cool comment', + created_by: { + email: null, + full_name: null, + username: 'elastic', + }, + owner: 'securitySolutionFixture', + space_ids: ['default'], + }); + }); + }); + + it('should backfill the activity index', async () => { + const postedCase = await createCase(supertest, postCaseReq, 200); + await updateCase({ + supertest, + params: { + cases: [ + { + id: postedCase.id, + version: postedCase.version, + tags: ['other'], + severity: CaseSeverity.MEDIUM, + category: 'categoryValue', + status: CaseStatuses['in-progress'], + }, + ], + }, + }); + + const caseToDelete = await createCase(supertest, getPostCaseRequest(), 200); + await deleteCases({ + supertest, + caseIDs: [caseToDelete.id], + }); + + await runActivityBackfillTask(supertest); + + let activityArray: any[] = []; + await retry.try(async () => { + const activityAnalytics = await esClient.search({ + index: '.internal.cases-activity', + }); + + // @ts-ignore + expect(activityAnalytics.hits.total?.value).to.be(5); + activityArray = activityAnalytics.hits.hits as unknown as any[]; + }); + + const tagsActivity = activityArray.filter((activity) => activity._source.type === 'tags'); + expect(tagsActivity.length).to.be(2); + + const categoryActivity = activityArray.find( + (activity) => activity._source.type === 'category' + ); + expect(categoryActivity?._source.owner).to.be('securitySolutionFixture'); + expect(categoryActivity?._source.action).to.be('update'); + expect(categoryActivity?._source.case_id).to.be(postedCase.id); + expect(categoryActivity?._source.payload?.category).to.be('categoryValue'); + + const severityActivity = activityArray.find( + (activity) => activity._source.type === 'severity' + ); + expect(severityActivity?._source.owner).to.be('securitySolutionFixture'); + expect(severityActivity?._source.action).to.be('update'); + expect(severityActivity?._source.case_id).to.be(postedCase.id); + expect(severityActivity?._source.payload?.severity).to.be('medium'); + + const statusActivity = activityArray.find((activity) => activity._source.type === 'status'); + expect(statusActivity?._source.owner).to.be('securitySolutionFixture'); + expect(statusActivity?._source.action).to.be('update'); + expect(statusActivity?._source.case_id).to.be(postedCase.id); + expect(statusActivity?._source.payload?.status).to.be('in-progress'); + }); + }); +}; diff --git a/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/creation.ts b/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/creation.ts new file mode 100644 index 0000000000000..2c3822aadae6d --- /dev/null +++ b/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/creation.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import type { FtrProviderContext } from '../../../../../../common/ftr_provider_context'; + +export default ({ getService }: FtrProviderContext): void => { + const esClient = getService('es'); + const retry = getService('retry'); + + describe('analytics indexes creation', () => { + const indexVersion = 1; + + it('cases index should be created with the correct mappings and scripts on startup', async () => { + const indexName = '.internal.cases'; + const painlessScriptId = 'cai_cases_script_1'; + const version = indexVersion; + + await retry.try(async () => { + expect( + await esClient.indices.exists({ + index: indexName, + }) + ).to.be(true); + }); + + const mappingDict = await esClient.indices.getMapping({ + index: indexName, + }); + + expect(mappingDict[indexName].mappings._meta?.mapping_version).to.be(version); + expect(mappingDict[indexName].mappings._meta?.painless_script_id).to.be(painlessScriptId); + + const painlessScript = await esClient.getScript({ + id: painlessScriptId, + }); + + expect(painlessScript.found).to.be(true); + }); + + it('activity index should be created with the correct mappings and scripts on startup', async () => { + const indexName = '.internal.cases-activity'; + const painlessScriptId = 'cai_activity_script_1'; + const version = indexVersion; + + await retry.try(async () => { + expect( + await esClient.indices.exists({ + index: indexName, + }) + ).to.be(true); + }); + + const mappingDict = await esClient.indices.getMapping({ + index: indexName, + }); + + expect(mappingDict[indexName].mappings._meta?.mapping_version).to.be(version); + expect(mappingDict[indexName].mappings._meta?.painless_script_id).to.be(painlessScriptId); + + const painlessScript = await esClient.getScript({ + id: painlessScriptId, + }); + + expect(painlessScript.found).to.be(true); + }); + + it('attachments index should be created with the correct mappings and scripts on startup', async () => { + const indexName = '.internal.cases-attachments'; + const painlessScriptId = 'cai_attachments_script_1'; + const version = indexVersion; + + await retry.try(async () => { + expect( + await esClient.indices.exists({ + index: indexName, + }) + ).to.be(true); + }); + + const mappingDict = await esClient.indices.getMapping({ + index: indexName, + }); + + expect(mappingDict[indexName].mappings._meta?.mapping_version).to.be(version); + expect(mappingDict[indexName].mappings._meta?.painless_script_id).to.be(painlessScriptId); + + const painlessScript = await esClient.getScript({ + id: painlessScriptId, + }); + + expect(painlessScript.found).to.be(true); + }); + + it('comments index should be created with the correct mappings and scripts on startup', async () => { + const indexName = '.internal.cases-comments'; + const painlessScriptId = 'cai_comments_script_1'; + const version = indexVersion; + + await retry.try(async () => { + expect( + await esClient.indices.exists({ + index: indexName, + }) + ).to.be(true); + }); + + const mappingDict = await esClient.indices.getMapping({ + index: indexName, + }); + + expect(mappingDict[indexName].mappings._meta?.mapping_version).to.be(version); + expect(mappingDict[indexName].mappings._meta?.painless_script_id).to.be(painlessScriptId); + + const painlessScript = await esClient.getScript({ + id: painlessScriptId, + }); + + expect(painlessScript.found).to.be(true); + }); + }); +}; diff --git a/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/synchronization.ts b/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/synchronization.ts new file mode 100644 index 0000000000000..d69361d1515fb --- /dev/null +++ b/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/cases/analytics_index/synchronization.ts @@ -0,0 +1,307 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { + CaseSeverity, + CaseStatuses, + CustomFieldTypes, +} from '@kbn/cases-plugin/common/types/domain'; +import { SECURITY_SOLUTION_OWNER } from '@kbn/cases-plugin/common/constants'; +import { + runActivitySynchronizationTask, + runAttachmentsSynchronizationTask, + runCasesSynchronizationTask, + runCommentsSynchronizationTask, +} from '../../../../../common/lib/api/analytics'; +import { + createCase, + createConfiguration, + createComment, + createFileAttachment, + deleteAllCaseItems, + deleteAllFiles, + getAuthWithSuperUser, + getConfigurationRequest, + updateCase, + deleteCases, + deleteAllCaseAnalyticsItems, +} from '../../../../../common/lib/api'; +import { + getPostCaseRequest, + postCaseReq, + postFileReq, + postCommentAlertReq, + postCommentUserReq, +} from '../../../../../common/lib/mock'; +import type { FtrProviderContext } from '../../../../../../common/ftr_provider_context'; + +export default ({ getService }: FtrProviderContext): void => { + const supertest = getService('supertest'); + const esClient = getService('es'); + const retry = getService('retry'); + const authSpace1 = getAuthWithSuperUser(); + + describe('analytics indexes synchronization task', () => { + beforeEach(async () => { + await deleteAllCaseAnalyticsItems(esClient); + await deleteAllCaseItems(esClient); + await deleteAllFiles({ + supertest, + auth: authSpace1, + }); + }); + + after(async () => { + await deleteAllCaseItems(esClient); + }); + + // This test passes locally but fails in the flaky test runner. + // Increasing the timeout did not work. + it.skip('should sync the cases index', async () => { + await createConfiguration( + supertest, + getConfigurationRequest({ + overrides: { + customFields: [ + { + key: 'test_custom_field', + label: 'text', + type: CustomFieldTypes.TEXT, + required: false, + }, + ], + }, + }) + ); + + const postCaseRequest = getPostCaseRequest({ + category: 'foobar', + customFields: [ + { + key: 'test_custom_field', + type: CustomFieldTypes.TEXT, + value: 'value', + }, + ], + }); + + const caseToBackfill = await createCase(supertest, postCaseRequest, 200); + + await runCasesSynchronizationTask(supertest); + + await retry.tryForTime(300000, async () => { + const caseAnalytics = await esClient.get({ + index: '.internal.cases', + id: `cases:${caseToBackfill.id}`, + }); + + expect(caseAnalytics.found).to.be(true); + + const { + '@timestamp': timestamp, + created_at: createdAt, + created_at_ms: createdAtMs, + ...analyticsFields + } = caseAnalytics._source as any; + + expect(timestamp).not.to.be(null); + expect(timestamp).not.to.be(undefined); + expect(createdAt).not.to.be(null); + expect(createdAt).not.to.be(undefined); + expect(createdAtMs).not.to.be(null); + expect(createdAtMs).not.to.be(undefined); + + expect(analyticsFields).to.eql({ + assignees: [], + category: 'foobar', + created_by: { + email: null, + full_name: null, + profile_uid: null, + username: 'elastic', + }, + custom_fields: [ + { + key: 'test_custom_field', + type: 'text', + value: 'value', + }, + ], + description: 'This is a brand new case of a bad meanie defacing data', + observables: [], + owner: 'securitySolutionFixture', + severity: 'low', + severity_sort: 0, + space_ids: ['default'], + status: 'open', + status_sort: 0, + tags: ['defacement'], + title: 'Super Bad Security Issue', + total_alerts: 0, + total_assignees: 0, + total_comments: 0, + }); + }); + }); + + it('should sync the cases attachments index', async () => { + const postedCase = await createCase( + supertest, + { ...postCaseReq, owner: SECURITY_SOLUTION_OWNER }, + 200, + authSpace1 + ); + + await createFileAttachment({ + supertest, + caseId: postedCase.id, + params: postFileReq, + auth: authSpace1, + }); + + const postedCaseWithAttachments = await createComment({ + supertest, + caseId: postedCase.id, + params: { + ...postCommentAlertReq, + alertId: 'test-id-2', + index: 'test-index-2', + owner: SECURITY_SOLUTION_OWNER, + }, + auth: authSpace1, + }); + + await runAttachmentsSynchronizationTask(supertest); + + await retry.try(async () => { + const firstAttachmentAnalytics = await esClient.get({ + index: '.internal.cases-attachments', + id: `cases-comments:${postedCaseWithAttachments.comments![0].id}`, + }); + + expect(firstAttachmentAnalytics.found).to.be(true); + }); + + const secondAttachmentAnalytics = await esClient.get({ + index: '.internal.cases-attachments', + id: `cases-comments:${postedCaseWithAttachments.comments![1].id}`, + }); + + expect(secondAttachmentAnalytics.found).to.be(true); + }); + + it('should sync the cases comments index', async () => { + const postedCase = await createCase(supertest, postCaseReq, 200); + const patchedCase = await createComment({ + supertest, + caseId: postedCase.id, + params: postCommentUserReq, + }); + + await runCommentsSynchronizationTask(supertest); + + await retry.try(async () => { + const commentAnalytics = await esClient.get({ + index: '.internal.cases-comments', + id: `cases-comments:${patchedCase.comments![0].id}`, + }); + + expect(commentAnalytics.found).to.be(true); + + const { + '@timestamp': timestamp, + created_at: createdAt, + case_id: caseId, + ...analyticsFields + } = commentAnalytics._source as any; + + expect(caseId).to.be(postedCase.id); + + expect(timestamp).not.to.be(null); + expect(timestamp).not.to.be(undefined); + expect(createdAt).not.to.be(null); + expect(createdAt).not.to.be(undefined); + + expect(analyticsFields).to.eql({ + comment: 'This is a cool comment', + created_by: { + email: null, + full_name: null, + username: 'elastic', + }, + owner: 'securitySolutionFixture', + space_ids: ['default'], + }); + }); + }); + + it('should sync the activity index', async () => { + const postedCase = await createCase(supertest, postCaseReq, 200); + await updateCase({ + supertest, + params: { + cases: [ + { + id: postedCase.id, + version: postedCase.version, + tags: ['other'], + severity: CaseSeverity.MEDIUM, + category: 'categoryValue', + status: CaseStatuses['in-progress'], + }, + ], + }, + }); + + const caseToDelete = await createCase(supertest, getPostCaseRequest(), 200, authSpace1); + await deleteCases({ + supertest, + caseIDs: [caseToDelete.id], + auth: authSpace1, + }); + + await runActivitySynchronizationTask(supertest); + + let activityArray: any[] = []; + await retry.try(async () => { + const activityAnalytics = await esClient.search({ + index: '.internal.cases-activity', + }); + + // @ts-ignore + expect(activityAnalytics.hits.total?.value).to.be(5); + activityArray = activityAnalytics.hits.hits as unknown as any[]; + }); + + const tagsActivity = activityArray.filter((activity) => activity._source.type === 'tags'); + expect(tagsActivity.length).to.be(2); + + const categoryActivity = activityArray.find( + (activity) => activity._source.type === 'category' + ); + expect(categoryActivity?._source.owner).to.be('securitySolutionFixture'); + expect(categoryActivity?._source.action).to.be('update'); + expect(categoryActivity?._source.case_id).to.be(postedCase.id); + expect(categoryActivity?._source.payload?.category).to.be('categoryValue'); + + const severityActivity = activityArray.find( + (activity) => activity._source.type === 'severity' + ); + expect(severityActivity?._source.owner).to.be('securitySolutionFixture'); + expect(severityActivity?._source.action).to.be('update'); + expect(severityActivity?._source.case_id).to.be(postedCase.id); + expect(severityActivity?._source.payload?.severity).to.be('medium'); + + const statusActivity = activityArray.find((activity) => activity._source.type === 'status'); + expect(statusActivity?._source.owner).to.be('securitySolutionFixture'); + expect(statusActivity?._source.action).to.be('update'); + expect(statusActivity?._source.case_id).to.be(postedCase.id); + expect(statusActivity?._source.payload?.status).to.be('in-progress'); + }); + }); +}; diff --git a/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/index.ts b/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/index.ts index c50ccb85af244..e8b3aad1c0179 100644 --- a/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/index.ts +++ b/x-pack/platform/test/cases_api_integration/security_and_spaces/tests/common/index.ts @@ -68,5 +68,12 @@ export default ({ loadTestFile }: FtrProviderContext): void => { // NOTE: Migrations are not included because they can inadvertently remove the .kibana indices which removes the users and spaces // which causes errors in any tests after them that relies on those + + /** + * Cases analytics + */ + loadTestFile(require.resolve('./cases/analytics_index/creation')); + loadTestFile(require.resolve('./cases/analytics_index/backfill')); + loadTestFile(require.resolve('./cases/analytics_index/synchronization')); }); }; diff --git a/x-pack/platform/test/cases_api_integration/spaces_only/README.md b/x-pack/platform/test/cases_api_integration/spaces_only/README.md new file mode 100644 index 0000000000000..4ff4149de084d --- /dev/null +++ b/x-pack/platform/test/cases_api_integration/spaces_only/README.md @@ -0,0 +1,15 @@ +# Example plugin functional tests + +This folder contains functional tests for the cases plugins. + +## Run the test + +To run these tests during development you can use the following commands: + +``` +# Start the test server (can continue running) +node scripts/functional_tests_server.js --config x-pack/test/cases_api_integration/spaces_only/config.ts + +# Start a test run +node scripts/functional_test_runner.js --config x-pack/test/cases_api_integration/spaces_only/config.ts +``` diff --git a/x-pack/test_serverless/api_integration/test_suites/common/search_xpack/search.ts b/x-pack/test_serverless/api_integration/test_suites/common/search_xpack/search.ts index bd7e38a6c7318..57c344eca8c20 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/search_xpack/search.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/search_xpack/search.ts @@ -75,6 +75,7 @@ export default function ({ getService }: FtrProviderContext) { .set(roleAuthc.apiKeyHeader) .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -104,6 +105,7 @@ export default function ({ getService }: FtrProviderContext) { .set(roleAuthc.apiKeyHeader) .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -133,6 +135,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -180,6 +183,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -218,6 +222,7 @@ export default function ({ getService }: FtrProviderContext) { .set(omit(svlCommonApi.getInternalRequestHeader(), 'kbn-xsrf')) .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -239,6 +244,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -262,6 +268,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -383,6 +390,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {}, @@ -430,6 +438,7 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'foo') .send({ params: { + index: 'search-api-test', body: { query: { match_all: {},