From a74e49db70001b63824e9ba4fe272a3ab3ab4340 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Wed, 9 Apr 2025 18:38:13 +0200 Subject: [PATCH 01/45] Support rrule for task scheduling --- .../server/lib/get_next_run_at.test.ts | 62 +++++++++++++++++++ .../server/lib/get_next_run_at.ts | 25 ++++++-- .../server/saved_objects/schemas/rrule.ts | 41 ++++++++++++ .../server/saved_objects/schemas/task.ts | 11 ++++ .../shared/task_manager/server/task.ts | 37 ++++++++++- .../shared/task_manager/server/task_store.ts | 57 ++++++++++++++++- 6 files changed, 226 insertions(+), 7 deletions(-) create mode 100644 x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts index efa7cf90ae15f..12232c6575058 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts @@ -55,4 +55,66 @@ describe('getNextRunAt', () => { ); expect(nextRunAt.getTime()).toBeGreaterThanOrEqual(testStart.getTime()); }); + + test('should use the rrule with a fixed time when it is given to calculate the next runAt', () => { + const now = new Date(); + const testStart = new Date(now.getTime() - 500); + const testRunAt = new Date(now.getTime() - 1000); + const nextRunAt = getNextRunAt( + taskManagerMock.createTask({ + schedule: { + rrule: { + freq: 3, // Daily + interval: 1, + tzid: 'UTC', + byhour: [12], + byminute: [15], + }, + }, + runAt: testRunAt, + startedAt: testStart, + }), + 0 + ); + + const currentDay = testStart.getUTCDay(); + const currentHour = testStart.getUTCHours(); + const currentSecond = testStart.getUTCSeconds(); + const currentMilliseconds = testStart.getUTCMilliseconds(); + + if (currentHour < 12) { + expect(nextRunAt.getUTCDay()).toBe(currentDay); + } else { + expect(nextRunAt.getUTCDay()).toBe(currentDay + 1); + } + expect(nextRunAt.getUTCHours()).toBe(12); + expect(nextRunAt.getUTCMinutes()).toBe(15); + expect(nextRunAt.getUTCSeconds()).toBe(currentSecond); + expect(nextRunAt.getUTCMilliseconds()).toBe(currentMilliseconds); + }); + + test('should use the rrule with a basic interval time when it is given to calculate the next runAt', () => { + const now = new Date(); + const testStart = now; + const testRunAt = new Date(now.getTime() - 1000); + const nextRunAt = getNextRunAt( + taskManagerMock.createTask({ + schedule: { + rrule: { + freq: 3, // Daily + interval: 1, + tzid: 'UTC', + }, + }, + runAt: testRunAt, + startedAt: testStart, + }), + 0 + ); + + const oneDay = 24 * 60 * 60 * 1000; + const expectedNextRunAt = new Date(testStart.getTime() + oneDay); + + expect(nextRunAt).toEqual(expectedNextRunAt); + }); }); diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts index a25960e61ee29..5c084d8429def 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { RRule } from '@kbn/rrule'; import { intervalFromDate } from './intervals'; import type { ConcreteTaskInstance } from '../task'; @@ -15,11 +16,25 @@ export function getNextRunAt( const taskDelay = startedAt!.getTime() - runAt.getTime(); const scheduleFromDate = taskDelay < taskDelayThresholdForPreciseScheduling ? runAt : startedAt!; - // Ensure we also don't schedule in the past by performing the Math.max with Date.now() - const nextCalculatedRunAt = Math.max( - intervalFromDate(scheduleFromDate, schedule!.interval)!.getTime(), - Date.now() - ); + const { rrule, interval } = schedule || {}; + + let nextRunAt: Date | undefined | null; + if (interval) { + nextRunAt = intervalFromDate(scheduleFromDate, interval); + } else if (rrule) { + const _rrule = new RRule({ + ...rrule, + dtstart: scheduleFromDate, + }); + // adding 1ms to ensure the next run is always in the future + // if scheduleFromDate is equal to now (very low possibility), the next run will be now again, which causes loops + nextRunAt = _rrule.after(new Date(Date.now() + 1)); + const nextCalculatedRunAt = Math.max(nextRunAt?.getTime() || 0, Date.now()); + return new Date(nextCalculatedRunAt); + } + + // Ensure we also don't schedule in the past by performing the Math.max with Date.now() + const nextCalculatedRunAt = Math.max(nextRunAt!.getTime(), Date.now()); return new Date(nextCalculatedRunAt); } diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts new file mode 100644 index 0000000000000..7e0ccd43ee85c --- /dev/null +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { Frequency } from '@kbn/rrule'; + +const rruleCommon = schema.object({ + freq: schema.literal(Frequency.MONTHLY), + interval: schema.number(), + tzid: schema.string({ defaultValue: 'UTC' }), +}); + +const rruleMonthly = rruleCommon.extends({ + freq: schema.literal(Frequency.MONTHLY), + byhour: schema.maybe(schema.arrayOf(schema.number())), + byminute: schema.maybe(schema.arrayOf(schema.number())), + byweekday: schema.maybe(schema.never()), + bymonthday: schema.maybe(schema.arrayOf(schema.number())), +}); + +const rruleWeekly = rruleCommon.extends({ + freq: schema.literal(Frequency.WEEKLY), + bbyhour: schema.maybe(schema.arrayOf(schema.number())), + byminute: schema.maybe(schema.arrayOf(schema.number())), + byweekday: schema.maybe(schema.arrayOf(schema.number())), // TODO: use Weekday enum + bymonthday: schema.maybe(schema.never()), +}); + +const rruleDaily = rruleCommon.extends({ + freq: schema.literal(Frequency.DAILY), + byhour: schema.maybe(schema.arrayOf(schema.number())), + byminute: schema.maybe(schema.arrayOf(schema.number())), + byweekday: schema.maybe(schema.never()), + bymonthday: schema.maybe(schema.never()), +}); + +export const rruleSchedule = schema.oneOf([rruleMonthly, rruleWeekly, rruleDaily]); diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts index af5b5cbede510..642804a6fcef9 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts @@ -7,6 +7,7 @@ import { schema } from '@kbn/config-schema'; import { isInterval } from '../../lib/intervals'; +import { rruleSchedule } from './rrule'; export function validateDuration(duration: string) { if (!isInterval(duration)) { @@ -63,4 +64,14 @@ export const taskSchemaV4 = taskSchemaV3.extends({ apiKeyCreatedByUser: schema.boolean(), }) ), + schedule: schema.maybe( + schema.oneOf([ + schema.object({ + interval: schema.string({ validate: validateDuration }), + }), + schema.object({ + rrule: rruleSchedule, + }), + ]) + ), }); diff --git a/x-pack/platform/plugins/shared/task_manager/server/task.ts b/x-pack/platform/plugins/shared/task_manager/server/task.ts index 160816f01f826..8379ca0c87bb5 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task.ts @@ -11,6 +11,7 @@ import type { ObjectType, TypeOf } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; import { isNumber } from 'lodash'; import type { KibanaRequest } from '@kbn/core/server'; +import type { Frequency, Weekday } from '@kbn/rrule'; import { isErr, tryAsResult } from './lib/result_type'; import type { Interval } from './lib/intervals'; import { isInterval, parseIntervalAsMillisecond } from './lib/intervals'; @@ -249,6 +250,40 @@ export interface IntervalSchedule { * An interval in minutes (e.g. '5m'). If specified, this is a recurring task. * */ interval: Interval; + rrule?: never; +} + +export interface RruleSchedule { + rrule: RruleMonthly | RruleWeekly | RruleDaily; + interval?: never; +} + +interface RruleCommon { + freq: Frequency; + interval: number; + tzid: string; +} + +interface RruleMonthly extends RruleCommon { + freq: Frequency.MONTHLY; + bymonthday?: number[]; + byhour?: number[]; + byminute?: number[]; + byweekday?: never; +} +interface RruleWeekly extends RruleCommon { + freq: Frequency.WEEKLY; + byweekday?: Weekday[]; + byhour?: number[]; + byminute?: number[]; + bymonthday?: never; +} +interface RruleDaily extends RruleCommon { + freq: Frequency.DAILY; + byhour?: number[]; + byminute?: number[]; + bymonthday?: never; + byweekday?: never; } export interface TaskUserScope { @@ -305,7 +340,7 @@ export interface TaskInstance { * * Currently, this supports a single format: an interval in minutes or seconds (e.g. '5m', '30s'). */ - schedule?: IntervalSchedule; + schedule?: IntervalSchedule | RruleSchedule; /** * A task-specific set of parameters, used by the task's run function to tailor diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_store.ts b/x-pack/platform/plugins/shared/task_manager/server/task_store.ts index 67fe82086daa6..d6b5f39441b03 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_store.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_store.ts @@ -36,6 +36,7 @@ import type { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-s import { decodeRequestVersion, encodeVersion } from '@kbn/core-saved-objects-base-server-internal'; import { nodeBuilder } from '@kbn/es-query'; +import { RRule } from '@kbn/rrule'; import type { RequestTimeoutsConfig } from './config'; import type { Result } from './lib/result_type'; import { asOk, asErr, unwrap } from './lib/result_type'; @@ -49,6 +50,7 @@ import type { PartialConcreteTaskInstance, PartialSerializedConcreteTaskInstance, ApiKeyOptions, + RruleSchedule, } from './task'; import { TaskStatus, TaskLifecycleResult } from './task'; @@ -149,6 +151,7 @@ export class TaskStore { private security: SecurityServiceStart; private canEncryptSavedObjects?: boolean; private spaces?: SpacesPluginStart; + private logger: Logger; /** * Constructs a new TaskStore. @@ -183,6 +186,7 @@ export class TaskStore { this.security = opts.security; this.spaces = opts.spaces; this.canEncryptSavedObjects = opts.canEncryptSavedObjects; + this.logger = opts.logger; } public registerEncryptedSavedObjectsClient(client: EncryptedSavedObjectsClient) { @@ -313,16 +317,26 @@ export class TaskStore { const id = taskInstance.id || v4(); const validatedTaskInstance = this.taskValidator.getValidatedTaskInstanceForUpdating(taskInstance); + + const runAt = getRunAt({ + taskInstance: validatedTaskInstance, + logger: this.logger, + }); + savedObject = await soClient.create( 'task', { ...taskInstanceToAttributes(validatedTaskInstance, id), ...(userScope ? { userScope } : {}), ...(apiKey ? { apiKey } : {}), + runAt, }, { id, refresh: false } ); - if (get(taskInstance, 'schedule.interval', null) == null) { + if ( + get(taskInstance, 'schedule.interval', null) == null && + get(taskInstance, 'schedule.rrule', null) == null + ) { this.adHocTaskCounter.increment(); } } catch (e) { @@ -354,12 +368,17 @@ export class TaskStore { this.definitions.ensureHas(taskInstance.taskType); const validatedTaskInstance = this.taskValidator.getValidatedTaskInstanceForUpdating(taskInstance); + const runAt = getRunAt({ + taskInstance: validatedTaskInstance, + logger: this.logger, + }); return { type: 'task', attributes: { ...taskInstanceToAttributes(validatedTaskInstance, id), ...(apiKey ? { apiKey } : {}), ...(userScope ? { userScope } : {}), + runAt, }, id, }; @@ -1060,3 +1079,39 @@ function ensureAggregationOnlyReturnsEnabledTaskObjects(opts: AggregationOpts): function isMGetSuccess(doc: estypes.MgetResponseItem): doc is estypes.GetGetResult { return (doc as estypes.GetGetResult).found !== undefined; } + +function rruleHasFixedTime(schedule: RruleSchedule['rrule']): boolean { + const keys = Object.keys(schedule); + const baseFields = ['freq', 'interval', 'tzid']; + + if ((keys.length === 2 || keys.length === 3) && keys.every((key) => baseFields.includes(key))) { + return false; + } + + return true; +} + +function getRunAt({ + taskInstance, + logger, +}: { + taskInstance: TaskInstance; + logger: Logger; +}): string { + const now = new Date(); + const nowString = now.toISOString(); + + if (taskInstance.schedule?.rrule && rruleHasFixedTime(taskInstance.schedule.rrule)) { + try { + const rrule = new RRule({ + ...taskInstance.schedule.rrule, + dtstart: now, + }); + return rrule.after(now)?.toISOString() || nowString; + } catch (e) { + logger.error(`runAt for the rrule with fixed time could not be calculated: ${e}`); + } + } + + return taskInstance.runAt?.toISOString() || nowString; +} From 6163f7e99c1853cea0f016e26cc92e3f2f60b99d Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Wed, 9 Apr 2025 18:41:41 +0200 Subject: [PATCH 02/45] remove redundant return --- .../plugins/shared/task_manager/server/lib/get_next_run_at.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts index 5c084d8429def..343d0fd0daa85 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts @@ -30,8 +30,6 @@ export function getNextRunAt( // adding 1ms to ensure the next run is always in the future // if scheduleFromDate is equal to now (very low possibility), the next run will be now again, which causes loops nextRunAt = _rrule.after(new Date(Date.now() + 1)); - const nextCalculatedRunAt = Math.max(nextRunAt?.getTime() || 0, Date.now()); - return new Date(nextCalculatedRunAt); } // Ensure we also don't schedule in the past by performing the Math.max with Date.now() From 20ecece9316d6706cd798f6bec742fae6962bfda Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Fri, 11 Apr 2025 17:54:58 +0200 Subject: [PATCH 03/45] add functional tests --- .../server/lib/get_first_run_at.test.ts | 262 ++++++++++++++++++ .../server/lib/get_first_run_at.ts | 49 ++++ .../server/lib/get_next_run_at.test.ts | 4 +- .../server/saved_objects/mappings.ts | 25 ++ .../server/saved_objects/schemas/rrule.ts | 42 ++- .../shared/task_manager/server/task.ts | 6 +- .../shared/task_manager/server/task_store.ts | 52 +--- .../sample_task_plugin/server/init_routes.ts | 12 +- .../sample_task_plugin/server/plugin.ts | 36 +++ .../check_registered_task_types.ts | 1 + .../task_manager/task_management.ts | 66 +++++ 11 files changed, 486 insertions(+), 69 deletions(-) create mode 100644 x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts create mode 100644 x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts new file mode 100644 index 0000000000000..78d6e6a67253b --- /dev/null +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts @@ -0,0 +1,262 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockLogger } from '../test_utils'; +import { getFirstRunAt } from './get_first_run_at'; + +describe('getFirstRunAt', () => { + const logger = mockLogger(); + + beforeEach(() => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2025-04-15T13:00:00Z')); + }); + + afterEach(() => { + jest.clearAllMocks(); + jest.clearAllTimers(); + }); + + test('should return runAt when provided', () => { + const runAt = new Date(Date.now() + 3000); + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + runAt, + }; + + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + expect(firstRunAt).toEqual(runAt.toISOString()); + }); + + test('should return now when there is neither runAt nor rrule provided ', () => { + const now = new Date(); + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + }; + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + expect(firstRunAt).toEqual(now.toISOString()); + }); + + test('should return now when an rrule with simple interval is provided ', () => { + const now = new Date(); + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 3, + interval: 1, + tzid: 'UTC', + }, + }, + }; + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + expect(firstRunAt).toEqual(now.toISOString()); + }); + + test('should return now when a simple interval is provided ', () => { + const now = new Date(); + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + + schedule: { + interval: '1m', + }, + }; + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + expect(firstRunAt).toEqual(now.toISOString()); + }); + + test('should return the calculated runAt when an rrule with fixed time is provided', () => { + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 3, + interval: 1, + tzid: 'UTC', + byhour: [12], + byminute: [15], + }, + }, + }; + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + // The next day from 2025-04-15 is 2025-04-16 + // The time is set to 12:15 + expect(firstRunAtDate).toEqual(new Date('2025-04-16T12:15:00Z')); + }); + + test('should return the calculated runAt when an rrule with weekly interval time is provided', () => { + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 2, // Weekly + interval: 1, + tzid: 'UTC', + byweekday: [1], // Monday + }, + }, + }; + + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + // The next Monday from 2025-04-15 is 2025-04-21 + // The time is set to now + expect(firstRunAtDate).toEqual(new Date('2025-04-21T13:00:00.000Z')); + }); + + test('should return the calculated runAt when an rrule with weekly fixed time interval time is provided', () => { + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 2, // Weekly + interval: 1, + tzid: 'UTC', + byweekday: [1], // Monday + byhour: [12], + byminute: [15], + }, + }, + }; + + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + // The next Monday from 2025-04-15 is 2025-04-21 + // The time is set to 12:15 + expect(firstRunAtDate).toEqual(new Date('2025-04-21T12:15:00.000Z')); + }); + + // bymonthday?: number[]; + // byhour?: number[]; + // byminute?: number[]; + // byweekday?: never; + + test('should return the calculated runAt when an rrule with monthly interval is provided', () => { + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 1, // Monthly + interval: 1, + tzid: 'UTC', + bymonthday: [3], + }, + }, + }; + + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + // The next month from 2025-04 is 2025-05 + // The day is set to 3 + // The time is set to now + expect(firstRunAtDate).toEqual(new Date('2025-05-03T13:00:00.000Z')); + }); + + test('should return the calculated runAt when an rrule with monthly interval with fixed time is provided', () => { + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 1, // Monthly + interval: 1, + tzid: 'UTC', + bymonthday: [3], + byhour: [12], + byminute: [17], + }, + }, + }; + + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + // The next month from 2025-04 is 2025-05 + // The day is set to 3 + // The time is set to 12:17 + expect(firstRunAtDate).toEqual(new Date('2025-05-03T12:17:00.000Z')); + }); + + test('should return the calculated runAt when an rrule with monthly interval with weekday is provided', () => { + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 1, // Monthly + interval: 1, + tzid: 'UTC', + byweekday: [3], // Wednesday + byhour: [12], + byminute: [17], + }, + }, + }; + + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + + // The next occurence is in the same month and is 2025-04-16 + // The time is set to 12:17 + expect(firstRunAtDate).toEqual(new Date('2025-04-16T12:17:00.000Z')); + }); + + test('should log an error and return now if rrule fails', () => { + const now = new Date(); + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 3, + interval: 1, + tzid: 'invalid-timezone', + bymonthday: [1], + }, + }, + }; + + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining('runAt for the rrule with fixed time could not be calculated') + ); + expect(firstRunAtDate).toEqual(now); + }); +}); diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts new file mode 100644 index 0000000000000..4b1f8e85aea7b --- /dev/null +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { RRule } from '@kbn/rrule'; +import type { Logger } from '@kbn/core/server'; +import type { RruleSchedule, TaskInstance } from '../task'; + +export function getFirstRunAt({ + taskInstance, + logger, +}: { + taskInstance: TaskInstance; + logger: Logger; +}): string { + if (taskInstance.runAt) { + return taskInstance.runAt.toISOString(); + } + const now = new Date(); + const nowString = now.toISOString(); + + if (taskInstance.schedule?.rrule && rruleHasFixedTime(taskInstance.schedule.rrule)) { + try { + const rrule = new RRule({ + ...taskInstance.schedule.rrule, + dtstart: now, + }); + return rrule.after(now)?.toISOString() || nowString; + } catch (e) { + logger.error(`runAt for the rrule with fixed time could not be calculated: ${e}`); + } + } + + return nowString; +} + +function rruleHasFixedTime(schedule: RruleSchedule['rrule']): boolean { + const keys = Object.keys(schedule); + const baseFields = ['freq', 'interval', 'tzid']; + + if (keys.length === 3 && keys.every((key) => baseFields.includes(key))) { + return false; + } + + return true; +} diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts index 12232c6575058..5adea96ffc4fe 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts @@ -85,7 +85,9 @@ describe('getNextRunAt', () => { if (currentHour < 12) { expect(nextRunAt.getUTCDay()).toBe(currentDay); } else { - expect(nextRunAt.getUTCDay()).toBe(currentDay + 1); + const tomorrow = new Date(); + tomorrow.setDate(tomorrow.getDate() + 1); + expect(nextRunAt.getUTCDay()).toBe(tomorrow.getUTCDay()); } expect(nextRunAt.getUTCHours()).toBe(12); expect(nextRunAt.getUTCMinutes()).toBe(15); diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/mappings.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/mappings.ts index d8981fab4511b..05ef59a0a7376 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/mappings.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/mappings.ts @@ -34,6 +34,31 @@ export const taskMappings: SavedObjectsTypeMappingDefinition = { interval: { type: 'keyword', }, + rrule: { + properties: { + freq: { + type: 'integer', + }, + interval: { + type: 'integer', + }, + tzid: { + type: 'keyword', + }, + bymonthday: { + type: 'integer', + }, + byweekday: { + type: 'integer', + }, + byhour: { + type: 'integer', + }, + byminute: { + type: 'integer', + }, + }, + }, }, }, attempts: { diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts index 7e0ccd43ee85c..0bd7141d011b2 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts @@ -7,35 +7,49 @@ import { schema } from '@kbn/config-schema'; import { Frequency } from '@kbn/rrule'; +import moment from 'moment-timezone'; + +function validateTimezone(timezone: string) { + if (moment.tz.zone(timezone) != null) { + return; + } + + return 'string is not a valid timezone: ' + timezone; +} const rruleCommon = schema.object({ - freq: schema.literal(Frequency.MONTHLY), + freq: schema.number(), interval: schema.number(), - tzid: schema.string({ defaultValue: 'UTC' }), + tzid: schema.string({ validate: validateTimezone, defaultValue: 'UTC' }), }); +const byminute = schema.maybe(schema.arrayOf(schema.number({ min: 0, max: 59 }))); +const byhour = schema.maybe(schema.arrayOf(schema.number({ min: 0, max: 23 }))); +const byweekday = schema.maybe(schema.arrayOf(schema.number({ min: 1, max: 7 }))); +const bymonthday = schema.maybe(schema.arrayOf(schema.number({ min: 1, max: 31 }))); + const rruleMonthly = rruleCommon.extends({ freq: schema.literal(Frequency.MONTHLY), - byhour: schema.maybe(schema.arrayOf(schema.number())), - byminute: schema.maybe(schema.arrayOf(schema.number())), - byweekday: schema.maybe(schema.never()), - bymonthday: schema.maybe(schema.arrayOf(schema.number())), + byhour, + byminute, + byweekday, + bymonthday, }); const rruleWeekly = rruleCommon.extends({ freq: schema.literal(Frequency.WEEKLY), - bbyhour: schema.maybe(schema.arrayOf(schema.number())), - byminute: schema.maybe(schema.arrayOf(schema.number())), - byweekday: schema.maybe(schema.arrayOf(schema.number())), // TODO: use Weekday enum - bymonthday: schema.maybe(schema.never()), + byhour, + byminute, + byweekday, + bymonthday: schema.never(), }); const rruleDaily = rruleCommon.extends({ freq: schema.literal(Frequency.DAILY), - byhour: schema.maybe(schema.arrayOf(schema.number())), - byminute: schema.maybe(schema.arrayOf(schema.number())), - byweekday: schema.maybe(schema.never()), - bymonthday: schema.maybe(schema.never()), + byhour, + byminute, + byweekday, + bymonthday: schema.never(), }); export const rruleSchedule = schema.oneOf([rruleMonthly, rruleWeekly, rruleDaily]); diff --git a/x-pack/platform/plugins/shared/task_manager/server/task.ts b/x-pack/platform/plugins/shared/task_manager/server/task.ts index 8379ca0c87bb5..0957003255b1d 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task.ts @@ -88,7 +88,7 @@ export type SuccessfulRunResult = { * continue to use which ever schedule it already has, and if no there is * no previous schedule then it will be treated as a single-run task. */ - schedule?: IntervalSchedule; + schedule?: IntervalSchedule | RruleSchedule; runAt?: never; } ); @@ -269,7 +269,7 @@ interface RruleMonthly extends RruleCommon { bymonthday?: number[]; byhour?: number[]; byminute?: number[]; - byweekday?: never; + byweekday?: Weekday[]; } interface RruleWeekly extends RruleCommon { freq: Frequency.WEEKLY; @@ -282,8 +282,8 @@ interface RruleDaily extends RruleCommon { freq: Frequency.DAILY; byhour?: number[]; byminute?: number[]; + byweekday?: Weekday[]; bymonthday?: never; - byweekday?: never; } export interface TaskUserScope { diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_store.ts b/x-pack/platform/plugins/shared/task_manager/server/task_store.ts index d6b5f39441b03..8d6fd5727b87c 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_store.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_store.ts @@ -36,7 +36,6 @@ import type { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-s import { decodeRequestVersion, encodeVersion } from '@kbn/core-saved-objects-base-server-internal'; import { nodeBuilder } from '@kbn/es-query'; -import { RRule } from '@kbn/rrule'; import type { RequestTimeoutsConfig } from './config'; import type { Result } from './lib/result_type'; import { asOk, asErr, unwrap } from './lib/result_type'; @@ -64,6 +63,7 @@ import { MsearchError } from './lib/msearch_error'; import { BulkUpdateError } from './lib/bulk_update_error'; import { TASK_SO_NAME } from './saved_objects'; import { getApiKeyAndUserScope } from './lib/api_key_utils'; +import { getFirstRunAt } from './lib/get_first_run_at'; export interface StoreOpts { esClient: ElasticsearchClient; @@ -318,18 +318,13 @@ export class TaskStore { const validatedTaskInstance = this.taskValidator.getValidatedTaskInstanceForUpdating(taskInstance); - const runAt = getRunAt({ - taskInstance: validatedTaskInstance, - logger: this.logger, - }); - savedObject = await soClient.create( 'task', { ...taskInstanceToAttributes(validatedTaskInstance, id), ...(userScope ? { userScope } : {}), ...(apiKey ? { apiKey } : {}), - runAt, + runAt: getFirstRunAt({ taskInstance: validatedTaskInstance, logger: this.logger }), }, { id, refresh: false } ); @@ -368,17 +363,14 @@ export class TaskStore { this.definitions.ensureHas(taskInstance.taskType); const validatedTaskInstance = this.taskValidator.getValidatedTaskInstanceForUpdating(taskInstance); - const runAt = getRunAt({ - taskInstance: validatedTaskInstance, - logger: this.logger, - }); + return { type: 'task', attributes: { ...taskInstanceToAttributes(validatedTaskInstance, id), ...(apiKey ? { apiKey } : {}), ...(userScope ? { userScope } : {}), - runAt, + runAt: getFirstRunAt({ taskInstance: validatedTaskInstance, logger: this.logger }), }, id, }; @@ -1079,39 +1071,3 @@ function ensureAggregationOnlyReturnsEnabledTaskObjects(opts: AggregationOpts): function isMGetSuccess(doc: estypes.MgetResponseItem): doc is estypes.GetGetResult { return (doc as estypes.GetGetResult).found !== undefined; } - -function rruleHasFixedTime(schedule: RruleSchedule['rrule']): boolean { - const keys = Object.keys(schedule); - const baseFields = ['freq', 'interval', 'tzid']; - - if ((keys.length === 2 || keys.length === 3) && keys.every((key) => baseFields.includes(key))) { - return false; - } - - return true; -} - -function getRunAt({ - taskInstance, - logger, -}: { - taskInstance: TaskInstance; - logger: Logger; -}): string { - const now = new Date(); - const nowString = now.toISOString(); - - if (taskInstance.schedule?.rrule && rruleHasFixedTime(taskInstance.schedule.rrule)) { - try { - const rrule = new RRule({ - ...taskInstance.schedule.rrule, - dtstart: now, - }); - return rrule.after(now)?.toISOString() || nowString; - } catch (e) { - logger.error(`runAt for the rrule with fixed time could not be calculated: ${e}`); - } - } - - return taskInstance.runAt?.toISOString() || nowString; -} diff --git a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts index e26b89742954b..6a1f1147b7733 100644 --- a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts +++ b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts @@ -17,6 +17,7 @@ import { import { EventEmitter } from 'events'; import { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import { BACKGROUND_TASK_NODE_SO_NAME } from '@kbn/task-manager-plugin/server/saved_objects'; +import { rruleSchedule } from '@kbn/task-manager-plugin/server/saved_objects/schemas/rrule'; const scope = 'testing'; const taskManagerQuery = { @@ -40,9 +41,14 @@ const taskSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), taskType: schema.string(), schedule: schema.maybe( - schema.object({ - interval: schema.string(), - }) + schema.oneOf([ + schema.object({ + interval: schema.string(), + }), + schema.object({ + rrule: rruleSchedule, + }), + ]) ), interval: schema.maybe(schema.string()), params: schema.recordOf(schema.string(), schema.any(), { defaultValue: {} }), diff --git a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts index 891df37e76158..8657b207688fe 100644 --- a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts +++ b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts @@ -107,6 +107,42 @@ export class SampleTaskManagerFixturePlugin }, }, }, + sampleRecurringTask: { + timeout: '1m', + title: 'Sample Recurring Task', + description: 'A sample recurring task for testing the task_manager.', + stateSchemaByVersion: { + 1: { + up: (state: Record) => ({ count: state.count }), + schema: schema.object({ + count: schema.maybe(schema.number()), + }), + }, + }, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => ({ + async run() { + const { params, state, schedule } = taskInstance; + + const [{ elasticsearch }] = await core.getStartServices(); + await elasticsearch.client.asInternalUser.index({ + index: '.kibana_task_manager_test_result', + document: { + type: 'task', + taskId: taskInstance.id, + params: JSON.stringify(params), + state: JSON.stringify(state), + ranAt: new Date(), + }, + refresh: true, + }); + + return { + state: {}, + schedule, + }; + }, + }), + }, singleAttemptSampleTask: { ...defaultSampleTaskConfig, title: 'Failing Sample Task', diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 73e2c951810dd..3537cb4a51468 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -27,6 +27,7 @@ export default function ({ getService }: FtrProviderContext) { 'sampleRecurringTaskWhichHangs', 'sampleRecurringTaskThatDeletesItself', 'sampleTask', + 'sampleRecurringTask', 'sampleTaskWithLimitedConcurrency', 'sampleTaskWithSingleConcurrency', 'singleAttemptSampleTask', diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index bf04d28dc2fa8..97b66d993e511 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -247,6 +247,72 @@ export default function ({ getService }: FtrProviderContext) { .send({ event: taskId, data }) .expect(200); } + it('should schedule a task with rrule', async () => { + const dailyTask = await scheduleTask({ + id: 'sample-recurring-task-id', + taskType: 'sampleRecurringTask', + schedule: { rrule: { freq: 3, tzid: 'UTC', interval: 1 } }, + params: {}, + }); + + await retry.try(async () => { + const history = await historyDocs(); + expect(history.length).to.eql(1); + }); + + await retry.try(async () => { + const task = await currentTask(dailyTask.id); + expect(task.status).to.be('idle'); + const runAt = new Date(task.runAt).getTime(); + const scheduledAt = new Date(task.scheduledAt).getTime(); + // scheduled to run 24 hours from now + expect(runAt).to.be(scheduledAt + 1000 * 60 * 60 * 24); + }); + }); + + it('should schedule a task with rrule with fixed time', async () => { + const dailyTask = await scheduleTask({ + id: 'sample-recurring-task-id', + taskType: 'sampleRecurringTask', + schedule: { + rrule: { freq: 3, tzid: 'UTC', interval: 1, byhour: [15], byminute: [27] }, + }, + params: {}, + }); + + await retry.try(async () => { + const task = await currentTask(dailyTask.id); + expect(task.status).to.be('idle'); + const runAt = new Date(task.runAt); + expect(runAt.getUTCHours()).to.be(15); + expect(runAt.getUTCMinutes()).to.be(27); + }); + + // should not run immediately as the task is scheduled to run at 15:27 UTC + expect((await historyDocs()).length).to.eql(0); + }); + + it('should not schedule a task with invalid rrule config', async () => { + await supertest + .post('/api/sample_tasks/schedule') + .set('kbn-xsrf', 'xxx') + .send({ + id: 'sample-recurring-task-id', + taskType: 'sampleRecurringTask', + schedule: { + rrule: { + freq: 3, + interval: 1, + byhour: [15], + byminute: [27], + // @ts-ignore + bymonthday: [1], // daily schedule cannot have bymonthday + }, + }, + params: {}, + }) + .expect(400); + }); it('should support middleware', async () => { const historyItem = random(1, 100); From c29a08e8342f15bacad13435fa011a7239fcf2e7 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 11 Apr 2025 16:08:51 +0000 Subject: [PATCH 04/45] [CI] Auto-commit changed files from 'node scripts/styled_components_mapping' --- x-pack/platform/plugins/shared/task_manager/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/platform/plugins/shared/task_manager/tsconfig.json b/x-pack/platform/plugins/shared/task_manager/tsconfig.json index 1b0d3514fa48b..13253f83efe25 100644 --- a/x-pack/platform/plugins/shared/task_manager/tsconfig.json +++ b/x-pack/platform/plugins/shared/task_manager/tsconfig.json @@ -34,6 +34,7 @@ "@kbn/spaces-plugin", "@kbn/encrypted-saved-objects-shared", "@kbn/core-saved-objects-api-server-mocks", + "@kbn/rrule", ], "exclude": ["target/**/*"] } From de20db10e9c4a4341981a5965c75304779459d9d Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 11 Apr 2025 16:21:46 +0000 Subject: [PATCH 05/45] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- x-pack/platform/plugins/shared/task_manager/server/task_store.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_store.ts b/x-pack/platform/plugins/shared/task_manager/server/task_store.ts index 8d6fd5727b87c..c1327b43b4a8f 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_store.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_store.ts @@ -49,7 +49,6 @@ import type { PartialConcreteTaskInstance, PartialSerializedConcreteTaskInstance, ApiKeyOptions, - RruleSchedule, } from './task'; import { TaskStatus, TaskLifecycleResult } from './task'; From 65f09602c0b818853d3a38e0acbe3c9d32617219 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Fri, 11 Apr 2025 19:11:48 +0200 Subject: [PATCH 06/45] fix linting --- .../plugins/shared/task_manager/server/task_store.ts | 1 - .../plugins/shared/task_manager/tsconfig.json | 1 + .../plugins/sample_task_plugin/server/init_routes.ts | 11 +++++++++-- .../test_suites/task_manager/task_management.ts | 4 +--- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_store.ts b/x-pack/platform/plugins/shared/task_manager/server/task_store.ts index 8d6fd5727b87c..c1327b43b4a8f 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_store.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_store.ts @@ -49,7 +49,6 @@ import type { PartialConcreteTaskInstance, PartialSerializedConcreteTaskInstance, ApiKeyOptions, - RruleSchedule, } from './task'; import { TaskStatus, TaskLifecycleResult } from './task'; diff --git a/x-pack/platform/plugins/shared/task_manager/tsconfig.json b/x-pack/platform/plugins/shared/task_manager/tsconfig.json index 1b0d3514fa48b..47b1512a3aede 100644 --- a/x-pack/platform/plugins/shared/task_manager/tsconfig.json +++ b/x-pack/platform/plugins/shared/task_manager/tsconfig.json @@ -34,6 +34,7 @@ "@kbn/spaces-plugin", "@kbn/encrypted-saved-objects-shared", "@kbn/core-saved-objects-api-server-mocks", + "@kbn/rrule" ], "exclude": ["target/**/*"] } diff --git a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts index 6a1f1147b7733..33e7200f2d210 100644 --- a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts +++ b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/init_routes.ts @@ -17,7 +17,6 @@ import { import { EventEmitter } from 'events'; import { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import { BACKGROUND_TASK_NODE_SO_NAME } from '@kbn/task-manager-plugin/server/saved_objects'; -import { rruleSchedule } from '@kbn/task-manager-plugin/server/saved_objects/schemas/rrule'; const scope = 'testing'; const taskManagerQuery = { @@ -46,7 +45,15 @@ const taskSchema = schema.object({ interval: schema.string(), }), schema.object({ - rrule: rruleSchedule, + rrule: schema.object({ + freq: schema.number(), + interval: schema.number(), + tzid: schema.string({ defaultValue: 'UTC' }), + byhour: schema.maybe(schema.arrayOf(schema.number({ min: 0, max: 23 }))), + byminute: schema.maybe(schema.arrayOf(schema.number({ min: 0, max: 59 }))), + byweekday: schema.maybe(schema.arrayOf(schema.number({ min: 1, max: 7 }))), + bymonthday: schema.maybe(schema.arrayOf(schema.number({ min: 1, max: 31 }))), + }), }), ]) ), diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index 97b66d993e511..d2a6b6acda297 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -303,10 +303,8 @@ export default function ({ getService }: FtrProviderContext) { rrule: { freq: 3, interval: 1, - byhour: [15], + byhour: [30], // invalid byminute: [27], - // @ts-ignore - bymonthday: [1], // daily schedule cannot have bymonthday }, }, params: {}, From 717cadb0b8fe92e12bde4768756e19c3687918de Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Fri, 11 Apr 2025 20:54:09 +0200 Subject: [PATCH 07/45] fix linter --- .../plugins/shared/alerting/server/task_runner/task_runner.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.ts index 59c0bf255d3ad..c6ae7a9756466 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.ts @@ -829,7 +829,8 @@ export class TaskRunner< let nextRun: string | null = null; if (taskSchedule) { - nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval }); + // rules cannot use rrule for scheduling yet + nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval! }); } const outcomeMsg = [ From 72b108e1ab1a857e6bae49a00ee9689e07dc28da Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 11 Apr 2025 19:09:26 +0000 Subject: [PATCH 08/45] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../current_fields.json | 8 ++++++ .../current_mappings.json | 25 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index b571428f176f9..1cbb1a1d9b874 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -1125,6 +1125,14 @@ "runAt", "schedule", "schedule.interval", + "schedule.rrule", + "schedule.rrule.byhour", + "schedule.rrule.byminute", + "schedule.rrule.bymonthday", + "schedule.rrule.byweekday", + "schedule.rrule.freq", + "schedule.rrule.interval", + "schedule.rrule.tzid", "scheduledAt", "scope", "status", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 95584ea3bdc4c..331b690852920 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -3723,6 +3723,31 @@ "properties": { "interval": { "type": "keyword" + }, + "rrule": { + "properties": { + "byhour": { + "type": "integer" + }, + "byminute": { + "type": "integer" + }, + "bymonthday": { + "type": "integer" + }, + "byweekday": { + "type": "integer" + }, + "freq": { + "type": "integer" + }, + "interval": { + "type": "integer" + }, + "tzid": { + "type": "keyword" + } + } } } }, From c9e71ba0be84bb179347d20b12d28ebf74170660 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:59:07 +0000 Subject: [PATCH 09/45] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index d8ecd92be2a1a..2afd9bb66eb0f 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -175,7 +175,7 @@ describe('checking migration metadata changes on all registered SO types', () => "synthetics-private-location": "8cecc9e4f39637d2f8244eb7985c0690ceab24be", "synthetics-privates-locations": "f53d799d5c9bc8454aaa32c6abc99a899b025d5c", "tag": "e2544392fe6563e215bb677abc8b01c2601ef2dc", - "task": "42a95c6563df655a3fd7dea412b5f9bd27cded4f", + "task": "56ee0eea47adb875d012fee2e764ac0143e6e2f3", "telemetry": "7b00bcf1c7b4f6db1192bb7405a6a63e78b699fd", "threshold-explorer-view": "175306806f9fc8e13fcc1c8953ec4ba89bda1b70", "ui-metric": "d227284528fd19904e9d972aea0a13716fc5fe24", From d05d9c89c204735b5913c92b373c8a3857bfc00a Mon Sep 17 00:00:00 2001 From: Ying Date: Mon, 14 Apr 2025 08:17:09 -0400 Subject: [PATCH 10/45] Allowing shared concurrency task types --- .../shared/task_manager/server/constants.ts | 2 + .../server/queries/task_claiming.ts | 4 +- .../lib/task_selector_by_capacity.test.ts | 128 +++++++++++++ .../lib/task_selector_by_capacity.ts | 38 +++- .../task_claimers/strategy_mget.test.ts | 172 ++++++++++++++++++ .../server/task_claimers/strategy_mget.ts | 37 +++- .../server/task_type_dictionary.test.ts | 51 +++++- .../server/task_type_dictionary.ts | 60 +++++- .../sample_task_plugin/server/plugin.ts | 30 +++ .../check_registered_task_types.ts | 2 + .../task_manager/task_management.ts | 49 +++++ 11 files changed, 545 insertions(+), 28 deletions(-) create mode 100644 x-pack/platform/plugins/shared/task_manager/server/task_claimers/lib/task_selector_by_capacity.test.ts diff --git a/x-pack/platform/plugins/shared/task_manager/server/constants.ts b/x-pack/platform/plugins/shared/task_manager/server/constants.ts index e9294b25e2859..86238e44ae642 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/constants.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/constants.ts @@ -11,6 +11,8 @@ export const CONCURRENCY_ALLOW_LIST_BY_TASK_TYPE: string[] = [ 'sampleTaskWithLimitedConcurrency', 'timedTaskWithSingleConcurrency', 'timedTaskWithLimitedConcurrency', + 'sampleTaskSharedConcurrencyType1', + 'sampleTaskSharedConcurrencyType2', // task types requiring a concurrency 'report:execute', diff --git a/x-pack/platform/plugins/shared/task_manager/server/queries/task_claiming.ts b/x-pack/platform/plugins/shared/task_manager/server/queries/task_claiming.ts index 9bc3efaeeadce..4051d2c68a194 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/queries/task_claiming.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/queries/task_claiming.ts @@ -198,14 +198,14 @@ export function isLimited( return batch.concurrency === BatchConcurrency.Limited; } -function asLimited(tasksType: string): LimitedBatch { +export function asLimited(tasksType: string): LimitedBatch { return { concurrency: BatchConcurrency.Limited, tasksTypes: tasksType, }; } -function asUnlimited(tasksTypes: Set): UnlimitedBatch { +export function asUnlimited(tasksTypes: Set): UnlimitedBatch { return { concurrency: BatchConcurrency.Unlimited, tasksTypes, diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/lib/task_selector_by_capacity.test.ts b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/lib/task_selector_by_capacity.test.ts new file mode 100644 index 0000000000000..e4c43901adaf0 --- /dev/null +++ b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/lib/task_selector_by_capacity.test.ts @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { v4 as uuidv4 } from 'uuid'; +import { asLimited, asUnlimited } from '../../queries/task_claiming'; +import { selectTasksByCapacity } from './task_selector_by_capacity'; +import type { ConcreteTaskInstance } from '../../task'; +import { TaskTypeDictionary } from '../../task_type_dictionary'; +import { mockLogger } from '../../test_utils'; + +jest.mock('../../constants', () => ({ + CONCURRENCY_ALLOW_LIST_BY_TASK_TYPE: [ + 'limitedTaskType', + 'sampleTaskSharedConcurrencyType1', + 'sampleTaskSharedConcurrencyType2', + ], +})); + +const taskManagerLogger = mockLogger(); +function mockInstance(instance: Partial = {}) { + return Object.assign( + { + id: uuidv4(), + taskType: 'bar', + sequenceNumber: 32, + primaryTerm: 32, + runAt: new Date(), + scheduledAt: new Date(), + startedAt: null, + retryAt: null, + attempts: 0, + params: {}, + scope: ['scope'], + state: {}, + status: 'idle', + user: 'example', + ownerId: null, + traceparent: '', + }, + instance + ); +} + +const taskDefinitions = new TaskTypeDictionary(taskManagerLogger); +taskDefinitions.registerTaskDefinitions({ + limitedTaskType: { + title: 'Limited Concurrency Task Type', + maxConcurrency: 1, + createTaskRunner: jest.fn(), + }, + taskType1: { + title: 'dernstraight', + createTaskRunner: jest.fn(), + }, + taskType2: { + title: 'yawn', + createTaskRunner: jest.fn(), + }, + sampleTaskSharedConcurrencyType1: { + title: 'Shared Concurrency Task Type 1', + maxConcurrency: 2, + createTaskRunner: jest.fn(), + }, + sampleTaskSharedConcurrencyType2: { + title: 'Shared Concurrency Task Type 2', + maxConcurrency: 2, + createTaskRunner: jest.fn(), + }, +}); + +describe('selectTasksByCapacity', () => { + it('should limit tasks by concurrency if limited', () => { + const batches = [ + asLimited('limitedTaskType'), + asUnlimited(new Set(['taskType1', 'taskType2'])), + ]; + const tasks = [ + mockInstance({ id: `id-1`, taskType: 'limitedTaskType' }), + mockInstance({ id: `id-2`, taskType: 'limitedTaskType' }), + ]; + + expect(selectTasksByCapacity({ definitions: taskDefinitions, tasks, batches })).toEqual([ + tasks[0], + ]); + }); + + it('should not limit tasks by concurrency if unlimited', () => { + const batches = [ + asLimited('limitedTaskType'), + asUnlimited(new Set(['taskType1', 'taskType2'])), + ]; + const tasks = [ + mockInstance({ id: `id-0`, taskType: 'taskType1' }), + mockInstance({ id: `id-1`, taskType: 'taskType1' }), + mockInstance({ id: `id-2`, taskType: 'taskType2' }), + mockInstance({ id: `id-3`, taskType: 'taskType1' }), + mockInstance({ id: `id-4`, taskType: 'taskType1' }), + mockInstance({ id: `id-5`, taskType: 'taskType1' }), + mockInstance({ id: `id-6`, taskType: 'taskType2' }), + mockInstance({ id: `id-7`, taskType: 'taskType2' }), + mockInstance({ id: `id-8`, taskType: 'taskType1' }), + mockInstance({ id: `id-9`, taskType: 'taskType1' }), + ]; + expect(selectTasksByCapacity({ definitions: taskDefinitions, tasks, batches })).toEqual(tasks); + }); + + it('should limit tasks by shared concurrency if limited', () => { + const batches = [ + asLimited('sampleTaskSharedConcurrencyType1'), + asLimited('sampleTaskSharedConcurrencyType2'), + asUnlimited(new Set(['taskType1', 'taskType2'])), + ]; + const tasks = [ + mockInstance({ id: `id-1`, taskType: 'sampleTaskSharedConcurrencyType1' }), + mockInstance({ id: `id-2`, taskType: 'sampleTaskSharedConcurrencyType2' }), + mockInstance({ id: `id-3`, taskType: 'sampleTaskSharedConcurrencyType1' }), + ]; + + expect(selectTasksByCapacity({ definitions: taskDefinitions, tasks, batches })).toEqual([ + tasks[0], + tasks[1], + ]); + }); +}); diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/lib/task_selector_by_capacity.ts b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/lib/task_selector_by_capacity.ts index 215952642f56e..0c0a57d7f309c 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/lib/task_selector_by_capacity.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/lib/task_selector_by_capacity.ts @@ -8,23 +8,34 @@ import type { ConcreteTaskInstance } from '../../task'; import type { TaskClaimingBatches } from '../../queries/task_claiming'; import { isLimited } from '../../queries/task_claiming'; +import { sharedConcurrencyTaskTypes, type TaskTypeDictionary } from '../../task_type_dictionary'; +interface SelectTasksByCapacityOpts { + definitions: TaskTypeDictionary; + tasks: ConcreteTaskInstance[]; + batches: TaskClaimingBatches; +} // given a list of tasks and capacity info, select the tasks that meet capacity -export function selectTasksByCapacity( - tasks: ConcreteTaskInstance[], - batches: TaskClaimingBatches -): ConcreteTaskInstance[] { +export function selectTasksByCapacity({ + definitions, + tasks, + batches, +}: SelectTasksByCapacityOpts): ConcreteTaskInstance[] { // create a map of task type - concurrency const limitedBatches = batches.filter(isLimited); - const limitedMap = new Map(); + const limitedMap = new Map(); for (const limitedBatch of limitedBatches) { - const { tasksTypes, concurrency } = limitedBatch; - limitedMap.set(tasksTypes, concurrency); + const { tasksTypes: taskType } = limitedBatch; + + // get concurrency from task definition + const taskDef = definitions.get(taskType); + limitedMap.set(taskType, taskDef?.maxConcurrency ?? null); } // apply the limited concurrency const result: ConcreteTaskInstance[] = []; for (const task of tasks) { + // get concurrency of this task type const concurrency = limitedMap.get(task.taskType); if (concurrency == null) { result.push(task); @@ -33,7 +44,18 @@ export function selectTasksByCapacity( if (concurrency > 0) { result.push(task); - limitedMap.set(task.taskType, concurrency - 1); + + // get any shared concurrency task types + const sharesConcurrencyWith = sharedConcurrencyTaskTypes(task.taskType); + if (sharesConcurrencyWith) { + for (const taskType of sharesConcurrencyWith) { + if (limitedMap.has(taskType)) { + limitedMap.set(taskType, concurrency - 1); + } + } + } else { + limitedMap.set(task.taskType, concurrency - 1); + } } } diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.test.ts b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.test.ts index e96d148a9cce1..8a508a174af59 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.test.ts @@ -48,6 +48,8 @@ jest.mock('../constants', () => ({ 'limitedToTwo', 'limitedToFive', 'yawn', + 'sampleTaskSharedConcurrencyType1', + 'sampleTaskSharedConcurrencyType2', ], })); @@ -86,6 +88,16 @@ taskDefinitions.registerTaskDefinitions({ maxConcurrency: 1, createTaskRunner: jest.fn(), }, + sampleTaskSharedConcurrencyType1: { + title: 'Shared Concurrency Task Type 1', + maxConcurrency: 2, + createTaskRunner: jest.fn(), + }, + sampleTaskSharedConcurrencyType2: { + title: 'Shared Concurrency Task Type 2', + maxConcurrency: 2, + createTaskRunner: jest.fn(), + }, }); const mockApmTrans = { @@ -892,6 +904,166 @@ describe('TaskClaiming', () => { expect(result.docs.length).toEqual(4); }); + test('should correctly handle shared concurrency tasks', async () => { + const store = taskStoreMock.create({ taskManagerId: 'test-test' }); + store.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); + + const fetchedTasks = [ + mockInstance({ id: `id-1`, taskType: 'report' }), + mockInstance({ id: `id-2`, taskType: 'report' }), + mockInstance({ id: `id-3`, taskType: 'yawn' }), + mockInstance({ id: `id-4`, taskType: 'yawn' }), + mockInstance({ id: `id-5`, taskType: 'report' }), + mockInstance({ id: `id-6`, taskType: 'yawn' }), + ]; + + const { versionMap, docLatestVersions } = getVersionMapsFromTasks(fetchedTasks); + store.msearch.mockResolvedValueOnce({ docs: fetchedTasks, versionMap }); + store.getDocVersions.mockResolvedValueOnce(docLatestVersions); + + store.bulkGet.mockResolvedValueOnce( + [fetchedTasks[0], fetchedTasks[1], fetchedTasks[2], fetchedTasks[4]].map(asOk) + ); + store.bulkPartialUpdate.mockResolvedValueOnce( + [fetchedTasks[0], fetchedTasks[1], fetchedTasks[2], fetchedTasks[4]].map( + getPartialUpdateResult + ) + ); + + const taskClaiming = new TaskClaiming({ + logger: taskManagerLogger, + strategy: CLAIM_STRATEGY_MGET, + definitions: taskDefinitions, + taskStore: store, + excludedTaskTypes: [], + maxAttempts: 2, + getAvailableCapacity: (type?: string) => { + if (type === 'sampleTaskSharedConcurrencyType1') { + return 2; + } else if (type === 'sampleTaskSharedConcurrencyType1') { + return 10; + } else { + return 10; + } + }, + taskPartitioner, + }); + + const resultOrErr = await taskClaiming.claimAvailableTasksIfCapacityIsAvailable({ + claimOwnershipUntil: new Date(), + }); + + if (!isOk(resultOrErr)) { + expect(resultOrErr).toBe(undefined); + } + + unwrap(resultOrErr) as ClaimOwnershipResult; + + expect(taskManagerLogger.debug).toHaveBeenCalledWith( + 'task claimer claimed: 4; stale: 0; conflicts: 0; missing: 0; capacity reached: 0; updateErrors: 0; getErrors: 0;', + { tags: ['taskClaiming', 'claimAvailableTasksMget'] } + ); + + const mockSearch = store.msearch.mock.calls[0][0]; + + // should be 3 searches + expect(mockSearch?.length).toEqual(3); + + // last search should be shared concurrency + expect(mockSearch?.[2].query).toMatchObject({ + bool: { + must: [ + { + bool: { + must: [{ term: { 'task.enabled': true } }], + }, + }, + { + bool: { + must: [ + { + terms: { + 'task.taskType': [ + 'sampleTaskSharedConcurrencyType1', + 'sampleTaskSharedConcurrencyType2', + ], + }, + }, + ], + }, + }, + { + bool: { + should: [ + { + bool: { + must: [ + { + term: { + 'task.status': 'idle', + }, + }, + { + range: { + 'task.runAt': { + lte: 'now', + }, + }, + }, + ], + }, + }, + { + bool: { + must: [ + { + bool: { + should: [ + { + term: { + 'task.status': 'running', + }, + }, + { + term: { + 'task.status': 'claiming', + }, + }, + ], + }, + }, + { + range: { + 'task.retryAt': { + lte: 'now', + }, + }, + }, + ], + }, + }, + ], + }, + }, + { + bool: { + must_not: [ + { + term: { + 'task.status': 'unrecognized', + }, + }, + ], + }, + }, + ], + }, + }); + // size of search should match the shared concurrency task with the + // lower capacity (* cost multiplier) + expect(mockSearch?.[2].size).toEqual(4); + }); + test('should handle individual errors when bulk getting the full task doc', async () => { const store = taskStoreMock.create({ taskManagerId: 'test-test' }); store.convertToSavedObjectIds.mockImplementation((ids) => ids.map((id) => `task:${id}`)); diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.ts b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.ts index b342760e64f34..015aaa0fcdf79 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.ts @@ -18,7 +18,7 @@ import apm from 'elastic-apm-node'; import type { Subject } from 'rxjs'; import { createWrappedLogger } from '../lib/wrapped_logger'; -import type { TaskTypeDictionary } from '../task_type_dictionary'; +import { sharedConcurrencyTaskTypes, type TaskTypeDictionary } from '../task_type_dictionary'; import type { TaskClaimerOpts, ClaimOwnershipResult } from '.'; import { getEmptyClaimOwnershipResult, getExcludedTaskTypes } from '.'; import type { @@ -144,7 +144,7 @@ async function claimAvailableTasks(opts: TaskClaimerOpts): Promise; + limitedTypes: Map; } interface BuildClaimPartitionsOpts { @@ -384,9 +384,30 @@ function buildClaimPartitions(opts: BuildClaimPartitionsOpts): ClaimPartitions { continue; } - const capacity = getCapacity(definition.type) / definition.cost; - if (capacity !== 0) { - result.limitedTypes.set(definition.type, capacity); + // task type has maxConcurrency defined + + const isSharingConcurrency = sharedConcurrencyTaskTypes(type); + if (isSharingConcurrency) { + let minCapacity = null; + for (const sharedType of isSharingConcurrency) { + const def = definitions.get(sharedType); + if (def) { + const capacity = getCapacity(def.type) / def.cost; + if (minCapacity == null) { + minCapacity = capacity; + } else if (capacity < minCapacity) { + minCapacity = capacity; + } + } + } + if (minCapacity) { + result.limitedTypes.set(isSharingConcurrency, minCapacity); + } + } else { + const capacity = getCapacity(definition.type) / definition.cost; + if (capacity !== 0) { + result.limitedTypes.set([definition.type], capacity); + } } } diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_type_dictionary.test.ts b/x-pack/platform/plugins/shared/task_manager/server/task_type_dictionary.test.ts index 7ca4042cda4f0..e2b22ab42bff2 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_type_dictionary.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_type_dictionary.test.ts @@ -13,7 +13,11 @@ import type { TaskDefinitionRegistry } from './task_type_dictionary'; import { sanitizeTaskDefinitions, TaskTypeDictionary } from './task_type_dictionary'; jest.mock('./constants', () => ({ - CONCURRENCY_ALLOW_LIST_BY_TASK_TYPE: ['foo'], + CONCURRENCY_ALLOW_LIST_BY_TASK_TYPE: [ + 'foo', + 'sampleTaskSharedConcurrencyType1', + 'sampleTaskSharedConcurrencyType2', + ], })); interface Opts { @@ -329,5 +333,50 @@ describe('taskTypeDictionary', () => { `"maxConcurrency setting isn't allowed for task type: foo2"` ); }); + + it('throws error when registering shared concurrency tasks with different max concurrencies', () => { + definitions.registerTaskDefinitions({ + sampleTaskSharedConcurrencyType1: { + title: 'Shared 1', + maxConcurrency: 2, + createTaskRunner: jest.fn(), + }, + }); + + expect(() => { + definitions.registerTaskDefinitions({ + sampleTaskSharedConcurrencyType2: { + title: 'Shared 2', + maxConcurrency: 1, + createTaskRunner: jest.fn(), + }, + }); + }).toThrowErrorMatchingInlineSnapshot( + `"Task type \\"sampleTaskSharedConcurrencyType2\\" shares concurrency limits with sampleTaskSharedConcurrencyType1 but has a different maxConcurrency."` + ); + }); + + it('throws error when registering shared concurrency tasks with different task costs', () => { + definitions.registerTaskDefinitions({ + sampleTaskSharedConcurrencyType1: { + title: 'Shared 1', + maxConcurrency: 1, + cost: TaskCost.ExtraLarge, + createTaskRunner: jest.fn(), + }, + }); + + expect(() => { + definitions.registerTaskDefinitions({ + sampleTaskSharedConcurrencyType2: { + title: 'Shared 2', + maxConcurrency: 1, + createTaskRunner: jest.fn(), + }, + }); + }).toThrowErrorMatchingInlineSnapshot( + `"Task type \\"sampleTaskSharedConcurrencyType2\\" shares concurrency limits with sampleTaskSharedConcurrencyType1 but has a different cost."` + ); + }); }); }); diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_type_dictionary.ts b/x-pack/platform/plugins/shared/task_manager/server/task_type_dictionary.ts index b70b568cf9e89..db738dadecf5f 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_type_dictionary.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_type_dictionary.ts @@ -35,6 +35,11 @@ export const REMOVED_TYPES: string[] = [ 'obs-ai-assistant:knowledge-base-migration', ]; +export const SHARED_CONCURRENCY_TASKS: string[][] = [ + // for testing + ['sampleTaskSharedConcurrencyType1', 'sampleTaskSharedConcurrencyType2'], +]; + /** * Defines a task which can be scheduled and run by the Kibana * task manager. @@ -156,15 +161,22 @@ export class TaskTypeDictionary { } for (const taskType of Object.keys(taskDefinitions)) { - if ( - taskDefinitions[taskType].maxConcurrency !== undefined && - !CONCURRENCY_ALLOW_LIST_BY_TASK_TYPE.includes(taskType) - ) { - // maxConcurrency is designed to limit how many tasks of the same type a single Kibana - // instance should run at a time. Meaning if you have 8 Kibanas running, you will still - // see up to 8 tasks running at a time but one per Kibana instance. This is helpful for - // reporting purposes but not for many other cases and are better off not setting this value. - throw new Error(`maxConcurrency setting isn't allowed for task type: ${taskType}`); + if (taskDefinitions[taskType].maxConcurrency !== undefined) { + if (!CONCURRENCY_ALLOW_LIST_BY_TASK_TYPE.includes(taskType)) { + // maxConcurrency is designed to limit how many tasks of the same type a single Kibana + // instance should run at a time. Meaning if you have 8 Kibanas running, you will still + // see up to 8 tasks running at a time but one per Kibana instance. This is helpful for + // reporting purposes but not for many other cases and are better off not setting this value. + throw new Error(`maxConcurrency setting isn't allowed for task type: ${taskType}`); + } + + // if this task type shares concurrency with another task type and both have been + // registered, throw an error if their maxConcurrency values are different + this.verifySharedConcurrencyAndCost( + taskType, + taskDefinitions[taskType].maxConcurrency!, + taskDefinitions[taskType].cost + ); } } @@ -176,6 +188,32 @@ export class TaskTypeDictionary { this.logger.error(`Could not sanitize task definitions: ${e.message}`); } } + + private verifySharedConcurrencyAndCost( + taskType: string, + maxConcurrency: number, + cost?: TaskCost + ) { + const shared = sharedConcurrencyTaskTypes(taskType); + + if (shared) { + const otherTaskTypes: string[] = shared.filter((type) => type !== taskType); + + for (const otherTaskType of otherTaskTypes) { + const otherTaskDef = this.definitions.get(otherTaskType); + if (otherTaskDef && otherTaskDef.maxConcurrency !== maxConcurrency) { + throw new Error( + `Task type "${taskType}" shares concurrency limits with ${otherTaskType} but has a different maxConcurrency.` + ); + } + if (otherTaskDef && otherTaskDef.cost !== cost) { + throw new Error( + `Task type "${taskType}" shares concurrency limits with ${otherTaskType} but has a different cost.` + ); + } + } + } + } } /** @@ -189,3 +227,7 @@ export function sanitizeTaskDefinitions(taskDefinitions: TaskDefinitionRegistry) return taskDefinitionSchema.validate({ type, ...rawDefinition }) as TaskDefinition; }); } + +export function sharedConcurrencyTaskTypes(taskType: string) { + return SHARED_CONCURRENCY_TASKS.find((tasks: string[]) => tasks.includes(taskType)); +} diff --git a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts index 891df37e76158..37b761a5cd39f 100644 --- a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts +++ b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts @@ -138,6 +138,36 @@ export class SampleTaskManagerFixturePlugin }, }, }, + sampleTaskSharedConcurrencyType1: { + ...defaultSampleTaskConfig, + title: 'Sample Task With Shared Concurrency 1', + maxConcurrency: 1, + timeout: '60s', + description: 'A sample task that shares concurrency with another task type.', + stateSchemaByVersion: { + 1: { + up: (state: Record) => ({ count: state.count }), + schema: schema.object({ + count: schema.maybe(schema.number()), + }), + }, + }, + }, + sampleTaskSharedConcurrencyType2: { + ...defaultSampleTaskConfig, + title: 'Sample Task With Shared Concurrency 2', + maxConcurrency: 1, + timeout: '60s', + description: 'A sample task that shares concurrency with another task type.', + stateSchemaByVersion: { + 1: { + up: (state: Record) => ({ count: state.count }), + schema: schema.object({ + count: schema.maybe(schema.number()), + }), + }, + }, + }, sampleTaskWithLimitedConcurrency: { ...defaultSampleTaskConfig, title: 'Sample Task With Max Concurrency of 2', diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index db09b1b17171d..d1c1022e8eed9 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -29,6 +29,8 @@ export default function ({ getService }: FtrProviderContext) { 'sampleTask', 'sampleTaskWithLimitedConcurrency', 'sampleTaskWithSingleConcurrency', + 'sampleTaskSharedConcurrencyType1', + 'sampleTaskSharedConcurrencyType2', 'singleAttemptSampleTask', 'timedTask', 'timedTaskWithLimitedConcurrency', diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index bf04d28dc2fa8..4a8bf8102c317 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -609,6 +609,55 @@ export default function ({ getService }: FtrProviderContext) { await releaseTasksWaitingForEventToComplete('releaseSecondWaveOfTasks'); }); + it('should only run as many instances of a task as its shared maxConcurrency will allow', async () => { + // should run as maxConcurrency on this taskType is 1 + const firstWithSharedConcurrency = await scheduleTask({ + taskType: 'sampleTaskSharedConcurrencyType1', + params: { + waitForEvent: 'releaseFirstWaveOfTasks', + }, + }); + + await retry.try(async () => { + expect((await historyDocs(firstWithSharedConcurrency.id)).length).to.eql(1); + }); + + // should not run as there is a task with shared concurrency running + const secondWithSharedConcurrency = await scheduleTask({ + taskType: 'sampleTaskSharedConcurrencyType2', + params: { + waitForEvent: 'releaseSecondWaveOfTasks', + }, + }); + + // schedule a task that should get picked up before the blocked task + const taskWithUnlimitedConcurrency = await scheduleTask({ + taskType: 'sampleTask', + params: {}, + }); + + await retry.try(async () => { + expect((await historyDocs(taskWithUnlimitedConcurrency.id)).length).to.eql(1); + expect((await currentTask(secondWithSharedConcurrency.id)).status).to.eql('idle'); + }); + + // release the running SingleConcurrency task and only one of the LimitedConcurrency tasks + await releaseTasksWaitingForEventToComplete('releaseFirstWaveOfTasks'); + + await retry.try(async () => { + // ensure the completed tasks were deleted + expect((await currentTaskError(firstWithSharedConcurrency.id)).message).to.eql( + `Saved object [task/${firstWithSharedConcurrency.id}] not found` + ); + + // ensure the blocked tasks begin running + expect((await currentTask(secondWithSharedConcurrency.id)).status).to.eql('running'); + }); + + // release blocked task + await releaseTasksWaitingForEventToComplete('releaseSecondWaveOfTasks'); + }); + it('should return a task run error result when trying to run a non-existent task', async () => { // runSoon should fail const failedRunSoonResult = await runTaskSoon({ From 249f14c78a43da337781d0013c9aa6ed84736174 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Tue, 15 Apr 2025 16:40:09 +0200 Subject: [PATCH 11/45] set second to [0] --- .../server/lib/get_first_run_at.test.ts | 64 +++++++++++++++---- .../server/lib/get_first_run_at.ts | 1 + 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts index 78d6e6a67253b..acaaa0224508e 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts @@ -13,7 +13,7 @@ describe('getFirstRunAt', () => { beforeEach(() => { jest.useFakeTimers(); - jest.setSystemTime(new Date('2025-04-15T13:00:00Z')); + jest.setSystemTime(new Date('2025-04-15T13:01:02Z')); }); afterEach(() => { @@ -105,7 +105,50 @@ describe('getFirstRunAt', () => { expect(firstRunAtDate).toEqual(new Date('2025-04-16T12:15:00Z')); }); - test('should return the calculated runAt when an rrule with weekly interval time is provided', () => { + test('should return the calculated runAt when an rrule only with byhour is provided', () => { + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 3, + interval: 1, + tzid: 'UTC', + byhour: [12], + }, + }, + }; + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + // The next day from 2025-04-15 is 2025-04-16 + // The hour is set to 12, default minute and second becomes 0 + expect(firstRunAtDate).toEqual(new Date('2025-04-16T12:00:00Z')); + }); + + test('should return the calculated runAt when an rrule byminute with byminute is provided', () => { + const taskInstance = { + id: 'id', + params: {}, + state: {}, + taskType: 'report', + schedule: { + rrule: { + freq: 3, + interval: 1, + tzid: 'UTC', + byminute: [17], + }, + }, + }; + const firstRunAt = getFirstRunAt({ taskInstance, logger }); + const firstRunAtDate = new Date(firstRunAt); + // The minute is set to 17, default second is 0, hour becomes the current hour + expect(firstRunAtDate).toEqual(new Date('2025-04-15T13:17:00Z')); + }); + + test('should return the calculated runAt when an rrule only with weekly interval time is provided', () => { const taskInstance = { id: 'id', params: {}, @@ -124,8 +167,8 @@ describe('getFirstRunAt', () => { const firstRunAt = getFirstRunAt({ taskInstance, logger }); const firstRunAtDate = new Date(firstRunAt); // The next Monday from 2025-04-15 is 2025-04-21 - // The time is set to now - expect(firstRunAtDate).toEqual(new Date('2025-04-21T13:00:00.000Z')); + // The time is set to midnight + expect(firstRunAtDate).toEqual(new Date('2025-04-21T00:00:00.000Z')); }); test('should return the calculated runAt when an rrule with weekly fixed time interval time is provided', () => { @@ -153,12 +196,7 @@ describe('getFirstRunAt', () => { expect(firstRunAtDate).toEqual(new Date('2025-04-21T12:15:00.000Z')); }); - // bymonthday?: number[]; - // byhour?: number[]; - // byminute?: number[]; - // byweekday?: never; - - test('should return the calculated runAt when an rrule with monthly interval is provided', () => { + test('should return the calculated runAt when an rrule only with monthly interval is provided', () => { const taskInstance = { id: 'id', params: {}, @@ -178,8 +216,8 @@ describe('getFirstRunAt', () => { const firstRunAtDate = new Date(firstRunAt); // The next month from 2025-04 is 2025-05 // The day is set to 3 - // The time is set to now - expect(firstRunAtDate).toEqual(new Date('2025-05-03T13:00:00.000Z')); + // The time is set to midnight + expect(firstRunAtDate).toEqual(new Date('2025-05-03T00:00:00.000Z')); }); test('should return the calculated runAt when an rrule with monthly interval with fixed time is provided', () => { @@ -204,7 +242,7 @@ describe('getFirstRunAt', () => { const firstRunAtDate = new Date(firstRunAt); // The next month from 2025-04 is 2025-05 // The day is set to 3 - // The time is set to 12:17 + // The time is set to 12:17:00 expect(firstRunAtDate).toEqual(new Date('2025-05-03T12:17:00.000Z')); }); diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts index 4b1f8e85aea7b..c3e06739992de 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts @@ -26,6 +26,7 @@ export function getFirstRunAt({ try { const rrule = new RRule({ ...taskInstance.schedule.rrule, + bysecond: [0], dtstart: now, }); return rrule.after(now)?.toISOString() || nowString; From 4a14142645c831d187ad65ea15e22ad220dd8333 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Wed, 16 Apr 2025 00:57:14 +0200 Subject: [PATCH 12/45] testing schedule is more resilient --- .../test_suites/task_manager/task_management.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index d2a6b6acda297..791319f7ce158 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -247,7 +247,7 @@ export default function ({ getService }: FtrProviderContext) { .send({ event: taskId, data }) .expect(200); } - it('should schedule a task with rrule', async () => { + it.only('should schedule a task with rrule', async () => { const dailyTask = await scheduleTask({ id: 'sample-recurring-task-id', taskType: 'sampleRecurringTask', @@ -266,7 +266,8 @@ export default function ({ getService }: FtrProviderContext) { const runAt = new Date(task.runAt).getTime(); const scheduledAt = new Date(task.scheduledAt).getTime(); // scheduled to run 24 hours from now - expect(runAt).to.be(scheduledAt + 1000 * 60 * 60 * 24); + expect(runAt).to.greaterThan(scheduledAt + 1000 * 59 * 60 * 24); + expect(runAt).to.lessThan(scheduledAt + 1000 * 61 * 60 * 24); }); }); From 4ec1bca024e431267fa62731975a7b463ee630d3 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Wed, 16 Apr 2025 01:31:11 +0200 Subject: [PATCH 13/45] remove "only" --- .../test_suites/task_manager/task_management.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index 791319f7ce158..ee60c8db09a1e 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -247,7 +247,7 @@ export default function ({ getService }: FtrProviderContext) { .send({ event: taskId, data }) .expect(200); } - it.only('should schedule a task with rrule', async () => { + it('should schedule a task with rrule', async () => { const dailyTask = await scheduleTask({ id: 'sample-recurring-task-id', taskType: 'sampleRecurringTask', From 850ec64c05b9201408c2f70fa6f19c45d0b237de Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Mon, 28 Apr 2025 15:20:59 +0200 Subject: [PATCH 14/45] throw when interval is not provided for the next run calculation --- .../shared/alerting/server/lib/next_run.ts | 5 +++- .../server/task_runner/task_runner.test.ts | 27 +++++++++++++++++++ .../server/task_runner/task_runner.ts | 4 +-- .../server/lib/get_first_run_at.test.ts | 4 +-- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/x-pack/platform/plugins/shared/alerting/server/lib/next_run.ts b/x-pack/platform/plugins/shared/alerting/server/lib/next_run.ts index 8ce87e9db6883..8a9f5e059753f 100644 --- a/x-pack/platform/plugins/shared/alerting/server/lib/next_run.ts +++ b/x-pack/platform/plugins/shared/alerting/server/lib/next_run.ts @@ -13,8 +13,11 @@ export const getNextRun = ({ interval, }: { startDate?: Date | null; - interval: string; + interval?: string; }) => { + if (!interval) { + throw new Error('Interval is required to calculate next run'); + } return moment(startDate || new Date()) .add(parseDuration(interval), 'ms') .toISOString(); diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.test.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.test.ts index 84bc0bc6c818a..b4308a7cca2f5 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.test.ts @@ -344,6 +344,33 @@ describe('Task Runner', () => { ).toHaveBeenCalled(); }); + test('throws error when schedule.interval is not provided', async () => { + mockGetAlertFromRaw.mockReturnValue({ ...mockedRuleTypeSavedObject, schedule: {} } as Rule); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValue({ + ...mockedRawRuleSO, + attributes: { ...mockedRawRuleSO.attributes, schedule: {} }, + }); + const taskRunner = new TaskRunner({ + ruleType, + taskInstance: { + ...mockedTaskInstance, + state: { + ...mockedTaskInstance.state, + previousStartedAt: new Date(Date.now() - 5 * 60 * 1000).toISOString(), + }, + // @ts-ignore + schedule: {}, + }, + context: taskRunnerFactoryInitializerParams, + inMemoryMetrics, + internalSavedObjectsRepository, + }); + + await expect(taskRunner.run()).rejects.toThrowErrorMatchingInlineSnapshot( + '"Interval is required to calculate next run"' + ); + }); + test('actionsPlugin.execute is called per alert alert that is scheduled', async () => { taskRunnerFactoryInitializerParams.actionsPlugin.isActionTypeEnabled.mockReturnValue(true); taskRunnerFactoryInitializerParams.actionsPlugin.isActionExecutable.mockReturnValue(true); diff --git a/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.ts b/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.ts index c6ae7a9756466..314900f28f0a2 100644 --- a/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.ts +++ b/x-pack/platform/plugins/shared/alerting/server/task_runner/task_runner.ts @@ -594,7 +594,7 @@ export class TaskRunner< nextRun = getNextRun({ startDate: startedAt, interval: schedule.value.interval }); } else if (taskSchedule) { // rules cannot use rrule for scheduling yet - nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval! }); + nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval }); } const { executionStatus, executionMetrics, lastRun, outcome } = processRunResults({ @@ -830,7 +830,7 @@ export class TaskRunner< let nextRun: string | null = null; if (taskSchedule) { // rules cannot use rrule for scheduling yet - nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval! }); + nextRun = getNextRun({ startDate: startedAt, interval: taskSchedule.interval }); } const outcomeMsg = [ diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts index acaaa0224508e..d3e562c67b723 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts @@ -105,7 +105,7 @@ describe('getFirstRunAt', () => { expect(firstRunAtDate).toEqual(new Date('2025-04-16T12:15:00Z')); }); - test('should return the calculated runAt when an rrule only with byhour is provided', () => { + test('should return the calculated runAt when an rrule with only byhour is provided', () => { const taskInstance = { id: 'id', params: {}, @@ -127,7 +127,7 @@ describe('getFirstRunAt', () => { expect(firstRunAtDate).toEqual(new Date('2025-04-16T12:00:00Z')); }); - test('should return the calculated runAt when an rrule byminute with byminute is provided', () => { + test('should return the calculated runAt when an rrule with byminute is provided', () => { const taskInstance = { id: 'id', params: {}, From 95cf333a391a01352d02a1cd152fc66f103a7ca3 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 28 Apr 2025 13:44:11 +0000 Subject: [PATCH 15/45] [CI] Auto-commit changed files from 'node scripts/jest_integration -u src/core/server/integration_tests/ci_checks' --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 742432dad96be..569a894786cd5 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -176,7 +176,7 @@ describe('checking migration metadata changes on all registered SO types', () => "synthetics-private-location": "27aaa44f792f70b734905e44e3e9b56bbeac7b86", "synthetics-privates-locations": "36036b881524108c7327fe14bd224c6e4d972cb5", "tag": "87f21f07df9cc37001b15a26e413c18f50d1fbfe", - "task": "56ee0eea47adb875d012fee2e764ac0143e6e2f3", + "task": "eacb6e8a7eea28a2fdc55fc1eda0d96471523157", "telemetry": "3b3b89cf411a2a2e60487cef6ccdbc5df691aeb9", "threshold-explorer-view": "5e2388a6835cec3c68c98b450cd267d66cce925f", "ui-metric": "410a8ad28e0f44b161c960ff0ce950c712b17c52", From d5227aa976c25e3b00d2cefd58e55d09a510a8b0 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Mon, 28 Apr 2025 16:47:58 +0200 Subject: [PATCH 16/45] update read me --- .../plugins/shared/task_manager/README.md | 132 +++++++++++++----- 1 file changed, 98 insertions(+), 34 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/README.md b/x-pack/platform/plugins/shared/task_manager/README.md index c5475c1b7d5df..76fb549e54b6e 100644 --- a/x-pack/platform/plugins/shared/task_manager/README.md +++ b/x-pack/platform/plugins/shared/task_manager/README.md @@ -131,7 +131,7 @@ export class Plugin { } public start(core: CoreStart, plugins: { taskManager }) { - + } } ``` @@ -161,12 +161,12 @@ When Kibana attempts to claim and run a task instance, it looks its definition u The task runner's `run` method is expected to return a promise that resolves to undefined or to an object that looks like the following: -|Property|Description|Type| -|---|---|---| -|runAt| Optional. If specified, this is used as the tasks' next `runAt`, overriding the default system scheduler. | Date ISO String | -|schedule| Optional. If specified, this is used as the tasks' new recurring schedule, overriding the default system scheduler and any existing schedule. | { interval: string } | -|error| Optional, an error object, logged out as a warning. The pressence of this property indicates that the task did not succeed.| Error | -|state| Optional, this will be passed into the next run of the task, if this is a recurring task. |Record| +| Property | Description | Type | +| -------- | --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | +| runAt | Optional. If specified, this is used as the tasks' next `runAt`, overriding the default system scheduler. | Date ISO String | +| schedule | Optional. If specified, this is used as the tasks' new recurring schedule, overriding the default system scheduler and any existing schedule. | { interval: string } | +| error | Optional, an error object, logged out as a warning. The pressence of this property indicates that the task did not succeed. | Error | +| state | Optional, this will be passed into the next run of the task, if this is a recurring task. | Record | ### Examples @@ -187,7 +187,7 @@ The task runner's `run` method is expected to return a promise that resolves to ```js { schedule: { interval: '30s' }, - + state: { anything: 'goes here', }, @@ -197,16 +197,19 @@ The task runner's `run` method is expected to return a promise that resolves to Other return values will result in a warning, but the system should continue to work. ### Task retries when the Task Runner fails + If a task runner throws an error, task manager will try to rerun the task shortly after (up to the task definition's `maxAttempts`). Normal tasks will wait a default amount of 5m before trying again and every subsequent attempt will add an additonal 5m cool off period to avoid a stampeding herd of failed tasks from storming Elasticsearch. Recurring tasks will also get retried, but instead of using the 5m interval for the retry, they will be retried on their next scheduled run. ### Force failing a task + If you wish to purposfully fail a task, you can throw an error of any kind and the retry logic will apply. If, on the other hand, you wish not only to fail the task, but you'd also like to indicate the Task Manager that it shouldn't retry the task, you can throw an Unrecoverable Error, using the `throwUnrecoverableError` helper function. For example: + ```js taskManager.registerTaskDefinitions({ myTask: { @@ -306,12 +309,15 @@ The data stored for a task instance looks something like this: The task manager mixin exposes a taskManager object on the Kibana server which plugins can use to manage scheduled tasks. Each method takes an optional `scope` argument and ensures that only tasks with the specified scope(s) will be affected. ### Overview + Interaction with the TaskManager Plugin is done via the Kibana Platform Plugin system. When developing your Plugin, you're asked to define a `setup` method and a `start` method. These methods are handed Kibana's Plugin APIs for these two stages, which means you'll have access to the following apis in these two stages: #### Setup + The _Setup_ Plugin api includes methods which configure Task Manager to support your Plugin's requirements, such as defining custom Middleware and Task Definitions. + ```js { addMiddleware: (middleware: Middleware) => { @@ -324,6 +330,7 @@ The _Setup_ Plugin api includes methods which configure Task Manager to support ``` #### Start + The _Start_ Plugin api allow you to use Task Manager to facilitate your Plugin's behaviour, such as scheduling tasks. ```js @@ -361,8 +368,9 @@ The _Start_ Plugin api allow you to use Task Manager to facilitate your Plugin's ### Detailed APIs #### schedule -Using `schedule` you can instruct TaskManger to schedule an instance of a TaskType at some point in the future. +Using `schedule` you can instruct TaskManger to schedule an instance of a TaskType at some point in the future. +Please check the [Schedule options](#schedule-options) for the scheduling config details ```js export class Plugin { @@ -395,27 +403,32 @@ export class Plugin { } } ``` -*results* then look something like this: + +_results_ then look something like this: ```json +{ + "searchAfter": ["233322"], + // Tasks is an array of task instances + "tasks": [ { - "searchAfter": ["233322"], - // Tasks is an array of task instances - "tasks": [{ - "id": "3242342", - "taskType": "reporting", - // etc - }] + "id": "3242342", + "taskType": "reporting" + // etc } + ] +} ``` #### ensureScheduling + When using the `schedule` api to schedule a Task you can provide a hard coded `id` on the Task. This tells TaskManager to use this `id` to identify the Task Instance rather than generate an `id` on its own. The danger is that in such a situation, a Task with that same `id` might already have been scheduled at some earlier point, and this would result in an error. In some cases, this is the expected behavior, but often you only care about ensuring the task has been _scheduled_ and don't need it to be scheduled a fresh. To achieve this you should use the `ensureScheduling` api which has the exact same behavior as `schedule`, except it allows the scheduling of a Task with an `id` that's already in assigned to another Task and it will assume that the existing Task is the one you wished to `schedule`, treating this as a successful operation. #### runSoon + Using `runSoon` you can instruct TaskManager to run an existing task as soon as possible by updating the next scheduled run date to be now ```js @@ -434,15 +447,17 @@ export class Plugin { // If running the task has failed, we throw an error with an appropriate message. // For example, if the requested task doesnt exist: `Error: failed to run task "91760f10-ba42-de9799" as it does not exist` // Or if, for example, the task is already running: `Error: failed to run task "91760f10-ba42-de9799" as it is currently running` - } + } } } ``` #### bulkDisable + Using `bulkDisable` you can instruct TaskManger to disable tasks by setting the `enabled` status of specific tasks to `false`. Example: + ```js export class Plugin { constructor() { @@ -460,15 +475,17 @@ export class Plugin { // But some updates of some tasks can be failed, due to OCC 409 conflict for example } catch(err: Error) { // if error is caught, means the whole method requested has failed and tasks weren't updated - } + } } } ``` #### bulkEnable + Using `bulkEnable` you can instruct TaskManger to enable tasks by setting the `enabled` status of specific tasks to `true`. Specify the `runSoon` parameter to run the task immediately on enable. Example: + ```js export class Plugin { constructor() { @@ -487,20 +504,23 @@ export class Plugin { // But some updates of some tasks can be failed, due to OCC 409 conflict for example } catch(err: Error) { // if error is caught, means the whole method requested has failed and tasks weren't updated - } + } } } ``` #### bulkUpdateSchedules + Using `bulkUpdatesSchedules` you can instruct TaskManger to update interval of tasks that are in `idle` status -(for the tasks which have `running` status, `schedule` and `runAt` will be recalculated after task run finishes). +(for the tasks which have `running` status, `schedule` and `runAt` will be recalculated after task run finishes). When interval updated, new `runAt` will be computed and task will be updated with that value, using formula + ``` newRunAt = oldRunAt - oldInterval + newInterval ``` Example: + ```js export class Plugin { constructor() { @@ -519,7 +539,7 @@ export class Plugin { // But some updates of some tasks can be failed, due to OCC 409 conflict for example } catch(err: Error) { // if error is caught, means the whole method requested has failed and tasks weren't updated - } + } } } ``` @@ -533,6 +553,7 @@ More custom access to the tasks can be done directly via Elasticsearch, though t The task manager exposes a middleware layer that allows modifying tasks before they are scheduled / persisted to the task manager index, and modifying tasks / the run context before a task is run. For example: + ```js export class Plugin { constructor() { @@ -568,12 +589,13 @@ export class Plugin { } public start(core: CoreStart, plugins: { taskManager }) { - + } } ``` ## Task Poller: polling for work + TaskManager used to work in a `pull` model, but it now needs to support both `push` and `pull`, so it has been remodeled internally to support a single `push` model. Task Manager's _push_ mechanism is driven by the following operations: @@ -596,11 +618,12 @@ Luckily, `Polling Interval` and `Task Scheduled` simply denote a request to "pol We achieve this model by buffering requests into a queue using a Set (which removes duplicated). As we don't want an unbounded queue in our system, we have limited the size of this queue (configurable by the `xpack.task_manager.request_capacity` config, defaulting to 1,000 requests) which forces us to throw an error once this cap is reached until the queue drain bellow the cap. Our current model, then, is this: + ``` - Polling Interval --> filter(availableWorkers > 0) - mapTo([]) -------\\ + Polling Interval --> filter(availableWorkers > 0) - mapTo([]) -------\\ Task Scheduled --> filter(availableWorkers > 0) - mapTo([]) --------||==>Set([]+[]+[`1`,`2`]) ==> work([`1`,`2`]) Run Task `1` Now --\ // - ----> buffer(availableWorkers > 0) -- [`1`,`2`] -// + ----> buffer(availableWorkers > 0) -- [`1`,`2`] -// Run Task `2` Now --/ ``` @@ -616,20 +639,61 @@ The task manager's public API is create / delete / list. Updates aren't directly - Unit tests: - Documentation: https://www.elastic.co/guide/en/kibana/current/development-tests.html#_unit_testing + Documentation: https://www.elastic.co/guide/en/kibana/current/development-tests.html#_unit_testing + + ``` + yarn test:jest x-pack/platform/plugins/shared/task_manager --watch + ``` - ``` - yarn test:jest x-pack/platform/plugins/shared/task_manager --watch - ``` - Integration tests: - ``` - node scripts/functional_tests_server.js --config x-pack/test/plugin_api_integration/config.ts - node scripts/functional_test_runner --config x-pack/test/plugin_api_integration/config.ts - ``` + ``` + node scripts/functional_tests_server.js --config x-pack/test/plugin_api_integration/config.ts + node scripts/functional_test_runner --config x-pack/test/plugin_api_integration/config.ts + ``` ## Monitoring Task Manager exposes runtime statistics which enable basic observability into its inner workings and makes it possible to monitor the system from external services. Public Documentation: https://www.elastic.co/guide/en/kibana/master/task-manager-health-monitoring.html -Developer Documentation: [./MONITORING](./MONITORING.MD) \ No newline at end of file +Developer Documentation: [./MONITORING](./MONITORING.MD) + +### Schedule options + +We keep the scheduling config under the schedule field. +And there are 2 different config options for scheduling a task: + +- `schedule.interval` + This is a basic interval string such as `1h`,`3m` or `7d` etc. + +- `schedule.rrule` + This is a subset of the rrule library. + We support only daily, weekly and monthly schedules so far. + +Monthly schedule options: + +``` + freq: 1, -> Enum for the monthly schedule is 1 + bymonthday: number[]; -> integer between 1 and 31 + byhour?: number[]; -> integer between 0 and 23 + byminute?: number[]; -> integer between 0 and 59 + byweekday?: Weekday[]; -> integer between 1 and 7. 1 is Monday. +``` + +Weekly schedule options: + +``` + freq: 2, -> Enum for the weekly schedule is 2 + byhour?: number[]; -> integer between 0 and 23 + byminute?: number[]; -> integer between 0 and 59 + byweekday?: Weekday[]; -> integer between 1 and 7. 1 is Monday. +``` + +Daily schedule options: + +``` + freq: 3, -> Enum for the daily schedule is 3 + byhour?: number[]; -> integer between 0 and 23 + byminute?: number[]; -> integer between 0 and 59 + byweekday?: Weekday[]; -> integer between 1 and 7. 1 is Monday. +``` From 62fe0727cb0540efafd316177d4403a266c79ad7 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Mon, 28 Apr 2025 16:54:08 +0200 Subject: [PATCH 17/45] update read me --- .../plugins/shared/task_manager/README.md | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/README.md b/x-pack/platform/plugins/shared/task_manager/README.md index 76fb549e54b6e..eb40a283d1ae2 100644 --- a/x-pack/platform/plugins/shared/task_manager/README.md +++ b/x-pack/platform/plugins/shared/task_manager/README.md @@ -673,27 +673,33 @@ And there are 2 different config options for scheduling a task: Monthly schedule options: ``` - freq: 1, -> Enum for the monthly schedule is 1 - bymonthday: number[]; -> integer between 1 and 31 - byhour?: number[]; -> integer between 0 and 23 - byminute?: number[]; -> integer between 0 and 59 - byweekday?: Weekday[]; -> integer between 1 and 7. 1 is Monday. + freq: 1, -> Enum for the monthly schedule is 1 (Required field) + interval: number; -> Any number. 1 means `every 1 month` (Required field) + tzid: string; -> Timezone e.g.: 'UTC' (Required field) + bymonthday: number[]; -> number between 1 and 31 + byhour?: number[]; -> number between 0 and 23 + byminute?: number[]; -> number between 0 and 59 + byweekday?: Weekday[]; -> number between 1 and 7. 1 is Monday. ``` Weekly schedule options: ``` - freq: 2, -> Enum for the weekly schedule is 2 - byhour?: number[]; -> integer between 0 and 23 - byminute?: number[]; -> integer between 0 and 59 - byweekday?: Weekday[]; -> integer between 1 and 7. 1 is Monday. + freq: 2, -> Enum for the weekly schedule is 2 (Required field) + interval: number; -> Any number. 1 means `every 1 week` (Required field) + tzid: string; -> Timezone e.g.: 'UTC' (Required field) + byhour?: number[]; -> number between 0 and 23 + byminute?: number[]; -> number between 0 and 59 + byweekday?: Weekday[]; -> number between 1 and 7. 1 is Monday. ``` Daily schedule options: ``` - freq: 3, -> Enum for the daily schedule is 3 - byhour?: number[]; -> integer between 0 and 23 - byminute?: number[]; -> integer between 0 and 59 - byweekday?: Weekday[]; -> integer between 1 and 7. 1 is Monday. + freq: 3, -> Enum for the daily schedule is 3 (Required field) + interval: number; -> Any number. 1 means `every 1 day` (Required field) + tzid: string; -> Timezone e.g.: 'UTC' (Required field) + byhour?: number[]; -> number between 0 and 23 + byminute?: number[]; -> number between 0 and 59 + byweekday?: Weekday[]; -> number between 1 and 7. 1 is Monday. ``` From 0f0685ed625d23d244ea2b5e38d3e20aa71a859a Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Mon, 28 Apr 2025 17:15:04 +0200 Subject: [PATCH 18/45] export Frequency and Weekday from TM --- .../platform/plugins/shared/task_manager/README.md | 12 ++++++------ .../plugins/shared/task_manager/server/index.ts | 2 ++ .../test_suites/task_manager/task_management.ts | 12 ++++++++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/README.md b/x-pack/platform/plugins/shared/task_manager/README.md index eb40a283d1ae2..0ceec029e5d11 100644 --- a/x-pack/platform/plugins/shared/task_manager/README.md +++ b/x-pack/platform/plugins/shared/task_manager/README.md @@ -673,33 +673,33 @@ And there are 2 different config options for scheduling a task: Monthly schedule options: ``` - freq: 1, -> Enum for the monthly schedule is 1 (Required field) + freq: Frequency.MONTHLY, -> Import the enum Frequency from TaskManager (Required field) interval: number; -> Any number. 1 means `every 1 month` (Required field) tzid: string; -> Timezone e.g.: 'UTC' (Required field) bymonthday: number[]; -> number between 1 and 31 byhour?: number[]; -> number between 0 and 23 byminute?: number[]; -> number between 0 and 59 - byweekday?: Weekday[]; -> number between 1 and 7. 1 is Monday. + byweekday?: Weekday[]; -> Import the enum Weekday from TaskManager. Weekday.MO is monday ``` Weekly schedule options: ``` - freq: 2, -> Enum for the weekly schedule is 2 (Required field) + freq: Frequency.WEEKLY, -> Import the enum Frequency from TaskManager (Required field) interval: number; -> Any number. 1 means `every 1 week` (Required field) tzid: string; -> Timezone e.g.: 'UTC' (Required field) byhour?: number[]; -> number between 0 and 23 byminute?: number[]; -> number between 0 and 59 - byweekday?: Weekday[]; -> number between 1 and 7. 1 is Monday. + byweekday?: Weekday[]; -> Import the enum Weekday from TaskManager. Weekday.MO is monday ``` Daily schedule options: ``` - freq: 3, -> Enum for the daily schedule is 3 (Required field) + freq: Frequency.DAILY, -> Import the enum Frequency from TaskManager (Required field) interval: number; -> Any number. 1 means `every 1 day` (Required field) tzid: string; -> Timezone e.g.: 'UTC' (Required field) byhour?: number[]; -> number between 0 and 23 byminute?: number[]; -> number between 0 and 59 - byweekday?: Weekday[]; -> number between 1 and 7. 1 is Monday. + byweekday?: Weekday[]; -> Import the enum Weekday from TaskManager. Weekday.MO is monday ``` diff --git a/x-pack/platform/plugins/shared/task_manager/server/index.ts b/x-pack/platform/plugins/shared/task_manager/server/index.ts index 37df3980b8c6f..6678d4c08ca18 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/index.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/index.ts @@ -23,6 +23,8 @@ export type { IntervalSchedule, } from './task'; +export { Frequency, Weekday } from '@kbn/rrule'; + export { TaskStatus, TaskPriority, TaskCost } from './task'; export type { TaskRegisterDefinition, TaskDefinitionRegistry } from './task_type_dictionary'; diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index ee60c8db09a1e..40eb7b1920b26 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -10,7 +10,11 @@ import { random } from 'lodash'; import expect from '@kbn/expect'; import type { estypes } from '@elastic/elasticsearch'; import { taskMappings as TaskManagerMapping } from '@kbn/task-manager-plugin/server/saved_objects/mappings'; -import { ConcreteTaskInstance, BulkUpdateTaskResult } from '@kbn/task-manager-plugin/server'; +import { + ConcreteTaskInstance, + BulkUpdateTaskResult, + Frequency, +} from '@kbn/task-manager-plugin/server'; import { FtrProviderContext } from '../../ftr_provider_context'; const { properties: taskManagerIndexMapping } = TaskManagerMapping; @@ -251,7 +255,7 @@ export default function ({ getService }: FtrProviderContext) { const dailyTask = await scheduleTask({ id: 'sample-recurring-task-id', taskType: 'sampleRecurringTask', - schedule: { rrule: { freq: 3, tzid: 'UTC', interval: 1 } }, + schedule: { rrule: { freq: Frequency.DAILY, tzid: 'UTC', interval: 1 } }, params: {}, }); @@ -276,7 +280,7 @@ export default function ({ getService }: FtrProviderContext) { id: 'sample-recurring-task-id', taskType: 'sampleRecurringTask', schedule: { - rrule: { freq: 3, tzid: 'UTC', interval: 1, byhour: [15], byminute: [27] }, + rrule: { freq: Frequency.DAILY, tzid: 'UTC', interval: 1, byhour: [15], byminute: [27] }, }, params: {}, }); @@ -302,7 +306,7 @@ export default function ({ getService }: FtrProviderContext) { taskType: 'sampleRecurringTask', schedule: { rrule: { - freq: 3, + freq: Frequency.DAILY, interval: 1, byhour: [30], // invalid byminute: [27], From 23d837da4d01ada574d7ed11995a8de77f43f867 Mon Sep 17 00:00:00 2001 From: Ying Date: Mon, 28 Apr 2025 17:02:48 -0400 Subject: [PATCH 19/45] Schedule report api --- .../current_fields.json | 1 + .../current_mappings.json | 4 + .../registration/type_registrations.test.ts | 1 + .../private/kbn-reporting/common/routes.ts | 1 + .../private/kbn-reporting/common/types.ts | 11 +- .../private/canvas/server/feature.test.ts | 8 +- .../plugins/private/canvas/server/feature.ts | 2 +- .../plugins/private/reporting/server/core.ts | 11 + .../private/reporting/server/plugin.test.ts | 15 + .../private/reporting/server/plugin.ts | 4 + .../routes/common/generate/request_handler.ts | 260 ------------ .../routes/common/jobs/get_job_routes.ts | 2 +- .../generate_request_handler.test.ts} | 76 ++-- .../generate_request_handler.ts | 130 ++++++ .../routes/common/request_handler/index.ts | 10 + .../common/request_handler/lib/index.ts | 9 + .../lib/transform_raw_scheduled_report.ts | 26 ++ .../transform_raw_scheduled_report_to_task.ts | 20 + .../common/request_handler/request_handler.ts | 205 +++++++++ .../schedule_request_handler.test.ts | 399 ++++++++++++++++++ .../schedule_request_handler.ts | 156 +++++++ .../private/reporting/server/routes/index.ts | 2 + .../generate/generate_from_jobparams.ts | 15 +- .../server/routes/internal/management/jobs.ts | 2 +- .../scheduling_from_jobparams.test.ts | 325 ++++++++++++++ .../schedule/schedule_from_jobparams.ts | 69 +++ .../routes/public/generate_from_jobparams.ts | 18 +- .../reporting/server/saved_objects/index.ts | 24 ++ .../saved_objects/scheduled_report/index.ts | 9 + .../scheduled_report/mappings.ts | 13 + .../scheduled_report/model_versions.ts | 19 + .../scheduled_report/schemas}/index.ts | 2 +- .../scheduled_report/schemas/latest.ts | 14 + .../scheduled_report/schemas/v1.ts | 40 ++ .../create_mock_reportingplugin.ts | 6 +- .../plugins/private/reporting/server/types.ts | 10 +- .../plugins/private/reporting/tsconfig.json | 1 + .../__snapshots__/oss_features.test.ts.snap | 32 +- .../shared/features/server/oss_features.ts | 8 +- .../shared/task_manager/server/index.ts | 3 +- .../server/saved_objects/index.ts | 3 + .../server/saved_objects/schemas/task.ts | 24 +- .../shared/task_manager/server/task.ts | 10 + .../security_roles_privileges.ts | 155 +++++++ .../services/scenarios.ts | 52 +++ .../spaces_only/telemetry/telemetry.ts | 1 + .../chat/platform_security/authorization.ts | 168 ++++++++ .../platform_security/authorization.ts | 168 ++++++++ .../search/platform_security/authorization.ts | 168 ++++++++ .../platform_security/authorization.ts | 48 +++ 50 files changed, 2419 insertions(+), 341 deletions(-) delete mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.ts rename x-pack/platform/plugins/private/reporting/server/routes/common/{generate/request_handler.test.ts => request_handler/generate_request_handler.test.ts} (82%) create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/index.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report_to_task.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/index.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/index.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/mappings.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/model_versions.ts rename x-pack/platform/plugins/private/reporting/server/{routes/common/generate => saved_objects/scheduled_report/schemas}/index.ts (76%) create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index d063966e90439..4b50f5dd85f61 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -915,6 +915,7 @@ "installCount", "unInstallCount" ], + "scheduled_report": [], "search": [ "description", "title" diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 65f9d60fba09e..7d0dbfc8bff49 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -3034,6 +3034,10 @@ } } }, + "scheduled_report": { + "dynamic": false, + "properties": {} + }, "search": { "dynamic": false, "properties": { diff --git a/src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts index d2288b8c33a2c..9cee0f2bbd203 100644 --- a/src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts @@ -122,6 +122,7 @@ const previouslyRegisteredTypes = [ 'query', 'rules-settings', 'sample-data-telemetry', + 'scheduled_report', 'search', 'search-session', 'search-telemetry', diff --git a/src/platform/packages/private/kbn-reporting/common/routes.ts b/src/platform/packages/private/kbn-reporting/common/routes.ts index 4fb56700cda28..92a6fa0cc5356 100644 --- a/src/platform/packages/private/kbn-reporting/common/routes.ts +++ b/src/platform/packages/private/kbn-reporting/common/routes.ts @@ -26,6 +26,7 @@ export const INTERNAL_ROUTES = { }, HEALTH: prefixInternalPath + '/_health', GENERATE_PREFIX: prefixInternalPath + '/generate', // exportTypeId is added to the final path + SCHEDULE_PREFIX: prefixInternalPath + '/schedule', // exportTypeId is added to the final path }; const prefixPublicPath = '/api/reporting'; diff --git a/src/platform/packages/private/kbn-reporting/common/types.ts b/src/platform/packages/private/kbn-reporting/common/types.ts index a7931e642f9ca..8df1df7c94a2e 100644 --- a/src/platform/packages/private/kbn-reporting/common/types.ts +++ b/src/platform/packages/private/kbn-reporting/common/types.ts @@ -11,7 +11,7 @@ import type { LayoutParams, PerformanceMetrics as ScreenshotMetrics, } from '@kbn/screenshotting-plugin/common'; -import type { ConcreteTaskInstance } from '@kbn/task-manager-plugin/server'; +import type { ConcreteTaskInstance, RruleSchedule } from '@kbn/task-manager-plugin/server'; import { JOB_STATUS } from './constants'; import type { LocatorParams } from './url'; @@ -202,6 +202,15 @@ export interface ReportApiJSON extends ReportSimple { index: string; } +export type ScheduledReportApiJSON = Omit< + ReportSource, + 'payload' | 'output' | 'attempts' | 'status' +> & { + id: string; + schedule: RruleSchedule; + payload: Omit; +}; + export interface LicenseCheckResults { enableLinks: boolean; showLinks: boolean; diff --git a/x-pack/platform/plugins/private/canvas/server/feature.test.ts b/x-pack/platform/plugins/private/canvas/server/feature.test.ts index 7f71bafd41700..6fde4788a9c89 100644 --- a/x-pack/platform/plugins/private/canvas/server/feature.test.ts +++ b/x-pack/platform/plugins/private/canvas/server/feature.test.ts @@ -107,7 +107,9 @@ it('Provides a feature declaration ', () => { "minimumLicense": "gold", "name": "Generate PDF reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -216,7 +218,9 @@ it(`Calls on Reporting whether to include Generate PDF as a sub-feature`, () => "minimumLicense": "gold", "name": "Generate PDF reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ diff --git a/x-pack/platform/plugins/private/canvas/server/feature.ts b/x-pack/platform/plugins/private/canvas/server/feature.ts index daa8a8fc4aa4f..4aa03b73425f4 100644 --- a/x-pack/platform/plugins/private/canvas/server/feature.ts +++ b/x-pack/platform/plugins/private/canvas/server/feature.ts @@ -68,7 +68,7 @@ export function getCanvasFeature(plugins: { reporting?: ReportingStart }): Kiban includeIn: 'all', management: { insightsAndAlerting: ['reporting'] }, minimumLicense: 'gold', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, api: ['generateReport'], ui: ['generatePdf'], }, diff --git a/x-pack/platform/plugins/private/reporting/server/core.ts b/x-pack/platform/plugins/private/reporting/server/core.ts index a29f4303ac1ca..0a51b5bd4ec27 100644 --- a/x-pack/platform/plugins/private/reporting/server/core.ts +++ b/x-pack/platform/plugins/private/reporting/server/core.ts @@ -23,6 +23,7 @@ import type { StatusServiceSetup, UiSettingsServiceStart, } from '@kbn/core/server'; +import { SECURITY_EXTENSION_ID } from '@kbn/core-saved-objects-server'; import type { PluginStart as DataPluginStart } from '@kbn/data-plugin/server'; import type { DiscoverServerPluginStart } from '@kbn/discover-plugin/server'; import type { FeaturesPluginSetup } from '@kbn/features-plugin/server'; @@ -54,6 +55,7 @@ import type { IReport, ReportingStore } from './lib/store'; import { ExecuteReportTask, ReportTaskParams } from './lib/tasks'; import type { ReportingPluginRouter } from './types'; import { EventTracker } from './usage'; +import { SCHEDULED_REPORT_SAVED_OBJECT_TYPE } from './saved_objects'; export interface ReportingInternalSetup { basePath: Pick; @@ -371,6 +373,15 @@ export class ReportingCore { return dataViews; } + public async getSoClient(request: KibanaRequest) { + const { savedObjects } = await this.getPluginStartDeps(); + const savedObjectsClient = savedObjects.getScopedClient(request, { + excludedExtensions: [SECURITY_EXTENSION_ID], + includedHiddenTypes: [SCHEDULED_REPORT_SAVED_OBJECT_TYPE], + }); + return savedObjectsClient; + } + public async getDataService() { const startDeps = await this.getPluginStartDeps(); return startDeps.data; diff --git a/x-pack/platform/plugins/private/reporting/server/plugin.test.ts b/x-pack/platform/plugins/private/reporting/server/plugin.test.ts index 2322794836989..df827abce22ae 100644 --- a/x-pack/platform/plugins/private/reporting/server/plugin.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/plugin.test.ts @@ -76,6 +76,21 @@ describe('Reporting Plugin', () => { ); }); + it('registers a saved object for scheduled reports', async () => { + plugin.setup(coreSetup, pluginSetup); + expect(coreSetup.savedObjects.registerType).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'scheduled_report', + namespaceType: 'multiple', + hidden: true, + indexPattern: '.kibana_alerting_cases', + management: { + importableAndExportable: false, + }, + }) + ); + }); + it('logs start issues', async () => { // wait for the setup phase background work plugin.setup(coreSetup, pluginSetup); diff --git a/x-pack/platform/plugins/private/reporting/server/plugin.ts b/x-pack/platform/plugins/private/reporting/server/plugin.ts index 12bfb3decb805..42d0889b08477 100644 --- a/x-pack/platform/plugins/private/reporting/server/plugin.ts +++ b/x-pack/platform/plugins/private/reporting/server/plugin.ts @@ -27,6 +27,7 @@ import type { import { ReportingRequestHandlerContext } from './types'; import { registerReportingEventTypes, registerReportingUsageCollector } from './usage'; import { registerFeatures } from './features'; +import { setupSavedObjects } from './saved_objects'; /* * @internal @@ -75,6 +76,9 @@ export class ReportingPlugin registerReportingUsageCollector(reportingCore, plugins.usageCollection); registerReportingEventTypes(core); + // Saved objects + setupSavedObjects(core.savedObjects); + // Routes registerRoutes(reportingCore, this.logger); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.ts deleted file mode 100644 index 2ba63a5d52655..0000000000000 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.ts +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import Boom from '@hapi/boom'; -import moment from 'moment'; - -import { schema, TypeOf } from '@kbn/config-schema'; -import type { KibanaRequest, KibanaResponseFactory, Logger } from '@kbn/core/server'; -import { i18n } from '@kbn/i18n'; -import { PUBLIC_ROUTES } from '@kbn/reporting-common'; -import type { BaseParams } from '@kbn/reporting-common/types'; -import { cryptoFactory } from '@kbn/reporting-server'; -import rison from '@kbn/rison'; - -import { type Counters, getCounters } from '..'; -import type { ReportingCore } from '../../..'; -import { checkParamsVersion } from '../../../lib'; -import { Report } from '../../../lib/store'; -import type { - ReportingJobResponse, - ReportingRequestHandlerContext, - ReportingUser, -} from '../../../types'; - -export const handleUnavailable = (res: KibanaResponseFactory) => { - return res.custom({ statusCode: 503, body: 'Not Available' }); -}; - -const validation = { - params: schema.object({ exportType: schema.string({ minLength: 2 }) }), - body: schema.nullable(schema.object({ jobParams: schema.maybe(schema.string()) })), - query: schema.nullable(schema.object({ jobParams: schema.string({ defaultValue: '' }) })), -}; - -/** - * Handles the common parts of requests to generate a report - * Serves report job handling in the context of the request to generate the report - */ -export class RequestHandler { - constructor( - private reporting: ReportingCore, - private user: ReportingUser, - private context: ReportingRequestHandlerContext, - private path: string, - private req: KibanaRequest< - TypeOf<(typeof validation)['params']>, - TypeOf<(typeof validation)['query']>, - TypeOf<(typeof validation)['body']> - >, - private res: KibanaResponseFactory, - private logger: Logger - ) {} - - private async encryptHeaders() { - const { encryptionKey } = this.reporting.getConfig(); - const crypto = cryptoFactory(encryptionKey); - return await crypto.encrypt(this.req.headers); - } - - public async enqueueJob(exportTypeId: string, jobParams: BaseParams) { - const { reporting, logger, context, req, user } = this; - - const exportType = reporting.getExportTypesRegistry().getById(exportTypeId); - - if (exportType == null) { - throw new Error(`Export type ${exportTypeId} does not exist in the registry!`); - } - - const store = await reporting.getStore(); - - if (!exportType.createJob) { - throw new Error(`Export type ${exportTypeId} is not a valid instance!`); - } - - // 1. Ensure the incoming params have a version field (should be set by the UI) - jobParams.version = checkParamsVersion(jobParams, logger); - - // 2. Encrypt request headers to store for the running report job to authenticate itself with Kibana - const headers = await this.encryptHeaders(); - - // 3. Create a payload object by calling exportType.createJob(), and adding some automatic parameters - const job = await exportType.createJob(jobParams, context, req); - - const payload = { - ...job, - headers, - title: job.title, - objectType: jobParams.objectType, - browserTimezone: jobParams.browserTimezone, - version: jobParams.version, - spaceId: reporting.getSpaceId(req, logger), - }; - - // 4. Add the report to ReportingStore to show as pending - const report = await store.addReport( - new Report({ - jobtype: exportType.jobType, - created_by: user ? user.username : false, - payload, - migration_version: jobParams.version, - meta: { - // telemetry fields - objectType: jobParams.objectType, - layout: jobParams.layout?.id, - isDeprecated: job.isDeprecated, - }, - }) - ); - logger.debug(`Successfully stored pending job: ${report._index}/${report._id}`); - - // 5. Schedule the report with Task Manager - const task = await reporting.scheduleTask(req, report.toReportTaskJSON()); - logger.info( - `Scheduled ${exportType.name} reporting task. Task ID: task:${task.id}. Report ID: ${report._id}` - ); - - // 6. Log the action with event log - reporting.getEventLogger(report, task).logScheduleTask(); - return report; - } - - public getJobParams(): BaseParams { - let jobParamsRison: null | string = null; - const req = this.req; - const res = this.res; - - if (req.body) { - const { jobParams: jobParamsPayload } = req.body; - jobParamsRison = jobParamsPayload ? jobParamsPayload : null; - } else if (req.query?.jobParams) { - const { jobParams: queryJobParams } = req.query; - if (queryJobParams) { - jobParamsRison = queryJobParams; - } else { - jobParamsRison = null; - } - } - - if (!jobParamsRison) { - throw res.customError({ - statusCode: 400, - body: 'A jobParams RISON string is required in the querystring or POST body', - }); - } - - let jobParams; - - try { - jobParams = rison.decode(jobParamsRison) as BaseParams | null; - if (!jobParams) { - throw res.customError({ - statusCode: 400, - body: 'Missing jobParams!', - }); - } - } catch (err) { - throw res.customError({ - statusCode: 400, - body: `invalid rison: ${jobParamsRison}`, - }); - } - - return jobParams; - } - - public static getValidation() { - return validation; - } - - public async handleGenerateRequest(exportTypeId: string, jobParams: BaseParams) { - const req = this.req; - const reporting = this.reporting; - - const counters = getCounters( - req.route.method, - this.path.replace(/{exportType}/, exportTypeId), - reporting.getUsageCounter() - ); - - // ensure the async dependencies are loaded - if (!this.context.reporting) { - return handleUnavailable(this.res); - } - - const licenseInfo = await this.reporting.getLicenseInfo(); - const licenseResults = licenseInfo[exportTypeId]; - - if (!licenseResults) { - return this.res.badRequest({ body: `Invalid export-type of ${exportTypeId}` }); - } - - if (!licenseResults.enableLinks) { - return this.res.forbidden({ body: licenseResults.message }); - } - - if (jobParams.browserTimezone && !moment.tz.zone(jobParams.browserTimezone)) { - return this.res.badRequest({ - body: `Invalid timezone "${jobParams.browserTimezone ?? ''}".`, - }); - } - - let report: Report | undefined; - try { - report = await this.enqueueJob(exportTypeId, jobParams); - const { basePath } = this.reporting.getServerInfo(); - const publicDownloadPath = basePath + PUBLIC_ROUTES.JOBS.DOWNLOAD_PREFIX; - - // return task manager's task information and the download URL - counters.usageCounter(); - const eventTracker = reporting.getEventTracker( - report._id, - exportTypeId, - jobParams.objectType - ); - eventTracker?.createReport({ - isDeprecated: Boolean(report.payload.isDeprecated), - isPublicApi: this.path.match(/internal/) === null, - }); - - return this.res.ok({ - headers: { 'content-type': 'application/json' }, - body: { - path: `${publicDownloadPath}/${report._id}`, - job: report.toApiJSON(), - }, - }); - } catch (err) { - return this.handleError(err, counters, report?.jobtype); - } - } - - private handleError(err: Error | Boom.Boom, counters: Counters, jobtype?: string) { - this.logger.error(err); - - if (err instanceof Boom.Boom) { - const statusCode = err.output.statusCode; - counters?.errorCounter(jobtype, statusCode); - - return this.res.customError({ - statusCode, - body: err.output.payload.message, - }); - } - - counters?.errorCounter(jobtype, 500); - - return this.res.customError({ - statusCode: 500, - body: - err?.message || - i18n.translate('xpack.reporting.errorHandler.unknownError', { - defaultMessage: 'Unknown error', - }), - }); - } -} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/jobs/get_job_routes.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/jobs/get_job_routes.ts index dab96944ea6e8..3f2aa4fd5e3b2 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/jobs/get_job_routes.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/jobs/get_job_routes.ts @@ -12,7 +12,7 @@ import { getCounters } from '..'; import { ReportingCore } from '../../..'; import { getContentStream } from '../../../lib'; import { ReportingRequestHandlerContext, ReportingUser } from '../../../types'; -import { handleUnavailable } from '../generate'; +import { handleUnavailable } from '../request_handler'; import { jobManagementPreRouting } from './job_management_pre_routing'; import { jobsQueryFactory } from './jobs_query'; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.test.ts similarity index 82% rename from x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.test.ts rename to x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.test.ts index 2ded83071d6a3..6f866a56ef4c5 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.test.ts @@ -22,7 +22,7 @@ import { ReportingRequestHandlerContext, ReportingSetup, } from '../../../types'; -import { RequestHandler } from './request_handler'; +import { GenerateRequestHandler } from './generate_request_handler'; jest.mock('@kbn/reporting-server/crypto', () => ({ cryptoFactory: () => ({ @@ -68,7 +68,7 @@ describe('Handle request to generate', () => { let mockContext: ReturnType; let mockRequest: ReturnType; let mockResponseFactory: ReturnType; - let requestHandler: RequestHandler; + let requestHandler: GenerateRequestHandler; beforeEach(async () => { reportingCore = await createMockReportingCore(createMockConfigSchema({})); @@ -91,20 +91,23 @@ describe('Handle request to generate', () => { mockContext = getMockContext(); mockContext.reporting = Promise.resolve({} as ReportingSetup); - requestHandler = new RequestHandler( - reportingCore, - { username: 'testymcgee' }, - mockContext, - '/api/reporting/test/generate/pdf', - mockRequest, - mockResponseFactory, - mockLogger - ); + requestHandler = new GenerateRequestHandler({ + reporting: reportingCore, + user: { username: 'testymcgee' }, + context: mockContext, + path: '/api/reporting/test/generate/pdf', + req: mockRequest, + res: mockResponseFactory, + logger: mockLogger, + }); }); describe('Enqueue Job', () => { test('creates a report object to queue', async () => { - const report = await requestHandler.enqueueJob('printablePdfV2', mockJobParams); + const report = await requestHandler.enqueueJob({ + exportTypeId: 'printablePdfV2', + jobParams: mockJobParams, + }); const { _id, created_at: _created_at, payload, ...snapObj } = report; expect(snapObj).toMatchInlineSnapshot(` @@ -157,7 +160,10 @@ describe('Handle request to generate', () => { test('provides a default kibana version field for older POST URLs', async () => { // how do we handle the printable_pdf endpoint that isn't migrating to the class instance of export types? (mockJobParams as unknown as { version?: string }).version = undefined; - const report = await requestHandler.enqueueJob('printablePdfV2', mockJobParams); + const report = await requestHandler.enqueueJob({ + exportTypeId: 'printablePdfV2', + jobParams: mockJobParams, + }); const { _id, created_at: _created_at, ...snapObj } = report; expect(snapObj.payload.version).toBe('7.14.0'); @@ -206,10 +212,14 @@ describe('Handle request to generate', () => { }); }); - describe('handleGenerateRequest', () => { + describe('handleRequest', () => { test('disallows invalid export type', async () => { - expect(await requestHandler.handleGenerateRequest('neanderthals', mockJobParams)) - .toMatchInlineSnapshot(` + expect( + await requestHandler.handleRequest({ + exportTypeId: 'neanderthals', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` Object { "body": "Invalid export-type of neanderthals", } @@ -224,8 +234,12 @@ describe('Handle request to generate', () => { }, })); - expect(await requestHandler.handleGenerateRequest('csv_searchsource', mockJobParams)) - .toMatchInlineSnapshot(` + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` Object { "body": "seeing this means the license isn't supported", } @@ -233,30 +247,26 @@ describe('Handle request to generate', () => { }); test('disallows invalid browser timezone', async () => { - (reportingCore.getLicenseInfo as jest.Mock) = jest.fn(() => ({ - csv_searchsource: { - enableLinks: false, - message: `seeing this means the license isn't supported`, - }, - })); - expect( - await requestHandler.handleGenerateRequest('csv_searchsource', { - ...mockJobParams, - browserTimezone: 'America/Amsterdam', + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: { + ...mockJobParams, + browserTimezone: 'America/Amsterdam', + }, }) ).toMatchInlineSnapshot(` Object { - "body": "seeing this means the license isn't supported", + "body": "Invalid timezone \\"America/Amsterdam\\".", } `); }); test('generates the download path', async () => { - const { body } = (await requestHandler.handleGenerateRequest( - 'csv_searchsource', - mockJobParams - )) as unknown as { body: ReportingJobResponse }; + const { body } = (await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + })) as unknown as { body: ReportingJobResponse }; expect(body.path).toMatch('/mock-server-basepath/api/reporting/jobs/download/mock-report-id'); }); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.ts new file mode 100644 index 0000000000000..98df5a62533e9 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { PUBLIC_ROUTES } from '@kbn/reporting-common'; +import { getCounters } from '..'; +import { Report, SavedReport } from '../../../lib/store'; +import type { ReportingJobResponse } from '../../../types'; +import { RequestHandler, RequestParams } from './request_handler'; + +const validation = { + params: schema.object({ exportType: schema.string({ minLength: 2 }) }), + body: schema.nullable(schema.object({ jobParams: schema.maybe(schema.string()) })), + query: schema.nullable(schema.object({ jobParams: schema.string({ defaultValue: '' }) })), +}; + +/** + * Handles the common parts of requests to generate a report + * Serves report job handling in the context of the request to generate the report + */ +export class GenerateRequestHandler extends RequestHandler< + (typeof validation)['params'], + (typeof validation)['query'], + (typeof validation)['body'], + SavedReport +> { + public static getValidation() { + return validation; + } + + public async enqueueJob(params: RequestParams) { + const { exportTypeId, jobParams } = params; + const { reporting, logger, req, user } = this.opts; + + const store = await reporting.getStore(); + const { version, job, jobType, name } = await this.createJob(exportTypeId, jobParams); + + // Encrypt request headers to store for the running report job to authenticate itself with Kibana + const headers = await this.encryptHeaders(); + + const payload = { + ...job, + headers, + title: job.title, + objectType: jobParams.objectType, + browserTimezone: jobParams.browserTimezone, + version, + spaceId: reporting.getSpaceId(req, logger), + }; + + // Add the report to ReportingStore to show as pending + const report = await store.addReport( + new Report({ + jobtype: jobType, + created_by: user ? user.username : false, + payload, + migration_version: version, + meta: { + // telemetry fields + objectType: jobParams.objectType, + layout: jobParams.layout?.id, + isDeprecated: job.isDeprecated, + }, + }) + ); + logger.debug(`Successfully stored pending job: ${report._index}/${report._id}`); + + // Schedule the report with Task Manager + const task = await reporting.scheduleTask(req, report.toReportTaskJSON()); + logger.info( + `Scheduled ${name} reporting task. Task ID: task:${task.id}. Report ID: ${report._id}` + ); + + // Log the action with event log + reporting.getEventLogger(report, task).logScheduleTask(); + return report; + } + + public async handleRequest(params: RequestParams) { + const { exportTypeId, jobParams } = params; + const { reporting, req, res, path } = this.opts; + + const counters = getCounters( + req.route.method, + path.replace(/{exportType}/, exportTypeId), + reporting.getUsageCounter() + ); + + const checkErrorResponse = await this.checkLicenseAndTimezone( + exportTypeId, + jobParams.browserTimezone + ); + if (checkErrorResponse) { + return checkErrorResponse; + } + + let report: Report | undefined; + try { + report = await this.enqueueJob(params); + const { basePath } = reporting.getServerInfo(); + const publicDownloadPath = basePath + PUBLIC_ROUTES.JOBS.DOWNLOAD_PREFIX; + + // return task manager's task information and the download URL + counters.usageCounter(); + const eventTracker = reporting.getEventTracker( + report._id, + exportTypeId, + jobParams.objectType + ); + eventTracker?.createReport({ + isDeprecated: Boolean(report.payload.isDeprecated), + isPublicApi: path.match(/internal/) === null, + }); + + return res.ok({ + headers: { 'content-type': 'application/json' }, + body: { + path: `${publicDownloadPath}/${report._id}`, + job: report.toApiJSON(), + }, + }); + } catch (err) { + return this.handleError(err, counters, report?.jobtype); + } + } +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/index.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/index.ts new file mode 100644 index 0000000000000..185b6ec86c37a --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { handleUnavailable } from './request_handler'; +export { GenerateRequestHandler } from './generate_request_handler'; +export { ScheduleRequestHandler } from './schedule_request_handler'; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts new file mode 100644 index 0000000000000..256f5046051f1 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { transformRawScheduledReportToReport } from './transform_raw_scheduled_report'; +export { transformRawScheduledReportToTaskParams } from './transform_raw_scheduled_report_to_task'; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts new file mode 100644 index 0000000000000..9727895036a64 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObject } from '@kbn/core/server'; +import { ScheduledReportApiJSON } from '@kbn/reporting-common/types'; +import { RawScheduledReport } from '../../../../saved_objects/scheduled_report/schemas/latest'; + +export function transformRawScheduledReportToReport( + rawScheduledReport: SavedObject +): ScheduledReportApiJSON { + const parsedPayload = JSON.parse(rawScheduledReport.attributes.payload); + return { + id: rawScheduledReport.id, + jobtype: rawScheduledReport.attributes.jobType, + created_at: rawScheduledReport.attributes.createdAt, + created_by: rawScheduledReport.attributes.createdBy as string | false, + payload: parsedPayload, + meta: rawScheduledReport.attributes.meta, + migration_version: rawScheduledReport.attributes.migrationVersion, + schedule: rawScheduledReport.attributes.schedule, + }; +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report_to_task.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report_to_task.ts new file mode 100644 index 0000000000000..b2d9598ef9bd2 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report_to_task.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObject } from '@kbn/core/server'; +import { ScheduledReportTaskParams } from '../../../../lib/tasks'; +import { RawScheduledReport } from '../../../../saved_objects/scheduled_report/schemas/latest'; + +export function transformRawScheduledReportToTaskParams( + rawScheduledReport: SavedObject +): ScheduledReportTaskParams { + return { + id: rawScheduledReport.id, + jobtype: rawScheduledReport.attributes.jobType, + schedule: rawScheduledReport.attributes.schedule, + }; +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts new file mode 100644 index 0000000000000..b086fc41912d6 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import moment from 'moment'; +import { schema, TypeOf } from '@kbn/config-schema'; +import type { + IKibanaResponse, + KibanaRequest, + KibanaResponseFactory, + Logger, +} from '@kbn/core/server'; +import { i18n } from '@kbn/i18n'; +import type { BaseParams } from '@kbn/reporting-common/types'; +import { cryptoFactory } from '@kbn/reporting-server'; +import rison from '@kbn/rison'; + +import { RruleSchedule } from '@kbn/task-manager-plugin/server'; +import { checkParamsVersion } from '../../../lib'; +import { type Counters } from '..'; +import type { ReportingCore } from '../../..'; +import type { ReportingRequestHandlerContext, ReportingUser } from '../../../types'; + +export const handleUnavailable = (res: KibanaResponseFactory) => { + return res.custom({ statusCode: 503, body: 'Not Available' }); +}; + +const ParamsValidation = schema.recordOf(schema.string(), schema.string()); +const QueryValidation = schema.nullable( + schema.recordOf(schema.string(), schema.maybe(schema.string())) +); +const BodyValidation = schema.nullable( + schema.recordOf(schema.string(), schema.maybe(schema.any())) +); + +interface ConstructorOpts< + Params extends typeof ParamsValidation, + Query extends typeof QueryValidation, + Body extends typeof BodyValidation +> { + reporting: ReportingCore; + user: ReportingUser; + context: ReportingRequestHandlerContext; + path: string; + req: KibanaRequest, TypeOf, TypeOf>; + res: KibanaResponseFactory; + logger: Logger; +} + +export interface RequestParams { + exportTypeId: string; + jobParams: BaseParams; + schedule?: RruleSchedule; +} + +/** + * Handles the common parts of requests to generate or schedule a report + * Serves report job handling in the context of the request to generate the report + */ +export abstract class RequestHandler< + Params extends typeof ParamsValidation, + Query extends typeof QueryValidation, + Body extends typeof BodyValidation, + Output extends Record +> { + constructor(protected readonly opts: ConstructorOpts) {} + + public static getValidation() { + throw new Error('getValidation() must be implemented in a subclass'); + } + + public abstract enqueueJob(params: RequestParams): Promise; + + public abstract handleRequest(params: RequestParams): Promise; + + public getJobParams(): BaseParams { + let jobParamsRison: null | string = null; + const req = this.opts.req; + const res = this.opts.res; + + if (req.body) { + const { jobParams: jobParamsPayload } = req.body; + jobParamsRison = jobParamsPayload ? jobParamsPayload : null; + } else if (req.query?.jobParams) { + const { jobParams: queryJobParams } = req.query; + if (queryJobParams) { + jobParamsRison = queryJobParams; + } else { + jobParamsRison = null; + } + } + + if (!jobParamsRison) { + throw res.customError({ + statusCode: 400, + body: 'A jobParams RISON string is required in the querystring or POST body', + }); + } + + let jobParams; + + try { + jobParams = rison.decode(jobParamsRison) as BaseParams | null; + if (!jobParams) { + throw res.customError({ + statusCode: 400, + body: 'Missing jobParams!', + }); + } + } catch (err) { + throw res.customError({ + statusCode: 400, + body: `invalid rison: ${jobParamsRison}`, + }); + } + + return jobParams; + } + + protected async createJob(exportTypeId: string, jobParams: BaseParams) { + const exportType = this.opts.reporting.getExportTypesRegistry().getById(exportTypeId); + + if (exportType == null) { + throw new Error(`Export type ${exportTypeId} does not exist in the registry!`); + } + + if (!exportType.createJob) { + throw new Error(`Export type ${exportTypeId} is not a valid instance!`); + } + + // 1. Ensure the incoming params have a version field (should be set by the UI) + const version = checkParamsVersion(jobParams, this.opts.logger); + + // 2. Create a payload object by calling exportType.createJob(), and adding some automatic parameters + const job = await exportType.createJob(jobParams, this.opts.context, this.opts.req); + + return { job, version, jobType: exportType.jobType, name: exportType.name }; + } + + protected async checkLicenseAndTimezone( + exportTypeId: string, + browserTimezone: string + ): Promise { + const { reporting, context, res } = this.opts; + + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + + const licenseInfo = await reporting.getLicenseInfo(); + const licenseResults = licenseInfo[exportTypeId]; + + if (!licenseResults) { + return res.badRequest({ body: `Invalid export-type of ${exportTypeId}` }); + } + + if (!licenseResults.enableLinks) { + return res.forbidden({ body: licenseResults.message }); + } + + if (browserTimezone && !moment.tz.zone(browserTimezone)) { + return res.badRequest({ + body: `Invalid timezone "${browserTimezone ?? ''}".`, + }); + } + + return null; + } + + protected async encryptHeaders() { + const { encryptionKey } = this.opts.reporting.getConfig(); + const crypto = cryptoFactory(encryptionKey); + return await crypto.encrypt(this.opts.req.headers); + } + + protected handleError(err: Error | Boom.Boom, counters?: Counters, jobtype?: string) { + this.opts.logger.error(err); + + if (err instanceof Boom.Boom) { + const statusCode = err.output.statusCode; + counters?.errorCounter(jobtype, statusCode); + + return this.opts.res.customError({ + statusCode, + body: err.output.payload.message, + }); + } + + counters?.errorCounter(jobtype, 500); + + return this.opts.res.customError({ + statusCode: 500, + body: + err?.message || + i18n.translate('xpack.reporting.errorHandler.unknownError', { + defaultMessage: 'Unknown error', + }), + }); + } +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts new file mode 100644 index 0000000000000..8611a3a27bea3 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts @@ -0,0 +1,399 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +jest.mock('uuid', () => ({ v4: () => 'mock-report-id' })); + +import rison from '@kbn/rison'; + +import { + FakeRawRequest, + KibanaRequest, + KibanaResponseFactory, + SavedObjectsClientContract, +} from '@kbn/core/server'; +import { coreMock, httpServerMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { JobParamsPDFV2, TaskPayloadPDFV2 } from '@kbn/reporting-export-types-pdf-common'; +import { createMockConfigSchema } from '@kbn/reporting-mocks-server'; +import { ReportingCore } from '../../..'; +import { createMockReportingCore } from '../../../test_helpers'; +import { ReportingRequestHandlerContext, ReportingSetup } from '../../../types'; +import { ScheduleRequestHandler } from './schedule_request_handler'; + +const getMockContext = () => + ({ + core: coreMock.createRequestHandlerContext(), + } as unknown as ReportingRequestHandlerContext); + +const getMockRequest = () => + ({ + url: { port: '5601', search: '', pathname: '/foo' }, + route: { path: '/foo', options: {} }, + } as KibanaRequest); + +const getMockResponseFactory = () => + ({ + ...httpServerMock.createResponseFactory(), + forbidden: (obj: unknown) => obj, + unauthorized: (obj: unknown) => obj, + customError: (err: unknown) => err, + } as unknown as KibanaResponseFactory); + +const mockLogger = loggingSystemMock.createLogger(); +const mockJobParams: JobParamsPDFV2 = { + browserTimezone: 'UTC', + objectType: 'cool_object_type', + title: 'cool_title', + version: 'unknown', + layout: { id: 'preserve_layout' }, + locatorParams: [], +}; + +const fakeRawRequest: FakeRawRequest = { + headers: { + authorization: `ApiKey skdjtq4u543yt3rhewrh`, + }, + path: '/', +}; + +describe('Handle request to schedule', () => { + let reportingCore: ReportingCore; + let mockContext: ReturnType; + let mockRequest: ReturnType; + let mockResponseFactory: ReturnType; + let requestHandler: ScheduleRequestHandler; + let soClient: SavedObjectsClientContract; + + beforeEach(async () => { + reportingCore = await createMockReportingCore(createMockConfigSchema({})); + + mockRequest = getMockRequest(); + + mockResponseFactory = getMockResponseFactory(); + (mockResponseFactory.ok as jest.Mock) = jest.fn((args: unknown) => args); + (mockResponseFactory.forbidden as jest.Mock) = jest.fn((args: unknown) => args); + (mockResponseFactory.badRequest as jest.Mock) = jest.fn((args: unknown) => args); + + mockContext = getMockContext(); + mockContext.reporting = Promise.resolve({} as ReportingSetup); + + soClient = await reportingCore.getSoClient(fakeRawRequest as unknown as KibanaRequest); + soClient.create = jest.fn().mockImplementation(async (_, opts) => { + return { + id: 'foo', + attributes: opts, + type: 'scheduled-report', + }; + }); + + requestHandler = new ScheduleRequestHandler({ + reporting: reportingCore, + user: { username: 'testymcgee' }, + context: mockContext, + path: '/api/reporting/test/generate/pdf', + // @ts-ignore + req: mockRequest, + res: mockResponseFactory, + logger: mockLogger, + }); + }); + + describe('enqueueJob', () => { + test('creates a scheduled report saved object', async () => { + const report = await requestHandler.enqueueJob({ + exportTypeId: 'printablePdfV2', + jobParams: mockJobParams, + schedule: { rrule: { freq: 1, interval: 2, tzid: 'UTC' } }, + }); + + const { id, created_at: _created_at, payload, ...snapObj } = report; + expect(snapObj).toMatchInlineSnapshot(` + Object { + "created_by": "testymcgee", + "jobtype": "printable_pdf_v2", + "meta": Object { + "isDeprecated": false, + "layout": "preserve_layout", + "objectType": "cool_object_type", + }, + "migration_version": "unknown", + "schedule": Object { + "rrule": Object { + "freq": 1, + "interval": 2, + "tzid": "UTC", + }, + }, + } + `); + const { forceNow, ...snapPayload } = payload as TaskPayloadPDFV2; + expect(snapPayload).toMatchInlineSnapshot(` + Object { + "browserTimezone": "UTC", + "isDeprecated": false, + "layout": Object { + "id": "preserve_layout", + }, + "locatorParams": Array [], + "objectType": "cool_object_type", + "title": "cool_title", + "version": "unknown", + } + `); + + expect(soClient.create).toHaveBeenCalledWith( + 'scheduled_report', + { + jobType: 'printable_pdf_v2', + createdAt: expect.any(String), + createdBy: 'testymcgee', + title: 'cool_title', + payload: expect.any(String), + schedule: { + rrule: { + freq: 1, + interval: 2, + }, + }, + migrationVersion: 'unknown', + meta: { + objectType: 'cool_object_type', + layout: 'preserve_layout', + isDeprecated: false, + }, + }, + {} + ); + }); + }); + + describe('getJobParams', () => { + test('parse jobParams from body', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { jobParams: rison.encode(mockJobParams) }; + expect(requestHandler.getJobParams()).toEqual(mockJobParams); + }); + + test('handles missing job params', () => { + let error: { statusCode: number; body: string } | undefined; + try { + requestHandler.getJobParams(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + }); + + test('handles null job params', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { jobParams: rison.encode(null) }; + requestHandler.getJobParams(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + }); + + test('handles invalid rison', () => { + let error: { statusCode: number; body: string } | undefined; + // @ts-ignore body is a read-only property + mockRequest.body = { jobParams: mockJobParams }; + try { + requestHandler.getJobParams(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + }); + }); + + describe('getSchedule', () => { + test('parse schedule from body', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + }; + expect(requestHandler.getSchedule()).toEqual({ rrule: { freq: 1, interval: 2 } }); + }); + + test('handles missing schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + + test('handles null schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: null, + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + + test('handles empty schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: {}, + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + + test('handles null rrule schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: null }, + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + + test('handles empty rrule schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: {} }, + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + }); + + describe('handleRequest', () => { + test('disallows invalid export type', async () => { + expect( + await requestHandler.handleRequest({ + exportTypeId: 'neanderthals', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "Invalid export-type of neanderthals", + } + `); + }); + + test('disallows unsupporting license', async () => { + (reportingCore.getLicenseInfo as jest.Mock) = jest.fn(() => ({ + csv_searchsource: { + enableLinks: false, + message: `seeing this means the license isn't supported`, + }, + })); + + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "seeing this means the license isn't supported", + } + `); + }); + + test('disallows invalid browser timezone', async () => { + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: { + ...mockJobParams, + browserTimezone: 'America/Amsterdam', + }, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "Invalid timezone \\"America/Amsterdam\\".", + } + `); + }); + + test('disallows scheduling when reportingHealth.hasPermanentEncryptionKey = false', async () => { + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ + isSufficientlySecure: true, + hasPermanentEncryptionKey: false, + }); + + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "Permanent encryption key must be set for scheduled reporting", + } + `); + }); + + test('disallows scheduling when reportingHealth.isSufficientlySecure=false', async () => { + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ + isSufficientlySecure: false, + hasPermanentEncryptionKey: true, + }); + + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "Security and API keys must be enabled for scheduled reporting", + } + `); + }); + }); +}); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts new file mode 100644 index 0000000000000..5e1d52267cec2 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +import { schema } from '@kbn/config-schema'; +import { ScheduledReportApiJSON } from '@kbn/reporting-common/types'; +import { isEmpty } from 'lodash'; +import { RruleSchedule, scheduleRruleSchema } from '@kbn/task-manager-plugin/server'; +import { rawNotificationSchema } from '../../../saved_objects/scheduled_report/schemas/v1'; +import { ScheduledReportingJobResponse } from '../../../types'; +import { RawScheduledReport } from '../../../saved_objects/scheduled_report/schemas/latest'; +import { SCHEDULED_REPORT_SAVED_OBJECT_TYPE } from '../../../saved_objects'; +import { RequestHandler, RequestParams } from './request_handler'; +import { transformRawScheduledReportToReport } from './lib'; + +const validation = { + params: schema.object({ exportType: schema.string({ minLength: 2 }) }), + body: schema.object({ + schedule: scheduleRruleSchema, + notification: schema.maybe(rawNotificationSchema), + jobParams: schema.string(), + }), + query: schema.nullable(schema.object({})), +}; + +/** + * Handles the common parts of requests to generate a report + * Serves report job handling in the context of the request to generate the report + */ +export class ScheduleRequestHandler extends RequestHandler< + (typeof validation)['params'], + (typeof validation)['query'], + (typeof validation)['body'], + ScheduledReportApiJSON +> { + public static getValidation() { + return validation; + } + + public getSchedule(): RruleSchedule { + let rruleDef: null | RruleSchedule['rrule'] = null; + const req = this.opts.req; + const res = this.opts.res; + + const { schedule } = req.body; + const { rrule } = schedule ?? {}; + rruleDef = rrule; + + if (isEmpty(rruleDef)) { + throw res.customError({ + statusCode: 400, + body: 'A schedule is required to create a scheduled report.', + }); + } + + return schedule; + } + + public async enqueueJob(params: RequestParams) { + const { exportTypeId, jobParams, schedule } = params; + const { reporting, logger, req, user } = this.opts; + + const soClient = await reporting.getSoClient(req); + const { version, job, jobType, name } = await this.createJob(exportTypeId, jobParams); + + const payload = { + ...job, + title: job.title, + objectType: jobParams.objectType, + browserTimezone: jobParams.browserTimezone, + version, + spaceId: reporting.getSpaceId(req, logger), + }; + + // TODO - extract saved object references before persisting + + const attributes = { + createdAt: moment.utc().toISOString(), + createdBy: user ? user.username : false, + enabled: true, + jobType, + meta: { + // telemetry fields + isDeprecated: job.isDeprecated, + layout: jobParams.layout?.id, + objectType: jobParams.objectType, + }, + migrationVersion: version, + title: job.title, + payload: JSON.stringify(payload), + schedule: schedule!, + }; + + // Create a scheduled report saved object + const report = await soClient.create( + SCHEDULED_REPORT_SAVED_OBJECT_TYPE, + attributes + ); + logger.debug(`Successfully created scheduled report: ${report.id}`); + + // Schedule the report with Task Manager + // const task = await reporting.scheduleRecurringTask( + // req, + // transformRawScheduledReportToTaskParams(report) + // ); + // logger.info( + // `Scheduled "${name}" reporting task. Task ID: task:${task.id}. Report ID: ${report.id}` + // ); + + return transformRawScheduledReportToReport(report); + } + + public async handleRequest(params: RequestParams) { + const { exportTypeId, jobParams } = params; + const { reporting, res } = this.opts; + + const checkErrorResponse = await this.checkLicenseAndTimezone( + exportTypeId, + jobParams.browserTimezone + ); + if (checkErrorResponse) { + return checkErrorResponse; + } + + // check that security requirements are met + const reportingHealth = await reporting.getHealthInfo(); + if (!reportingHealth.hasPermanentEncryptionKey) { + return res.forbidden({ + body: `Permanent encryption key must be set for scheduled reporting`, + }); + } + if (!reportingHealth.isSufficientlySecure) { + return res.forbidden({ + body: `Security and API keys must be enabled for scheduled reporting`, + }); + } + + let report: ScheduledReportApiJSON | undefined; + try { + report = await this.enqueueJob(params); + return res.ok({ + headers: { 'content-type': 'application/json' }, + body: { + job: report, + }, + }); + } catch (err) { + return this.handleError(err, undefined, report?.jobtype); + } + } +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/index.ts b/x-pack/platform/plugins/private/reporting/server/routes/index.ts index f9fbd12802e26..31414587802f4 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/index.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/index.ts @@ -11,6 +11,7 @@ import { registerDeprecationsRoutes } from './internal/deprecations/deprecations import { registerDiagnosticRoutes } from './internal/diagnostic'; import { registerHealthRoute } from './internal/health'; import { registerGenerationRoutesInternal } from './internal/generate/generate_from_jobparams'; +import { registerScheduleRoutesInternal } from './internal/schedule/schedule_from_jobparams'; import { registerJobInfoRoutesInternal } from './internal/management/jobs'; import { registerGenerationRoutesPublic } from './public/generate_from_jobparams'; import { registerJobInfoRoutesPublic } from './public/jobs'; @@ -20,6 +21,7 @@ export function registerRoutes(reporting: ReportingCore, logger: Logger) { registerHealthRoute(reporting, logger); registerDiagnosticRoutes(reporting, logger); registerGenerationRoutesInternal(reporting, logger); + registerScheduleRoutesInternal(reporting, logger); registerJobInfoRoutesInternal(reporting); registerGenerationRoutesPublic(reporting, logger); registerJobInfoRoutesPublic(reporting); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/generate/generate_from_jobparams.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/generate/generate_from_jobparams.ts index bd26c88bf6a0a..2d762ebf1cdb7 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/generate/generate_from_jobparams.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/generate/generate_from_jobparams.ts @@ -10,7 +10,7 @@ import type { Logger } from '@kbn/core/server'; import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import type { ReportingCore } from '../../..'; import { authorizedUserPreRouting } from '../../common'; -import { RequestHandler } from '../../common/generate'; +import { GenerateRequestHandler } from '../../common/request_handler'; const { GENERATE_PREFIX } = INTERNAL_ROUTES; @@ -30,7 +30,7 @@ export function registerGenerationRoutesInternal(reporting: ReportingCore, logge requiredPrivileges: kibanaAccessControlTags, }, }, - validate: RequestHandler.getValidation(), + validate: GenerateRequestHandler.getValidation(), options: { tags: kibanaAccessControlTags.map((accessControlTag) => `access:${accessControlTag}`), access: 'internal', @@ -38,17 +38,20 @@ export function registerGenerationRoutesInternal(reporting: ReportingCore, logge }, authorizedUserPreRouting(reporting, async (user, context, req, res) => { try { - const requestHandler = new RequestHandler( + const requestHandler = new GenerateRequestHandler({ reporting, user, context, path, req, res, - logger - ); + logger, + }); const jobParams = requestHandler.getJobParams(); - return await requestHandler.handleGenerateRequest(req.params.exportType, jobParams); + return await requestHandler.handleRequest({ + exportTypeId: req.params.exportType, + jobParams, + }); } catch (err) { if (err instanceof KibanaResponse) { return err; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/management/jobs.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/management/jobs.ts index 492d579727c51..bfb4c3a59312f 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/management/jobs.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/management/jobs.ts @@ -10,7 +10,7 @@ import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import { ROUTE_TAG_CAN_REDIRECT } from '@kbn/security-plugin/server'; import { ReportingCore } from '../../..'; import { authorizedUserPreRouting, getCounters } from '../../common'; -import { handleUnavailable } from '../../common/generate'; +import { handleUnavailable } from '../../common/request_handler'; import { commonJobsRouteHandlerFactory, jobManagementPreRouting, diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts new file mode 100644 index 0000000000000..8fa8414e9ea41 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts @@ -0,0 +1,325 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import rison from '@kbn/rison'; +import { BehaviorSubject } from 'rxjs'; +import supertest from 'supertest'; + +import { setupServer } from '@kbn/core-test-helpers-test-utils'; +import { coreMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import { INTERNAL_ROUTES } from '@kbn/reporting-common'; +import { PdfExportType } from '@kbn/reporting-export-types-pdf'; +import { createMockConfigSchema } from '@kbn/reporting-mocks-server'; +import { ExportTypesRegistry } from '@kbn/reporting-server/export_types_registry'; + +import { ReportingCore } from '../../../..'; +import { reportingMock } from '../../../../mocks'; +import { + createMockPluginSetup, + createMockPluginStart, + createMockReportingCore, +} from '../../../../test_helpers'; +import { ReportingRequestHandlerContext } from '../../../../types'; +import { registerScheduleRoutesInternal } from '../schedule_from_jobparams'; +import { FakeRawRequest, KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server'; + +type SetupServerReturn = Awaited>; + +const fakeRawRequest: FakeRawRequest = { + headers: { + authorization: `ApiKey skdjtq4u543yt3rhewrh`, + }, + path: '/', +}; + +describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { + const reportingSymbol = Symbol('reporting'); + let server: SetupServerReturn['server']; + let httpSetup: SetupServerReturn['httpSetup']; + let mockExportTypesRegistry: ExportTypesRegistry; + let reportingCore: ReportingCore; + let soClient: SavedObjectsClientContract; + + const mockConfigSchema = createMockConfigSchema({ + queue: { indexInterval: 'year', timeout: 10000, pollEnabled: true }, + }); + + const mockLogger = loggingSystemMock.createLogger(); + const mockCoreSetup = coreMock.createSetup(); + + const mockPdfExportType = new PdfExportType( + mockCoreSetup, + mockConfigSchema, + mockLogger, + coreMock.createPluginInitializerContext(mockConfigSchema) + ); + + beforeEach(async () => { + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext( + reportingSymbol, + 'reporting', + () => reportingMock.createStart() + ); + + const mockSetupDeps = createMockPluginSetup({ + security: { license: { isEnabled: () => true, getFeature: () => true } }, + router: httpSetup.createRouter(''), + }); + + const mockStartDeps = await createMockPluginStart( + { + licensing: { + ...licensingMock.createStart(), + license$: new BehaviorSubject({ + isActive: true, + isAvailable: true, + type: 'gold', + getFeature: () => true, + }), + }, + securityService: { + authc: { + apiKeys: { areAPIKeysEnabled: () => true }, + getCurrentUser: () => ({ id: '123', roles: ['superuser'], username: 'Tom Riddle' }), + }, + }, + }, + mockConfigSchema + ); + + reportingCore = await createMockReportingCore(mockConfigSchema, mockSetupDeps, mockStartDeps); + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ + isSufficientlySecure: true, + hasPermanentEncryptionKey: true, + }); + + mockExportTypesRegistry = new ExportTypesRegistry(); + mockExportTypesRegistry.register(mockPdfExportType); + + soClient = await reportingCore.getSoClient(fakeRawRequest as unknown as KibanaRequest); + soClient.create = jest.fn().mockImplementation(async (_, opts) => { + return { + id: 'foo', + attributes: opts, + type: 'scheduled-report', + }; + }); + }); + + afterEach(async () => { + await server.stop(); + }); + + it('returns 400 if there are no job params', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + '"[request body]: expected a plain object value, but found [null] instead."' + ) + ); + }); + + it('returns 400 if job params body is invalid', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ jobParams: `foo:`, schedule: { rrule: { freq: 1, interval: 2 } } }) + .expect(400) + .then(({ body }) => expect(body.message).toMatchInlineSnapshot('"invalid rison: foo:"')); + }); + + it('returns 400 export type is invalid', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/TonyHawksProSkater2`) + .send({ + schedule: { rrule: { freq: 1, interval: 2 } }, + jobParams: rison.encode({ title: `abc` }), + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot('"Invalid export-type of TonyHawksProSkater2"') + ); + }); + + it('returns 400 on invalid browser timezone', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot(`"Invalid timezone \\"America/Amsterdam\\"."`) + ); + }); + + it('returns 400 on invalid rrule', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), + schedule: { rrule: { freq: 6, interval: 2 } }, + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot(`"Invalid timezone \\"America/Amsterdam\\"."`) + ); + }); + + it('returns 400 on invalid notification list', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), + schedule: { rrule: { freq: 6, interval: 2 } }, + notification: { + email: { + to: 'single@email.com', + }, + }, + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + `"[request body.notification.email.to]: could not parse array value from json input"` + ) + ); + }); + + it('returns 403 on when no permanent encryption key', async () => { + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValueOnce({ + isSufficientlySecure: true, + hasPermanentEncryptionKey: false, + }); + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(403) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + `"Permanent encryption key must be set for scheduled reporting"` + ) + ); + }); + + it('returns 403 on when not sufficiently secure', async () => { + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValueOnce({ + isSufficientlySecure: false, + hasPermanentEncryptionKey: true, + }); + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(403) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + `"Security and API keys must be enabled for scheduled reporting"` + ) + ); + }); + + it('returns 500 if job handler throws an error', async () => { + soClient.create = jest.fn().mockRejectedValue('silly'); + + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(500); + }); + + it(`returns 200 if job handler doesn't error`, async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ + title: `abc`, + layout: { id: 'test' }, + objectType: 'canvas workpad', + }), + notification: { + email: { + to: ['single@email.com'], + }, + }, + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(200) + .then(({ body }) => { + expect(body).toMatchObject({ + job: { + created_by: 'Tom Riddle', + id: 'foo', + jobtype: 'printable_pdf_v2', + payload: { + forceNow: expect.any(String), + isDeprecated: false, + layout: { + id: 'test', + }, + objectType: 'canvas workpad', + title: 'abc', + version: '7.14.0', + }, + schedule: { rrule: { freq: 1, interval: 2 } }, + }, + }); + }); + }); +}); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts new file mode 100644 index 0000000000000..79c408104444b --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaResponse } from '@kbn/core-http-router-server-internal'; +import type { Logger } from '@kbn/core/server'; +import { INTERNAL_ROUTES } from '@kbn/reporting-common'; +import type { ReportingCore } from '../../..'; +import { authorizedUserPreRouting } from '../../common'; +import { ScheduleRequestHandler } from '../../common/request_handler'; + +const { SCHEDULE_PREFIX } = INTERNAL_ROUTES; + +export function registerScheduleRoutesInternal(reporting: ReportingCore, logger: Logger) { + const setupDeps = reporting.getPluginSetupDeps(); + const { router } = setupDeps; + + const kibanaAccessControlTags = ['generateReport']; + + const registerInternalPostScheduleEndpoint = () => { + const path = `${SCHEDULE_PREFIX}/{exportType}`; + router.post( + { + path, + security: { + authz: { + requiredPrivileges: kibanaAccessControlTags, + }, + }, + validate: ScheduleRequestHandler.getValidation(), + options: { + tags: kibanaAccessControlTags.map((accessControlTag) => `access:${accessControlTag}`), + access: 'internal', + }, + }, + authorizedUserPreRouting(reporting, async (user, context, req, res) => { + try { + const requestHandler = new ScheduleRequestHandler({ + reporting, + user, + context, + path, + req, + res, + logger, + }); + const jobParams = requestHandler.getJobParams(); + const schedule = requestHandler.getSchedule(); + + return await requestHandler.handleRequest({ + exportTypeId: req.params.exportType, + jobParams, + schedule, + }); + } catch (err) { + if (err instanceof KibanaResponse) { + return err; + } + throw err; + } + }) + ); + }; + + registerInternalPostScheduleEndpoint(); +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/public/generate_from_jobparams.ts b/x-pack/platform/plugins/private/reporting/server/routes/public/generate_from_jobparams.ts index f547faa9cab52..34507cf79d56c 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/public/generate_from_jobparams.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/public/generate_from_jobparams.ts @@ -10,7 +10,7 @@ import type { Logger } from '@kbn/core/server'; import { PUBLIC_ROUTES } from '@kbn/reporting-common'; import type { ReportingCore } from '../..'; import { authorizedUserPreRouting } from '../common'; -import { RequestHandler } from '../common/generate'; +import { GenerateRequestHandler } from '../common/request_handler'; export function registerGenerationRoutesPublic(reporting: ReportingCore, logger: Logger) { const setupDeps = reporting.getPluginSetupDeps(); @@ -28,7 +28,7 @@ export function registerGenerationRoutesPublic(reporting: ReportingCore, logger: requiredPrivileges: kibanaAccessControlTags, }, }, - validate: RequestHandler.getValidation(), + validate: GenerateRequestHandler.getValidation(), options: { tags: kibanaAccessControlTags.map((controlAccessTag) => `access:${controlAccessTag}`), access: 'public', @@ -36,19 +36,19 @@ export function registerGenerationRoutesPublic(reporting: ReportingCore, logger: }, authorizedUserPreRouting(reporting, async (user, context, req, res) => { try { - const requestHandler = new RequestHandler( + const requestHandler = new GenerateRequestHandler({ reporting, user, context, path, req, res, - logger - ); - return await requestHandler.handleGenerateRequest( - req.params.exportType, - requestHandler.getJobParams() - ); + logger, + }); + return await requestHandler.handleRequest({ + exportTypeId: req.params.exportType, + jobParams: requestHandler.getJobParams(), + }); } catch (err) { if (err instanceof KibanaResponse) { return err; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/index.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/index.ts new file mode 100644 index 0000000000000..df34d38fc50d8 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsServiceSetup } from '@kbn/core/server'; +import { ALERTING_CASES_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { scheduledReportMappings, scheduledReportModelVersions } from './scheduled_report'; + +export const SCHEDULED_REPORT_SAVED_OBJECT_TYPE = 'scheduled_report'; + +export function setupSavedObjects(savedObjects: SavedObjectsServiceSetup) { + savedObjects.registerType({ + name: SCHEDULED_REPORT_SAVED_OBJECT_TYPE, + indexPattern: ALERTING_CASES_SAVED_OBJECT_INDEX, + hidden: true, + namespaceType: 'multiple', + mappings: scheduledReportMappings, + management: { importableAndExportable: false }, + modelVersions: scheduledReportModelVersions, + }); +} diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/index.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/index.ts new file mode 100644 index 0000000000000..285297b977eef --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { scheduledReportMappings } from './mappings'; +export { scheduledReportModelVersions } from './model_versions'; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/mappings.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/mappings.ts new file mode 100644 index 0000000000000..8a464d3aad666 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/mappings.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsTypeMappingDefinition } from '@kbn/core/server'; + +export const scheduledReportMappings: SavedObjectsTypeMappingDefinition = { + dynamic: false, + properties: {}, +}; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/model_versions.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/model_versions.ts new file mode 100644 index 0000000000000..4123d20974d6c --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/model_versions.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsModelVersionMap } from '@kbn/core-saved-objects-server'; +import { rawScheduledReportSchemaV1 } from './schemas'; + +export const scheduledReportModelVersions: SavedObjectsModelVersionMap = { + '1': { + changes: [], + schemas: { + forwardCompatibility: rawScheduledReportSchemaV1.extends({}, { unknowns: 'ignore' }), + create: rawScheduledReportSchemaV1, + }, + }, +}; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/index.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/index.ts similarity index 76% rename from x-pack/platform/plugins/private/reporting/server/routes/common/generate/index.ts rename to x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/index.ts index a16ddf1204b8f..6df4417bb6cef 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/index.ts +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { handleUnavailable, RequestHandler } from './request_handler'; +export { rawScheduledReportSchema as rawScheduledReportSchemaV1 } from './v1'; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts new file mode 100644 index 0000000000000..f6d1877cd90bd --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import type { rawNotificationSchema, rawScheduledReportSchema } from './v1'; + +type Mutable = { -readonly [P in keyof T]: T[P] extends object ? Mutable : T[P] }; + +export type RawNotification = Mutable>; +export type RawScheduledReport = Mutable>; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts new file mode 100644 index 0000000000000..ef407977051b9 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { scheduleRruleSchema } from '@kbn/task-manager-plugin/server'; + +const rawLayoutIdSchema = schema.oneOf([ + schema.literal('preserve_layout'), + schema.literal('print'), + schema.literal('canvas'), +]); + +export const rawNotificationSchema = schema.object({ + email: schema.maybe( + schema.object({ + to: schema.arrayOf(schema.string()), + }) + ), +}); + +export const rawScheduledReportSchema = schema.object({ + createdAt: schema.string(), + createdBy: schema.oneOf([schema.string(), schema.boolean()]), + enabled: schema.boolean(), + jobType: schema.string(), + meta: schema.object({ + isDeprecated: schema.maybe(schema.boolean()), + layout: schema.maybe(rawLayoutIdSchema), + objectType: schema.string(), + }), + migrationVersion: schema.string(), + notification: schema.maybe(rawNotificationSchema), + payload: schema.string(), + schedule: scheduleRruleSchema, + title: schema.string(), +}); diff --git a/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts index 4d4b386c1020f..94e6096e41bcd 100644 --- a/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -15,6 +15,7 @@ import { docLinksServiceMock, elasticsearchServiceMock, loggingSystemMock, + savedObjectsClientMock, statusServiceMock, } from '@kbn/core/server/mocks'; import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; @@ -39,7 +40,7 @@ export const createMockPluginSetup = ( setupMock: Partial> ): ReportingInternalSetup => { return { - encryptedSavedObjects: encryptedSavedObjectsMock.createSetup(), + encryptedSavedObjects: encryptedSavedObjectsMock.createSetup({ canEncrypt: true }), features: featuresPluginMock.createSetup(), basePath: { set: jest.fn() }, router: { get: jest.fn(), post: jest.fn(), put: jest.fn(), delete: jest.fn() }, @@ -55,6 +56,7 @@ export const createMockPluginSetup = ( const coreSetupMock = coreMock.createSetup(); const coreStartMock = coreMock.createStart(); const logger = loggingSystemMock.createLogger(); +const savedObjectsClient = savedObjectsClientMock.create(); const createMockReportingStore = async (config: ReportingConfigType) => { const mockConfigSchema = createMockConfigSchema(config); @@ -70,7 +72,7 @@ export const createMockPluginStart = async ( return { analytics: coreSetupMock.analytics, esClient: elasticsearchServiceMock.createClusterClient(), - savedObjects: { getScopedClient: jest.fn() }, + savedObjects: { getScopedClient: jest.fn().mockReturnValue(savedObjectsClient) }, uiSettings: { asScopedToClient: () => ({ get: jest.fn() }) }, discover: discoverPluginMock.createStartContract(), data: dataPluginMock.createStartContract(), diff --git a/x-pack/platform/plugins/private/reporting/server/types.ts b/x-pack/platform/plugins/private/reporting/server/types.ts index 4d4b25177ab1e..79e1cc5012f07 100644 --- a/x-pack/platform/plugins/private/reporting/server/types.ts +++ b/x-pack/platform/plugins/private/reporting/server/types.ts @@ -12,7 +12,7 @@ import type { DiscoverServerPluginStart } from '@kbn/discover-plugin/server'; import type { FeaturesPluginSetup } from '@kbn/features-plugin/server'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/server'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; -import type { UrlOrUrlLocatorTuple } from '@kbn/reporting-common/types'; +import type { ScheduledReportApiJSON, UrlOrUrlLocatorTuple } from '@kbn/reporting-common/types'; import type { ReportApiJSON } from '@kbn/reporting-common/types'; import type { ReportingConfigType } from '@kbn/reporting-server'; import type { ScreenshotModePluginSetup } from '@kbn/screenshot-mode-plugin/server'; @@ -90,6 +90,14 @@ export interface ReportingJobResponse { job: ReportApiJSON; } +export interface ScheduledReportingJobResponse { + /** + * Details of a new report job that was requested + * @public + */ + job: ScheduledReportApiJSON; +} + export interface PdfScreenshotOptions extends Omit { urls: UrlOrUrlLocatorTuple[]; } diff --git a/x-pack/platform/plugins/private/reporting/tsconfig.json b/x-pack/platform/plugins/private/reporting/tsconfig.json index ddde2d34d6d7f..814205e381988 100644 --- a/x-pack/platform/plugins/private/reporting/tsconfig.json +++ b/x-pack/platform/plugins/private/reporting/tsconfig.json @@ -52,6 +52,7 @@ "@kbn/react-kibana-mount", "@kbn/core-security-common", "@kbn/core-http-server-utils", + "@kbn/core-saved-objects-server", ], "exclude": [ "target/**/*", diff --git a/x-pack/platform/plugins/shared/features/server/__snapshots__/oss_features.test.ts.snap b/x-pack/platform/plugins/shared/features/server/__snapshots__/oss_features.test.ts.snap index b8b5910d83921..01a0046f28e8a 100644 --- a/x-pack/platform/plugins/shared/features/server/__snapshots__/oss_features.test.ts.snap +++ b/x-pack/platform/plugins/shared/features/server/__snapshots__/oss_features.test.ts.snap @@ -479,7 +479,9 @@ Array [ }, ], "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -570,7 +572,9 @@ Array [ }, "name": "Generate CSV reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -646,7 +650,9 @@ Array [ }, ], "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -706,7 +712,9 @@ Array [ "minimumLicense": "gold", "name": "Generate PDF or PNG reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -822,7 +830,9 @@ Array [ }, ], "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -850,7 +860,9 @@ Array [ }, ], "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -942,7 +954,9 @@ Array [ "minimumLicense": "gold", "name": "Generate PDF or PNG reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -962,7 +976,9 @@ Array [ }, "name": "Generate CSV reports from Discover session panels", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ diff --git a/x-pack/platform/plugins/shared/features/server/oss_features.ts b/x-pack/platform/plugins/shared/features/server/oss_features.ts index 39e016301b0ef..7f3f815453cfb 100644 --- a/x-pack/platform/plugins/shared/features/server/oss_features.ts +++ b/x-pack/platform/plugins/shared/features/server/oss_features.ts @@ -802,7 +802,7 @@ const reportingFeatures: { defaultMessage: 'Generate CSV reports', }), includeIn: 'all', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, management: { insightsAndAlerting: ['reporting'] }, api: ['generateReport'], ui: ['generateCsv'], @@ -830,7 +830,7 @@ const reportingFeatures: { ), includeIn: 'all', minimumLicense: 'gold', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, management: { insightsAndAlerting: ['reporting'] }, api: ['generateReport'], ui: ['generateScreenshot'], @@ -844,7 +844,7 @@ const reportingFeatures: { defaultMessage: 'Generate CSV reports from Discover session panels', }), includeIn: 'all', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, management: { insightsAndAlerting: ['reporting'] }, api: ['downloadCsv'], ui: ['downloadCsv'], @@ -872,7 +872,7 @@ const reportingFeatures: { ), includeIn: 'all', minimumLicense: 'gold', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, management: { insightsAndAlerting: ['reporting'] }, api: ['generateReport'], ui: ['generateScreenshot'], diff --git a/x-pack/platform/plugins/shared/task_manager/server/index.ts b/x-pack/platform/plugins/shared/task_manager/server/index.ts index 6678d4c08ca18..ce36cf68809dc 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/index.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/index.ts @@ -23,7 +23,8 @@ export type { IntervalSchedule, } from './task'; -export { Frequency, Weekday } from '@kbn/rrule'; +export type { RruleSchedule } from './saved_objects'; +export { scheduleRruleSchema } from './saved_objects'; export { TaskStatus, TaskPriority, TaskCost } from './task'; diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts index 9994426fb0831..377fd4bc210d2 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts @@ -14,6 +14,9 @@ import { getOldestIdleActionTask } from '../queries/oldest_idle_action_task'; import { TASK_MANAGER_INDEX } from '../constants'; import { backgroundTaskNodeModelVersions, taskModelVersions } from './model_versions'; +export type { RruleSchedule } from './schemas/task'; +export { scheduleRruleSchema } from './schemas/task'; + export const TASK_SO_NAME = 'task'; export const BACKGROUND_TASK_NODE_SO_NAME = 'background-task-node'; diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts index 642804a6fcef9..0ed04b3471817 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { TypeOf } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; import { isInterval } from '../../lib/intervals'; import { rruleSchedule } from './rrule'; @@ -55,6 +56,14 @@ export const taskSchemaV3 = taskSchemaV2.extends({ priority: schema.maybe(schema.number()), }); +export const scheduleIntervalSchema = schema.object({ + interval: schema.string({ validate: validateDuration }), +}); + +export const scheduleRruleSchema = schema.object({ + rrule: rruleSchedule, +}); + export const taskSchemaV4 = taskSchemaV3.extends({ apiKey: schema.maybe(schema.string()), userScope: schema.maybe( @@ -64,14 +73,9 @@ export const taskSchemaV4 = taskSchemaV3.extends({ apiKeyCreatedByUser: schema.boolean(), }) ), - schedule: schema.maybe( - schema.oneOf([ - schema.object({ - interval: schema.string({ validate: validateDuration }), - }), - schema.object({ - rrule: rruleSchedule, - }), - ]) - ), + schedule: schema.maybe(schema.oneOf([scheduleIntervalSchema, scheduleRruleSchema])), }); + +type Mutable = { -readonly [P in keyof T]: T[P] extends object ? Mutable : T[P] }; + +export type RruleSchedule = Mutable>; diff --git a/x-pack/platform/plugins/shared/task_manager/server/task.ts b/x-pack/platform/plugins/shared/task_manager/server/task.ts index 2e22f5be2b4a5..45731462d6e60 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task.ts @@ -264,12 +264,22 @@ export interface RruleSchedule { interval?: never; } +export type Rrule = RruleMonthly | RruleWeekly | RruleDaily | RruleMinutely; + interface RruleCommon { freq: Frequency; interval: number; tzid: string; } +interface RruleMinutely extends RruleCommon { + freq: Frequency.MINUTELY; + bymonthday?: number[]; + byhour?: number[]; + byminute?: number[]; + byweekday?: Weekday[]; +} + interface RruleMonthly extends RruleCommon { freq: Frequency.MONTHLY; bymonthday?: number[]; diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts b/x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts index 0b5237a9051d6..5604ff058c34e 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts @@ -162,6 +162,161 @@ export default function ({ getService }: FtrProviderContext) { }); }); + describe('Dashboard: Schedule PDF report', () => { + it('does not allow user that does not have the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.DATA_ANALYST_USERNAME, + reportingAPI.DATA_ANALYST_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF disallowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'dashboard', + version: '7.14.0', + } + ); + expect(res.status).to.eql(403); + }); + + it('does allow user with the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.REPORTING_USER_USERNAME, + reportingAPI.REPORTING_USER_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF allowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'dashboard', + version: '7.14.0', + } + ); + expect(res.status).to.eql(200); + + const soResult = await reportingAPI.getScheduledReportSO(res.body.job.id); + expect(soResult.status).to.eql(200); + expect(soResult.body._source.scheduled_report.title).to.eql('test PDF allowed'); + }); + }); + + describe('Visualize: Schedule PDF report', () => { + it('does not allow user that does not have the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.DATA_ANALYST_USERNAME, + reportingAPI.DATA_ANALYST_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF disallowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'visualization', + version: '7.14.0', + } + ); + expect(res.status).to.eql(403); + }); + + it('does allow user with the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.REPORTING_USER_USERNAME, + reportingAPI.REPORTING_USER_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF allowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'visualization', + version: '7.14.0', + } + ); + expect(res.status).to.eql(200); + + const soResult = await reportingAPI.getScheduledReportSO(res.body.job.id); + expect(soResult.status).to.eql(200); + expect(soResult.body._source.scheduled_report.title).to.eql('test PDF allowed'); + }); + }); + + describe('Canvas: Schedule PDF report', () => { + it('does not allow user that does not have the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.DATA_ANALYST_USERNAME, + reportingAPI.DATA_ANALYST_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF disallowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'canvas', + version: '7.14.0', + } + ); + expect(res.status).to.eql(403); + }); + + it('does allow user with the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.REPORTING_USER_USERNAME, + reportingAPI.REPORTING_USER_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF allowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'canvas', + version: '7.14.0', + } + ); + expect(res.status).to.eql(200); + + const soResult = await reportingAPI.getScheduledReportSO(res.body.job.id); + expect(soResult.status).to.eql(200); + expect(soResult.body._source.scheduled_report.title).to.eql('test PDF allowed'); + }); + }); + + describe('Discover: Schedule CSV report', () => { + it('does not allow user that does not have the role-based privilege', async () => { + const res = await reportingAPI.scheduleCsv( + { + browserTimezone: 'UTC', + searchSource: {} as SerializedSearchSourceFields, + objectType: 'search', + title: 'test disallowed', + version: '7.14.0', + }, + reportingAPI.DATA_ANALYST_USERNAME, + reportingAPI.DATA_ANALYST_PASSWORD + ); + expect(res.status).to.eql(403); + }); + + it('does allow user with the role-based privilege', async () => { + const res = await reportingAPI.scheduleCsv( + { + browserTimezone: 'UTC', + title: 'allowed search', + objectType: 'search', + searchSource: { + version: true, + fields: [{ field: '*', include_unmapped: true }], + index: '5193f870-d861-11e9-a311-0fa548c5f953', + } as unknown as SerializedSearchSourceFields, + columns: [], + version: '7.13.0', + }, + reportingAPI.REPORTING_USER_USERNAME, + reportingAPI.REPORTING_USER_PASSWORD + ); + expect(res.status).to.eql(200); + + const soResult = await reportingAPI.getScheduledReportSO(res.body.job.id); + expect(soResult.status).to.eql(200); + expect(soResult.body._source.scheduled_report.title).to.eql('allowed search'); + }); + }); + // This tests the same API as x-pack/test/api_integration/apis/security/privileges.ts, but it uses the non-deprecated config it('should register reporting privileges with the security privileges API', async () => { await supertest diff --git a/x-pack/test/reporting_api_integration/services/scenarios.ts b/x-pack/test/reporting_api_integration/services/scenarios.ts index 0800647d2abef..adcd3151aef20 100644 --- a/x-pack/test/reporting_api_integration/services/scenarios.ts +++ b/x-pack/test/reporting_api_integration/services/scenarios.ts @@ -15,6 +15,8 @@ import { REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY, } from '@kbn/reporting-server'; import rison from '@kbn/rison'; +import { ALERTING_CASES_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { RruleSchedule } from '@kbn/task-manager-plugin/server'; import { FtrProviderContext } from '../ftr_provider_context'; function removeWhitespace(str: string) { @@ -150,6 +152,19 @@ export function createScenarios({ getService }: Pick { + const jobParams = rison.encode(job); + return await supertestWithoutAuth + .post(`/internal/reporting/schedule/printablePdfV2`) + .auth(username, password) + .set('kbn-xsrf', 'xxx') + .send({ jobParams, schedule }); + }; const generatePng = async (username: string, password: string, job: JobParamsPNGV2) => { const jobParams = rison.encode(job); return await supertestWithoutAuth @@ -158,6 +173,19 @@ export function createScenarios({ getService }: Pick { + const jobParams = rison.encode(job); + return await supertestWithoutAuth + .post(`/internal/reporting/schedule/pngV2`) + .auth(username, password) + .set('kbn-xsrf', 'xxx') + .send({ jobParams, schedule }); + }; const generateCsv = async ( job: JobParamsCSV, username = 'elastic', @@ -171,6 +199,20 @@ export function createScenarios({ getService }: Pick { + const jobParams = rison.encode(job); + + return await supertestWithoutAuth + .post(`/internal/reporting/schedule/csv_searchsource`) + .auth(username, password) + .set('kbn-xsrf', 'xxx') + .send({ jobParams, schedule }); + }; const postJob = async ( apiPath: string, @@ -263,6 +305,12 @@ export function createScenarios({ getService }: Pick { + return await esSupertest.get( + `/${ALERTING_CASES_SAVED_OBJECT_INDEX}/_doc/scheduled_report:${id}` + ); + }; + return { logTaskManagerHealth, initEcommerce, @@ -281,6 +329,9 @@ export function createScenarios({ getService }: Pick Date: Mon, 28 Apr 2025 17:03:32 -0400 Subject: [PATCH 20/45] Schedule report api --- .../routes/common/request_handler/schedule_request_handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts index 5e1d52267cec2..4658d1e85f499 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts @@ -66,7 +66,7 @@ export class ScheduleRequestHandler extends RequestHandler< const { reporting, logger, req, user } = this.opts; const soClient = await reporting.getSoClient(req); - const { version, job, jobType, name } = await this.createJob(exportTypeId, jobParams); + const { version, job, jobType } = await this.createJob(exportTypeId, jobParams); const payload = { ...job, From f5bb665a629829b439fb8e10e0e1307848bdb470 Mon Sep 17 00:00:00 2001 From: Ying Date: Mon, 28 Apr 2025 17:28:03 -0400 Subject: [PATCH 21/45] Cleanup --- .../common/request_handler/lib/index.ts | 1 - .../transform_raw_scheduled_report_to_task.ts | 20 --------- .../schedule_request_handler.test.ts | 45 +++++++++---------- .../schedule_request_handler.ts | 13 ++---- .../scheduling_from_jobparams.test.ts | 11 +++-- .../server/saved_objects/schemas/rrule.ts | 10 ++++- 6 files changed, 41 insertions(+), 59 deletions(-) delete mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report_to_task.ts diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts index 256f5046051f1..d80e6e2e93b0d 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts @@ -6,4 +6,3 @@ */ export { transformRawScheduledReportToReport } from './transform_raw_scheduled_report'; -export { transformRawScheduledReportToTaskParams } from './transform_raw_scheduled_report_to_task'; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report_to_task.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report_to_task.ts deleted file mode 100644 index b2d9598ef9bd2..0000000000000 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report_to_task.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { SavedObject } from '@kbn/core/server'; -import { ScheduledReportTaskParams } from '../../../../lib/tasks'; -import { RawScheduledReport } from '../../../../saved_objects/scheduled_report/schemas/latest'; - -export function transformRawScheduledReportToTaskParams( - rawScheduledReport: SavedObject -): ScheduledReportTaskParams { - return { - id: rawScheduledReport.id, - jobtype: rawScheduledReport.attributes.jobType, - schedule: rawScheduledReport.attributes.schedule, - }; -} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts index 8611a3a27bea3..0bec76c48ad77 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts @@ -16,7 +16,7 @@ import { SavedObjectsClientContract, } from '@kbn/core/server'; import { coreMock, httpServerMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { JobParamsPDFV2, TaskPayloadPDFV2 } from '@kbn/reporting-export-types-pdf-common'; +import { JobParamsPDFV2 } from '@kbn/reporting-export-types-pdf-common'; import { createMockConfigSchema } from '@kbn/reporting-mocks-server'; import { ReportingCore } from '../../..'; import { createMockReportingCore } from '../../../test_helpers'; @@ -129,8 +129,7 @@ describe('Handle request to schedule', () => { }, } `); - const { forceNow, ...snapPayload } = payload as TaskPayloadPDFV2; - expect(snapPayload).toMatchInlineSnapshot(` + expect(payload).toMatchInlineSnapshot(` Object { "browserTimezone": "UTC", "isDeprecated": false, @@ -144,29 +143,27 @@ describe('Handle request to schedule', () => { } `); - expect(soClient.create).toHaveBeenCalledWith( - 'scheduled_report', - { - jobType: 'printable_pdf_v2', - createdAt: expect.any(String), - createdBy: 'testymcgee', - title: 'cool_title', - payload: expect.any(String), - schedule: { - rrule: { - freq: 1, - interval: 2, - }, - }, - migrationVersion: 'unknown', - meta: { - objectType: 'cool_object_type', - layout: 'preserve_layout', - isDeprecated: false, + expect(soClient.create).toHaveBeenCalledWith('scheduled_report', { + jobType: 'printable_pdf_v2', + createdAt: expect.any(String), + createdBy: 'testymcgee', + title: 'cool_title', + enabled: true, + payload: JSON.stringify(payload), + schedule: { + rrule: { + freq: 1, + interval: 2, + tzid: 'UTC', }, }, - {} - ); + migrationVersion: 'unknown', + meta: { + objectType: 'cool_object_type', + layout: 'preserve_layout', + isDeprecated: false, + }, + }); }); }); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts index 4658d1e85f499..b81c789c44c95 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts @@ -9,7 +9,7 @@ import moment from 'moment'; import { schema } from '@kbn/config-schema'; import { ScheduledReportApiJSON } from '@kbn/reporting-common/types'; -import { isEmpty } from 'lodash'; +import { isEmpty, omit } from 'lodash'; import { RruleSchedule, scheduleRruleSchema } from '@kbn/task-manager-plugin/server'; import { rawNotificationSchema } from '../../../saved_objects/scheduled_report/schemas/v1'; import { ScheduledReportingJobResponse } from '../../../types'; @@ -92,7 +92,7 @@ export class ScheduleRequestHandler extends RequestHandler< }, migrationVersion: version, title: job.title, - payload: JSON.stringify(payload), + payload: JSON.stringify(omit(payload, 'forceNow')), schedule: schedule!, }; @@ -103,14 +103,7 @@ export class ScheduleRequestHandler extends RequestHandler< ); logger.debug(`Successfully created scheduled report: ${report.id}`); - // Schedule the report with Task Manager - // const task = await reporting.scheduleRecurringTask( - // req, - // transformRawScheduledReportToTaskParams(report) - // ); - // logger.info( - // `Scheduled "${name}" reporting task. Task ID: task:${task.id}. Report ID: ${report.id}` - // ); + // TODO - Schedule the report with Task Manager return transformRawScheduledReportToReport(report); } diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts index 8fa8414e9ea41..7352b835d91a4 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts @@ -190,7 +190,13 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { }) .expect(400) .then(({ body }) => - expect(body.message).toMatchInlineSnapshot(`"Invalid timezone \\"America/Amsterdam\\"."`) + expect(body.message).toMatchInlineSnapshot(` + "[request body.schedule.rrule]: types that failed validation: + - [request body.schedule.rrule.0.freq]: expected value to equal [1] + - [request body.schedule.rrule.1.freq]: expected value to equal [2] + - [request body.schedule.rrule.2.freq]: expected value to equal [3] + - [request body.schedule.rrule.3.freq]: expected value to equal [5]" + `) ); }); @@ -203,7 +209,7 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) .send({ jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), - schedule: { rrule: { freq: 6, interval: 2 } }, + schedule: { rrule: { freq: 1, interval: 2 } }, notification: { email: { to: 'single@email.com', @@ -308,7 +314,6 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { id: 'foo', jobtype: 'printable_pdf_v2', payload: { - forceNow: expect.any(String), isDeprecated: false, layout: { id: 'test', diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts index 0bd7141d011b2..82f2da7b4512a 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts @@ -28,6 +28,14 @@ const byhour = schema.maybe(schema.arrayOf(schema.number({ min: 0, max: 23 }))); const byweekday = schema.maybe(schema.arrayOf(schema.number({ min: 1, max: 7 }))); const bymonthday = schema.maybe(schema.arrayOf(schema.number({ min: 1, max: 31 }))); +const rruleMinutely = rruleCommon.extends({ + freq: schema.literal(Frequency.MINUTELY), + byhour, + byminute, + byweekday, + bymonthday, +}); + const rruleMonthly = rruleCommon.extends({ freq: schema.literal(Frequency.MONTHLY), byhour, @@ -52,4 +60,4 @@ const rruleDaily = rruleCommon.extends({ bymonthday: schema.never(), }); -export const rruleSchedule = schema.oneOf([rruleMonthly, rruleWeekly, rruleDaily]); +export const rruleSchedule = schema.oneOf([rruleMonthly, rruleWeekly, rruleDaily, rruleMinutely]); From f039e518d58828a0823ebe744d912dd2d09a5b3e Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Tue, 29 Apr 2025 03:24:51 +0200 Subject: [PATCH 22/45] add rrule examples --- .../plugins/shared/task_manager/README.md | 79 ++++++++++++++++++- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/README.md b/x-pack/platform/plugins/shared/task_manager/README.md index 0ceec029e5d11..9ff2f3c5bbb41 100644 --- a/x-pack/platform/plugins/shared/task_manager/README.md +++ b/x-pack/platform/plugins/shared/task_manager/README.md @@ -672,7 +672,7 @@ And there are 2 different config options for scheduling a task: Monthly schedule options: -``` +```typescript freq: Frequency.MONTHLY, -> Import the enum Frequency from TaskManager (Required field) interval: number; -> Any number. 1 means `every 1 month` (Required field) tzid: string; -> Timezone e.g.: 'UTC' (Required field) @@ -684,7 +684,7 @@ Monthly schedule options: Weekly schedule options: -``` +```typescript freq: Frequency.WEEKLY, -> Import the enum Frequency from TaskManager (Required field) interval: number; -> Any number. 1 means `every 1 week` (Required field) tzid: string; -> Timezone e.g.: 'UTC' (Required field) @@ -695,7 +695,7 @@ Weekly schedule options: Daily schedule options: -``` +```typescript freq: Frequency.DAILY, -> Import the enum Frequency from TaskManager (Required field) interval: number; -> Any number. 1 means `every 1 day` (Required field) tzid: string; -> Timezone e.g.: 'UTC' (Required field) @@ -703,3 +703,76 @@ Daily schedule options: byminute?: number[]; -> number between 0 and 59 byweekday?: Weekday[]; -> Import the enum Weekday from TaskManager. Weekday.MO is monday ``` + +Examples: + +Every day at current time: + +```js + schedule: { + rrule: { + freq: Frequency.DAILY, + tzid: 'UTC', + interval: 1 + } + } +``` + +Every day at 13:15: + +```js + schedule: { + rrule: { + freq: Frequency.DAILY, + tzid: 'UTC', + interval: 1, + byhour: [13], + byminute: [15] + } + } +``` + +Every Monday at 17:30 + +```js + schedule: { + rrule: { + freq: Frequency.DAILY, + tzid: 'UTC', + interval: 1, + byhour: [17], + byminute: [30] + byweekday: Weekday.MO + } + } +``` + +Every 2 weeks on Friday at 08:45 + +```js + schedule: { + rrule: { + freq: Frequency.WEEKLY, + tzid: 'UTC', + interval: 2, + byhour: [08], + byminute: [45] + byweekday: Weekday.FR + } + } +``` + +Every Month on 1st, 15th and 30th at 12:10 and 18:10 + +```js + schedule: { + rrule: { + freq: Frequency.MONTHLY, + tzid: 'UTC', + interval: 1, + byhour: [12,18], + byminute: [10] + bymonthday: [1,15,30] + } + } +``` From e75e7968a9fd6c88802fce33ec720115f01af301 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Tue, 29 Apr 2025 03:25:46 +0200 Subject: [PATCH 23/45] fix rrule example --- x-pack/platform/plugins/shared/task_manager/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/task_manager/README.md b/x-pack/platform/plugins/shared/task_manager/README.md index 9ff2f3c5bbb41..c324ad8f391ee 100644 --- a/x-pack/platform/plugins/shared/task_manager/README.md +++ b/x-pack/platform/plugins/shared/task_manager/README.md @@ -755,7 +755,7 @@ Every 2 weeks on Friday at 08:45 freq: Frequency.WEEKLY, tzid: 'UTC', interval: 2, - byhour: [08], + byhour: [8], byminute: [45] byweekday: Weekday.FR } From 424d7338fa3623b4e577608f6711fdaf0f657202 Mon Sep 17 00:00:00 2001 From: Ying Date: Tue, 29 Apr 2025 17:57:49 -0400 Subject: [PATCH 24/45] PR feedback --- .../server/task_claimers/strategy_mget.test.ts | 2 -- .../task_manager/server/task_claimers/strategy_mget.ts | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.test.ts b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.test.ts index 8a508a174af59..ce31c4e6a842f 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.test.ts @@ -940,8 +940,6 @@ describe('TaskClaiming', () => { getAvailableCapacity: (type?: string) => { if (type === 'sampleTaskSharedConcurrencyType1') { return 2; - } else if (type === 'sampleTaskSharedConcurrencyType1') { - return 10; } else { return 10; } diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.ts b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.ts index 015aaa0fcdf79..c833ba1f0dcf6 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_claimers/strategy_mget.ts @@ -330,7 +330,7 @@ async function searchAvailableTasks({ // Task must be enabled EnabledTask, // Specific task type - OneOfTaskTypes('task.taskType', types), + OneOfTaskTypes('task.taskType', types.split(',')), // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), @@ -356,7 +356,7 @@ async function searchAvailableTasks({ interface ClaimPartitions { unlimitedTypes: string[]; - limitedTypes: Map; + limitedTypes: Map; } interface BuildClaimPartitionsOpts { @@ -401,12 +401,12 @@ function buildClaimPartitions(opts: BuildClaimPartitionsOpts): ClaimPartitions { } } if (minCapacity) { - result.limitedTypes.set(isSharingConcurrency, minCapacity); + result.limitedTypes.set(isSharingConcurrency.join(','), minCapacity); } } else { const capacity = getCapacity(definition.type) / definition.cost; if (capacity !== 0) { - result.limitedTypes.set([definition.type], capacity); + result.limitedTypes.set(definition.type, capacity); } } } From 2d6dfab27bfaf4b135054d3390836066e7bae5a0 Mon Sep 17 00:00:00 2001 From: Ying Date: Wed, 30 Apr 2025 14:15:32 -0400 Subject: [PATCH 25/45] Fixing types --- .../test_suites/task_manager/task_management.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index ec49129acd6b3..e3c57d01a55a4 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -9,12 +9,9 @@ import moment from 'moment'; import { random } from 'lodash'; import expect from '@kbn/expect'; import type { estypes } from '@elastic/elasticsearch'; +import type { Frequency } from '@kbn/rrule'; import { taskMappings as TaskManagerMapping } from '@kbn/task-manager-plugin/server/saved_objects/mappings'; -import { - ConcreteTaskInstance, - BulkUpdateTaskResult, - Frequency, -} from '@kbn/task-manager-plugin/server'; +import { ConcreteTaskInstance, BulkUpdateTaskResult } from '@kbn/task-manager-plugin/server'; import { FtrProviderContext } from '../../ftr_provider_context'; const { properties: taskManagerIndexMapping } = TaskManagerMapping; From b98f26c7b3d935abacbe944d7b0d60cd5f9cb22d Mon Sep 17 00:00:00 2001 From: Ying Date: Wed, 30 Apr 2025 14:31:28 -0400 Subject: [PATCH 26/45] Fixing types --- x-pack/test/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 30b931e5d79c4..79e98d2d144a6 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -189,5 +189,6 @@ "@kbn/es-errors", "@kbn/content-packs-schema", "@kbn/lock-manager", + "@kbn/rrule", ] } From 3fd59a14c1330e4ffc1be144c75e415934d0df49 Mon Sep 17 00:00:00 2001 From: Ying Date: Wed, 30 Apr 2025 14:42:35 -0400 Subject: [PATCH 27/45] Cleanup --- .../plugins/shared/task_manager/server/index.ts | 1 + .../server/saved_objects/schemas/rrule.ts | 10 +--------- .../plugins/shared/task_manager/server/task.ts | 11 +---------- .../test_suites/task_manager/task_management.ts | 7 +++++-- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/index.ts b/x-pack/platform/plugins/shared/task_manager/server/index.ts index ce36cf68809dc..21bbda675d02c 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/index.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/index.ts @@ -23,6 +23,7 @@ export type { IntervalSchedule, } from './task'; +export { Frequency, Weekday } from '@kbn/rrule'; export type { RruleSchedule } from './saved_objects'; export { scheduleRruleSchema } from './saved_objects'; diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts index 82f2da7b4512a..0bd7141d011b2 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/rrule.ts @@ -28,14 +28,6 @@ const byhour = schema.maybe(schema.arrayOf(schema.number({ min: 0, max: 23 }))); const byweekday = schema.maybe(schema.arrayOf(schema.number({ min: 1, max: 7 }))); const bymonthday = schema.maybe(schema.arrayOf(schema.number({ min: 1, max: 31 }))); -const rruleMinutely = rruleCommon.extends({ - freq: schema.literal(Frequency.MINUTELY), - byhour, - byminute, - byweekday, - bymonthday, -}); - const rruleMonthly = rruleCommon.extends({ freq: schema.literal(Frequency.MONTHLY), byhour, @@ -60,4 +52,4 @@ const rruleDaily = rruleCommon.extends({ bymonthday: schema.never(), }); -export const rruleSchedule = schema.oneOf([rruleMonthly, rruleWeekly, rruleDaily, rruleMinutely]); +export const rruleSchedule = schema.oneOf([rruleMonthly, rruleWeekly, rruleDaily]); diff --git a/x-pack/platform/plugins/shared/task_manager/server/task.ts b/x-pack/platform/plugins/shared/task_manager/server/task.ts index 45731462d6e60..277049facb1a9 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task.ts @@ -264,22 +264,13 @@ export interface RruleSchedule { interval?: never; } -export type Rrule = RruleMonthly | RruleWeekly | RruleDaily | RruleMinutely; +export type Rrule = RruleMonthly | RruleWeekly | RruleDaily; interface RruleCommon { freq: Frequency; interval: number; tzid: string; } - -interface RruleMinutely extends RruleCommon { - freq: Frequency.MINUTELY; - bymonthday?: number[]; - byhour?: number[]; - byminute?: number[]; - byweekday?: Weekday[]; -} - interface RruleMonthly extends RruleCommon { freq: Frequency.MONTHLY; bymonthday?: number[]; diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index e3c57d01a55a4..ec49129acd6b3 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -9,9 +9,12 @@ import moment from 'moment'; import { random } from 'lodash'; import expect from '@kbn/expect'; import type { estypes } from '@elastic/elasticsearch'; -import type { Frequency } from '@kbn/rrule'; import { taskMappings as TaskManagerMapping } from '@kbn/task-manager-plugin/server/saved_objects/mappings'; -import { ConcreteTaskInstance, BulkUpdateTaskResult } from '@kbn/task-manager-plugin/server'; +import { + ConcreteTaskInstance, + BulkUpdateTaskResult, + Frequency, +} from '@kbn/task-manager-plugin/server'; import { FtrProviderContext } from '../../ftr_provider_context'; const { properties: taskManagerIndexMapping } = TaskManagerMapping; From 03701622a4afeb551511c49b3846ba17f7523d75 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 30 Apr 2025 18:55:50 +0000 Subject: [PATCH 28/45] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/test/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 79e98d2d144a6..30b931e5d79c4 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -189,6 +189,5 @@ "@kbn/es-errors", "@kbn/content-packs-schema", "@kbn/lock-manager", - "@kbn/rrule", ] } From 0277de99e078550e5a33fb2fa538ac5429f72c30 Mon Sep 17 00:00:00 2001 From: Ying Date: Wed, 30 Apr 2025 16:19:02 -0400 Subject: [PATCH 29/45] Fixing tests --- .../integration_tests/scheduling_from_jobparams.test.ts | 3 +-- .../spaces_api_integration/spaces_only/telemetry/telemetry.ts | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts index 7352b835d91a4..8014379401589 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts @@ -194,8 +194,7 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { "[request body.schedule.rrule]: types that failed validation: - [request body.schedule.rrule.0.freq]: expected value to equal [1] - [request body.schedule.rrule.1.freq]: expected value to equal [2] - - [request body.schedule.rrule.2.freq]: expected value to equal [3] - - [request body.schedule.rrule.3.freq]: expected value to equal [5]" + - [request body.schedule.rrule.2.freq]: expected value to equal [3]" `) ); }); diff --git a/x-pack/test/spaces_api_integration/spaces_only/telemetry/telemetry.ts b/x-pack/test/spaces_api_integration/spaces_only/telemetry/telemetry.ts index 7f5216cce27b3..3be649a4c07c3 100644 --- a/x-pack/test/spaces_api_integration/spaces_only/telemetry/telemetry.ts +++ b/x-pack/test/spaces_api_integration/spaces_only/telemetry/telemetry.ts @@ -69,7 +69,6 @@ export default function ({ getService }: FtrProviderContext) { generalCases: 0, generalCasesV2: 0, generalCasesV3: 0, - manageReporting: 0, maps: 2, maps_v2: 2, canvas: 2, From fbb3f859af1bd0f165f089d13868a2d7e2f62356 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Fri, 2 May 2025 17:30:36 +0200 Subject: [PATCH 30/45] remove mappings --- .../server/saved_objects/mappings.ts | 25 ---------------- .../model_versions/task_model_versions.ts | 29 ------------------- 2 files changed, 54 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/mappings.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/mappings.ts index 05ef59a0a7376..d8981fab4511b 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/mappings.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/mappings.ts @@ -34,31 +34,6 @@ export const taskMappings: SavedObjectsTypeMappingDefinition = { interval: { type: 'keyword', }, - rrule: { - properties: { - freq: { - type: 'integer', - }, - interval: { - type: 'integer', - }, - tzid: { - type: 'keyword', - }, - bymonthday: { - type: 'integer', - }, - byweekday: { - type: 'integer', - }, - byhour: { - type: 'integer', - }, - byminute: { - type: 'integer', - }, - }, - }, }, }, attempts: { diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/model_versions/task_model_versions.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/model_versions/task_model_versions.ts index d99b6fc158f39..a14009d82fa50 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/model_versions/task_model_versions.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/model_versions/task_model_versions.ts @@ -69,35 +69,6 @@ export const taskModelVersions: SavedObjectsModelVersionMap = { apiKeyId: { type: 'keyword' }, }, }, - schedule: { - properties: { - rrule: { - properties: { - freq: { - type: 'integer', - }, - interval: { - type: 'integer', - }, - tzid: { - type: 'keyword', - }, - bymonthday: { - type: 'integer', - }, - byweekday: { - type: 'integer', - }, - byhour: { - type: 'integer', - }, - byminute: { - type: 'integer', - }, - }, - }, - }, - }, }, }, ], From f36105831bd00ba1366ecad283e2c696f168d555 Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Fri, 2 May 2025 18:44:17 +0200 Subject: [PATCH 31/45] remove mappings --- .../current_fields.json | 355 +++--------------- .../current_mappings.json | 25 -- 2 files changed, 62 insertions(+), 318 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 6c037af2ab1a9..795679d1801b0 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -1,11 +1,6 @@ { - "action": [ - "actionTypeId", - "name" - ], - "action_task_params": [ - "apiKeyId" - ], + "action": ["actionTypeId", "name"], + "action_task_params": ["apiKeyId"], "ad_hoc_run_params": [ "apiKeyId", "createdAt", @@ -86,10 +81,7 @@ "updatedAt", "updatedBy" ], - "api_key_pending_invalidation": [ - "apiKeyId", - "createdAt" - ], + "api_key_pending_invalidation": ["apiKeyId", "createdAt"], "apm-custom-dashboards": [ "dashboardSavedObjectId", "kuery", @@ -97,44 +89,16 @@ "serviceNameFilterEnabled" ], "apm-indices": [], - "apm-server-schema": [ - "schemaJson" - ], - "apm-service-group": [ - "color", - "description", - "groupName", - "kuery" - ], + "apm-server-schema": ["schemaJson"], + "apm-service-group": ["color", "description", "groupName", "kuery"], "apm-telemetry": [], "app_search_telemetry": [], - "application_usage_daily": [ - "timestamp" - ], + "application_usage_daily": ["timestamp"], "application_usage_totals": [], - "background-task-node": [ - "id", - "last_seen" - ], - "canvas-element": [ - "@created", - "@timestamp", - "content", - "help", - "image", - "name" - ], - "canvas-workpad": [ - "@created", - "@timestamp", - "name" - ], - "canvas-workpad-template": [ - "help", - "name", - "tags", - "template_key" - ], + "background-task-node": ["id", "last_seen"], + "canvas-element": ["@created", "@timestamp", "content", "help", "image", "name"], + "canvas-workpad": ["@created", "@timestamp", "name"], + "canvas-workpad-template": ["help", "name", "tags", "template_key"], "cases": [ "assignees", "assignees.uid", @@ -208,21 +172,9 @@ "type", "updated_at" ], - "cases-configure": [ - "closure_type", - "created_at", - "owner" - ], - "cases-connector-mappings": [ - "owner" - ], - "cases-rules": [ - "counter", - "createdAt", - "rules", - "rules.id", - "updatedAt" - ], + "cases-configure": ["closure_type", "created_at", "owner"], + "cases-connector-mappings": ["owner"], + "cases-rules": ["counter", "createdAt", "rules", "rules.id", "updatedAt"], "cases-telemetry": [], "cases-user-actions": [ "action", @@ -243,16 +195,9 @@ ], "cloud": [], "cloud-security-posture-settings": [], - "config": [ - "buildNum" - ], - "config-global": [ - "buildNum" - ], - "connector_token": [ - "connectorId", - "tokenType" - ], + "config": ["buildNum"], + "config-global": ["buildNum"], + "connector_token": ["connectorId", "tokenType"], "core-usage-stats": [], "csp-rule-template": [ "metadata", @@ -292,15 +237,8 @@ "version" ], "dynamic-config-overrides": [], - "endpoint:unified-user-artifact-manifest": [ - "artifactIds", - "policyId", - "semanticVersion" - ], - "endpoint:user-artifact-manifest": [ - "artifacts", - "schemaVersion" - ], + "endpoint:unified-user-artifact-manifest": ["artifactIds", "policyId", "semanticVersion"], + "endpoint:user-artifact-manifest": ["artifacts", "schemaVersion"], "enterprise_search_telemetry": [], "entity-definition": [ "description", @@ -316,9 +254,7 @@ "type", "version" ], - "entity-discovery-api-key": [ - "apiKey" - ], + "entity-discovery-api-key": ["apiKey"], "entity-engine-status": [ "fieldHistoryLength", "filter", @@ -366,13 +302,8 @@ "package_name", "package_version" ], - "event-annotation-group": [ - "description", - "title" - ], - "event_loop_delays_daily": [ - "lastUpdatedAt" - ], + "event-annotation-group": ["description", "title"], + "event_loop_delays_daily": ["lastUpdatedAt"], "exception-list": [ "_tags", "comments", @@ -451,12 +382,7 @@ "updated_by", "version" ], - "favorites": [ - "favoriteIds", - "favoriteMetadata", - "type", - "userId" - ], + "favorites": ["favoriteIds", "favoriteMetadata", "type", "userId"], "file": [ "FileKind", "Meta", @@ -470,16 +396,8 @@ "size", "user" ], - "file-upload-usage-collection-telemetry": [ - "file_upload", - "file_upload.index_creation_count" - ], - "fileShare": [ - "created", - "name", - "token", - "valid_until" - ], + "file-upload-usage-collection-telemetry": ["file_upload", "file_upload.index_creation_count"], + "fileShare": ["created", "name", "token", "valid_until"], "fleet-agent-policies": [ "advanced_settings", "agent_features", @@ -549,9 +467,7 @@ "updated_by", "vars" ], - "fleet-preconfiguration-deletion-record": [ - "id" - ], + "fleet-preconfiguration-deletion-record": ["id"], "fleet-proxy": [ "certificate", "certificate_authorities", @@ -561,17 +477,9 @@ "proxy_headers", "url" ], - "fleet-setup-lock": [ - "started_at", - "status", - "uuid" - ], + "fleet-setup-lock": ["started_at", "status", "uuid"], "fleet-space-settings": [], - "fleet-uninstall-tokens": [ - "namespaces", - "policy_id", - "token_plain" - ], + "fleet-uninstall-tokens": ["namespaces", "policy_id", "token_plain"], "graph-workspace": [ "description", "kibanaSavedObjectMeta", @@ -583,24 +491,15 @@ "version", "wsState" ], - "guided-onboarding-guide-state": [ - "guideId", - "isActive" - ], + "guided-onboarding-guide-state": ["guideId", "isActive"], "guided-onboarding-plugin-state": [], - "index-pattern": [ - "name", - "title", - "type" - ], + "index-pattern": ["name", "title", "type"], "infra-custom-dashboards": [ "assetType", "dashboardFilterAssetIdEnabled", "dashboardSavedObjectId" ], - "infrastructure-monitoring-log-view": [ - "name" - ], + "infrastructure-monitoring-log-view": ["name"], "infrastructure-ui-source": [], "ingest-agent-policies": [ "advanced_settings", @@ -637,13 +536,7 @@ "updated_at", "updated_by" ], - "ingest-download-sources": [ - "host", - "is_default", - "name", - "proxy_id", - "source_id" - ], + "ingest-download-sources": ["host", "is_default", "name", "proxy_id", "source_id"], "ingest-outputs": [ "allow_edit", "auth_type", @@ -756,30 +649,10 @@ "targetNamespace", "targetType" ], - "lens": [ - "description", - "state", - "title", - "visualizationType" - ], - "lens-ui-telemetry": [ - "count", - "date", - "name", - "type" - ], - "links": [ - "description", - "links", - "title" - ], - "maintenance-window": [ - "enabled", - "events", - "expirationDate", - "title", - "updatedAt" - ], + "lens": ["description", "state", "title", "visualizationType"], + "lens-ui-telemetry": ["count", "date", "name", "type"], + "links": ["description", "links", "title"], + "maintenance-window": ["enabled", "events", "expirationDate", "title", "updatedAt"], "map": [ "bounds", "description", @@ -791,11 +664,7 @@ ], "metrics-data-source": [], "metrics-explorer-view": [], - "ml-job": [ - "datafeed_id", - "job_id", - "type" - ], + "ml-job": ["datafeed_id", "job_id", "type"], "ml-module": [ "datafeeds", "defaultIndexPattern", @@ -808,20 +677,9 @@ "title", "type" ], - "ml-trained-model": [ - "job", - "job.create_time", - "job.job_id", - "model_id" - ], - "monitoring-telemetry": [ - "reportedClusterUuids" - ], - "observability-onboarding-state": [ - "progress", - "state", - "type" - ], + "ml-trained-model": ["job", "job.create_time", "job.job_id", "model_id"], + "monitoring-telemetry": ["reportedClusterUuids"], + "observability-onboarding-state": ["progress", "state", "type"], "onechat_workflow": [ "access_control", "access_control.public", @@ -829,10 +687,7 @@ "user_id", "workflow_id" ], - "osquery-manager-usage-metric": [ - "count", - "errors" - ], + "osquery-manager-usage-metric": ["count", "errors"], "osquery-pack": [ "created_at", "created_by", @@ -880,12 +735,8 @@ "updated_by", "version" ], - "policy-settings-protection-updates-note": [ - "note" - ], - "privilege-monitoring-status": [ - "status" - ], + "policy-settings-protection-updates-note": ["note"], + "privilege-monitoring-status": ["status"], "product-doc-install-status": [ "index_name", "installation_status", @@ -893,11 +744,7 @@ "product_name", "product_version" ], - "query": [ - "description", - "title", - "titleKeyword" - ], + "query": ["description", "title", "titleKeyword"], "risk-engine-configuration": [ "alertSampleSizePerShard", "dataViewId", @@ -911,28 +758,12 @@ "range.end", "range.start" ], - "rules-settings": [ - "flapping" - ], - "sample-data-telemetry": [ - "installCount", - "unInstallCount" - ], - "search": [ - "description", - "title" - ], - "search-session": [ - "created", - "realmName", - "realmType", - "sessionId", - "username" - ], + "rules-settings": ["flapping"], + "sample-data-telemetry": ["installCount", "unInstallCount"], + "search": ["description", "title"], + "search-session": ["created", "realmName", "realmType", "sessionId", "username"], "search-telemetry": [], - "search_playground": [ - "name" - ], + "search_playground": ["name"], "security-ai-prompt": [ "description", "model", @@ -942,15 +773,8 @@ "promptId", "provider" ], - "security-rule": [ - "rule_id", - "version" - ], - "security-solution-signals-migration": [ - "sourceIndex", - "updated", - "version" - ], + "security-rule": ["rule_id", "version"], + "security-solution-signals-migration": ["sourceIndex", "updated", "version"], "siem-detection-engine-rule-actions": [ "actions", "actions.actionRef", @@ -1061,21 +885,8 @@ "updated", "updatedBy" ], - "siem-ui-timeline-note": [ - "created", - "createdBy", - "eventId", - "note", - "updated", - "updatedBy" - ], - "siem-ui-timeline-pinned-event": [ - "created", - "createdBy", - "eventId", - "updated", - "updatedBy" - ], + "siem-ui-timeline-note": ["created", "createdBy", "eventId", "note", "updated", "updatedBy"], + "siem-ui-timeline-pinned-event": ["created", "createdBy", "eventId", "updated", "updatedBy"], "slo": [ "budgetingMethod", "description", @@ -1089,11 +900,7 @@ "version" ], "slo-settings": [], - "space": [ - "disabledFeatures", - "name", - "solution" - ], + "space": ["disabledFeatures", "name", "solution"], "spaces-usage-stats": [], "synthetics-dynamic-settings": [], "synthetics-monitor": [ @@ -1126,11 +933,7 @@ "synthetics-param": [], "synthetics-private-location": [], "synthetics-privates-locations": [], - "tag": [ - "color", - "description", - "name" - ], + "tag": ["color", "description", "name"], "task": [ "attempts", "enabled", @@ -1141,14 +944,6 @@ "runAt", "schedule", "schedule.interval", - "schedule.rrule", - "schedule.rrule.byhour", - "schedule.rrule.byminute", - "schedule.rrule.bymonthday", - "schedule.rrule.byweekday", - "schedule.rrule.freq", - "schedule.rrule.interval", - "schedule.rrule.tzid", "scheduledAt", "scope", "status", @@ -1158,41 +953,15 @@ ], "telemetry": [], "threshold-explorer-view": [], - "ui-metric": [ - "count" - ], - "upgrade-assistant-ml-upgrade-operation": [ - "snapshotId" - ], - "upgrade-assistant-reindex-operation": [ - "indexName", - "status" - ], + "ui-metric": ["count"], + "upgrade-assistant-ml-upgrade-operation": ["snapshotId"], + "upgrade-assistant-reindex-operation": ["indexName", "status"], "uptime-dynamic-settings": [], - "uptime-synthetics-api-key": [ - "apiKey" - ], - "url": [ - "accessDate", - "createDate", - "slug" - ], - "usage-counter": [ - "count", - "counterName", - "counterType", - "domainId", - "source" - ], - "usage-counters": [ - "domainId" - ], - "visualization": [ - "description", - "kibanaSavedObjectMeta", - "title", - "version" - ], + "uptime-synthetics-api-key": ["apiKey"], + "url": ["accessDate", "createDate", "slug"], + "usage-counter": ["count", "counterName", "counterType", "domainId", "source"], + "usage-counters": ["domainId"], + "visualization": ["description", "kibanaSavedObjectMeta", "title", "version"], "workchat_agent": [ "access_control", "access_control.public", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 637c87f4f61f0..063f0e3f6f854 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -3777,31 +3777,6 @@ "properties": { "interval": { "type": "keyword" - }, - "rrule": { - "properties": { - "byhour": { - "type": "integer" - }, - "byminute": { - "type": "integer" - }, - "bymonthday": { - "type": "integer" - }, - "byweekday": { - "type": "integer" - }, - "freq": { - "type": "integer" - }, - "interval": { - "type": "integer" - }, - "tzid": { - "type": "keyword" - } - } } } }, From 7099229b223ab5d10625dbd6b90991aa714b9559 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 2 May 2025 16:58:50 +0000 Subject: [PATCH 32/45] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- .../current_fields.json | 347 ++++++++++++++---- 1 file changed, 285 insertions(+), 62 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 795679d1801b0..fa63eee1e8430 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -1,6 +1,11 @@ { - "action": ["actionTypeId", "name"], - "action_task_params": ["apiKeyId"], + "action": [ + "actionTypeId", + "name" + ], + "action_task_params": [ + "apiKeyId" + ], "ad_hoc_run_params": [ "apiKeyId", "createdAt", @@ -81,7 +86,10 @@ "updatedAt", "updatedBy" ], - "api_key_pending_invalidation": ["apiKeyId", "createdAt"], + "api_key_pending_invalidation": [ + "apiKeyId", + "createdAt" + ], "apm-custom-dashboards": [ "dashboardSavedObjectId", "kuery", @@ -89,16 +97,44 @@ "serviceNameFilterEnabled" ], "apm-indices": [], - "apm-server-schema": ["schemaJson"], - "apm-service-group": ["color", "description", "groupName", "kuery"], + "apm-server-schema": [ + "schemaJson" + ], + "apm-service-group": [ + "color", + "description", + "groupName", + "kuery" + ], "apm-telemetry": [], "app_search_telemetry": [], - "application_usage_daily": ["timestamp"], + "application_usage_daily": [ + "timestamp" + ], "application_usage_totals": [], - "background-task-node": ["id", "last_seen"], - "canvas-element": ["@created", "@timestamp", "content", "help", "image", "name"], - "canvas-workpad": ["@created", "@timestamp", "name"], - "canvas-workpad-template": ["help", "name", "tags", "template_key"], + "background-task-node": [ + "id", + "last_seen" + ], + "canvas-element": [ + "@created", + "@timestamp", + "content", + "help", + "image", + "name" + ], + "canvas-workpad": [ + "@created", + "@timestamp", + "name" + ], + "canvas-workpad-template": [ + "help", + "name", + "tags", + "template_key" + ], "cases": [ "assignees", "assignees.uid", @@ -172,9 +208,21 @@ "type", "updated_at" ], - "cases-configure": ["closure_type", "created_at", "owner"], - "cases-connector-mappings": ["owner"], - "cases-rules": ["counter", "createdAt", "rules", "rules.id", "updatedAt"], + "cases-configure": [ + "closure_type", + "created_at", + "owner" + ], + "cases-connector-mappings": [ + "owner" + ], + "cases-rules": [ + "counter", + "createdAt", + "rules", + "rules.id", + "updatedAt" + ], "cases-telemetry": [], "cases-user-actions": [ "action", @@ -195,9 +243,16 @@ ], "cloud": [], "cloud-security-posture-settings": [], - "config": ["buildNum"], - "config-global": ["buildNum"], - "connector_token": ["connectorId", "tokenType"], + "config": [ + "buildNum" + ], + "config-global": [ + "buildNum" + ], + "connector_token": [ + "connectorId", + "tokenType" + ], "core-usage-stats": [], "csp-rule-template": [ "metadata", @@ -237,8 +292,15 @@ "version" ], "dynamic-config-overrides": [], - "endpoint:unified-user-artifact-manifest": ["artifactIds", "policyId", "semanticVersion"], - "endpoint:user-artifact-manifest": ["artifacts", "schemaVersion"], + "endpoint:unified-user-artifact-manifest": [ + "artifactIds", + "policyId", + "semanticVersion" + ], + "endpoint:user-artifact-manifest": [ + "artifacts", + "schemaVersion" + ], "enterprise_search_telemetry": [], "entity-definition": [ "description", @@ -254,7 +316,9 @@ "type", "version" ], - "entity-discovery-api-key": ["apiKey"], + "entity-discovery-api-key": [ + "apiKey" + ], "entity-engine-status": [ "fieldHistoryLength", "filter", @@ -302,8 +366,13 @@ "package_name", "package_version" ], - "event-annotation-group": ["description", "title"], - "event_loop_delays_daily": ["lastUpdatedAt"], + "event-annotation-group": [ + "description", + "title" + ], + "event_loop_delays_daily": [ + "lastUpdatedAt" + ], "exception-list": [ "_tags", "comments", @@ -382,7 +451,12 @@ "updated_by", "version" ], - "favorites": ["favoriteIds", "favoriteMetadata", "type", "userId"], + "favorites": [ + "favoriteIds", + "favoriteMetadata", + "type", + "userId" + ], "file": [ "FileKind", "Meta", @@ -396,8 +470,16 @@ "size", "user" ], - "file-upload-usage-collection-telemetry": ["file_upload", "file_upload.index_creation_count"], - "fileShare": ["created", "name", "token", "valid_until"], + "file-upload-usage-collection-telemetry": [ + "file_upload", + "file_upload.index_creation_count" + ], + "fileShare": [ + "created", + "name", + "token", + "valid_until" + ], "fleet-agent-policies": [ "advanced_settings", "agent_features", @@ -467,7 +549,9 @@ "updated_by", "vars" ], - "fleet-preconfiguration-deletion-record": ["id"], + "fleet-preconfiguration-deletion-record": [ + "id" + ], "fleet-proxy": [ "certificate", "certificate_authorities", @@ -477,9 +561,17 @@ "proxy_headers", "url" ], - "fleet-setup-lock": ["started_at", "status", "uuid"], + "fleet-setup-lock": [ + "started_at", + "status", + "uuid" + ], "fleet-space-settings": [], - "fleet-uninstall-tokens": ["namespaces", "policy_id", "token_plain"], + "fleet-uninstall-tokens": [ + "namespaces", + "policy_id", + "token_plain" + ], "graph-workspace": [ "description", "kibanaSavedObjectMeta", @@ -491,15 +583,24 @@ "version", "wsState" ], - "guided-onboarding-guide-state": ["guideId", "isActive"], + "guided-onboarding-guide-state": [ + "guideId", + "isActive" + ], "guided-onboarding-plugin-state": [], - "index-pattern": ["name", "title", "type"], + "index-pattern": [ + "name", + "title", + "type" + ], "infra-custom-dashboards": [ "assetType", "dashboardFilterAssetIdEnabled", "dashboardSavedObjectId" ], - "infrastructure-monitoring-log-view": ["name"], + "infrastructure-monitoring-log-view": [ + "name" + ], "infrastructure-ui-source": [], "ingest-agent-policies": [ "advanced_settings", @@ -536,7 +637,13 @@ "updated_at", "updated_by" ], - "ingest-download-sources": ["host", "is_default", "name", "proxy_id", "source_id"], + "ingest-download-sources": [ + "host", + "is_default", + "name", + "proxy_id", + "source_id" + ], "ingest-outputs": [ "allow_edit", "auth_type", @@ -649,10 +756,30 @@ "targetNamespace", "targetType" ], - "lens": ["description", "state", "title", "visualizationType"], - "lens-ui-telemetry": ["count", "date", "name", "type"], - "links": ["description", "links", "title"], - "maintenance-window": ["enabled", "events", "expirationDate", "title", "updatedAt"], + "lens": [ + "description", + "state", + "title", + "visualizationType" + ], + "lens-ui-telemetry": [ + "count", + "date", + "name", + "type" + ], + "links": [ + "description", + "links", + "title" + ], + "maintenance-window": [ + "enabled", + "events", + "expirationDate", + "title", + "updatedAt" + ], "map": [ "bounds", "description", @@ -664,7 +791,11 @@ ], "metrics-data-source": [], "metrics-explorer-view": [], - "ml-job": ["datafeed_id", "job_id", "type"], + "ml-job": [ + "datafeed_id", + "job_id", + "type" + ], "ml-module": [ "datafeeds", "defaultIndexPattern", @@ -677,9 +808,20 @@ "title", "type" ], - "ml-trained-model": ["job", "job.create_time", "job.job_id", "model_id"], - "monitoring-telemetry": ["reportedClusterUuids"], - "observability-onboarding-state": ["progress", "state", "type"], + "ml-trained-model": [ + "job", + "job.create_time", + "job.job_id", + "model_id" + ], + "monitoring-telemetry": [ + "reportedClusterUuids" + ], + "observability-onboarding-state": [ + "progress", + "state", + "type" + ], "onechat_workflow": [ "access_control", "access_control.public", @@ -687,7 +829,10 @@ "user_id", "workflow_id" ], - "osquery-manager-usage-metric": ["count", "errors"], + "osquery-manager-usage-metric": [ + "count", + "errors" + ], "osquery-pack": [ "created_at", "created_by", @@ -735,8 +880,12 @@ "updated_by", "version" ], - "policy-settings-protection-updates-note": ["note"], - "privilege-monitoring-status": ["status"], + "policy-settings-protection-updates-note": [ + "note" + ], + "privilege-monitoring-status": [ + "status" + ], "product-doc-install-status": [ "index_name", "installation_status", @@ -744,7 +893,11 @@ "product_name", "product_version" ], - "query": ["description", "title", "titleKeyword"], + "query": [ + "description", + "title", + "titleKeyword" + ], "risk-engine-configuration": [ "alertSampleSizePerShard", "dataViewId", @@ -758,12 +911,28 @@ "range.end", "range.start" ], - "rules-settings": ["flapping"], - "sample-data-telemetry": ["installCount", "unInstallCount"], - "search": ["description", "title"], - "search-session": ["created", "realmName", "realmType", "sessionId", "username"], + "rules-settings": [ + "flapping" + ], + "sample-data-telemetry": [ + "installCount", + "unInstallCount" + ], + "search": [ + "description", + "title" + ], + "search-session": [ + "created", + "realmName", + "realmType", + "sessionId", + "username" + ], "search-telemetry": [], - "search_playground": ["name"], + "search_playground": [ + "name" + ], "security-ai-prompt": [ "description", "model", @@ -773,8 +942,15 @@ "promptId", "provider" ], - "security-rule": ["rule_id", "version"], - "security-solution-signals-migration": ["sourceIndex", "updated", "version"], + "security-rule": [ + "rule_id", + "version" + ], + "security-solution-signals-migration": [ + "sourceIndex", + "updated", + "version" + ], "siem-detection-engine-rule-actions": [ "actions", "actions.actionRef", @@ -885,8 +1061,21 @@ "updated", "updatedBy" ], - "siem-ui-timeline-note": ["created", "createdBy", "eventId", "note", "updated", "updatedBy"], - "siem-ui-timeline-pinned-event": ["created", "createdBy", "eventId", "updated", "updatedBy"], + "siem-ui-timeline-note": [ + "created", + "createdBy", + "eventId", + "note", + "updated", + "updatedBy" + ], + "siem-ui-timeline-pinned-event": [ + "created", + "createdBy", + "eventId", + "updated", + "updatedBy" + ], "slo": [ "budgetingMethod", "description", @@ -900,7 +1089,11 @@ "version" ], "slo-settings": [], - "space": ["disabledFeatures", "name", "solution"], + "space": [ + "disabledFeatures", + "name", + "solution" + ], "spaces-usage-stats": [], "synthetics-dynamic-settings": [], "synthetics-monitor": [ @@ -933,7 +1126,11 @@ "synthetics-param": [], "synthetics-private-location": [], "synthetics-privates-locations": [], - "tag": ["color", "description", "name"], + "tag": [ + "color", + "description", + "name" + ], "task": [ "attempts", "enabled", @@ -953,15 +1150,41 @@ ], "telemetry": [], "threshold-explorer-view": [], - "ui-metric": ["count"], - "upgrade-assistant-ml-upgrade-operation": ["snapshotId"], - "upgrade-assistant-reindex-operation": ["indexName", "status"], + "ui-metric": [ + "count" + ], + "upgrade-assistant-ml-upgrade-operation": [ + "snapshotId" + ], + "upgrade-assistant-reindex-operation": [ + "indexName", + "status" + ], "uptime-dynamic-settings": [], - "uptime-synthetics-api-key": ["apiKey"], - "url": ["accessDate", "createDate", "slug"], - "usage-counter": ["count", "counterName", "counterType", "domainId", "source"], - "usage-counters": ["domainId"], - "visualization": ["description", "kibanaSavedObjectMeta", "title", "version"], + "uptime-synthetics-api-key": [ + "apiKey" + ], + "url": [ + "accessDate", + "createDate", + "slug" + ], + "usage-counter": [ + "count", + "counterName", + "counterType", + "domainId", + "source" + ], + "usage-counters": [ + "domainId" + ], + "visualization": [ + "description", + "kibanaSavedObjectMeta", + "title", + "version" + ], "workchat_agent": [ "access_control", "access_control.public", From 3afe5b2f6decd5fe27756661fbc8eb20aff300b6 Mon Sep 17 00:00:00 2001 From: Ying Date: Fri, 2 May 2025 15:13:29 -0400 Subject: [PATCH 33/45] Fix --- .../task_manager/server/saved_objects/schemas/task.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts index b33c66183e8fd..365cc506cb123 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts @@ -64,16 +64,6 @@ export const taskSchemaV4 = taskSchemaV3.extends({ apiKeyCreatedByUser: schema.boolean(), }) ), - schedule: schema.maybe( - schema.oneOf([ - schema.object({ - interval: schema.string({ validate: validateDuration }), - }), - schema.object({ - rrule: rruleSchedule, - }), - ]) - ), }); export const taskSchemaV5 = taskSchemaV4.extends({ From 5398116d11111d6624880943cfda988e87a37cdb Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Fri, 2 May 2025 22:08:47 +0200 Subject: [PATCH 34/45] remove schema from v4 --- .../task_manager/server/saved_objects/schemas/task.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts index b33c66183e8fd..365cc506cb123 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts @@ -64,16 +64,6 @@ export const taskSchemaV4 = taskSchemaV3.extends({ apiKeyCreatedByUser: schema.boolean(), }) ), - schedule: schema.maybe( - schema.oneOf([ - schema.object({ - interval: schema.string({ validate: validateDuration }), - }), - schema.object({ - rrule: rruleSchedule, - }), - ]) - ), }); export const taskSchemaV5 = taskSchemaV4.extends({ From 85cb01588c87f66a186dbd0afd7c788ba3c5bf6e Mon Sep 17 00:00:00 2001 From: Ying Date: Mon, 5 May 2025 09:45:46 -0400 Subject: [PATCH 35/45] Updating types --- .../lib/transform_raw_scheduled_report.ts | 4 ++-- .../request_handler/schedule_request_handler.ts | 5 ++--- .../scheduled_report/schemas/latest.ts | 6 ++---- .../plugins/private/reporting/server/types.ts | 6 ++++++ .../plugins/shared/task_manager/server/index.ts | 2 +- .../server/lib/get_first_run_at.test.ts | 6 +++--- .../task_manager/server/saved_objects/index.ts | 1 - .../server/saved_objects/schemas/task.ts | 3 --- .../plugins/shared/task_manager/server/task.ts | 13 ++++++------- 9 files changed, 22 insertions(+), 24 deletions(-) diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts index 9727895036a64..f386fe790dfbc 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts @@ -7,10 +7,10 @@ import { SavedObject } from '@kbn/core/server'; import { ScheduledReportApiJSON } from '@kbn/reporting-common/types'; -import { RawScheduledReport } from '../../../../saved_objects/scheduled_report/schemas/latest'; +import { ScheduledReportType } from '../../../../types'; export function transformRawScheduledReportToReport( - rawScheduledReport: SavedObject + rawScheduledReport: SavedObject ): ScheduledReportApiJSON { const parsedPayload = JSON.parse(rawScheduledReport.attributes.payload); return { diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts index b81c789c44c95..80a3006876073 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts @@ -12,8 +12,7 @@ import { ScheduledReportApiJSON } from '@kbn/reporting-common/types'; import { isEmpty, omit } from 'lodash'; import { RruleSchedule, scheduleRruleSchema } from '@kbn/task-manager-plugin/server'; import { rawNotificationSchema } from '../../../saved_objects/scheduled_report/schemas/v1'; -import { ScheduledReportingJobResponse } from '../../../types'; -import { RawScheduledReport } from '../../../saved_objects/scheduled_report/schemas/latest'; +import { ScheduledReportType, ScheduledReportingJobResponse } from '../../../types'; import { SCHEDULED_REPORT_SAVED_OBJECT_TYPE } from '../../../saved_objects'; import { RequestHandler, RequestParams } from './request_handler'; import { transformRawScheduledReportToReport } from './lib'; @@ -97,7 +96,7 @@ export class ScheduleRequestHandler extends RequestHandler< }; // Create a scheduled report saved object - const report = await soClient.create( + const report = await soClient.create( SCHEDULED_REPORT_SAVED_OBJECT_TYPE, attributes ); diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts index f6d1877cd90bd..6f684f9d7cbd7 100644 --- a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts @@ -8,7 +8,5 @@ import type { TypeOf } from '@kbn/config-schema'; import type { rawNotificationSchema, rawScheduledReportSchema } from './v1'; -type Mutable = { -readonly [P in keyof T]: T[P] extends object ? Mutable : T[P] }; - -export type RawNotification = Mutable>; -export type RawScheduledReport = Mutable>; +export type RawNotification = TypeOf; +export type RawScheduledReport = TypeOf; diff --git a/x-pack/platform/plugins/private/reporting/server/types.ts b/x-pack/platform/plugins/private/reporting/server/types.ts index 79e1cc5012f07..f0fb06f065433 100644 --- a/x-pack/platform/plugins/private/reporting/server/types.ts +++ b/x-pack/platform/plugins/private/reporting/server/types.ts @@ -24,6 +24,7 @@ import type { import type { SecurityPluginSetup } from '@kbn/security-plugin/server'; import type { SpacesPluginSetup } from '@kbn/spaces-plugin/server'; import type { + RruleSchedule, TaskManagerSetupContract, TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; @@ -32,6 +33,7 @@ import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import { ExportTypesRegistry } from '@kbn/reporting-server/export_types_registry'; import type { AuthenticatedUser } from '@kbn/core-security-common'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { RawScheduledReport } from './saved_objects/scheduled_report/schemas/latest'; /** * Plugin Setup Contract @@ -98,6 +100,10 @@ export interface ScheduledReportingJobResponse { job: ScheduledReportApiJSON; } +export type ScheduledReportType = Omit & { + schedule: RruleSchedule; +}; + export interface PdfScreenshotOptions extends Omit { urls: UrlOrUrlLocatorTuple[]; } diff --git a/x-pack/platform/plugins/shared/task_manager/server/index.ts b/x-pack/platform/plugins/shared/task_manager/server/index.ts index 21bbda675d02c..a20f3bf83b41f 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/index.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/index.ts @@ -24,9 +24,9 @@ export type { } from './task'; export { Frequency, Weekday } from '@kbn/rrule'; -export type { RruleSchedule } from './saved_objects'; export { scheduleRruleSchema } from './saved_objects'; +export type { RruleSchedule } from './task'; export { TaskStatus, TaskPriority, TaskCost } from './task'; export type { TaskRegisterDefinition, TaskDefinitionRegistry } from './task_type_dictionary'; diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts index d3e562c67b723..e030aa9c61e8a 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts @@ -159,7 +159,7 @@ describe('getFirstRunAt', () => { freq: 2, // Weekly interval: 1, tzid: 'UTC', - byweekday: [1], // Monday + byweekday: ['1'], // Monday }, }, }; @@ -182,7 +182,7 @@ describe('getFirstRunAt', () => { freq: 2, // Weekly interval: 1, tzid: 'UTC', - byweekday: [1], // Monday + byweekday: ['MO'], // Monday byhour: [12], byminute: [15], }, @@ -257,7 +257,7 @@ describe('getFirstRunAt', () => { freq: 1, // Monthly interval: 1, tzid: 'UTC', - byweekday: [3], // Wednesday + byweekday: ['3'], // Wednesday byhour: [12], byminute: [17], }, diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts index 377fd4bc210d2..75b200acc38b1 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts @@ -14,7 +14,6 @@ import { getOldestIdleActionTask } from '../queries/oldest_idle_action_task'; import { TASK_MANAGER_INDEX } from '../constants'; import { backgroundTaskNodeModelVersions, taskModelVersions } from './model_versions'; -export type { RruleSchedule } from './schemas/task'; export { scheduleRruleSchema } from './schemas/task'; export const TASK_SO_NAME = 'task'; diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts index 299b20722c268..874f374355bbb 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type { TypeOf } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; import { isInterval } from '../../lib/intervals'; import { rruleSchedule } from './rrule'; @@ -78,5 +77,3 @@ export const taskSchemaV4 = taskSchemaV3.extends({ export const taskSchemaV5 = taskSchemaV4.extends({ schedule: schema.maybe(schema.oneOf([scheduleIntervalSchema, scheduleRruleSchema])), }); - -export type RruleSchedule = TypeOf; diff --git a/x-pack/platform/plugins/shared/task_manager/server/task.ts b/x-pack/platform/plugins/shared/task_manager/server/task.ts index 277049facb1a9..b9a59e3452267 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task.ts @@ -11,7 +11,7 @@ import type { ObjectType, TypeOf } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; import { isNumber } from 'lodash'; import type { KibanaRequest } from '@kbn/core/server'; -import type { Frequency, Weekday } from '@kbn/rrule'; +import type { Frequency } from '@kbn/rrule'; import { isErr, tryAsResult } from './lib/result_type'; import type { Interval } from './lib/intervals'; import { isInterval, parseIntervalAsMillisecond } from './lib/intervals'; @@ -259,13 +259,12 @@ export interface IntervalSchedule { rrule?: never; } +export type Rrule = RruleMonthly | RruleWeekly | RruleDaily; export interface RruleSchedule { - rrule: RruleMonthly | RruleWeekly | RruleDaily; + rrule: Rrule; interval?: never; } -export type Rrule = RruleMonthly | RruleWeekly | RruleDaily; - interface RruleCommon { freq: Frequency; interval: number; @@ -276,11 +275,11 @@ interface RruleMonthly extends RruleCommon { bymonthday?: number[]; byhour?: number[]; byminute?: number[]; - byweekday?: Weekday[]; + byweekday?: string[]; } interface RruleWeekly extends RruleCommon { freq: Frequency.WEEKLY; - byweekday?: Weekday[]; + byweekday?: string[]; byhour?: number[]; byminute?: number[]; bymonthday?: never; @@ -289,7 +288,7 @@ interface RruleDaily extends RruleCommon { freq: Frequency.DAILY; byhour?: number[]; byminute?: number[]; - byweekday?: Weekday[]; + byweekday?: string[]; bymonthday?: never; } From 0253ced12c0420013ebf84f9c2a3fc3718f4249e Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Mon, 5 May 2025 17:44:32 +0200 Subject: [PATCH 36/45] add explicit error log when the next run at cannot be calculated --- .../server/lib/get_first_run_at.ts | 6 ++- .../server/lib/get_next_run_at.test.ts | 50 +++++++++++++++++-- .../server/lib/get_next_run_at.ts | 38 ++++++++------ 3 files changed, 74 insertions(+), 20 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts index c3e06739992de..8d436a2ba1c58 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.ts @@ -38,11 +38,15 @@ export function getFirstRunAt({ return nowString; } +// This function checks if the rrule has fixed time by checking if it has any fields other than +// 'freq', 'interval', and 'tzid'. If it does, it means the rrule has fixed time. +// The first run of a rule that has a fixed time has to be on the expected time, +// therefore should be calculated using the rrule library, otherwise it can be `now`. function rruleHasFixedTime(schedule: RruleSchedule['rrule']): boolean { const keys = Object.keys(schedule); const baseFields = ['freq', 'interval', 'tzid']; - if (keys.length === 3 && keys.every((key) => baseFields.includes(key))) { + if (keys.length === baseFields.length && keys.every((key) => baseFields.includes(key))) { return false; } diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts index 5adea96ffc4fe..3acbe89b57e24 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.test.ts @@ -8,8 +8,13 @@ import { taskManagerMock } from '../mocks'; import { getNextRunAt } from './get_next_run_at'; +import { loggerMock } from '@kbn/logging-mocks'; +const mockLogger = loggerMock.create(); describe('getNextRunAt', () => { + afterEach(() => { + jest.clearAllMocks(); + }); test('should use startedAt when the task delay is greater than the threshold', () => { const now = new Date(); // Use time in the past to ensure the task delay calculation isn't relative to "now" @@ -21,7 +26,8 @@ describe('getNextRunAt', () => { runAt: fiveSecondsAgo, startedAt: fourSecondsAgo, }), - 500 + 500, + mockLogger ); expect(nextRunAt).toEqual(new Date(fourSecondsAgo.getTime() + 60000)); }); @@ -37,7 +43,8 @@ describe('getNextRunAt', () => { runAt: fiveSecondsAgo, startedAt: aBitLessThanFiveSecondsAgo, }), - 500 + 500, + mockLogger ); expect(nextRunAt).toEqual(new Date(fiveSecondsAgo.getTime() + 60000)); }); @@ -51,7 +58,8 @@ describe('getNextRunAt', () => { runAt: fiveMinsAgo, startedAt: fiveMinsAgo, }), - 0 + 0, + mockLogger ); expect(nextRunAt.getTime()).toBeGreaterThanOrEqual(testStart.getTime()); }); @@ -74,7 +82,8 @@ describe('getNextRunAt', () => { runAt: testRunAt, startedAt: testStart, }), - 0 + 0, + mockLogger ); const currentDay = testStart.getUTCDay(); @@ -111,7 +120,8 @@ describe('getNextRunAt', () => { runAt: testRunAt, startedAt: testStart, }), - 0 + 0, + mockLogger ); const oneDay = 24 * 60 * 60 * 1000; @@ -119,4 +129,34 @@ describe('getNextRunAt', () => { expect(nextRunAt).toEqual(expectedNextRunAt); }); + + test('should throw an error if the next runAt cannot be calculated', () => { + const now = new Date(); + const testStart = now; + const testRunAt = new Date(now.getTime() - 1000); + + expect(() => + getNextRunAt( + taskManagerMock.createTask({ + schedule: { + rrule: { + freq: 3, // Daily + interval: 1, + tzid: 'UTC', + // @ts-ignore + count: 1, // Invalid field for rrule + }, + }, + runAt: testRunAt, + startedAt: testStart, + }), + 0, + mockLogger + ) + ).toThrow(`Cannot read properties of null (reading 'getTime')`); + + expect(mockLogger.error).toHaveBeenCalledWith( + "The next runAt for the task with a fixed time schedule could not be calculated: TypeError: Cannot read properties of null (reading 'getTime')" + ); + }); }); diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts index 343d0fd0daa85..1f41914fd2a24 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_next_run_at.ts @@ -6,33 +6,43 @@ */ import { RRule } from '@kbn/rrule'; +import type { Logger } from '@kbn/core/server'; import { intervalFromDate } from './intervals'; import type { ConcreteTaskInstance } from '../task'; export function getNextRunAt( { runAt, startedAt, schedule }: Pick, - taskDelayThresholdForPreciseScheduling: number = 0 + taskDelayThresholdForPreciseScheduling: number = 0, + logger: Logger ): Date { const taskDelay = startedAt!.getTime() - runAt.getTime(); const scheduleFromDate = taskDelay < taskDelayThresholdForPreciseScheduling ? runAt : startedAt!; const { rrule, interval } = schedule || {}; + let nextCalculatedRunAt: number; - let nextRunAt: Date | undefined | null; + try { + let nextRunAt: Date | undefined | null; - if (interval) { - nextRunAt = intervalFromDate(scheduleFromDate, interval); - } else if (rrule) { - const _rrule = new RRule({ - ...rrule, - dtstart: scheduleFromDate, - }); - // adding 1ms to ensure the next run is always in the future - // if scheduleFromDate is equal to now (very low possibility), the next run will be now again, which causes loops - nextRunAt = _rrule.after(new Date(Date.now() + 1)); + if (interval) { + nextRunAt = intervalFromDate(scheduleFromDate, interval); + } else if (rrule) { + const _rrule = new RRule({ + ...rrule, + dtstart: scheduleFromDate, + }); + // adding 1ms to ensure the next run is always in the future + // if scheduleFromDate is equal to now (very low possibility), the next run will be now again, which causes loops + nextRunAt = _rrule.after(new Date(Date.now() + 1)); + } + // Ensure we also don't schedule in the past by performing the Math.max with Date.now + nextCalculatedRunAt = Math.max(nextRunAt!.getTime(), Date.now()); + } catch (e) { + logger.error( + `The next runAt for the task with a fixed time schedule could not be calculated: ${e}` + ); + throw e; } - // Ensure we also don't schedule in the past by performing the Math.max with Date.now() - const nextCalculatedRunAt = Math.max(nextRunAt!.getTime(), Date.now()); return new Date(nextCalculatedRunAt); } From 3834eaf9eae4c073ccdb72553d796b8d50bfacf8 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 5 May 2025 15:59:06 +0000 Subject: [PATCH 37/45] [CI] Auto-commit changed files from 'node scripts/styled_components_mapping' --- x-pack/platform/plugins/shared/task_manager/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/task_manager/tsconfig.json b/x-pack/platform/plugins/shared/task_manager/tsconfig.json index c2687806e7f2a..69567b61f6dde 100644 --- a/x-pack/platform/plugins/shared/task_manager/tsconfig.json +++ b/x-pack/platform/plugins/shared/task_manager/tsconfig.json @@ -35,7 +35,8 @@ "@kbn/encrypted-saved-objects-shared", "@kbn/core-saved-objects-api-server-mocks", "@kbn/core-http-server-utils", - "@kbn/rrule" + "@kbn/rrule", + "@kbn/logging-mocks" ], "exclude": ["target/**/*"] } From c7e26f0d0c4bb0dab373dd15907cdb973bd97e7b Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Mon, 5 May 2025 18:36:42 +0200 Subject: [PATCH 38/45] add missing logger --- .../shared/task_manager/server/task_running/task_runner.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_running/task_runner.ts b/x-pack/platform/plugins/shared/task_manager/server/task_running/task_runner.ts index f34d83dc4dec5..d103e4a981858 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_running/task_runner.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_running/task_runner.ts @@ -676,7 +676,8 @@ export class TaskManagerRunner implements TaskRunner { startedAt: this.instance.task.startedAt, schedule: updatedTaskSchedule, }, - this.getPollInterval() + this.getPollInterval(), + this.logger ), state, schedule: updatedTaskSchedule, From 1e3d3e961cdb355388c1af03935a8c8f0940636e Mon Sep 17 00:00:00 2001 From: Ying Date: Mon, 5 May 2025 12:53:44 -0400 Subject: [PATCH 39/45] Handling notification --- .../private/kbn-reporting/common/types.ts | 11 +- .../plugins/private/reporting/kibana.jsonc | 1 + .../plugins/private/reporting/server/core.ts | 9 ++ .../lib/transform_raw_scheduled_report.ts | 4 +- .../common/request_handler/request_handler.ts | 2 + .../schedule_request_handler.test.ts | 147 ++++++++++++++++++ .../schedule_request_handler.ts | 32 +++- .../scheduling_from_jobparams.test.ts | 24 +++ .../schedule/schedule_from_jobparams.ts | 2 + .../scheduled_report/schemas/v1.ts | 2 +- .../create_mock_reportingplugin.ts | 7 + .../plugins/private/reporting/server/types.ts | 19 ++- .../plugins/private/reporting/tsconfig.json | 1 + 13 files changed, 243 insertions(+), 18 deletions(-) diff --git a/src/platform/packages/private/kbn-reporting/common/types.ts b/src/platform/packages/private/kbn-reporting/common/types.ts index 8df1df7c94a2e..a7931e642f9ca 100644 --- a/src/platform/packages/private/kbn-reporting/common/types.ts +++ b/src/platform/packages/private/kbn-reporting/common/types.ts @@ -11,7 +11,7 @@ import type { LayoutParams, PerformanceMetrics as ScreenshotMetrics, } from '@kbn/screenshotting-plugin/common'; -import type { ConcreteTaskInstance, RruleSchedule } from '@kbn/task-manager-plugin/server'; +import type { ConcreteTaskInstance } from '@kbn/task-manager-plugin/server'; import { JOB_STATUS } from './constants'; import type { LocatorParams } from './url'; @@ -202,15 +202,6 @@ export interface ReportApiJSON extends ReportSimple { index: string; } -export type ScheduledReportApiJSON = Omit< - ReportSource, - 'payload' | 'output' | 'attempts' | 'status' -> & { - id: string; - schedule: RruleSchedule; - payload: Omit; -}; - export interface LicenseCheckResults { enableLinks: boolean; showLinks: boolean; diff --git a/x-pack/platform/plugins/private/reporting/kibana.jsonc b/x-pack/platform/plugins/private/reporting/kibana.jsonc index ce6b130018af0..e22f4605cbd60 100644 --- a/x-pack/platform/plugins/private/reporting/kibana.jsonc +++ b/x-pack/platform/plugins/private/reporting/kibana.jsonc @@ -16,6 +16,7 @@ "reporting" ], "requiredPlugins": [ + "actions", "data", "discover", "encryptedSavedObjects", diff --git a/x-pack/platform/plugins/private/reporting/server/core.ts b/x-pack/platform/plugins/private/reporting/server/core.ts index 0a51b5bd4ec27..bc8c232eafb1b 100644 --- a/x-pack/platform/plugins/private/reporting/server/core.ts +++ b/x-pack/platform/plugins/private/reporting/server/core.ts @@ -24,6 +24,7 @@ import type { UiSettingsServiceStart, } from '@kbn/core/server'; import { SECURITY_EXTENSION_ID } from '@kbn/core-saved-objects-server'; +import type { PluginSetupContract as ActionsPluginSetupContract } from '@kbn/actions-plugin/server'; import type { PluginStart as DataPluginStart } from '@kbn/data-plugin/server'; import type { DiscoverServerPluginStart } from '@kbn/discover-plugin/server'; import type { FeaturesPluginSetup } from '@kbn/features-plugin/server'; @@ -58,6 +59,7 @@ import { EventTracker } from './usage'; import { SCHEDULED_REPORT_SAVED_OBJECT_TYPE } from './saved_objects'; export interface ReportingInternalSetup { + actions: ActionsPluginSetupContract; basePath: Pick; docLinks: DocLinksServiceSetup; encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; @@ -353,6 +355,13 @@ export class ReportingCore { ); } + public validateNotificationEmails(emails: string[]): string | undefined { + const pluginSetupDeps = this.getPluginSetupDeps(); + return pluginSetupDeps.actions + .getActionsConfigurationUtilities() + .validateEmailAddresses(emails); + } + /* * Gives synchronous access to the setupDeps */ diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts index f386fe790dfbc..3006ff48bfad3 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts @@ -6,8 +6,7 @@ */ import { SavedObject } from '@kbn/core/server'; -import { ScheduledReportApiJSON } from '@kbn/reporting-common/types'; -import { ScheduledReportType } from '../../../../types'; +import { ScheduledReportApiJSON, ScheduledReportType } from '../../../../types'; export function transformRawScheduledReportToReport( rawScheduledReport: SavedObject @@ -22,5 +21,6 @@ export function transformRawScheduledReportToReport( meta: rawScheduledReport.attributes.meta, migration_version: rawScheduledReport.attributes.migrationVersion, schedule: rawScheduledReport.attributes.schedule, + notification: rawScheduledReport.attributes.notification, }; } diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts index b086fc41912d6..55bffafc82552 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts @@ -20,6 +20,7 @@ import { cryptoFactory } from '@kbn/reporting-server'; import rison from '@kbn/rison'; import { RruleSchedule } from '@kbn/task-manager-plugin/server'; +import { RawNotification } from '../../../saved_objects/scheduled_report/schemas/latest'; import { checkParamsVersion } from '../../../lib'; import { type Counters } from '..'; import type { ReportingCore } from '../../..'; @@ -55,6 +56,7 @@ export interface RequestParams { exportTypeId: string; jobParams: BaseParams; schedule?: RruleSchedule; + notification?: RawNotification; } /** diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts index 0bec76c48ad77..91aa5d8851eb2 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts @@ -120,6 +120,7 @@ describe('Handle request to schedule', () => { "objectType": "cool_object_type", }, "migration_version": "unknown", + "notification": undefined, "schedule": Object { "rrule": Object { "freq": 1, @@ -165,6 +166,79 @@ describe('Handle request to schedule', () => { }, }); }); + + test('creates a scheduled report saved object with notification', async () => { + const report = await requestHandler.enqueueJob({ + exportTypeId: 'printablePdfV2', + jobParams: mockJobParams, + schedule: { rrule: { freq: 1, interval: 2, tzid: 'UTC' } }, + notification: { email: { to: ['a@b.com'] } }, + }); + + const { id, created_at: _created_at, payload, ...snapObj } = report; + expect(snapObj).toMatchInlineSnapshot(` + Object { + "created_by": "testymcgee", + "jobtype": "printable_pdf_v2", + "meta": Object { + "isDeprecated": false, + "layout": "preserve_layout", + "objectType": "cool_object_type", + }, + "migration_version": "unknown", + "notification": Object { + "email": Object { + "to": Array [ + "a@b.com", + ], + }, + }, + "schedule": Object { + "rrule": Object { + "freq": 1, + "interval": 2, + "tzid": "UTC", + }, + }, + } + `); + expect(payload).toMatchInlineSnapshot(` + Object { + "browserTimezone": "UTC", + "isDeprecated": false, + "layout": Object { + "id": "preserve_layout", + }, + "locatorParams": Array [], + "objectType": "cool_object_type", + "title": "cool_title", + "version": "unknown", + } + `); + + expect(soClient.create).toHaveBeenCalledWith('scheduled_report', { + jobType: 'printable_pdf_v2', + createdAt: expect.any(String), + createdBy: 'testymcgee', + title: 'cool_title', + enabled: true, + payload: JSON.stringify(payload), + schedule: { + rrule: { + freq: 1, + interval: 2, + tzid: 'UTC', + }, + }, + migrationVersion: 'unknown', + meta: { + objectType: 'cool_object_type', + layout: 'preserve_layout', + isDeprecated: false, + }, + notification: { email: { to: ['a@b.com'] } }, + }); + }); }); describe('getJobParams', () => { @@ -307,6 +381,79 @@ describe('Handle request to schedule', () => { }); }); + describe('getNotification', () => { + test('parse notification from body', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: { to: ['a@b.com'] } }, + }; + expect(requestHandler.getNotification()).toEqual({ email: { to: ['a@b.com'] } }); + }); + + test('returns undefined if notification object is empty', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: {}, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + + test('returns undefined if notification object is null', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: null, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + + test('returns undefined if notification.email object is empty', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: {} }, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + + test('returns undefined if notification.email object is null', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: null }, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + + test('handles invalid email address', () => { + jest + .spyOn(reportingCore, 'validateNotificationEmails') + .mockReturnValueOnce('not valid emails: foo'); + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: { to: ['foo'] } }, + }; + requestHandler.getNotification(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('Invalid email address(es): not valid emails: foo'); + }); + }); + describe('handleRequest', () => { test('disallows invalid export type', async () => { expect( diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts index 80a3006876073..b1735de8515ee 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts @@ -8,11 +8,15 @@ import moment from 'moment'; import { schema } from '@kbn/config-schema'; -import { ScheduledReportApiJSON } from '@kbn/reporting-common/types'; import { isEmpty, omit } from 'lodash'; import { RruleSchedule, scheduleRruleSchema } from '@kbn/task-manager-plugin/server'; +import { RawNotification } from '../../../saved_objects/scheduled_report/schemas/latest'; import { rawNotificationSchema } from '../../../saved_objects/scheduled_report/schemas/v1'; -import { ScheduledReportType, ScheduledReportingJobResponse } from '../../../types'; +import { + ScheduledReportApiJSON, + ScheduledReportType, + ScheduledReportingJobResponse, +} from '../../../types'; import { SCHEDULED_REPORT_SAVED_OBJECT_TYPE } from '../../../saved_objects'; import { RequestHandler, RequestParams } from './request_handler'; import { transformRawScheduledReportToReport } from './lib'; @@ -60,8 +64,29 @@ export class ScheduleRequestHandler extends RequestHandler< return schedule; } + public getNotification(): RawNotification | undefined { + const { reporting, req, res } = this.opts; + + const { notification } = req.body; + if (isEmpty(notification) || isEmpty(notification.email)) { + return undefined; + } + + if (notification && notification.email && notification.email.to) { + const invalidEmails = reporting.validateNotificationEmails(notification.email.to); + if (invalidEmails) { + throw res.customError({ + statusCode: 400, + body: `Invalid email address(es): ${invalidEmails}`, + }); + } + } + + return notification; + } + public async enqueueJob(params: RequestParams) { - const { exportTypeId, jobParams, schedule } = params; + const { exportTypeId, jobParams, schedule, notification } = params; const { reporting, logger, req, user } = this.opts; const soClient = await reporting.getSoClient(req); @@ -90,6 +115,7 @@ export class ScheduleRequestHandler extends RequestHandler< objectType: jobParams.objectType, }, migrationVersion: version, + ...(notification ? { notification } : {}), title: job.title, payload: JSON.stringify(omit(payload, 'forceNow')), schedule: schedule!, diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts index 8014379401589..b7205c3cfb7c4 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts @@ -223,6 +223,30 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { ); }); + it('returns 400 on empty notification list', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { + email: { + to: [], + }, + }, + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + `"[request body.notification.email.to]: array size is [0], but cannot be smaller than [1]"` + ) + ); + }); + it('returns 403 on when no permanent encryption key', async () => { jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValueOnce({ isSufficientlySecure: true, diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts index 79c408104444b..0c501c4321898 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts @@ -49,11 +49,13 @@ export function registerScheduleRoutesInternal(reporting: ReportingCore, logger: }); const jobParams = requestHandler.getJobParams(); const schedule = requestHandler.getSchedule(); + const notification = requestHandler.getNotification(); return await requestHandler.handleRequest({ exportTypeId: req.params.exportType, jobParams, schedule, + notification, }); } catch (err) { if (err instanceof KibanaResponse) { diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts index ef407977051b9..22768f0f2e333 100644 --- a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts @@ -17,7 +17,7 @@ const rawLayoutIdSchema = schema.oneOf([ export const rawNotificationSchema = schema.object({ email: schema.maybe( schema.object({ - to: schema.arrayOf(schema.string()), + to: schema.arrayOf(schema.string(), { minSize: 1 }), }) ), }); diff --git a/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts index 94e6096e41bcd..199dc23a94327 100644 --- a/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -18,6 +18,7 @@ import { savedObjectsClientMock, statusServiceMock, } from '@kbn/core/server/mocks'; +import { actionsMock } from '@kbn/actions-plugin/server/mocks'; import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; import { discoverPluginMock } from '@kbn/discover-plugin/server/mocks'; import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks'; @@ -40,6 +41,12 @@ export const createMockPluginSetup = ( setupMock: Partial> ): ReportingInternalSetup => { return { + actions: { + ...actionsMock.createSetup(), + getActionsConfigurationUtilities: jest.fn().mockReturnValue({ + validateEmailAddresses: jest.fn(), + }), + }, encryptedSavedObjects: encryptedSavedObjectsMock.createSetup({ canEncrypt: true }), features: featuresPluginMock.createSetup(), basePath: { set: jest.fn() }, diff --git a/x-pack/platform/plugins/private/reporting/server/types.ts b/x-pack/platform/plugins/private/reporting/server/types.ts index f0fb06f065433..c4d238b3e90fc 100644 --- a/x-pack/platform/plugins/private/reporting/server/types.ts +++ b/x-pack/platform/plugins/private/reporting/server/types.ts @@ -12,7 +12,7 @@ import type { DiscoverServerPluginStart } from '@kbn/discover-plugin/server'; import type { FeaturesPluginSetup } from '@kbn/features-plugin/server'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/server'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; -import type { ScheduledReportApiJSON, UrlOrUrlLocatorTuple } from '@kbn/reporting-common/types'; +import type { ReportSource, UrlOrUrlLocatorTuple } from '@kbn/reporting-common/types'; import type { ReportApiJSON } from '@kbn/reporting-common/types'; import type { ReportingConfigType } from '@kbn/reporting-server'; import type { ScreenshotModePluginSetup } from '@kbn/screenshot-mode-plugin/server'; @@ -29,11 +29,15 @@ import type { TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; +import type { PluginSetupContract as ActionsPluginSetupContract } from '@kbn/actions-plugin/server'; import { ExportTypesRegistry } from '@kbn/reporting-server/export_types_registry'; import type { AuthenticatedUser } from '@kbn/core-security-common'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; -import { RawScheduledReport } from './saved_objects/scheduled_report/schemas/latest'; +import { + RawNotification, + RawScheduledReport, +} from './saved_objects/scheduled_report/schemas/latest'; /** * Plugin Setup Contract @@ -51,6 +55,7 @@ export type ReportingUser = { username: AuthenticatedUser['username'] } | false; export type ScrollConfig = ReportingConfigType['csv']['scroll']; export interface ReportingSetupDeps { + actions: ActionsPluginSetupContract; encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; features: FeaturesPluginSetup; screenshotMode: ScreenshotModePluginSetup; @@ -92,6 +97,16 @@ export interface ReportingJobResponse { job: ReportApiJSON; } +export type ScheduledReportApiJSON = Omit< + ReportSource, + 'payload' | 'output' | 'attempts' | 'status' +> & { + id: string; + schedule: RruleSchedule; + notification?: RawNotification; + payload: Omit; +}; + export interface ScheduledReportingJobResponse { /** * Details of a new report job that was requested diff --git a/x-pack/platform/plugins/private/reporting/tsconfig.json b/x-pack/platform/plugins/private/reporting/tsconfig.json index 814205e381988..51c75b5b431a0 100644 --- a/x-pack/platform/plugins/private/reporting/tsconfig.json +++ b/x-pack/platform/plugins/private/reporting/tsconfig.json @@ -5,6 +5,7 @@ }, "include": ["common/**/*", "public/**/*", "server/**/*", "../../../../../typings/**/*"], "kbn_references": [ + "@kbn/actions-plugin", "@kbn/core", "@kbn/data-plugin", "@kbn/discover-plugin", From bbea40521311c48e9d8e739515cdaed44c60425d Mon Sep 17 00:00:00 2001 From: Ersin Erdal Date: Mon, 5 May 2025 20:29:40 +0200 Subject: [PATCH 40/45] fix unit test --- .../task_manager/server/task_running/task_runner.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/plugins/shared/task_manager/server/task_running/task_runner.test.ts b/x-pack/platform/plugins/shared/task_manager/server/task_running/task_runner.test.ts index 160c6f6a87564..8344835a674ba 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task_running/task_runner.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task_running/task_runner.test.ts @@ -1168,7 +1168,7 @@ describe('TaskManagerRunner', () => { const schedule = { interval: '1m', }; - const { instance, runner, store } = await readyToRunStageSetup({ + const { instance, runner, store, logger } = await readyToRunStageSetup({ instance: { status: TaskStatus.Running, startedAt: new Date(), @@ -1195,7 +1195,8 @@ describe('TaskManagerRunner', () => { expect(getNextRunAtSpy).toHaveBeenCalledWith( expect.objectContaining({ schedule }), - expect.any(Number) + expect.any(Number), + logger ); }); From 2bd0d5d2d2b8510e6ddf1bdb92963a40d871ac6c Mon Sep 17 00:00:00 2001 From: Ying Date: Wed, 7 May 2025 09:36:07 -0400 Subject: [PATCH 41/45] Adding schedule reports API --- .../current_fields.json | 3 + .../current_mappings.json | 8 + .../registration/type_registrations.test.ts | 1 + .../private/kbn-reporting/common/routes.ts | 1 + .../private/canvas/server/feature.test.ts | 8 +- .../plugins/private/canvas/server/feature.ts | 2 +- .../plugins/private/reporting/kibana.jsonc | 1 + .../plugins/private/reporting/server/core.ts | 20 + .../private/reporting/server/plugin.test.ts | 15 + .../private/reporting/server/plugin.ts | 4 + .../routes/common/generate/request_handler.ts | 260 --------- .../routes/common/jobs/get_job_routes.ts | 2 +- .../generate_request_handler.test.ts} | 76 +-- .../generate_request_handler.ts | 130 +++++ .../routes/common/request_handler/index.ts | 10 + .../common/request_handler/lib/index.ts | 8 + .../lib/transform_raw_scheduled_report.ts | 26 + .../common/request_handler/request_handler.ts | 207 +++++++ .../schedule_request_handler.test.ts | 543 ++++++++++++++++++ .../schedule_request_handler.ts | 174 ++++++ .../private/reporting/server/routes/index.ts | 2 + .../generate/generate_from_jobparams.ts | 15 +- .../server/routes/internal/management/jobs.ts | 2 +- .../scheduling_from_jobparams.test.ts | 353 ++++++++++++ .../schedule/schedule_from_jobparams.ts | 71 +++ .../routes/public/generate_from_jobparams.ts | 18 +- .../reporting/server/saved_objects/index.ts | 24 + .../saved_objects/scheduled_report/index.ts | 9 + .../scheduled_report/mappings.ts | 17 + .../scheduled_report/model_versions.ts | 19 + .../scheduled_report/schemas}/index.ts | 2 +- .../scheduled_report/schemas/latest.ts | 12 + .../scheduled_report/schemas/v1.ts | 40 ++ .../create_mock_reportingplugin.ts | 13 +- .../plugins/private/reporting/server/types.ts | 31 +- .../plugins/private/reporting/tsconfig.json | 2 + .../__snapshots__/oss_features.test.ts.snap | 32 +- .../shared/features/server/oss_features.ts | 8 +- .../shared/task_manager/server/index.ts | 2 + .../server/lib/get_first_run_at.test.ts | 6 +- .../server/saved_objects/index.ts | 2 + .../server/saved_objects/schemas/task.ts | 19 +- .../shared/task_manager/server/task.ts | 12 +- .../security_roles_privileges.ts | 155 +++++ .../services/scenarios.ts | 52 ++ .../chat/platform_security/authorization.ts | 168 ++++++ .../platform_security/authorization.ts | 168 ++++++ .../search/platform_security/authorization.ts | 168 ++++++ .../platform_security/authorization.ts | 48 ++ 49 files changed, 2621 insertions(+), 348 deletions(-) delete mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.ts rename x-pack/platform/plugins/private/reporting/server/routes/common/{generate/request_handler.test.ts => request_handler/generate_request_handler.test.ts} (82%) create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/index.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/index.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/index.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/mappings.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/model_versions.ts rename x-pack/platform/plugins/private/reporting/server/{routes/common/generate => saved_objects/scheduled_report/schemas}/index.ts (76%) create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts create mode 100644 x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index fa63eee1e8430..70f1f6e75ff80 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -918,6 +918,9 @@ "installCount", "unInstallCount" ], + "scheduled_report": [ + "createdBy" + ], "search": [ "description", "title" diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 063f0e3f6f854..06a7de98e4ff6 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -3045,6 +3045,14 @@ } } }, + "scheduled_report": { + "dynamic": false, + "properties": { + "createdBy": { + "type": "keyword" + } + } + }, "search": { "dynamic": false, "properties": { diff --git a/src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts index 47254ed205309..fa951d0fa52ee 100644 --- a/src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts +++ b/src/core/server/integration_tests/saved_objects/registration/type_registrations.test.ts @@ -122,6 +122,7 @@ const previouslyRegisteredTypes = [ 'query', 'rules-settings', 'sample-data-telemetry', + 'scheduled_report', 'search', 'search-session', 'search-telemetry', diff --git a/src/platform/packages/private/kbn-reporting/common/routes.ts b/src/platform/packages/private/kbn-reporting/common/routes.ts index 4fb56700cda28..92a6fa0cc5356 100644 --- a/src/platform/packages/private/kbn-reporting/common/routes.ts +++ b/src/platform/packages/private/kbn-reporting/common/routes.ts @@ -26,6 +26,7 @@ export const INTERNAL_ROUTES = { }, HEALTH: prefixInternalPath + '/_health', GENERATE_PREFIX: prefixInternalPath + '/generate', // exportTypeId is added to the final path + SCHEDULE_PREFIX: prefixInternalPath + '/schedule', // exportTypeId is added to the final path }; const prefixPublicPath = '/api/reporting'; diff --git a/x-pack/platform/plugins/private/canvas/server/feature.test.ts b/x-pack/platform/plugins/private/canvas/server/feature.test.ts index 7f71bafd41700..6fde4788a9c89 100644 --- a/x-pack/platform/plugins/private/canvas/server/feature.test.ts +++ b/x-pack/platform/plugins/private/canvas/server/feature.test.ts @@ -107,7 +107,9 @@ it('Provides a feature declaration ', () => { "minimumLicense": "gold", "name": "Generate PDF reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -216,7 +218,9 @@ it(`Calls on Reporting whether to include Generate PDF as a sub-feature`, () => "minimumLicense": "gold", "name": "Generate PDF reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ diff --git a/x-pack/platform/plugins/private/canvas/server/feature.ts b/x-pack/platform/plugins/private/canvas/server/feature.ts index daa8a8fc4aa4f..4aa03b73425f4 100644 --- a/x-pack/platform/plugins/private/canvas/server/feature.ts +++ b/x-pack/platform/plugins/private/canvas/server/feature.ts @@ -68,7 +68,7 @@ export function getCanvasFeature(plugins: { reporting?: ReportingStart }): Kiban includeIn: 'all', management: { insightsAndAlerting: ['reporting'] }, minimumLicense: 'gold', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, api: ['generateReport'], ui: ['generatePdf'], }, diff --git a/x-pack/platform/plugins/private/reporting/kibana.jsonc b/x-pack/platform/plugins/private/reporting/kibana.jsonc index ce6b130018af0..e22f4605cbd60 100644 --- a/x-pack/platform/plugins/private/reporting/kibana.jsonc +++ b/x-pack/platform/plugins/private/reporting/kibana.jsonc @@ -16,6 +16,7 @@ "reporting" ], "requiredPlugins": [ + "actions", "data", "discover", "encryptedSavedObjects", diff --git a/x-pack/platform/plugins/private/reporting/server/core.ts b/x-pack/platform/plugins/private/reporting/server/core.ts index a29f4303ac1ca..bc8c232eafb1b 100644 --- a/x-pack/platform/plugins/private/reporting/server/core.ts +++ b/x-pack/platform/plugins/private/reporting/server/core.ts @@ -23,6 +23,8 @@ import type { StatusServiceSetup, UiSettingsServiceStart, } from '@kbn/core/server'; +import { SECURITY_EXTENSION_ID } from '@kbn/core-saved-objects-server'; +import type { PluginSetupContract as ActionsPluginSetupContract } from '@kbn/actions-plugin/server'; import type { PluginStart as DataPluginStart } from '@kbn/data-plugin/server'; import type { DiscoverServerPluginStart } from '@kbn/discover-plugin/server'; import type { FeaturesPluginSetup } from '@kbn/features-plugin/server'; @@ -54,8 +56,10 @@ import type { IReport, ReportingStore } from './lib/store'; import { ExecuteReportTask, ReportTaskParams } from './lib/tasks'; import type { ReportingPluginRouter } from './types'; import { EventTracker } from './usage'; +import { SCHEDULED_REPORT_SAVED_OBJECT_TYPE } from './saved_objects'; export interface ReportingInternalSetup { + actions: ActionsPluginSetupContract; basePath: Pick; docLinks: DocLinksServiceSetup; encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; @@ -351,6 +355,13 @@ export class ReportingCore { ); } + public validateNotificationEmails(emails: string[]): string | undefined { + const pluginSetupDeps = this.getPluginSetupDeps(); + return pluginSetupDeps.actions + .getActionsConfigurationUtilities() + .validateEmailAddresses(emails); + } + /* * Gives synchronous access to the setupDeps */ @@ -371,6 +382,15 @@ export class ReportingCore { return dataViews; } + public async getSoClient(request: KibanaRequest) { + const { savedObjects } = await this.getPluginStartDeps(); + const savedObjectsClient = savedObjects.getScopedClient(request, { + excludedExtensions: [SECURITY_EXTENSION_ID], + includedHiddenTypes: [SCHEDULED_REPORT_SAVED_OBJECT_TYPE], + }); + return savedObjectsClient; + } + public async getDataService() { const startDeps = await this.getPluginStartDeps(); return startDeps.data; diff --git a/x-pack/platform/plugins/private/reporting/server/plugin.test.ts b/x-pack/platform/plugins/private/reporting/server/plugin.test.ts index 2322794836989..df827abce22ae 100644 --- a/x-pack/platform/plugins/private/reporting/server/plugin.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/plugin.test.ts @@ -76,6 +76,21 @@ describe('Reporting Plugin', () => { ); }); + it('registers a saved object for scheduled reports', async () => { + plugin.setup(coreSetup, pluginSetup); + expect(coreSetup.savedObjects.registerType).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'scheduled_report', + namespaceType: 'multiple', + hidden: true, + indexPattern: '.kibana_alerting_cases', + management: { + importableAndExportable: false, + }, + }) + ); + }); + it('logs start issues', async () => { // wait for the setup phase background work plugin.setup(coreSetup, pluginSetup); diff --git a/x-pack/platform/plugins/private/reporting/server/plugin.ts b/x-pack/platform/plugins/private/reporting/server/plugin.ts index 12bfb3decb805..42d0889b08477 100644 --- a/x-pack/platform/plugins/private/reporting/server/plugin.ts +++ b/x-pack/platform/plugins/private/reporting/server/plugin.ts @@ -27,6 +27,7 @@ import type { import { ReportingRequestHandlerContext } from './types'; import { registerReportingEventTypes, registerReportingUsageCollector } from './usage'; import { registerFeatures } from './features'; +import { setupSavedObjects } from './saved_objects'; /* * @internal @@ -75,6 +76,9 @@ export class ReportingPlugin registerReportingUsageCollector(reportingCore, plugins.usageCollection); registerReportingEventTypes(core); + // Saved objects + setupSavedObjects(core.savedObjects); + // Routes registerRoutes(reportingCore, this.logger); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.ts deleted file mode 100644 index 2ba63a5d52655..0000000000000 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.ts +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import Boom from '@hapi/boom'; -import moment from 'moment'; - -import { schema, TypeOf } from '@kbn/config-schema'; -import type { KibanaRequest, KibanaResponseFactory, Logger } from '@kbn/core/server'; -import { i18n } from '@kbn/i18n'; -import { PUBLIC_ROUTES } from '@kbn/reporting-common'; -import type { BaseParams } from '@kbn/reporting-common/types'; -import { cryptoFactory } from '@kbn/reporting-server'; -import rison from '@kbn/rison'; - -import { type Counters, getCounters } from '..'; -import type { ReportingCore } from '../../..'; -import { checkParamsVersion } from '../../../lib'; -import { Report } from '../../../lib/store'; -import type { - ReportingJobResponse, - ReportingRequestHandlerContext, - ReportingUser, -} from '../../../types'; - -export const handleUnavailable = (res: KibanaResponseFactory) => { - return res.custom({ statusCode: 503, body: 'Not Available' }); -}; - -const validation = { - params: schema.object({ exportType: schema.string({ minLength: 2 }) }), - body: schema.nullable(schema.object({ jobParams: schema.maybe(schema.string()) })), - query: schema.nullable(schema.object({ jobParams: schema.string({ defaultValue: '' }) })), -}; - -/** - * Handles the common parts of requests to generate a report - * Serves report job handling in the context of the request to generate the report - */ -export class RequestHandler { - constructor( - private reporting: ReportingCore, - private user: ReportingUser, - private context: ReportingRequestHandlerContext, - private path: string, - private req: KibanaRequest< - TypeOf<(typeof validation)['params']>, - TypeOf<(typeof validation)['query']>, - TypeOf<(typeof validation)['body']> - >, - private res: KibanaResponseFactory, - private logger: Logger - ) {} - - private async encryptHeaders() { - const { encryptionKey } = this.reporting.getConfig(); - const crypto = cryptoFactory(encryptionKey); - return await crypto.encrypt(this.req.headers); - } - - public async enqueueJob(exportTypeId: string, jobParams: BaseParams) { - const { reporting, logger, context, req, user } = this; - - const exportType = reporting.getExportTypesRegistry().getById(exportTypeId); - - if (exportType == null) { - throw new Error(`Export type ${exportTypeId} does not exist in the registry!`); - } - - const store = await reporting.getStore(); - - if (!exportType.createJob) { - throw new Error(`Export type ${exportTypeId} is not a valid instance!`); - } - - // 1. Ensure the incoming params have a version field (should be set by the UI) - jobParams.version = checkParamsVersion(jobParams, logger); - - // 2. Encrypt request headers to store for the running report job to authenticate itself with Kibana - const headers = await this.encryptHeaders(); - - // 3. Create a payload object by calling exportType.createJob(), and adding some automatic parameters - const job = await exportType.createJob(jobParams, context, req); - - const payload = { - ...job, - headers, - title: job.title, - objectType: jobParams.objectType, - browserTimezone: jobParams.browserTimezone, - version: jobParams.version, - spaceId: reporting.getSpaceId(req, logger), - }; - - // 4. Add the report to ReportingStore to show as pending - const report = await store.addReport( - new Report({ - jobtype: exportType.jobType, - created_by: user ? user.username : false, - payload, - migration_version: jobParams.version, - meta: { - // telemetry fields - objectType: jobParams.objectType, - layout: jobParams.layout?.id, - isDeprecated: job.isDeprecated, - }, - }) - ); - logger.debug(`Successfully stored pending job: ${report._index}/${report._id}`); - - // 5. Schedule the report with Task Manager - const task = await reporting.scheduleTask(req, report.toReportTaskJSON()); - logger.info( - `Scheduled ${exportType.name} reporting task. Task ID: task:${task.id}. Report ID: ${report._id}` - ); - - // 6. Log the action with event log - reporting.getEventLogger(report, task).logScheduleTask(); - return report; - } - - public getJobParams(): BaseParams { - let jobParamsRison: null | string = null; - const req = this.req; - const res = this.res; - - if (req.body) { - const { jobParams: jobParamsPayload } = req.body; - jobParamsRison = jobParamsPayload ? jobParamsPayload : null; - } else if (req.query?.jobParams) { - const { jobParams: queryJobParams } = req.query; - if (queryJobParams) { - jobParamsRison = queryJobParams; - } else { - jobParamsRison = null; - } - } - - if (!jobParamsRison) { - throw res.customError({ - statusCode: 400, - body: 'A jobParams RISON string is required in the querystring or POST body', - }); - } - - let jobParams; - - try { - jobParams = rison.decode(jobParamsRison) as BaseParams | null; - if (!jobParams) { - throw res.customError({ - statusCode: 400, - body: 'Missing jobParams!', - }); - } - } catch (err) { - throw res.customError({ - statusCode: 400, - body: `invalid rison: ${jobParamsRison}`, - }); - } - - return jobParams; - } - - public static getValidation() { - return validation; - } - - public async handleGenerateRequest(exportTypeId: string, jobParams: BaseParams) { - const req = this.req; - const reporting = this.reporting; - - const counters = getCounters( - req.route.method, - this.path.replace(/{exportType}/, exportTypeId), - reporting.getUsageCounter() - ); - - // ensure the async dependencies are loaded - if (!this.context.reporting) { - return handleUnavailable(this.res); - } - - const licenseInfo = await this.reporting.getLicenseInfo(); - const licenseResults = licenseInfo[exportTypeId]; - - if (!licenseResults) { - return this.res.badRequest({ body: `Invalid export-type of ${exportTypeId}` }); - } - - if (!licenseResults.enableLinks) { - return this.res.forbidden({ body: licenseResults.message }); - } - - if (jobParams.browserTimezone && !moment.tz.zone(jobParams.browserTimezone)) { - return this.res.badRequest({ - body: `Invalid timezone "${jobParams.browserTimezone ?? ''}".`, - }); - } - - let report: Report | undefined; - try { - report = await this.enqueueJob(exportTypeId, jobParams); - const { basePath } = this.reporting.getServerInfo(); - const publicDownloadPath = basePath + PUBLIC_ROUTES.JOBS.DOWNLOAD_PREFIX; - - // return task manager's task information and the download URL - counters.usageCounter(); - const eventTracker = reporting.getEventTracker( - report._id, - exportTypeId, - jobParams.objectType - ); - eventTracker?.createReport({ - isDeprecated: Boolean(report.payload.isDeprecated), - isPublicApi: this.path.match(/internal/) === null, - }); - - return this.res.ok({ - headers: { 'content-type': 'application/json' }, - body: { - path: `${publicDownloadPath}/${report._id}`, - job: report.toApiJSON(), - }, - }); - } catch (err) { - return this.handleError(err, counters, report?.jobtype); - } - } - - private handleError(err: Error | Boom.Boom, counters: Counters, jobtype?: string) { - this.logger.error(err); - - if (err instanceof Boom.Boom) { - const statusCode = err.output.statusCode; - counters?.errorCounter(jobtype, statusCode); - - return this.res.customError({ - statusCode, - body: err.output.payload.message, - }); - } - - counters?.errorCounter(jobtype, 500); - - return this.res.customError({ - statusCode: 500, - body: - err?.message || - i18n.translate('xpack.reporting.errorHandler.unknownError', { - defaultMessage: 'Unknown error', - }), - }); - } -} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/jobs/get_job_routes.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/jobs/get_job_routes.ts index dab96944ea6e8..3f2aa4fd5e3b2 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/jobs/get_job_routes.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/jobs/get_job_routes.ts @@ -12,7 +12,7 @@ import { getCounters } from '..'; import { ReportingCore } from '../../..'; import { getContentStream } from '../../../lib'; import { ReportingRequestHandlerContext, ReportingUser } from '../../../types'; -import { handleUnavailable } from '../generate'; +import { handleUnavailable } from '../request_handler'; import { jobManagementPreRouting } from './job_management_pre_routing'; import { jobsQueryFactory } from './jobs_query'; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.test.ts similarity index 82% rename from x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.test.ts rename to x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.test.ts index 2ded83071d6a3..6f866a56ef4c5 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/request_handler.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.test.ts @@ -22,7 +22,7 @@ import { ReportingRequestHandlerContext, ReportingSetup, } from '../../../types'; -import { RequestHandler } from './request_handler'; +import { GenerateRequestHandler } from './generate_request_handler'; jest.mock('@kbn/reporting-server/crypto', () => ({ cryptoFactory: () => ({ @@ -68,7 +68,7 @@ describe('Handle request to generate', () => { let mockContext: ReturnType; let mockRequest: ReturnType; let mockResponseFactory: ReturnType; - let requestHandler: RequestHandler; + let requestHandler: GenerateRequestHandler; beforeEach(async () => { reportingCore = await createMockReportingCore(createMockConfigSchema({})); @@ -91,20 +91,23 @@ describe('Handle request to generate', () => { mockContext = getMockContext(); mockContext.reporting = Promise.resolve({} as ReportingSetup); - requestHandler = new RequestHandler( - reportingCore, - { username: 'testymcgee' }, - mockContext, - '/api/reporting/test/generate/pdf', - mockRequest, - mockResponseFactory, - mockLogger - ); + requestHandler = new GenerateRequestHandler({ + reporting: reportingCore, + user: { username: 'testymcgee' }, + context: mockContext, + path: '/api/reporting/test/generate/pdf', + req: mockRequest, + res: mockResponseFactory, + logger: mockLogger, + }); }); describe('Enqueue Job', () => { test('creates a report object to queue', async () => { - const report = await requestHandler.enqueueJob('printablePdfV2', mockJobParams); + const report = await requestHandler.enqueueJob({ + exportTypeId: 'printablePdfV2', + jobParams: mockJobParams, + }); const { _id, created_at: _created_at, payload, ...snapObj } = report; expect(snapObj).toMatchInlineSnapshot(` @@ -157,7 +160,10 @@ describe('Handle request to generate', () => { test('provides a default kibana version field for older POST URLs', async () => { // how do we handle the printable_pdf endpoint that isn't migrating to the class instance of export types? (mockJobParams as unknown as { version?: string }).version = undefined; - const report = await requestHandler.enqueueJob('printablePdfV2', mockJobParams); + const report = await requestHandler.enqueueJob({ + exportTypeId: 'printablePdfV2', + jobParams: mockJobParams, + }); const { _id, created_at: _created_at, ...snapObj } = report; expect(snapObj.payload.version).toBe('7.14.0'); @@ -206,10 +212,14 @@ describe('Handle request to generate', () => { }); }); - describe('handleGenerateRequest', () => { + describe('handleRequest', () => { test('disallows invalid export type', async () => { - expect(await requestHandler.handleGenerateRequest('neanderthals', mockJobParams)) - .toMatchInlineSnapshot(` + expect( + await requestHandler.handleRequest({ + exportTypeId: 'neanderthals', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` Object { "body": "Invalid export-type of neanderthals", } @@ -224,8 +234,12 @@ describe('Handle request to generate', () => { }, })); - expect(await requestHandler.handleGenerateRequest('csv_searchsource', mockJobParams)) - .toMatchInlineSnapshot(` + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` Object { "body": "seeing this means the license isn't supported", } @@ -233,30 +247,26 @@ describe('Handle request to generate', () => { }); test('disallows invalid browser timezone', async () => { - (reportingCore.getLicenseInfo as jest.Mock) = jest.fn(() => ({ - csv_searchsource: { - enableLinks: false, - message: `seeing this means the license isn't supported`, - }, - })); - expect( - await requestHandler.handleGenerateRequest('csv_searchsource', { - ...mockJobParams, - browserTimezone: 'America/Amsterdam', + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: { + ...mockJobParams, + browserTimezone: 'America/Amsterdam', + }, }) ).toMatchInlineSnapshot(` Object { - "body": "seeing this means the license isn't supported", + "body": "Invalid timezone \\"America/Amsterdam\\".", } `); }); test('generates the download path', async () => { - const { body } = (await requestHandler.handleGenerateRequest( - 'csv_searchsource', - mockJobParams - )) as unknown as { body: ReportingJobResponse }; + const { body } = (await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + })) as unknown as { body: ReportingJobResponse }; expect(body.path).toMatch('/mock-server-basepath/api/reporting/jobs/download/mock-report-id'); }); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.ts new file mode 100644 index 0000000000000..98df5a62533e9 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/generate_request_handler.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { PUBLIC_ROUTES } from '@kbn/reporting-common'; +import { getCounters } from '..'; +import { Report, SavedReport } from '../../../lib/store'; +import type { ReportingJobResponse } from '../../../types'; +import { RequestHandler, RequestParams } from './request_handler'; + +const validation = { + params: schema.object({ exportType: schema.string({ minLength: 2 }) }), + body: schema.nullable(schema.object({ jobParams: schema.maybe(schema.string()) })), + query: schema.nullable(schema.object({ jobParams: schema.string({ defaultValue: '' }) })), +}; + +/** + * Handles the common parts of requests to generate a report + * Serves report job handling in the context of the request to generate the report + */ +export class GenerateRequestHandler extends RequestHandler< + (typeof validation)['params'], + (typeof validation)['query'], + (typeof validation)['body'], + SavedReport +> { + public static getValidation() { + return validation; + } + + public async enqueueJob(params: RequestParams) { + const { exportTypeId, jobParams } = params; + const { reporting, logger, req, user } = this.opts; + + const store = await reporting.getStore(); + const { version, job, jobType, name } = await this.createJob(exportTypeId, jobParams); + + // Encrypt request headers to store for the running report job to authenticate itself with Kibana + const headers = await this.encryptHeaders(); + + const payload = { + ...job, + headers, + title: job.title, + objectType: jobParams.objectType, + browserTimezone: jobParams.browserTimezone, + version, + spaceId: reporting.getSpaceId(req, logger), + }; + + // Add the report to ReportingStore to show as pending + const report = await store.addReport( + new Report({ + jobtype: jobType, + created_by: user ? user.username : false, + payload, + migration_version: version, + meta: { + // telemetry fields + objectType: jobParams.objectType, + layout: jobParams.layout?.id, + isDeprecated: job.isDeprecated, + }, + }) + ); + logger.debug(`Successfully stored pending job: ${report._index}/${report._id}`); + + // Schedule the report with Task Manager + const task = await reporting.scheduleTask(req, report.toReportTaskJSON()); + logger.info( + `Scheduled ${name} reporting task. Task ID: task:${task.id}. Report ID: ${report._id}` + ); + + // Log the action with event log + reporting.getEventLogger(report, task).logScheduleTask(); + return report; + } + + public async handleRequest(params: RequestParams) { + const { exportTypeId, jobParams } = params; + const { reporting, req, res, path } = this.opts; + + const counters = getCounters( + req.route.method, + path.replace(/{exportType}/, exportTypeId), + reporting.getUsageCounter() + ); + + const checkErrorResponse = await this.checkLicenseAndTimezone( + exportTypeId, + jobParams.browserTimezone + ); + if (checkErrorResponse) { + return checkErrorResponse; + } + + let report: Report | undefined; + try { + report = await this.enqueueJob(params); + const { basePath } = reporting.getServerInfo(); + const publicDownloadPath = basePath + PUBLIC_ROUTES.JOBS.DOWNLOAD_PREFIX; + + // return task manager's task information and the download URL + counters.usageCounter(); + const eventTracker = reporting.getEventTracker( + report._id, + exportTypeId, + jobParams.objectType + ); + eventTracker?.createReport({ + isDeprecated: Boolean(report.payload.isDeprecated), + isPublicApi: path.match(/internal/) === null, + }); + + return res.ok({ + headers: { 'content-type': 'application/json' }, + body: { + path: `${publicDownloadPath}/${report._id}`, + job: report.toApiJSON(), + }, + }); + } catch (err) { + return this.handleError(err, counters, report?.jobtype); + } + } +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/index.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/index.ts new file mode 100644 index 0000000000000..185b6ec86c37a --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { handleUnavailable } from './request_handler'; +export { GenerateRequestHandler } from './generate_request_handler'; +export { ScheduleRequestHandler } from './schedule_request_handler'; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts new file mode 100644 index 0000000000000..d80e6e2e93b0d --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { transformRawScheduledReportToReport } from './transform_raw_scheduled_report'; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts new file mode 100644 index 0000000000000..3006ff48bfad3 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/lib/transform_raw_scheduled_report.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObject } from '@kbn/core/server'; +import { ScheduledReportApiJSON, ScheduledReportType } from '../../../../types'; + +export function transformRawScheduledReportToReport( + rawScheduledReport: SavedObject +): ScheduledReportApiJSON { + const parsedPayload = JSON.parse(rawScheduledReport.attributes.payload); + return { + id: rawScheduledReport.id, + jobtype: rawScheduledReport.attributes.jobType, + created_at: rawScheduledReport.attributes.createdAt, + created_by: rawScheduledReport.attributes.createdBy as string | false, + payload: parsedPayload, + meta: rawScheduledReport.attributes.meta, + migration_version: rawScheduledReport.attributes.migrationVersion, + schedule: rawScheduledReport.attributes.schedule, + notification: rawScheduledReport.attributes.notification, + }; +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts new file mode 100644 index 0000000000000..55bffafc82552 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/request_handler.ts @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import moment from 'moment'; +import { schema, TypeOf } from '@kbn/config-schema'; +import type { + IKibanaResponse, + KibanaRequest, + KibanaResponseFactory, + Logger, +} from '@kbn/core/server'; +import { i18n } from '@kbn/i18n'; +import type { BaseParams } from '@kbn/reporting-common/types'; +import { cryptoFactory } from '@kbn/reporting-server'; +import rison from '@kbn/rison'; + +import { RruleSchedule } from '@kbn/task-manager-plugin/server'; +import { RawNotification } from '../../../saved_objects/scheduled_report/schemas/latest'; +import { checkParamsVersion } from '../../../lib'; +import { type Counters } from '..'; +import type { ReportingCore } from '../../..'; +import type { ReportingRequestHandlerContext, ReportingUser } from '../../../types'; + +export const handleUnavailable = (res: KibanaResponseFactory) => { + return res.custom({ statusCode: 503, body: 'Not Available' }); +}; + +const ParamsValidation = schema.recordOf(schema.string(), schema.string()); +const QueryValidation = schema.nullable( + schema.recordOf(schema.string(), schema.maybe(schema.string())) +); +const BodyValidation = schema.nullable( + schema.recordOf(schema.string(), schema.maybe(schema.any())) +); + +interface ConstructorOpts< + Params extends typeof ParamsValidation, + Query extends typeof QueryValidation, + Body extends typeof BodyValidation +> { + reporting: ReportingCore; + user: ReportingUser; + context: ReportingRequestHandlerContext; + path: string; + req: KibanaRequest, TypeOf, TypeOf>; + res: KibanaResponseFactory; + logger: Logger; +} + +export interface RequestParams { + exportTypeId: string; + jobParams: BaseParams; + schedule?: RruleSchedule; + notification?: RawNotification; +} + +/** + * Handles the common parts of requests to generate or schedule a report + * Serves report job handling in the context of the request to generate the report + */ +export abstract class RequestHandler< + Params extends typeof ParamsValidation, + Query extends typeof QueryValidation, + Body extends typeof BodyValidation, + Output extends Record +> { + constructor(protected readonly opts: ConstructorOpts) {} + + public static getValidation() { + throw new Error('getValidation() must be implemented in a subclass'); + } + + public abstract enqueueJob(params: RequestParams): Promise; + + public abstract handleRequest(params: RequestParams): Promise; + + public getJobParams(): BaseParams { + let jobParamsRison: null | string = null; + const req = this.opts.req; + const res = this.opts.res; + + if (req.body) { + const { jobParams: jobParamsPayload } = req.body; + jobParamsRison = jobParamsPayload ? jobParamsPayload : null; + } else if (req.query?.jobParams) { + const { jobParams: queryJobParams } = req.query; + if (queryJobParams) { + jobParamsRison = queryJobParams; + } else { + jobParamsRison = null; + } + } + + if (!jobParamsRison) { + throw res.customError({ + statusCode: 400, + body: 'A jobParams RISON string is required in the querystring or POST body', + }); + } + + let jobParams; + + try { + jobParams = rison.decode(jobParamsRison) as BaseParams | null; + if (!jobParams) { + throw res.customError({ + statusCode: 400, + body: 'Missing jobParams!', + }); + } + } catch (err) { + throw res.customError({ + statusCode: 400, + body: `invalid rison: ${jobParamsRison}`, + }); + } + + return jobParams; + } + + protected async createJob(exportTypeId: string, jobParams: BaseParams) { + const exportType = this.opts.reporting.getExportTypesRegistry().getById(exportTypeId); + + if (exportType == null) { + throw new Error(`Export type ${exportTypeId} does not exist in the registry!`); + } + + if (!exportType.createJob) { + throw new Error(`Export type ${exportTypeId} is not a valid instance!`); + } + + // 1. Ensure the incoming params have a version field (should be set by the UI) + const version = checkParamsVersion(jobParams, this.opts.logger); + + // 2. Create a payload object by calling exportType.createJob(), and adding some automatic parameters + const job = await exportType.createJob(jobParams, this.opts.context, this.opts.req); + + return { job, version, jobType: exportType.jobType, name: exportType.name }; + } + + protected async checkLicenseAndTimezone( + exportTypeId: string, + browserTimezone: string + ): Promise { + const { reporting, context, res } = this.opts; + + // ensure the async dependencies are loaded + if (!context.reporting) { + return handleUnavailable(res); + } + + const licenseInfo = await reporting.getLicenseInfo(); + const licenseResults = licenseInfo[exportTypeId]; + + if (!licenseResults) { + return res.badRequest({ body: `Invalid export-type of ${exportTypeId}` }); + } + + if (!licenseResults.enableLinks) { + return res.forbidden({ body: licenseResults.message }); + } + + if (browserTimezone && !moment.tz.zone(browserTimezone)) { + return res.badRequest({ + body: `Invalid timezone "${browserTimezone ?? ''}".`, + }); + } + + return null; + } + + protected async encryptHeaders() { + const { encryptionKey } = this.opts.reporting.getConfig(); + const crypto = cryptoFactory(encryptionKey); + return await crypto.encrypt(this.opts.req.headers); + } + + protected handleError(err: Error | Boom.Boom, counters?: Counters, jobtype?: string) { + this.opts.logger.error(err); + + if (err instanceof Boom.Boom) { + const statusCode = err.output.statusCode; + counters?.errorCounter(jobtype, statusCode); + + return this.opts.res.customError({ + statusCode, + body: err.output.payload.message, + }); + } + + counters?.errorCounter(jobtype, 500); + + return this.opts.res.customError({ + statusCode: 500, + body: + err?.message || + i18n.translate('xpack.reporting.errorHandler.unknownError', { + defaultMessage: 'Unknown error', + }), + }); + } +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts new file mode 100644 index 0000000000000..91aa5d8851eb2 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts @@ -0,0 +1,543 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +jest.mock('uuid', () => ({ v4: () => 'mock-report-id' })); + +import rison from '@kbn/rison'; + +import { + FakeRawRequest, + KibanaRequest, + KibanaResponseFactory, + SavedObjectsClientContract, +} from '@kbn/core/server'; +import { coreMock, httpServerMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { JobParamsPDFV2 } from '@kbn/reporting-export-types-pdf-common'; +import { createMockConfigSchema } from '@kbn/reporting-mocks-server'; +import { ReportingCore } from '../../..'; +import { createMockReportingCore } from '../../../test_helpers'; +import { ReportingRequestHandlerContext, ReportingSetup } from '../../../types'; +import { ScheduleRequestHandler } from './schedule_request_handler'; + +const getMockContext = () => + ({ + core: coreMock.createRequestHandlerContext(), + } as unknown as ReportingRequestHandlerContext); + +const getMockRequest = () => + ({ + url: { port: '5601', search: '', pathname: '/foo' }, + route: { path: '/foo', options: {} }, + } as KibanaRequest); + +const getMockResponseFactory = () => + ({ + ...httpServerMock.createResponseFactory(), + forbidden: (obj: unknown) => obj, + unauthorized: (obj: unknown) => obj, + customError: (err: unknown) => err, + } as unknown as KibanaResponseFactory); + +const mockLogger = loggingSystemMock.createLogger(); +const mockJobParams: JobParamsPDFV2 = { + browserTimezone: 'UTC', + objectType: 'cool_object_type', + title: 'cool_title', + version: 'unknown', + layout: { id: 'preserve_layout' }, + locatorParams: [], +}; + +const fakeRawRequest: FakeRawRequest = { + headers: { + authorization: `ApiKey skdjtq4u543yt3rhewrh`, + }, + path: '/', +}; + +describe('Handle request to schedule', () => { + let reportingCore: ReportingCore; + let mockContext: ReturnType; + let mockRequest: ReturnType; + let mockResponseFactory: ReturnType; + let requestHandler: ScheduleRequestHandler; + let soClient: SavedObjectsClientContract; + + beforeEach(async () => { + reportingCore = await createMockReportingCore(createMockConfigSchema({})); + + mockRequest = getMockRequest(); + + mockResponseFactory = getMockResponseFactory(); + (mockResponseFactory.ok as jest.Mock) = jest.fn((args: unknown) => args); + (mockResponseFactory.forbidden as jest.Mock) = jest.fn((args: unknown) => args); + (mockResponseFactory.badRequest as jest.Mock) = jest.fn((args: unknown) => args); + + mockContext = getMockContext(); + mockContext.reporting = Promise.resolve({} as ReportingSetup); + + soClient = await reportingCore.getSoClient(fakeRawRequest as unknown as KibanaRequest); + soClient.create = jest.fn().mockImplementation(async (_, opts) => { + return { + id: 'foo', + attributes: opts, + type: 'scheduled-report', + }; + }); + + requestHandler = new ScheduleRequestHandler({ + reporting: reportingCore, + user: { username: 'testymcgee' }, + context: mockContext, + path: '/api/reporting/test/generate/pdf', + // @ts-ignore + req: mockRequest, + res: mockResponseFactory, + logger: mockLogger, + }); + }); + + describe('enqueueJob', () => { + test('creates a scheduled report saved object', async () => { + const report = await requestHandler.enqueueJob({ + exportTypeId: 'printablePdfV2', + jobParams: mockJobParams, + schedule: { rrule: { freq: 1, interval: 2, tzid: 'UTC' } }, + }); + + const { id, created_at: _created_at, payload, ...snapObj } = report; + expect(snapObj).toMatchInlineSnapshot(` + Object { + "created_by": "testymcgee", + "jobtype": "printable_pdf_v2", + "meta": Object { + "isDeprecated": false, + "layout": "preserve_layout", + "objectType": "cool_object_type", + }, + "migration_version": "unknown", + "notification": undefined, + "schedule": Object { + "rrule": Object { + "freq": 1, + "interval": 2, + "tzid": "UTC", + }, + }, + } + `); + expect(payload).toMatchInlineSnapshot(` + Object { + "browserTimezone": "UTC", + "isDeprecated": false, + "layout": Object { + "id": "preserve_layout", + }, + "locatorParams": Array [], + "objectType": "cool_object_type", + "title": "cool_title", + "version": "unknown", + } + `); + + expect(soClient.create).toHaveBeenCalledWith('scheduled_report', { + jobType: 'printable_pdf_v2', + createdAt: expect.any(String), + createdBy: 'testymcgee', + title: 'cool_title', + enabled: true, + payload: JSON.stringify(payload), + schedule: { + rrule: { + freq: 1, + interval: 2, + tzid: 'UTC', + }, + }, + migrationVersion: 'unknown', + meta: { + objectType: 'cool_object_type', + layout: 'preserve_layout', + isDeprecated: false, + }, + }); + }); + + test('creates a scheduled report saved object with notification', async () => { + const report = await requestHandler.enqueueJob({ + exportTypeId: 'printablePdfV2', + jobParams: mockJobParams, + schedule: { rrule: { freq: 1, interval: 2, tzid: 'UTC' } }, + notification: { email: { to: ['a@b.com'] } }, + }); + + const { id, created_at: _created_at, payload, ...snapObj } = report; + expect(snapObj).toMatchInlineSnapshot(` + Object { + "created_by": "testymcgee", + "jobtype": "printable_pdf_v2", + "meta": Object { + "isDeprecated": false, + "layout": "preserve_layout", + "objectType": "cool_object_type", + }, + "migration_version": "unknown", + "notification": Object { + "email": Object { + "to": Array [ + "a@b.com", + ], + }, + }, + "schedule": Object { + "rrule": Object { + "freq": 1, + "interval": 2, + "tzid": "UTC", + }, + }, + } + `); + expect(payload).toMatchInlineSnapshot(` + Object { + "browserTimezone": "UTC", + "isDeprecated": false, + "layout": Object { + "id": "preserve_layout", + }, + "locatorParams": Array [], + "objectType": "cool_object_type", + "title": "cool_title", + "version": "unknown", + } + `); + + expect(soClient.create).toHaveBeenCalledWith('scheduled_report', { + jobType: 'printable_pdf_v2', + createdAt: expect.any(String), + createdBy: 'testymcgee', + title: 'cool_title', + enabled: true, + payload: JSON.stringify(payload), + schedule: { + rrule: { + freq: 1, + interval: 2, + tzid: 'UTC', + }, + }, + migrationVersion: 'unknown', + meta: { + objectType: 'cool_object_type', + layout: 'preserve_layout', + isDeprecated: false, + }, + notification: { email: { to: ['a@b.com'] } }, + }); + }); + }); + + describe('getJobParams', () => { + test('parse jobParams from body', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { jobParams: rison.encode(mockJobParams) }; + expect(requestHandler.getJobParams()).toEqual(mockJobParams); + }); + + test('handles missing job params', () => { + let error: { statusCode: number; body: string } | undefined; + try { + requestHandler.getJobParams(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + }); + + test('handles null job params', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { jobParams: rison.encode(null) }; + requestHandler.getJobParams(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + }); + + test('handles invalid rison', () => { + let error: { statusCode: number; body: string } | undefined; + // @ts-ignore body is a read-only property + mockRequest.body = { jobParams: mockJobParams }; + try { + requestHandler.getJobParams(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + }); + }); + + describe('getSchedule', () => { + test('parse schedule from body', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + }; + expect(requestHandler.getSchedule()).toEqual({ rrule: { freq: 1, interval: 2 } }); + }); + + test('handles missing schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + + test('handles null schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: null, + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + + test('handles empty schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: {}, + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + + test('handles null rrule schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: null }, + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + + test('handles empty rrule schedule', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: {} }, + }; + requestHandler.getSchedule(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('A schedule is required to create a scheduled report.'); + }); + }); + + describe('getNotification', () => { + test('parse notification from body', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: { to: ['a@b.com'] } }, + }; + expect(requestHandler.getNotification()).toEqual({ email: { to: ['a@b.com'] } }); + }); + + test('returns undefined if notification object is empty', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: {}, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + + test('returns undefined if notification object is null', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: null, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + + test('returns undefined if notification.email object is empty', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: {} }, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + + test('returns undefined if notification.email object is null', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: null }, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + + test('handles invalid email address', () => { + jest + .spyOn(reportingCore, 'validateNotificationEmails') + .mockReturnValueOnce('not valid emails: foo'); + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: { to: ['foo'] } }, + }; + requestHandler.getNotification(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe('Invalid email address(es): not valid emails: foo'); + }); + }); + + describe('handleRequest', () => { + test('disallows invalid export type', async () => { + expect( + await requestHandler.handleRequest({ + exportTypeId: 'neanderthals', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "Invalid export-type of neanderthals", + } + `); + }); + + test('disallows unsupporting license', async () => { + (reportingCore.getLicenseInfo as jest.Mock) = jest.fn(() => ({ + csv_searchsource: { + enableLinks: false, + message: `seeing this means the license isn't supported`, + }, + })); + + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "seeing this means the license isn't supported", + } + `); + }); + + test('disallows invalid browser timezone', async () => { + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: { + ...mockJobParams, + browserTimezone: 'America/Amsterdam', + }, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "Invalid timezone \\"America/Amsterdam\\".", + } + `); + }); + + test('disallows scheduling when reportingHealth.hasPermanentEncryptionKey = false', async () => { + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ + isSufficientlySecure: true, + hasPermanentEncryptionKey: false, + }); + + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "Permanent encryption key must be set for scheduled reporting", + } + `); + }); + + test('disallows scheduling when reportingHealth.isSufficientlySecure=false', async () => { + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ + isSufficientlySecure: false, + hasPermanentEncryptionKey: true, + }); + + expect( + await requestHandler.handleRequest({ + exportTypeId: 'csv_searchsource', + jobParams: mockJobParams, + }) + ).toMatchInlineSnapshot(` + Object { + "body": "Security and API keys must be enabled for scheduled reporting", + } + `); + }); + }); +}); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts new file mode 100644 index 0000000000000..b1735de8515ee --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +import { schema } from '@kbn/config-schema'; +import { isEmpty, omit } from 'lodash'; +import { RruleSchedule, scheduleRruleSchema } from '@kbn/task-manager-plugin/server'; +import { RawNotification } from '../../../saved_objects/scheduled_report/schemas/latest'; +import { rawNotificationSchema } from '../../../saved_objects/scheduled_report/schemas/v1'; +import { + ScheduledReportApiJSON, + ScheduledReportType, + ScheduledReportingJobResponse, +} from '../../../types'; +import { SCHEDULED_REPORT_SAVED_OBJECT_TYPE } from '../../../saved_objects'; +import { RequestHandler, RequestParams } from './request_handler'; +import { transformRawScheduledReportToReport } from './lib'; + +const validation = { + params: schema.object({ exportType: schema.string({ minLength: 2 }) }), + body: schema.object({ + schedule: scheduleRruleSchema, + notification: schema.maybe(rawNotificationSchema), + jobParams: schema.string(), + }), + query: schema.nullable(schema.object({})), +}; + +/** + * Handles the common parts of requests to generate a report + * Serves report job handling in the context of the request to generate the report + */ +export class ScheduleRequestHandler extends RequestHandler< + (typeof validation)['params'], + (typeof validation)['query'], + (typeof validation)['body'], + ScheduledReportApiJSON +> { + public static getValidation() { + return validation; + } + + public getSchedule(): RruleSchedule { + let rruleDef: null | RruleSchedule['rrule'] = null; + const req = this.opts.req; + const res = this.opts.res; + + const { schedule } = req.body; + const { rrule } = schedule ?? {}; + rruleDef = rrule; + + if (isEmpty(rruleDef)) { + throw res.customError({ + statusCode: 400, + body: 'A schedule is required to create a scheduled report.', + }); + } + + return schedule; + } + + public getNotification(): RawNotification | undefined { + const { reporting, req, res } = this.opts; + + const { notification } = req.body; + if (isEmpty(notification) || isEmpty(notification.email)) { + return undefined; + } + + if (notification && notification.email && notification.email.to) { + const invalidEmails = reporting.validateNotificationEmails(notification.email.to); + if (invalidEmails) { + throw res.customError({ + statusCode: 400, + body: `Invalid email address(es): ${invalidEmails}`, + }); + } + } + + return notification; + } + + public async enqueueJob(params: RequestParams) { + const { exportTypeId, jobParams, schedule, notification } = params; + const { reporting, logger, req, user } = this.opts; + + const soClient = await reporting.getSoClient(req); + const { version, job, jobType } = await this.createJob(exportTypeId, jobParams); + + const payload = { + ...job, + title: job.title, + objectType: jobParams.objectType, + browserTimezone: jobParams.browserTimezone, + version, + spaceId: reporting.getSpaceId(req, logger), + }; + + // TODO - extract saved object references before persisting + + const attributes = { + createdAt: moment.utc().toISOString(), + createdBy: user ? user.username : false, + enabled: true, + jobType, + meta: { + // telemetry fields + isDeprecated: job.isDeprecated, + layout: jobParams.layout?.id, + objectType: jobParams.objectType, + }, + migrationVersion: version, + ...(notification ? { notification } : {}), + title: job.title, + payload: JSON.stringify(omit(payload, 'forceNow')), + schedule: schedule!, + }; + + // Create a scheduled report saved object + const report = await soClient.create( + SCHEDULED_REPORT_SAVED_OBJECT_TYPE, + attributes + ); + logger.debug(`Successfully created scheduled report: ${report.id}`); + + // TODO - Schedule the report with Task Manager + + return transformRawScheduledReportToReport(report); + } + + public async handleRequest(params: RequestParams) { + const { exportTypeId, jobParams } = params; + const { reporting, res } = this.opts; + + const checkErrorResponse = await this.checkLicenseAndTimezone( + exportTypeId, + jobParams.browserTimezone + ); + if (checkErrorResponse) { + return checkErrorResponse; + } + + // check that security requirements are met + const reportingHealth = await reporting.getHealthInfo(); + if (!reportingHealth.hasPermanentEncryptionKey) { + return res.forbidden({ + body: `Permanent encryption key must be set for scheduled reporting`, + }); + } + if (!reportingHealth.isSufficientlySecure) { + return res.forbidden({ + body: `Security and API keys must be enabled for scheduled reporting`, + }); + } + + let report: ScheduledReportApiJSON | undefined; + try { + report = await this.enqueueJob(params); + return res.ok({ + headers: { 'content-type': 'application/json' }, + body: { + job: report, + }, + }); + } catch (err) { + return this.handleError(err, undefined, report?.jobtype); + } + } +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/index.ts b/x-pack/platform/plugins/private/reporting/server/routes/index.ts index f9fbd12802e26..31414587802f4 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/index.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/index.ts @@ -11,6 +11,7 @@ import { registerDeprecationsRoutes } from './internal/deprecations/deprecations import { registerDiagnosticRoutes } from './internal/diagnostic'; import { registerHealthRoute } from './internal/health'; import { registerGenerationRoutesInternal } from './internal/generate/generate_from_jobparams'; +import { registerScheduleRoutesInternal } from './internal/schedule/schedule_from_jobparams'; import { registerJobInfoRoutesInternal } from './internal/management/jobs'; import { registerGenerationRoutesPublic } from './public/generate_from_jobparams'; import { registerJobInfoRoutesPublic } from './public/jobs'; @@ -20,6 +21,7 @@ export function registerRoutes(reporting: ReportingCore, logger: Logger) { registerHealthRoute(reporting, logger); registerDiagnosticRoutes(reporting, logger); registerGenerationRoutesInternal(reporting, logger); + registerScheduleRoutesInternal(reporting, logger); registerJobInfoRoutesInternal(reporting); registerGenerationRoutesPublic(reporting, logger); registerJobInfoRoutesPublic(reporting); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/generate/generate_from_jobparams.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/generate/generate_from_jobparams.ts index bd26c88bf6a0a..2d762ebf1cdb7 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/generate/generate_from_jobparams.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/generate/generate_from_jobparams.ts @@ -10,7 +10,7 @@ import type { Logger } from '@kbn/core/server'; import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import type { ReportingCore } from '../../..'; import { authorizedUserPreRouting } from '../../common'; -import { RequestHandler } from '../../common/generate'; +import { GenerateRequestHandler } from '../../common/request_handler'; const { GENERATE_PREFIX } = INTERNAL_ROUTES; @@ -30,7 +30,7 @@ export function registerGenerationRoutesInternal(reporting: ReportingCore, logge requiredPrivileges: kibanaAccessControlTags, }, }, - validate: RequestHandler.getValidation(), + validate: GenerateRequestHandler.getValidation(), options: { tags: kibanaAccessControlTags.map((accessControlTag) => `access:${accessControlTag}`), access: 'internal', @@ -38,17 +38,20 @@ export function registerGenerationRoutesInternal(reporting: ReportingCore, logge }, authorizedUserPreRouting(reporting, async (user, context, req, res) => { try { - const requestHandler = new RequestHandler( + const requestHandler = new GenerateRequestHandler({ reporting, user, context, path, req, res, - logger - ); + logger, + }); const jobParams = requestHandler.getJobParams(); - return await requestHandler.handleGenerateRequest(req.params.exportType, jobParams); + return await requestHandler.handleRequest({ + exportTypeId: req.params.exportType, + jobParams, + }); } catch (err) { if (err instanceof KibanaResponse) { return err; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/management/jobs.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/management/jobs.ts index 492d579727c51..bfb4c3a59312f 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/management/jobs.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/management/jobs.ts @@ -10,7 +10,7 @@ import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import { ROUTE_TAG_CAN_REDIRECT } from '@kbn/security-plugin/server'; import { ReportingCore } from '../../..'; import { authorizedUserPreRouting, getCounters } from '../../common'; -import { handleUnavailable } from '../../common/generate'; +import { handleUnavailable } from '../../common/request_handler'; import { commonJobsRouteHandlerFactory, jobManagementPreRouting, diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts new file mode 100644 index 0000000000000..b7205c3cfb7c4 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts @@ -0,0 +1,353 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import rison from '@kbn/rison'; +import { BehaviorSubject } from 'rxjs'; +import supertest from 'supertest'; + +import { setupServer } from '@kbn/core-test-helpers-test-utils'; +import { coreMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; +import { INTERNAL_ROUTES } from '@kbn/reporting-common'; +import { PdfExportType } from '@kbn/reporting-export-types-pdf'; +import { createMockConfigSchema } from '@kbn/reporting-mocks-server'; +import { ExportTypesRegistry } from '@kbn/reporting-server/export_types_registry'; + +import { ReportingCore } from '../../../..'; +import { reportingMock } from '../../../../mocks'; +import { + createMockPluginSetup, + createMockPluginStart, + createMockReportingCore, +} from '../../../../test_helpers'; +import { ReportingRequestHandlerContext } from '../../../../types'; +import { registerScheduleRoutesInternal } from '../schedule_from_jobparams'; +import { FakeRawRequest, KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server'; + +type SetupServerReturn = Awaited>; + +const fakeRawRequest: FakeRawRequest = { + headers: { + authorization: `ApiKey skdjtq4u543yt3rhewrh`, + }, + path: '/', +}; + +describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { + const reportingSymbol = Symbol('reporting'); + let server: SetupServerReturn['server']; + let httpSetup: SetupServerReturn['httpSetup']; + let mockExportTypesRegistry: ExportTypesRegistry; + let reportingCore: ReportingCore; + let soClient: SavedObjectsClientContract; + + const mockConfigSchema = createMockConfigSchema({ + queue: { indexInterval: 'year', timeout: 10000, pollEnabled: true }, + }); + + const mockLogger = loggingSystemMock.createLogger(); + const mockCoreSetup = coreMock.createSetup(); + + const mockPdfExportType = new PdfExportType( + mockCoreSetup, + mockConfigSchema, + mockLogger, + coreMock.createPluginInitializerContext(mockConfigSchema) + ); + + beforeEach(async () => { + ({ server, httpSetup } = await setupServer(reportingSymbol)); + httpSetup.registerRouteHandlerContext( + reportingSymbol, + 'reporting', + () => reportingMock.createStart() + ); + + const mockSetupDeps = createMockPluginSetup({ + security: { license: { isEnabled: () => true, getFeature: () => true } }, + router: httpSetup.createRouter(''), + }); + + const mockStartDeps = await createMockPluginStart( + { + licensing: { + ...licensingMock.createStart(), + license$: new BehaviorSubject({ + isActive: true, + isAvailable: true, + type: 'gold', + getFeature: () => true, + }), + }, + securityService: { + authc: { + apiKeys: { areAPIKeysEnabled: () => true }, + getCurrentUser: () => ({ id: '123', roles: ['superuser'], username: 'Tom Riddle' }), + }, + }, + }, + mockConfigSchema + ); + + reportingCore = await createMockReportingCore(mockConfigSchema, mockSetupDeps, mockStartDeps); + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ + isSufficientlySecure: true, + hasPermanentEncryptionKey: true, + }); + + mockExportTypesRegistry = new ExportTypesRegistry(); + mockExportTypesRegistry.register(mockPdfExportType); + + soClient = await reportingCore.getSoClient(fakeRawRequest as unknown as KibanaRequest); + soClient.create = jest.fn().mockImplementation(async (_, opts) => { + return { + id: 'foo', + attributes: opts, + type: 'scheduled-report', + }; + }); + }); + + afterEach(async () => { + await server.stop(); + }); + + it('returns 400 if there are no job params', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + '"[request body]: expected a plain object value, but found [null] instead."' + ) + ); + }); + + it('returns 400 if job params body is invalid', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ jobParams: `foo:`, schedule: { rrule: { freq: 1, interval: 2 } } }) + .expect(400) + .then(({ body }) => expect(body.message).toMatchInlineSnapshot('"invalid rison: foo:"')); + }); + + it('returns 400 export type is invalid', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/TonyHawksProSkater2`) + .send({ + schedule: { rrule: { freq: 1, interval: 2 } }, + jobParams: rison.encode({ title: `abc` }), + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot('"Invalid export-type of TonyHawksProSkater2"') + ); + }); + + it('returns 400 on invalid browser timezone', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot(`"Invalid timezone \\"America/Amsterdam\\"."`) + ); + }); + + it('returns 400 on invalid rrule', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), + schedule: { rrule: { freq: 6, interval: 2 } }, + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot(` + "[request body.schedule.rrule]: types that failed validation: + - [request body.schedule.rrule.0.freq]: expected value to equal [1] + - [request body.schedule.rrule.1.freq]: expected value to equal [2] + - [request body.schedule.rrule.2.freq]: expected value to equal [3]" + `) + ); + }); + + it('returns 400 on invalid notification list', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { + email: { + to: 'single@email.com', + }, + }, + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + `"[request body.notification.email.to]: could not parse array value from json input"` + ) + ); + }); + + it('returns 400 on empty notification list', async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ browserTimezone: 'America/Amsterdam', title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { + email: { + to: [], + }, + }, + }) + .expect(400) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + `"[request body.notification.email.to]: array size is [0], but cannot be smaller than [1]"` + ) + ); + }); + + it('returns 403 on when no permanent encryption key', async () => { + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValueOnce({ + isSufficientlySecure: true, + hasPermanentEncryptionKey: false, + }); + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(403) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + `"Permanent encryption key must be set for scheduled reporting"` + ) + ); + }); + + it('returns 403 on when not sufficiently secure', async () => { + jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValueOnce({ + isSufficientlySecure: false, + hasPermanentEncryptionKey: true, + }); + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(403) + .then(({ body }) => + expect(body.message).toMatchInlineSnapshot( + `"Security and API keys must be enabled for scheduled reporting"` + ) + ); + }); + + it('returns 500 if job handler throws an error', async () => { + soClient.create = jest.fn().mockRejectedValue('silly'); + + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ title: `abc` }), + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(500); + }); + + it(`returns 200 if job handler doesn't error`, async () => { + registerScheduleRoutesInternal(reportingCore, mockLogger); + + await server.start(); + + await supertest(httpSetup.server.listener) + .post(`${INTERNAL_ROUTES.SCHEDULE_PREFIX}/printablePdfV2`) + .send({ + jobParams: rison.encode({ + title: `abc`, + layout: { id: 'test' }, + objectType: 'canvas workpad', + }), + notification: { + email: { + to: ['single@email.com'], + }, + }, + schedule: { rrule: { freq: 1, interval: 2 } }, + }) + .expect(200) + .then(({ body }) => { + expect(body).toMatchObject({ + job: { + created_by: 'Tom Riddle', + id: 'foo', + jobtype: 'printable_pdf_v2', + payload: { + isDeprecated: false, + layout: { + id: 'test', + }, + objectType: 'canvas workpad', + title: 'abc', + version: '7.14.0', + }, + schedule: { rrule: { freq: 1, interval: 2 } }, + }, + }); + }); + }); +}); diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts new file mode 100644 index 0000000000000..0c501c4321898 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/schedule_from_jobparams.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaResponse } from '@kbn/core-http-router-server-internal'; +import type { Logger } from '@kbn/core/server'; +import { INTERNAL_ROUTES } from '@kbn/reporting-common'; +import type { ReportingCore } from '../../..'; +import { authorizedUserPreRouting } from '../../common'; +import { ScheduleRequestHandler } from '../../common/request_handler'; + +const { SCHEDULE_PREFIX } = INTERNAL_ROUTES; + +export function registerScheduleRoutesInternal(reporting: ReportingCore, logger: Logger) { + const setupDeps = reporting.getPluginSetupDeps(); + const { router } = setupDeps; + + const kibanaAccessControlTags = ['generateReport']; + + const registerInternalPostScheduleEndpoint = () => { + const path = `${SCHEDULE_PREFIX}/{exportType}`; + router.post( + { + path, + security: { + authz: { + requiredPrivileges: kibanaAccessControlTags, + }, + }, + validate: ScheduleRequestHandler.getValidation(), + options: { + tags: kibanaAccessControlTags.map((accessControlTag) => `access:${accessControlTag}`), + access: 'internal', + }, + }, + authorizedUserPreRouting(reporting, async (user, context, req, res) => { + try { + const requestHandler = new ScheduleRequestHandler({ + reporting, + user, + context, + path, + req, + res, + logger, + }); + const jobParams = requestHandler.getJobParams(); + const schedule = requestHandler.getSchedule(); + const notification = requestHandler.getNotification(); + + return await requestHandler.handleRequest({ + exportTypeId: req.params.exportType, + jobParams, + schedule, + notification, + }); + } catch (err) { + if (err instanceof KibanaResponse) { + return err; + } + throw err; + } + }) + ); + }; + + registerInternalPostScheduleEndpoint(); +} diff --git a/x-pack/platform/plugins/private/reporting/server/routes/public/generate_from_jobparams.ts b/x-pack/platform/plugins/private/reporting/server/routes/public/generate_from_jobparams.ts index f547faa9cab52..34507cf79d56c 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/public/generate_from_jobparams.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/public/generate_from_jobparams.ts @@ -10,7 +10,7 @@ import type { Logger } from '@kbn/core/server'; import { PUBLIC_ROUTES } from '@kbn/reporting-common'; import type { ReportingCore } from '../..'; import { authorizedUserPreRouting } from '../common'; -import { RequestHandler } from '../common/generate'; +import { GenerateRequestHandler } from '../common/request_handler'; export function registerGenerationRoutesPublic(reporting: ReportingCore, logger: Logger) { const setupDeps = reporting.getPluginSetupDeps(); @@ -28,7 +28,7 @@ export function registerGenerationRoutesPublic(reporting: ReportingCore, logger: requiredPrivileges: kibanaAccessControlTags, }, }, - validate: RequestHandler.getValidation(), + validate: GenerateRequestHandler.getValidation(), options: { tags: kibanaAccessControlTags.map((controlAccessTag) => `access:${controlAccessTag}`), access: 'public', @@ -36,19 +36,19 @@ export function registerGenerationRoutesPublic(reporting: ReportingCore, logger: }, authorizedUserPreRouting(reporting, async (user, context, req, res) => { try { - const requestHandler = new RequestHandler( + const requestHandler = new GenerateRequestHandler({ reporting, user, context, path, req, res, - logger - ); - return await requestHandler.handleGenerateRequest( - req.params.exportType, - requestHandler.getJobParams() - ); + logger, + }); + return await requestHandler.handleRequest({ + exportTypeId: req.params.exportType, + jobParams: requestHandler.getJobParams(), + }); } catch (err) { if (err instanceof KibanaResponse) { return err; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/index.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/index.ts new file mode 100644 index 0000000000000..df34d38fc50d8 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectsServiceSetup } from '@kbn/core/server'; +import { ALERTING_CASES_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { scheduledReportMappings, scheduledReportModelVersions } from './scheduled_report'; + +export const SCHEDULED_REPORT_SAVED_OBJECT_TYPE = 'scheduled_report'; + +export function setupSavedObjects(savedObjects: SavedObjectsServiceSetup) { + savedObjects.registerType({ + name: SCHEDULED_REPORT_SAVED_OBJECT_TYPE, + indexPattern: ALERTING_CASES_SAVED_OBJECT_INDEX, + hidden: true, + namespaceType: 'multiple', + mappings: scheduledReportMappings, + management: { importableAndExportable: false }, + modelVersions: scheduledReportModelVersions, + }); +} diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/index.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/index.ts new file mode 100644 index 0000000000000..285297b977eef --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { scheduledReportMappings } from './mappings'; +export { scheduledReportModelVersions } from './model_versions'; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/mappings.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/mappings.ts new file mode 100644 index 0000000000000..26520db4d22ab --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/mappings.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsTypeMappingDefinition } from '@kbn/core/server'; + +export const scheduledReportMappings: SavedObjectsTypeMappingDefinition = { + dynamic: false, + properties: { + createdBy: { + type: 'keyword', + }, + }, +}; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/model_versions.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/model_versions.ts new file mode 100644 index 0000000000000..4123d20974d6c --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/model_versions.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsModelVersionMap } from '@kbn/core-saved-objects-server'; +import { rawScheduledReportSchemaV1 } from './schemas'; + +export const scheduledReportModelVersions: SavedObjectsModelVersionMap = { + '1': { + changes: [], + schemas: { + forwardCompatibility: rawScheduledReportSchemaV1.extends({}, { unknowns: 'ignore' }), + create: rawScheduledReportSchemaV1, + }, + }, +}; diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/index.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/index.ts similarity index 76% rename from x-pack/platform/plugins/private/reporting/server/routes/common/generate/index.ts rename to x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/index.ts index a16ddf1204b8f..6df4417bb6cef 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/generate/index.ts +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { handleUnavailable, RequestHandler } from './request_handler'; +export { rawScheduledReportSchema as rawScheduledReportSchemaV1 } from './v1'; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts new file mode 100644 index 0000000000000..6f684f9d7cbd7 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/latest.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import type { rawNotificationSchema, rawScheduledReportSchema } from './v1'; + +export type RawNotification = TypeOf; +export type RawScheduledReport = TypeOf; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts new file mode 100644 index 0000000000000..22768f0f2e333 --- /dev/null +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { scheduleRruleSchema } from '@kbn/task-manager-plugin/server'; + +const rawLayoutIdSchema = schema.oneOf([ + schema.literal('preserve_layout'), + schema.literal('print'), + schema.literal('canvas'), +]); + +export const rawNotificationSchema = schema.object({ + email: schema.maybe( + schema.object({ + to: schema.arrayOf(schema.string(), { minSize: 1 }), + }) + ), +}); + +export const rawScheduledReportSchema = schema.object({ + createdAt: schema.string(), + createdBy: schema.oneOf([schema.string(), schema.boolean()]), + enabled: schema.boolean(), + jobType: schema.string(), + meta: schema.object({ + isDeprecated: schema.maybe(schema.boolean()), + layout: schema.maybe(rawLayoutIdSchema), + objectType: schema.string(), + }), + migrationVersion: schema.string(), + notification: schema.maybe(rawNotificationSchema), + payload: schema.string(), + schedule: scheduleRruleSchema, + title: schema.string(), +}); diff --git a/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts index 4d4b386c1020f..199dc23a94327 100644 --- a/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/platform/plugins/private/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -15,8 +15,10 @@ import { docLinksServiceMock, elasticsearchServiceMock, loggingSystemMock, + savedObjectsClientMock, statusServiceMock, } from '@kbn/core/server/mocks'; +import { actionsMock } from '@kbn/actions-plugin/server/mocks'; import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; import { discoverPluginMock } from '@kbn/discover-plugin/server/mocks'; import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks'; @@ -39,7 +41,13 @@ export const createMockPluginSetup = ( setupMock: Partial> ): ReportingInternalSetup => { return { - encryptedSavedObjects: encryptedSavedObjectsMock.createSetup(), + actions: { + ...actionsMock.createSetup(), + getActionsConfigurationUtilities: jest.fn().mockReturnValue({ + validateEmailAddresses: jest.fn(), + }), + }, + encryptedSavedObjects: encryptedSavedObjectsMock.createSetup({ canEncrypt: true }), features: featuresPluginMock.createSetup(), basePath: { set: jest.fn() }, router: { get: jest.fn(), post: jest.fn(), put: jest.fn(), delete: jest.fn() }, @@ -55,6 +63,7 @@ export const createMockPluginSetup = ( const coreSetupMock = coreMock.createSetup(); const coreStartMock = coreMock.createStart(); const logger = loggingSystemMock.createLogger(); +const savedObjectsClient = savedObjectsClientMock.create(); const createMockReportingStore = async (config: ReportingConfigType) => { const mockConfigSchema = createMockConfigSchema(config); @@ -70,7 +79,7 @@ export const createMockPluginStart = async ( return { analytics: coreSetupMock.analytics, esClient: elasticsearchServiceMock.createClusterClient(), - savedObjects: { getScopedClient: jest.fn() }, + savedObjects: { getScopedClient: jest.fn().mockReturnValue(savedObjectsClient) }, uiSettings: { asScopedToClient: () => ({ get: jest.fn() }) }, discover: discoverPluginMock.createStartContract(), data: dataPluginMock.createStartContract(), diff --git a/x-pack/platform/plugins/private/reporting/server/types.ts b/x-pack/platform/plugins/private/reporting/server/types.ts index 4d4b25177ab1e..c4d238b3e90fc 100644 --- a/x-pack/platform/plugins/private/reporting/server/types.ts +++ b/x-pack/platform/plugins/private/reporting/server/types.ts @@ -12,7 +12,7 @@ import type { DiscoverServerPluginStart } from '@kbn/discover-plugin/server'; import type { FeaturesPluginSetup } from '@kbn/features-plugin/server'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/server'; import type { LicensingPluginStart } from '@kbn/licensing-plugin/server'; -import type { UrlOrUrlLocatorTuple } from '@kbn/reporting-common/types'; +import type { ReportSource, UrlOrUrlLocatorTuple } from '@kbn/reporting-common/types'; import type { ReportApiJSON } from '@kbn/reporting-common/types'; import type { ReportingConfigType } from '@kbn/reporting-server'; import type { ScreenshotModePluginSetup } from '@kbn/screenshot-mode-plugin/server'; @@ -24,14 +24,20 @@ import type { import type { SecurityPluginSetup } from '@kbn/security-plugin/server'; import type { SpacesPluginSetup } from '@kbn/spaces-plugin/server'; import type { + RruleSchedule, TaskManagerSetupContract, TaskManagerStartContract, } from '@kbn/task-manager-plugin/server'; import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; +import type { PluginSetupContract as ActionsPluginSetupContract } from '@kbn/actions-plugin/server'; import { ExportTypesRegistry } from '@kbn/reporting-server/export_types_registry'; import type { AuthenticatedUser } from '@kbn/core-security-common'; import { EncryptedSavedObjectsPluginSetup } from '@kbn/encrypted-saved-objects-plugin/server'; +import { + RawNotification, + RawScheduledReport, +} from './saved_objects/scheduled_report/schemas/latest'; /** * Plugin Setup Contract @@ -49,6 +55,7 @@ export type ReportingUser = { username: AuthenticatedUser['username'] } | false; export type ScrollConfig = ReportingConfigType['csv']['scroll']; export interface ReportingSetupDeps { + actions: ActionsPluginSetupContract; encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; features: FeaturesPluginSetup; screenshotMode: ScreenshotModePluginSetup; @@ -90,6 +97,28 @@ export interface ReportingJobResponse { job: ReportApiJSON; } +export type ScheduledReportApiJSON = Omit< + ReportSource, + 'payload' | 'output' | 'attempts' | 'status' +> & { + id: string; + schedule: RruleSchedule; + notification?: RawNotification; + payload: Omit; +}; + +export interface ScheduledReportingJobResponse { + /** + * Details of a new report job that was requested + * @public + */ + job: ScheduledReportApiJSON; +} + +export type ScheduledReportType = Omit & { + schedule: RruleSchedule; +}; + export interface PdfScreenshotOptions extends Omit { urls: UrlOrUrlLocatorTuple[]; } diff --git a/x-pack/platform/plugins/private/reporting/tsconfig.json b/x-pack/platform/plugins/private/reporting/tsconfig.json index ddde2d34d6d7f..51c75b5b431a0 100644 --- a/x-pack/platform/plugins/private/reporting/tsconfig.json +++ b/x-pack/platform/plugins/private/reporting/tsconfig.json @@ -5,6 +5,7 @@ }, "include": ["common/**/*", "public/**/*", "server/**/*", "../../../../../typings/**/*"], "kbn_references": [ + "@kbn/actions-plugin", "@kbn/core", "@kbn/data-plugin", "@kbn/discover-plugin", @@ -52,6 +53,7 @@ "@kbn/react-kibana-mount", "@kbn/core-security-common", "@kbn/core-http-server-utils", + "@kbn/core-saved-objects-server", ], "exclude": [ "target/**/*", diff --git a/x-pack/platform/plugins/shared/features/server/__snapshots__/oss_features.test.ts.snap b/x-pack/platform/plugins/shared/features/server/__snapshots__/oss_features.test.ts.snap index b8b5910d83921..01a0046f28e8a 100644 --- a/x-pack/platform/plugins/shared/features/server/__snapshots__/oss_features.test.ts.snap +++ b/x-pack/platform/plugins/shared/features/server/__snapshots__/oss_features.test.ts.snap @@ -479,7 +479,9 @@ Array [ }, ], "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -570,7 +572,9 @@ Array [ }, "name": "Generate CSV reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -646,7 +650,9 @@ Array [ }, ], "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -706,7 +712,9 @@ Array [ "minimumLicense": "gold", "name": "Generate PDF or PNG reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -822,7 +830,9 @@ Array [ }, ], "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -850,7 +860,9 @@ Array [ }, ], "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -942,7 +954,9 @@ Array [ "minimumLicense": "gold", "name": "Generate PDF or PNG reports", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ @@ -962,7 +976,9 @@ Array [ }, "name": "Generate CSV reports from Discover session panels", "savedObject": Object { - "all": Array [], + "all": Array [ + "scheduled_report", + ], "read": Array [], }, "ui": Array [ diff --git a/x-pack/platform/plugins/shared/features/server/oss_features.ts b/x-pack/platform/plugins/shared/features/server/oss_features.ts index 39e016301b0ef..7f3f815453cfb 100644 --- a/x-pack/platform/plugins/shared/features/server/oss_features.ts +++ b/x-pack/platform/plugins/shared/features/server/oss_features.ts @@ -802,7 +802,7 @@ const reportingFeatures: { defaultMessage: 'Generate CSV reports', }), includeIn: 'all', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, management: { insightsAndAlerting: ['reporting'] }, api: ['generateReport'], ui: ['generateCsv'], @@ -830,7 +830,7 @@ const reportingFeatures: { ), includeIn: 'all', minimumLicense: 'gold', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, management: { insightsAndAlerting: ['reporting'] }, api: ['generateReport'], ui: ['generateScreenshot'], @@ -844,7 +844,7 @@ const reportingFeatures: { defaultMessage: 'Generate CSV reports from Discover session panels', }), includeIn: 'all', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, management: { insightsAndAlerting: ['reporting'] }, api: ['downloadCsv'], ui: ['downloadCsv'], @@ -872,7 +872,7 @@ const reportingFeatures: { ), includeIn: 'all', minimumLicense: 'gold', - savedObject: { all: [], read: [] }, + savedObject: { all: ['scheduled_report'], read: [] }, management: { insightsAndAlerting: ['reporting'] }, api: ['generateReport'], ui: ['generateScreenshot'], diff --git a/x-pack/platform/plugins/shared/task_manager/server/index.ts b/x-pack/platform/plugins/shared/task_manager/server/index.ts index 6678d4c08ca18..a20f3bf83b41f 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/index.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/index.ts @@ -24,7 +24,9 @@ export type { } from './task'; export { Frequency, Weekday } from '@kbn/rrule'; +export { scheduleRruleSchema } from './saved_objects'; +export type { RruleSchedule } from './task'; export { TaskStatus, TaskPriority, TaskCost } from './task'; export type { TaskRegisterDefinition, TaskDefinitionRegistry } from './task_type_dictionary'; diff --git a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts index d3e562c67b723..e030aa9c61e8a 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/lib/get_first_run_at.test.ts @@ -159,7 +159,7 @@ describe('getFirstRunAt', () => { freq: 2, // Weekly interval: 1, tzid: 'UTC', - byweekday: [1], // Monday + byweekday: ['1'], // Monday }, }, }; @@ -182,7 +182,7 @@ describe('getFirstRunAt', () => { freq: 2, // Weekly interval: 1, tzid: 'UTC', - byweekday: [1], // Monday + byweekday: ['MO'], // Monday byhour: [12], byminute: [15], }, @@ -257,7 +257,7 @@ describe('getFirstRunAt', () => { freq: 1, // Monthly interval: 1, tzid: 'UTC', - byweekday: [3], // Wednesday + byweekday: ['3'], // Wednesday byhour: [12], byminute: [17], }, diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts index 9994426fb0831..75b200acc38b1 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/index.ts @@ -14,6 +14,8 @@ import { getOldestIdleActionTask } from '../queries/oldest_idle_action_task'; import { TASK_MANAGER_INDEX } from '../constants'; import { backgroundTaskNodeModelVersions, taskModelVersions } from './model_versions'; +export { scheduleRruleSchema } from './schemas/task'; + export const TASK_SO_NAME = 'task'; export const BACKGROUND_TASK_NODE_SO_NAME = 'background-task-node'; diff --git a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts index 365cc506cb123..874f374355bbb 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/saved_objects/schemas/task.ts @@ -55,6 +55,14 @@ export const taskSchemaV3 = taskSchemaV2.extends({ priority: schema.maybe(schema.number()), }); +export const scheduleIntervalSchema = schema.object({ + interval: schema.string({ validate: validateDuration }), +}); + +export const scheduleRruleSchema = schema.object({ + rrule: rruleSchedule, +}); + export const taskSchemaV4 = taskSchemaV3.extends({ apiKey: schema.maybe(schema.string()), userScope: schema.maybe( @@ -67,14 +75,5 @@ export const taskSchemaV4 = taskSchemaV3.extends({ }); export const taskSchemaV5 = taskSchemaV4.extends({ - schedule: schema.maybe( - schema.oneOf([ - schema.object({ - interval: schema.string({ validate: validateDuration }), - }), - schema.object({ - rrule: rruleSchedule, - }), - ]) - ), + schedule: schema.maybe(schema.oneOf([scheduleIntervalSchema, scheduleRruleSchema])), }); diff --git a/x-pack/platform/plugins/shared/task_manager/server/task.ts b/x-pack/platform/plugins/shared/task_manager/server/task.ts index 2e22f5be2b4a5..b9a59e3452267 100644 --- a/x-pack/platform/plugins/shared/task_manager/server/task.ts +++ b/x-pack/platform/plugins/shared/task_manager/server/task.ts @@ -11,7 +11,7 @@ import type { ObjectType, TypeOf } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; import { isNumber } from 'lodash'; import type { KibanaRequest } from '@kbn/core/server'; -import type { Frequency, Weekday } from '@kbn/rrule'; +import type { Frequency } from '@kbn/rrule'; import { isErr, tryAsResult } from './lib/result_type'; import type { Interval } from './lib/intervals'; import { isInterval, parseIntervalAsMillisecond } from './lib/intervals'; @@ -259,8 +259,9 @@ export interface IntervalSchedule { rrule?: never; } +export type Rrule = RruleMonthly | RruleWeekly | RruleDaily; export interface RruleSchedule { - rrule: RruleMonthly | RruleWeekly | RruleDaily; + rrule: Rrule; interval?: never; } @@ -269,17 +270,16 @@ interface RruleCommon { interval: number; tzid: string; } - interface RruleMonthly extends RruleCommon { freq: Frequency.MONTHLY; bymonthday?: number[]; byhour?: number[]; byminute?: number[]; - byweekday?: Weekday[]; + byweekday?: string[]; } interface RruleWeekly extends RruleCommon { freq: Frequency.WEEKLY; - byweekday?: Weekday[]; + byweekday?: string[]; byhour?: number[]; byminute?: number[]; bymonthday?: never; @@ -288,7 +288,7 @@ interface RruleDaily extends RruleCommon { freq: Frequency.DAILY; byhour?: number[]; byminute?: number[]; - byweekday?: Weekday[]; + byweekday?: string[]; bymonthday?: never; } diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts b/x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts index 0b5237a9051d6..5604ff058c34e 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/security_roles_privileges.ts @@ -162,6 +162,161 @@ export default function ({ getService }: FtrProviderContext) { }); }); + describe('Dashboard: Schedule PDF report', () => { + it('does not allow user that does not have the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.DATA_ANALYST_USERNAME, + reportingAPI.DATA_ANALYST_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF disallowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'dashboard', + version: '7.14.0', + } + ); + expect(res.status).to.eql(403); + }); + + it('does allow user with the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.REPORTING_USER_USERNAME, + reportingAPI.REPORTING_USER_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF allowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'dashboard', + version: '7.14.0', + } + ); + expect(res.status).to.eql(200); + + const soResult = await reportingAPI.getScheduledReportSO(res.body.job.id); + expect(soResult.status).to.eql(200); + expect(soResult.body._source.scheduled_report.title).to.eql('test PDF allowed'); + }); + }); + + describe('Visualize: Schedule PDF report', () => { + it('does not allow user that does not have the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.DATA_ANALYST_USERNAME, + reportingAPI.DATA_ANALYST_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF disallowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'visualization', + version: '7.14.0', + } + ); + expect(res.status).to.eql(403); + }); + + it('does allow user with the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.REPORTING_USER_USERNAME, + reportingAPI.REPORTING_USER_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF allowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'visualization', + version: '7.14.0', + } + ); + expect(res.status).to.eql(200); + + const soResult = await reportingAPI.getScheduledReportSO(res.body.job.id); + expect(soResult.status).to.eql(200); + expect(soResult.body._source.scheduled_report.title).to.eql('test PDF allowed'); + }); + }); + + describe('Canvas: Schedule PDF report', () => { + it('does not allow user that does not have the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.DATA_ANALYST_USERNAME, + reportingAPI.DATA_ANALYST_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF disallowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'canvas', + version: '7.14.0', + } + ); + expect(res.status).to.eql(403); + }); + + it('does allow user with the role-based privilege', async () => { + const res = await reportingAPI.schedulePdf( + reportingAPI.REPORTING_USER_USERNAME, + reportingAPI.REPORTING_USER_PASSWORD, + { + browserTimezone: 'UTC', + title: 'test PDF allowed', + layout: { id: 'preserve_layout' }, + locatorParams: [{ id: 'canvas', version: '7.14.0', params: {} }], + objectType: 'canvas', + version: '7.14.0', + } + ); + expect(res.status).to.eql(200); + + const soResult = await reportingAPI.getScheduledReportSO(res.body.job.id); + expect(soResult.status).to.eql(200); + expect(soResult.body._source.scheduled_report.title).to.eql('test PDF allowed'); + }); + }); + + describe('Discover: Schedule CSV report', () => { + it('does not allow user that does not have the role-based privilege', async () => { + const res = await reportingAPI.scheduleCsv( + { + browserTimezone: 'UTC', + searchSource: {} as SerializedSearchSourceFields, + objectType: 'search', + title: 'test disallowed', + version: '7.14.0', + }, + reportingAPI.DATA_ANALYST_USERNAME, + reportingAPI.DATA_ANALYST_PASSWORD + ); + expect(res.status).to.eql(403); + }); + + it('does allow user with the role-based privilege', async () => { + const res = await reportingAPI.scheduleCsv( + { + browserTimezone: 'UTC', + title: 'allowed search', + objectType: 'search', + searchSource: { + version: true, + fields: [{ field: '*', include_unmapped: true }], + index: '5193f870-d861-11e9-a311-0fa548c5f953', + } as unknown as SerializedSearchSourceFields, + columns: [], + version: '7.13.0', + }, + reportingAPI.REPORTING_USER_USERNAME, + reportingAPI.REPORTING_USER_PASSWORD + ); + expect(res.status).to.eql(200); + + const soResult = await reportingAPI.getScheduledReportSO(res.body.job.id); + expect(soResult.status).to.eql(200); + expect(soResult.body._source.scheduled_report.title).to.eql('allowed search'); + }); + }); + // This tests the same API as x-pack/test/api_integration/apis/security/privileges.ts, but it uses the non-deprecated config it('should register reporting privileges with the security privileges API', async () => { await supertest diff --git a/x-pack/test/reporting_api_integration/services/scenarios.ts b/x-pack/test/reporting_api_integration/services/scenarios.ts index 0800647d2abef..adcd3151aef20 100644 --- a/x-pack/test/reporting_api_integration/services/scenarios.ts +++ b/x-pack/test/reporting_api_integration/services/scenarios.ts @@ -15,6 +15,8 @@ import { REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY, } from '@kbn/reporting-server'; import rison from '@kbn/rison'; +import { ALERTING_CASES_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { RruleSchedule } from '@kbn/task-manager-plugin/server'; import { FtrProviderContext } from '../ftr_provider_context'; function removeWhitespace(str: string) { @@ -150,6 +152,19 @@ export function createScenarios({ getService }: Pick { + const jobParams = rison.encode(job); + return await supertestWithoutAuth + .post(`/internal/reporting/schedule/printablePdfV2`) + .auth(username, password) + .set('kbn-xsrf', 'xxx') + .send({ jobParams, schedule }); + }; const generatePng = async (username: string, password: string, job: JobParamsPNGV2) => { const jobParams = rison.encode(job); return await supertestWithoutAuth @@ -158,6 +173,19 @@ export function createScenarios({ getService }: Pick { + const jobParams = rison.encode(job); + return await supertestWithoutAuth + .post(`/internal/reporting/schedule/pngV2`) + .auth(username, password) + .set('kbn-xsrf', 'xxx') + .send({ jobParams, schedule }); + }; const generateCsv = async ( job: JobParamsCSV, username = 'elastic', @@ -171,6 +199,20 @@ export function createScenarios({ getService }: Pick { + const jobParams = rison.encode(job); + + return await supertestWithoutAuth + .post(`/internal/reporting/schedule/csv_searchsource`) + .auth(username, password) + .set('kbn-xsrf', 'xxx') + .send({ jobParams, schedule }); + }; const postJob = async ( apiPath: string, @@ -263,6 +305,12 @@ export function createScenarios({ getService }: Pick { + return await esSupertest.get( + `/${ALERTING_CASES_SAVED_OBJECT_INDEX}/_doc/scheduled_report:${id}` + ); + }; + return { logTaskManagerHealth, initEcommerce, @@ -281,6 +329,9 @@ export function createScenarios({ getService }: Pick Date: Thu, 8 May 2025 13:18:47 -0400 Subject: [PATCH 42/45] Fixing types --- .../common/request_handler/schedule_request_handler.test.ts | 2 ++ .../integration_tests/scheduling_from_jobparams.test.ts | 3 +++ 2 files changed, 5 insertions(+) diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts index 91aa5d8851eb2..d6648b2b079f5 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts @@ -508,6 +508,7 @@ describe('Handle request to schedule', () => { jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ isSufficientlySecure: true, hasPermanentEncryptionKey: false, + areNotificationsEnabled: true, }); expect( @@ -526,6 +527,7 @@ describe('Handle request to schedule', () => { jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ isSufficientlySecure: false, hasPermanentEncryptionKey: true, + areNotificationsEnabled: true, }); expect( diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts index b7205c3cfb7c4..031c33ff6642f 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts @@ -97,6 +97,7 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValue({ isSufficientlySecure: true, hasPermanentEncryptionKey: true, + areNotificationsEnabled: true, }); mockExportTypesRegistry = new ExportTypesRegistry(); @@ -251,6 +252,7 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValueOnce({ isSufficientlySecure: true, hasPermanentEncryptionKey: false, + areNotificationsEnabled: false, }); registerScheduleRoutesInternal(reportingCore, mockLogger); @@ -274,6 +276,7 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { jest.spyOn(reportingCore, 'getHealthInfo').mockResolvedValueOnce({ isSufficientlySecure: false, hasPermanentEncryptionKey: true, + areNotificationsEnabled: false, }); registerScheduleRoutesInternal(reportingCore, mockLogger); From bc09318eb6d5e16d38e947f2b8cc8e2b7bdd40c2 Mon Sep 17 00:00:00 2001 From: Ying Date: Thu, 8 May 2025 14:06:02 -0400 Subject: [PATCH 43/45] Adding cc,bcc fields and validation for max recipients --- .../schedule_request_handler.test.ts | 70 +++++++++++++++++++ .../schedule_request_handler.ts | 35 +++++++--- .../scheduled_report/schemas/v1.ts | 2 + 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts index d6648b2b079f5..c873a66f45e2e 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts @@ -422,6 +422,16 @@ describe('Handle request to schedule', () => { expect(requestHandler.getNotification()).toBeUndefined(); }); + test('returns undefined if notification.email arrays are all empty', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: { to: [], cc: [], bcc: [] } }, + }; + expect(requestHandler.getNotification()).toBeUndefined(); + }); + test('returns undefined if notification.email object is null', () => { // @ts-ignore body is a read-only property mockRequest.body = { @@ -452,6 +462,66 @@ describe('Handle request to schedule', () => { expect(error?.statusCode).toBe(400); expect(error?.body).toBe('Invalid email address(es): not valid emails: foo'); }); + + test('handles too many recipients', () => { + let error: { statusCode: number; body: string } | undefined; + try { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { + email: { + to: [ + '1@elastic.co', + '2@elastic.co', + '3@elastic.co', + '4@elastic.co', + '5@elastic.co', + '6@elastic.co', + '7@elastic.co', + ], + cc: [ + '8@elastic.co', + '9@elastic.co', + '10@elastic.co', + '11@elastic.co', + '12@elastic.co', + '13@elastic.co', + '14@elastic.co', + '15@elastic.co', + '16@elastic.co', + '17@elastic.co', + ], + bcc: [ + '18@elastic.co', + '19@elastic.co', + '20@elastic.co', + '21@elastic.co', + '22@elastic.co', + '23@elastic.co', + '24@elastic.co', + '25@elastic.co', + '26@elastic.co', + '27@elastic.co', + '28@elastic.co', + '29@elastic.co', + '30@elastic.co', + '31@elastic.co', + ], + }, + }, + }; + requestHandler.getNotification(); + } catch (err) { + error = err; + } + + expect(error?.statusCode).toBe(400); + expect(error?.body).toBe( + 'Maximum number of recipients exceeded: cannot specify more than 30 recipients.' + ); + }); }); describe('handleRequest', () => { diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts index b1735de8515ee..3ac2a26d285a9 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.ts @@ -21,6 +21,10 @@ import { SCHEDULED_REPORT_SAVED_OBJECT_TYPE } from '../../../saved_objects'; import { RequestHandler, RequestParams } from './request_handler'; import { transformRawScheduledReportToReport } from './lib'; +// Using the limit specified in the cloud email service limits +// https://www.elastic.co/docs/explore-analyze/alerts-cases/watcher/enable-watcher#cloud-email-service-limits +const MAX_ALLOWED_EMAILS = 30; + const validation = { params: schema.object({ exportType: schema.string({ minLength: 2 }) }), body: schema.object({ @@ -72,14 +76,29 @@ export class ScheduleRequestHandler extends RequestHandler< return undefined; } - if (notification && notification.email && notification.email.to) { - const invalidEmails = reporting.validateNotificationEmails(notification.email.to); - if (invalidEmails) { - throw res.customError({ - statusCode: 400, - body: `Invalid email address(es): ${invalidEmails}`, - }); - } + const allEmails = new Set([ + ...(notification.email.to || []), + ...(notification.email.bcc || []), + ...(notification.email.cc || []), + ]); + + if (allEmails.size === 0) { + return undefined; + } + + if (allEmails.size > MAX_ALLOWED_EMAILS) { + throw res.customError({ + statusCode: 400, + body: `Maximum number of recipients exceeded: cannot specify more than ${MAX_ALLOWED_EMAILS} recipients.`, + }); + } + + const invalidEmails = reporting.validateNotificationEmails([...allEmails]); + if (invalidEmails) { + throw res.customError({ + statusCode: 400, + body: `Invalid email address(es): ${invalidEmails}`, + }); } return notification; diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts index 22768f0f2e333..37a8315d28082 100644 --- a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts @@ -18,6 +18,8 @@ export const rawNotificationSchema = schema.object({ email: schema.maybe( schema.object({ to: schema.arrayOf(schema.string(), { minSize: 1 }), + bcc: schema.maybe(schema.arrayOf(schema.string())), + cc: schema.maybe(schema.arrayOf(schema.string())), }) ), }); From b6af77043838059fb8cc8417f473fa0010466dc2 Mon Sep 17 00:00:00 2001 From: Ying Date: Tue, 20 May 2025 09:43:24 -0400 Subject: [PATCH 44/45] Making migration_version optional --- .../server/saved_objects/scheduled_report/schemas/v1.ts | 2 +- x-pack/platform/plugins/private/reporting/server/types.ts | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts index 37a8315d28082..2dcfd2abb685c 100644 --- a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts @@ -34,7 +34,7 @@ export const rawScheduledReportSchema = schema.object({ layout: schema.maybe(rawLayoutIdSchema), objectType: schema.string(), }), - migrationVersion: schema.string(), + migrationVersion: schema.maybe(schema.string()), notification: schema.maybe(rawNotificationSchema), payload: schema.string(), schedule: scheduleRruleSchema, diff --git a/x-pack/platform/plugins/private/reporting/server/types.ts b/x-pack/platform/plugins/private/reporting/server/types.ts index 028059ae565a2..ceeed3e955c95 100644 --- a/x-pack/platform/plugins/private/reporting/server/types.ts +++ b/x-pack/platform/plugins/private/reporting/server/types.ts @@ -101,12 +101,13 @@ export interface ReportingJobResponse { export type ScheduledReportApiJSON = Omit< ReportSource, - 'payload' | 'output' | 'attempts' | 'status' + 'attempts' | 'migration_version' | 'output' | 'payload' | 'status' > & { id: string; - schedule: RruleSchedule; + migration_version?: string; notification?: RawNotification; payload: Omit; + schedule: RruleSchedule; }; export interface ScheduledReportingJobResponse { From 55c7011cd023dabc5c4fea4b47f3d2db6a47fde6 Mon Sep 17 00:00:00 2001 From: Ying Date: Tue, 20 May 2025 10:04:25 -0400 Subject: [PATCH 45/45] Fixing email validation --- .../schedule_request_handler.test.ts | 10 ++++++++ .../scheduling_from_jobparams.test.ts | 4 +-- .../scheduled_report/schemas/v1.ts | 25 +++++++++++++++---- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts index c873a66f45e2e..418242ecb57b1 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/common/request_handler/schedule_request_handler.test.ts @@ -392,6 +392,16 @@ describe('Handle request to schedule', () => { expect(requestHandler.getNotification()).toEqual({ email: { to: ['a@b.com'] } }); }); + test('parse notification from body when no to defined', () => { + // @ts-ignore body is a read-only property + mockRequest.body = { + jobParams: rison.encode(mockJobParams), + schedule: { rrule: { freq: 1, interval: 2 } }, + notification: { email: { bcc: ['a@b.com'] } }, + }; + expect(requestHandler.getNotification()).toEqual({ email: { bcc: ['a@b.com'] } }); + }); + test('returns undefined if notification object is empty', () => { // @ts-ignore body is a read-only property mockRequest.body = { diff --git a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts index 031c33ff6642f..23f2229090234 100644 --- a/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts +++ b/x-pack/platform/plugins/private/reporting/server/routes/internal/schedule/integration_tests/scheduling_from_jobparams.test.ts @@ -243,7 +243,7 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { .expect(400) .then(({ body }) => expect(body.message).toMatchInlineSnapshot( - `"[request body.notification.email.to]: array size is [0], but cannot be smaller than [1]"` + `"[request body.notification.email]: At least one email address is required"` ) ); }); @@ -327,7 +327,7 @@ describe(`POST ${INTERNAL_ROUTES.SCHEDULE_PREFIX}`, () => { }), notification: { email: { - to: ['single@email.com'], + bcc: ['single@email.com'], }, }, schedule: { rrule: { freq: 1, interval: 2 } }, diff --git a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts index 2dcfd2abb685c..f79b49d3eeed6 100644 --- a/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts +++ b/x-pack/platform/plugins/private/reporting/server/saved_objects/scheduled_report/schemas/v1.ts @@ -16,11 +16,26 @@ const rawLayoutIdSchema = schema.oneOf([ export const rawNotificationSchema = schema.object({ email: schema.maybe( - schema.object({ - to: schema.arrayOf(schema.string(), { minSize: 1 }), - bcc: schema.maybe(schema.arrayOf(schema.string())), - cc: schema.maybe(schema.arrayOf(schema.string())), - }) + schema.object( + { + to: schema.maybe(schema.arrayOf(schema.string())), + bcc: schema.maybe(schema.arrayOf(schema.string())), + cc: schema.maybe(schema.arrayOf(schema.string())), + }, + { + validate: (value) => { + const allEmails = new Set([ + ...(value.to || []), + ...(value.bcc || []), + ...(value.cc || []), + ]); + + if (allEmails.size === 0) { + return 'At least one email address is required'; + } + }, + } + ) ), });