diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0a2b260b39daf..656d9ff31850d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -706,6 +706,7 @@ src/platform/plugins/shared/ui_actions_enhanced @elastic/appex-sharedux src/platform/plugins/shared/unified_doc_viewer @elastic/kibana-data-discovery src/platform/plugins/shared/unified_search @elastic/kibana-presentation src/platform/plugins/shared/usage_collection @elastic/kibana-core +src/platform/plugins/shared/unused_urls_cleanup @elastic/appex-sharedux src/platform/plugins/shared/vis_types/timeseries @elastic/kibana-visualizations src/platform/plugins/shared/visualizations @elastic/kibana-visualizations src/platform/test diff --git a/package.json b/package.json index 62abe21d18cdd..00bdcb008b1f0 100644 --- a/package.json +++ b/package.json @@ -1004,6 +1004,7 @@ "@kbn/unified-tabs-examples-plugin": "link:examples/unified_tabs_examples", "@kbn/unsaved-changes-badge": "link:src/platform/packages/private/kbn-unsaved-changes-badge", "@kbn/unsaved-changes-prompt": "link:src/platform/packages/shared/kbn-unsaved-changes-prompt", + "@kbn/unused-urls-cleanup": "link:src/platform/plugins/shared/unused_urls_cleanup", "@kbn/upgrade-assistant": "link:x-pack/platform/packages/private/upgrade-assistant", "@kbn/upgrade-assistant-plugin": "link:x-pack/platform/plugins/private/upgrade_assistant", "@kbn/uptime-plugin": "link:x-pack/solutions/observability/plugins/uptime", diff --git a/src/platform/plugins/shared/unused_urls_cleanup/README.md b/src/platform/plugins/shared/unused_urls_cleanup/README.md new file mode 100755 index 0000000000000..30404ce4c5463 --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/README.md @@ -0,0 +1 @@ +TODO \ No newline at end of file diff --git a/src/platform/plugins/shared/unused_urls_cleanup/kibana.jsonc b/src/platform/plugins/shared/unused_urls_cleanup/kibana.jsonc new file mode 100644 index 0000000000000..dbed25cd994d2 --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/kibana.jsonc @@ -0,0 +1,21 @@ +{ + "type": "plugin", + "id": "@kbn/unused-urls-cleanup", + "owner": [ + "@elastic/appex-sharedux" + ], + "group": "platform", + "visibility": "private", + "description": "Background task responsible for deleting saved objects of type 'url' which are unused.", + "plugin": { + "id": "unusedUrlsCleanup", + "browser": false, + "server": true, + "requiredPlugins": [ + "taskManager" + ], + "configPath": "unused_urls_cleanup", + } +} + + diff --git a/src/platform/plugins/shared/unused_urls_cleanup/server/config.ts b/src/platform/plugins/shared/unused_urls_cleanup/server/config.ts new file mode 100644 index 0000000000000..28b15c298a915 --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/server/config.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { schema, TypeOf } from '@kbn/config-schema'; +import type { PluginConfigDescriptor } from '@kbn/core/server'; +import { DEFAULT_MAX_AGE } from '@kbn/unused-urls-cleanup/server/constants'; + +export const configSchema = schema.object({ + maxAge: schema.string({ + // TODO: Possibly disable this for new installations + defaultValue: DEFAULT_MAX_AGE, + validate: (value) => { + const rangeRegex = /\d+[yMwdhms]/; + if (!rangeRegex.test(value)) { + return `Invalid value: ${value}. Expected format: , where unit is one of y, M, w, d, h, m, s.`; + } + }, + }), +}); + +export type UnusedUrlsCleanupPluginConfig = TypeOf; + +export const config: PluginConfigDescriptor = { + schema: configSchema, +}; diff --git a/src/platform/plugins/shared/unused_urls_cleanup/server/constants.ts b/src/platform/plugins/shared/unused_urls_cleanup/server/constants.ts new file mode 100644 index 0000000000000..0453754b04484 --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/server/constants.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { TaskInstanceWithId } from '@kbn/task-manager-plugin/server/task'; + +export const TASK_ID = 'unusedUrlsCleanupTask'; +export const TASK_SCHEDULE_INTERVAL = '30s'; // TODO: Change this to 1 week +export const SAVED_OBJECT_TYPE = 'url'; +export const PIT_KEEP_ALIVE = '10m'; +export const MAX_PAGE_SIZE = 10000; +export const DEFAULT_MAX_AGE = '1y'; +export const DELETE_UNUSED_URLS_TASK: TaskInstanceWithId = { + id: TASK_ID, + taskType: TASK_ID, + params: {}, + state: {}, + schedule: { + interval: TASK_SCHEDULE_INTERVAL, + }, +}; diff --git a/src/platform/plugins/shared/unused_urls_cleanup/server/index.ts b/src/platform/plugins/shared/unused_urls_cleanup/server/index.ts new file mode 100644 index 0000000000000..bbdee3daf87ea --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/server/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { PluginInitializerContext } from '@kbn/core/server'; + +export type { UnusedUrlsCleanupPluginSetup, UnusedUrlsCleanupPluginStart } from './types'; +export type { UnusedUrlsCleanupPluginConfig } from './config'; +export { config, configSchema } from './config'; +export { + TASK_ID, + TASK_SCHEDULE_INTERVAL, + SAVED_OBJECT_TYPE, + PIT_KEEP_ALIVE, + MAX_PAGE_SIZE, + DEFAULT_MAX_AGE, + DELETE_UNUSED_URLS_TASK, +} from './constants'; + +export async function plugin(initializerContext: PluginInitializerContext) { + const { UnusedUrlsCleanupPlugin } = await import('./plugin'); + return new UnusedUrlsCleanupPlugin(initializerContext); +} diff --git a/src/platform/plugins/shared/unused_urls_cleanup/server/lib/index.ts b/src/platform/plugins/shared/unused_urls_cleanup/server/lib/index.ts new file mode 100644 index 0000000000000..4c2669f1171f7 --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/server/lib/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +export * from './saved_objects'; diff --git a/src/platform/plugins/shared/unused_urls_cleanup/server/lib/saved_objects.ts b/src/platform/plugins/shared/unused_urls_cleanup/server/lib/saved_objects.ts new file mode 100644 index 0000000000000..9d13eb7893837 --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/server/lib/saved_objects.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { SortResults } from '@elastic/elasticsearch/lib/api/types'; +import { ISavedObjectsRepository, SavedObjectsFindResult } from '@kbn/core/server'; +import { Logger } from '@kbn/logging'; +import { + MAX_PAGE_SIZE, + PIT_KEEP_ALIVE, + SAVED_OBJECT_TYPE, +} from '@kbn/unused-urls-cleanup/server/constants'; + +export const deleteUnusedUrls = async ({ + savedObjectsRepository, + unusedUrls, + logger, +}: { + savedObjectsRepository: ISavedObjectsRepository; + unusedUrls: Array<{ id: string; type: string }>; + logger: Logger; +}) => { + const total = unusedUrls.length; + logger.info(`Deleting ${total} unused URL(s)`); + + try { + await savedObjectsRepository.bulkDelete(unusedUrls, { + refresh: 'wait_for', + }); + + logger.info(`Succesfully deleted ${total} unused URL(s)`); + } catch (e) { + logger.error(`Failed to delete unused URL(s): ${e.message}`); + } +}; + +export const fetchAllUnusedUrls = async ({ + savedObjectsRepository, + filter, + logger, +}: { + savedObjectsRepository: ISavedObjectsRepository; + filter: string; + logger: Logger; +}) => { + const results: SavedObjectsFindResult[] = []; + + const { id: pitId } = await savedObjectsRepository.openPointInTimeForType(SAVED_OBJECT_TYPE, { + keepAlive: PIT_KEEP_ALIVE, + }); + + try { + let searchAfter: SortResults | undefined; + let hasMore = true; + + while (hasMore) { + const response = await savedObjectsRepository.find({ + type: SAVED_OBJECT_TYPE, + filter, + pit: { id: pitId, keepAlive: PIT_KEEP_ALIVE }, + searchAfter, + perPage: MAX_PAGE_SIZE, + }); + + results.push(...response.saved_objects); + hasMore = response.saved_objects.length === MAX_PAGE_SIZE; + + if (hasMore) { + searchAfter = response.saved_objects[response.saved_objects.length - 1].sort; + } + } + } catch (e) { + logger.error(`Failed to fetch unused URLs: ${e.message}`); + } finally { + await savedObjectsRepository.closePointInTime(pitId); + } + + return results.map(({ id }) => ({ + id, + type: SAVED_OBJECT_TYPE, + })); +}; + +export const runDeleteUnusedUrlsTask = async ({ + savedObjectsRepository, + filter, + logger, +}: { + savedObjectsRepository: ISavedObjectsRepository; + filter: string; + logger: Logger; +}) => { + try { + logger.info('Unused URLs cleanup started'); + + const unusedUrls = await fetchAllUnusedUrls({ + savedObjectsRepository, + filter, + logger, + }); + + logger.info(`Found ${unusedUrls.length} unused URL(s)`); + + if (unusedUrls.length > 0) { + await deleteUnusedUrls({ + savedObjectsRepository, + unusedUrls, + logger, + }); + } + } catch (e) { + logger.error(`Failed to run: ${e.message}`); + } +}; diff --git a/src/platform/plugins/shared/unused_urls_cleanup/server/plugin.ts b/src/platform/plugins/shared/unused_urls_cleanup/server/plugin.ts new file mode 100644 index 0000000000000..c3ad518200dc9 --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/server/plugin.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { + PluginInitializerContext, + CoreSetup, + CoreStart, + Plugin, + Logger, +} from '@kbn/core/server'; +import { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; +import { TASK_ID, DELETE_UNUSED_URLS_TASK } from '@kbn/unused-urls-cleanup/server/constants'; +import { runDeleteUnusedUrlsTask } from '@kbn/unused-urls-cleanup/server/lib'; +import type { UnusedUrlsCleanupPluginSetup, UnusedUrlsCleanupPluginStart } from './types'; +import type { UnusedUrlsCleanupPluginConfig } from './config'; + +export class UnusedUrlsCleanupPlugin implements Plugin { + private readonly logger: Logger; + private readonly config: UnusedUrlsCleanupPluginConfig; + private taskManagerSetup: TaskManagerSetupContract | undefined; + + constructor(initializerContext: PluginInitializerContext) { + this.logger = initializerContext.logger.get(); + this.config = initializerContext.config.get(); + } + + public setup(_core: CoreSetup, { taskManager }: UnusedUrlsCleanupPluginSetup) { + this.taskManagerSetup = taskManager; + } + + public start(core: CoreStart, { taskManager }: UnusedUrlsCleanupPluginStart) { + const { + logger, + taskManagerSetup, + config: { maxAge }, + } = this; + + if (!taskManagerSetup) { + logger.error('taskManagerSetup is not defined'); + return; + } + + const savedObjectsRepository = core.savedObjects.createInternalRepository(); + const filter = `url.attributes.accessDate <= now-${maxAge}`; + + taskManagerSetup.registerTaskDefinitions({ + [TASK_ID]: { + title: 'Unused URLs Cleanup', + description: `Deletes unused (unaccessed for 1 year - configurable via unused_urls_cleanup.maxAge config) saved objects of type 'url' once a week.`, + createTaskRunner: () => { + return { + async run() { + runDeleteUnusedUrlsTask({ + savedObjectsRepository, + filter, + logger, + }); + }, + }; + }, + }, + }); + + taskManager.ensureScheduled(DELETE_UNUSED_URLS_TASK); + } +} diff --git a/src/platform/plugins/shared/unused_urls_cleanup/server/types.ts b/src/platform/plugins/shared/unused_urls_cleanup/server/types.ts new file mode 100644 index 0000000000000..9762112e76e3d --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/server/types.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; + +export interface UnusedUrlsCleanupPluginSetup { + taskManager: TaskManagerSetupContract; +} + +export interface UnusedUrlsCleanupPluginStart { + taskManager: TaskManagerStartContract; +} diff --git a/src/platform/plugins/shared/unused_urls_cleanup/tsconfig.json b/src/platform/plugins/shared/unused_urls_cleanup/tsconfig.json new file mode 100644 index 0000000000000..e95e9956ffd26 --- /dev/null +++ b/src/platform/plugins/shared/unused_urls_cleanup/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types" + }, + "include": [ + "server/**/*", + ], + "exclude": [ + "target/**/*", + ], + "kbn_references": [ + "@kbn/core", + "@kbn/task-manager-plugin", + "@kbn/config-schema", + ] +} diff --git a/tsconfig.base.json b/tsconfig.base.json index 4a07954a904f2..6b088be37565f 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -2062,6 +2062,8 @@ "@kbn/unsaved-changes-badge/*": ["src/platform/packages/private/kbn-unsaved-changes-badge/*"], "@kbn/unsaved-changes-prompt": ["src/platform/packages/shared/kbn-unsaved-changes-prompt"], "@kbn/unsaved-changes-prompt/*": ["src/platform/packages/shared/kbn-unsaved-changes-prompt/*"], + "@kbn/unused-urls-cleanup": ["src/platform/plugins/shared/unused_urls_cleanup"], + "@kbn/unused-urls-cleanup/*": ["src/platform/plugins/shared/unused_urls_cleanup/*"], "@kbn/upgrade-assistant": ["x-pack/platform/packages/private/upgrade-assistant"], "@kbn/upgrade-assistant/*": ["x-pack/platform/packages/private/upgrade-assistant/*"], "@kbn/upgrade-assistant-plugin": ["x-pack/platform/plugins/private/upgrade_assistant"], diff --git a/yarn.lock b/yarn.lock index 2fae0c2b6ee7b..fc0fdf6c9818b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7819,6 +7819,10 @@ version "0.0.0" uid "" +"@kbn/unused-urls-cleanup@link:src/platform/plugins/shared/unused_urls_cleanup": + version "0.0.0" + uid "" + "@kbn/upgrade-assistant-plugin@link:x-pack/platform/plugins/private/upgrade_assistant": version "0.0.0" uid ""