Add unused-urls-cleanup plugin

.github/CODEOWNERS

@@ -706,6 +706,7 @@ src/platform/plugins/shared/ui_actions_enhanced @elastic/appex-sharedux
 src/platform/plugins/shared/unified_doc_viewer @elastic/kibana-data-discovery
 src/platform/plugins/shared/unified_search @elastic/kibana-presentation
 src/platform/plugins/shared/usage_collection @elastic/kibana-core
+src/platform/plugins/shared/unused_urls_cleanup @elastic/appex-sharedux
 src/platform/plugins/shared/vis_types/timeseries @elastic/kibana-visualizations
 src/platform/plugins/shared/visualizations @elastic/kibana-visualizations
 src/platform/test

package.json

@@ -1004,6 +1004,7 @@
     "@kbn/unified-tabs-examples-plugin": "link:examples/unified_tabs_examples",
     "@kbn/unsaved-changes-badge": "link:src/platform/packages/private/kbn-unsaved-changes-badge",
     "@kbn/unsaved-changes-prompt": "link:src/platform/packages/shared/kbn-unsaved-changes-prompt",
+    "@kbn/unused-urls-cleanup": "link:src/platform/plugins/shared/unused_urls_cleanup",
     "@kbn/upgrade-assistant": "link:x-pack/platform/packages/private/upgrade-assistant",
     "@kbn/upgrade-assistant-plugin": "link:x-pack/platform/plugins/private/upgrade_assistant",
     "@kbn/uptime-plugin": "link:x-pack/solutions/observability/plugins/uptime",

src/platform/plugins/shared/unused_urls_cleanup/README.md

@@ -0,0 +1 @@
+TODO

src/platform/plugins/shared/unused_urls_cleanup/kibana.jsonc

@@ -0,0 +1,21 @@
+{
+  "type": "plugin",
+  "id": "@kbn/unused-urls-cleanup",
+  "owner": [
+    "@elastic/appex-sharedux"
+  ],
+  "group": "platform",
+  "visibility": "private",
+  "description": "Background task responsible for deleting saved objects of type 'url' which are unused.",
+  "plugin": {
+    "id": "unusedUrlsCleanup",
+    "browser": false,
+    "server": true,
+    "requiredPlugins": [
+      "taskManager"
+    ],
+    "configPath": "unused_urls_cleanup",
+  }
+}
+
+

src/platform/plugins/shared/unused_urls_cleanup/server/config.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+import type { PluginConfigDescriptor } from '@kbn/core/server';
+import { DEFAULT_MAX_AGE } from '@kbn/unused-urls-cleanup/server/constants';
+
+export const configSchema = schema.object({
+  maxAge: schema.string({
+    // TODO: Possibly disable this for new installations
+    defaultValue: DEFAULT_MAX_AGE,
+    validate: (value) => {
+      const rangeRegex = /\d+[yMwdhms]/;
+      if (!rangeRegex.test(value)) {
+        return `Invalid value: ${value}. Expected format: <number><unit>, where unit is one of y, M, w, d, h, m, s.`;
+      }
+    },
+  }),
+});
+
+export type UnusedUrlsCleanupPluginConfig = TypeOf<typeof configSchema>;
+
+export const config: PluginConfigDescriptor<UnusedUrlsCleanupPluginConfig> = {
+  schema: configSchema,
+};

src/platform/plugins/shared/unused_urls_cleanup/server/constants.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { TaskInstanceWithId } from '@kbn/task-manager-plugin/server/task';
+
+export const TASK_ID = 'unusedUrlsCleanupTask';
+export const TASK_SCHEDULE_INTERVAL = '30s'; // TODO: Change this to 1 week
+export const SAVED_OBJECT_TYPE = 'url';
+export const PIT_KEEP_ALIVE = '10m';
+export const MAX_PAGE_SIZE = 10000;
+export const DEFAULT_MAX_AGE = '1y';
+export const DELETE_UNUSED_URLS_TASK: TaskInstanceWithId = {
+  id: TASK_ID,
+  taskType: TASK_ID,
+  params: {},
+  state: {},
+  schedule: {
+    interval: TASK_SCHEDULE_INTERVAL,
+  },
+};

src/platform/plugins/shared/unused_urls_cleanup/server/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { PluginInitializerContext } from '@kbn/core/server';
+
+export type { UnusedUrlsCleanupPluginSetup, UnusedUrlsCleanupPluginStart } from './types';
+export type { UnusedUrlsCleanupPluginConfig } from './config';
+export { config, configSchema } from './config';
+export {
+  TASK_ID,
+  TASK_SCHEDULE_INTERVAL,
+  SAVED_OBJECT_TYPE,
+  PIT_KEEP_ALIVE,
+  MAX_PAGE_SIZE,
+  DEFAULT_MAX_AGE,
+  DELETE_UNUSED_URLS_TASK,
+} from './constants';
+
+export async function plugin(initializerContext: PluginInitializerContext) {
+  const { UnusedUrlsCleanupPlugin } = await import('./plugin');
+  return new UnusedUrlsCleanupPlugin(initializerContext);
+}

src/platform/plugins/shared/unused_urls_cleanup/server/lib/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+export * from './saved_objects';

src/platform/plugins/shared/unused_urls_cleanup/server/lib/saved_objects.ts

@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { SortResults } from '@elastic/elasticsearch/lib/api/types';
+import { ISavedObjectsRepository, SavedObjectsFindResult } from '@kbn/core/server';
+import { Logger } from '@kbn/logging';
+import {
+  MAX_PAGE_SIZE,
+  PIT_KEEP_ALIVE,
+  SAVED_OBJECT_TYPE,
+} from '@kbn/unused-urls-cleanup/server/constants';
+
+export const deleteUnusedUrls = async ({
+  savedObjectsRepository,
+  unusedUrls,
+  logger,
+}: {
+  savedObjectsRepository: ISavedObjectsRepository;
+  unusedUrls: Array<{ id: string; type: string }>;
+  logger: Logger;
+}) => {
+  const total = unusedUrls.length;
+  logger.info(`Deleting ${total} unused URL(s)`);
+
+  try {
+    await savedObjectsRepository.bulkDelete(unusedUrls, {
+      refresh: 'wait_for',
+    });
+
+    logger.info(`Succesfully deleted ${total} unused URL(s)`);
+  } catch (e) {
+    logger.error(`Failed to delete unused URL(s): ${e.message}`);
+  }
+};
+
+export const fetchAllUnusedUrls = async ({
+  savedObjectsRepository,
+  filter,
+  logger,
+}: {
+  savedObjectsRepository: ISavedObjectsRepository;
+  filter: string;
+  logger: Logger;
+}) => {
+  const results: SavedObjectsFindResult[] = [];
+
+  const { id: pitId } = await savedObjectsRepository.openPointInTimeForType(SAVED_OBJECT_TYPE, {
+    keepAlive: PIT_KEEP_ALIVE,
+  });
+
+  try {
+    let searchAfter: SortResults | undefined;
+    let hasMore = true;
+
+    while (hasMore) {
+      const response = await savedObjectsRepository.find({
+        type: SAVED_OBJECT_TYPE,
+        filter,
+        pit: { id: pitId, keepAlive: PIT_KEEP_ALIVE },
+        searchAfter,
+        perPage: MAX_PAGE_SIZE,
+      });
+
+      results.push(...response.saved_objects);
+      hasMore = response.saved_objects.length === MAX_PAGE_SIZE;
+
+      if (hasMore) {
+        searchAfter = response.saved_objects[response.saved_objects.length - 1].sort;
+      }
+    }
+  } catch (e) {
+    logger.error(`Failed to fetch unused URLs: ${e.message}`);
+  } finally {
+    await savedObjectsRepository.closePointInTime(pitId);
+  }
+
+  return results.map(({ id }) => ({
+    id,
+    type: SAVED_OBJECT_TYPE,
+  }));
+};
+
+export const runDeleteUnusedUrlsTask = async ({
+  savedObjectsRepository,
+  filter,
+  logger,
+}: {
+  savedObjectsRepository: ISavedObjectsRepository;
+  filter: string;
+  logger: Logger;
+}) => {
+  try {
+    logger.info('Unused URLs cleanup started');
+
+    const unusedUrls = await fetchAllUnusedUrls({
+      savedObjectsRepository,
+      filter,
+      logger,
+    });
+
+    logger.info(`Found ${unusedUrls.length} unused URL(s)`);
+
+    if (unusedUrls.length > 0) {
+      await deleteUnusedUrls({
+        savedObjectsRepository,
+        unusedUrls,
+        logger,
+      });
+    }
+  } catch (e) {
+    logger.error(`Failed to run: ${e.message}`);
+  }
+};

src/platform/plugins/shared/unused_urls_cleanup/server/plugin.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type {
+  PluginInitializerContext,
+  CoreSetup,
+  CoreStart,
+  Plugin,
+  Logger,
+} from '@kbn/core/server';
+import { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server';
+import { TASK_ID, DELETE_UNUSED_URLS_TASK } from '@kbn/unused-urls-cleanup/server/constants';
+import { runDeleteUnusedUrlsTask } from '@kbn/unused-urls-cleanup/server/lib';
+import type { UnusedUrlsCleanupPluginSetup, UnusedUrlsCleanupPluginStart } from './types';
+import type { UnusedUrlsCleanupPluginConfig } from './config';
+
+export class UnusedUrlsCleanupPlugin implements Plugin {
+  private readonly logger: Logger;
+  private readonly config: UnusedUrlsCleanupPluginConfig;
+  private taskManagerSetup: TaskManagerSetupContract | undefined;
+
+  constructor(initializerContext: PluginInitializerContext) {
+    this.logger = initializerContext.logger.get();
+    this.config = initializerContext.config.get<UnusedUrlsCleanupPluginConfig>();
+  }
+
+  public setup(_core: CoreSetup, { taskManager }: UnusedUrlsCleanupPluginSetup) {
+    this.taskManagerSetup = taskManager;
+  }
+
+  public start(core: CoreStart, { taskManager }: UnusedUrlsCleanupPluginStart) {
+    const {
+      logger,
+      taskManagerSetup,
+      config: { maxAge },
+    } = this;
+
+    if (!taskManagerSetup) {
+      logger.error('taskManagerSetup is not defined');
+      return;
+    }
+
+    const savedObjectsRepository = core.savedObjects.createInternalRepository();
+    const filter = `url.attributes.accessDate <= now-${maxAge}`;
+
+    taskManagerSetup.registerTaskDefinitions({
+      [TASK_ID]: {
+        title: 'Unused URLs Cleanup',
+        description: `Deletes unused (unaccessed for 1 year - configurable via unused_urls_cleanup.maxAge config) saved objects of type 'url' once a week.`,
+        createTaskRunner: () => {
+          return {
+            async run() {
+              runDeleteUnusedUrlsTask({
+                savedObjectsRepository,
+                filter,
+                logger,
+              });
+            },
+          };
+        },
+      },
+    });
+
+    taskManager.ensureScheduled(DELETE_UNUSED_URLS_TASK);
+  }
+}

src/platform/plugins/shared/unused_urls_cleanup/server/types.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import {
+  TaskManagerSetupContract,
+  TaskManagerStartContract,
+} from '@kbn/task-manager-plugin/server';
+
+export interface UnusedUrlsCleanupPluginSetup {
+  taskManager: TaskManagerSetupContract;
+}
+
+export interface UnusedUrlsCleanupPluginStart {
+  taskManager: TaskManagerStartContract;
+}

src/platform/plugins/shared/unused_urls_cleanup/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types"
+  },
+  "include": [
+    "server/**/*",
+  ],
+  "exclude": [
+    "target/**/*",
+  ],
+  "kbn_references": [
+    "@kbn/core",
+    "@kbn/task-manager-plugin",
+    "@kbn/config-schema",
+  ]
+}

tsconfig.base.json

@@ -2062,6 +2062,8 @@
       "@kbn/unsaved-changes-badge/*": ["src/platform/packages/private/kbn-unsaved-changes-badge/*"],
       "@kbn/unsaved-changes-prompt": ["src/platform/packages/shared/kbn-unsaved-changes-prompt"],
       "@kbn/unsaved-changes-prompt/*": ["src/platform/packages/shared/kbn-unsaved-changes-prompt/*"],
+      "@kbn/unused-urls-cleanup": ["src/platform/plugins/shared/unused_urls_cleanup"],
+      "@kbn/unused-urls-cleanup/*": ["src/platform/plugins/shared/unused_urls_cleanup/*"],
       "@kbn/upgrade-assistant": ["x-pack/platform/packages/private/upgrade-assistant"],
       "@kbn/upgrade-assistant/*": ["x-pack/platform/packages/private/upgrade-assistant/*"],
       "@kbn/upgrade-assistant-plugin": ["x-pack/platform/plugins/private/upgrade_assistant"],

yarn.lock

@@ -7819,6 +7819,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/unused-urls-cleanup@link:src/platform/plugins/shared/unused_urls_cleanup":
+  version "0.0.0"
+  uid ""
+
 "@kbn/upgrade-assistant-plugin@link:x-pack/platform/plugins/private/upgrade_assistant":
   version "0.0.0"
   uid ""