diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 0ab1c89c1d8f7..215a4f3a4ebb4 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -370,6 +370,10 @@ and actions. |The features plugin enhance Kibana with a per-feature privilege system. +|{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload] +|WARNING: Missing README. + + |{kib-repo}blob/{branch}/x-pack/plugins/fleet/README.md[fleet] |Fleet needs to have Elasticsearch API keys enabled, and also to have TLS enabled on kibana, (if you want to run Kibana without TLS you can provide the following config flag --xpack.fleet.agents.tlsCheckDisabled=false) diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 950fdf9405b75..14cd7141ac9e2 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -4097,6 +4097,9 @@ "xpackDashboardMode:roles": { "type": "keyword" }, + "securitySolution:ipReputationLinks": { + "type": "keyword" + }, "visualize:enableLabs": { "type": "boolean" }, @@ -4115,9 +4118,6 @@ "visualization:tileMap:maxPrecision": { "type": "long" }, - "securitySolution:ipReputationLinks": { - "type": "keyword" - }, "csv:separator": { "type": "keyword" }, diff --git a/x-pack/plugins/ml/common/constants/file_datavisualizer.ts b/x-pack/plugins/file_upload/common/constants.ts similarity index 100% rename from x-pack/plugins/ml/common/constants/file_datavisualizer.ts rename to x-pack/plugins/file_upload/common/constants.ts diff --git a/x-pack/plugins/file_upload/common/index.ts b/x-pack/plugins/file_upload/common/index.ts new file mode 100644 index 0000000000000..6c1725d61c059 --- /dev/null +++ b/x-pack/plugins/file_upload/common/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './constants'; +export * from './types'; diff --git a/x-pack/plugins/file_upload/common/types.ts b/x-pack/plugins/file_upload/common/types.ts new file mode 100644 index 0000000000000..229983f1c535a --- /dev/null +++ b/x-pack/plugins/file_upload/common/types.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface ImportResponse { + success: boolean; + id: string; + index?: string; + pipelineId?: string; + docCount: number; + failures: ImportFailure[]; + error?: any; + ingestError?: boolean; +} + +export interface ImportFailure { + item: number; + reason: string; + doc: ImportDoc; +} + +export interface Doc { + message: string; +} + +export type ImportDoc = Doc | string; + +export interface Settings { + pipeline?: string; + index: string; + body: any[]; + [key: string]: any; +} + +export interface Mappings { + _meta?: { + created_by: string; + }; + properties: { + [key: string]: any; + }; +} + +export interface IngestPipelineWrapper { + id: string; + pipeline: IngestPipeline; +} + +export interface IngestPipeline { + description: string; + processors: any[]; +} diff --git a/x-pack/plugins/file_upload/jest.config.js b/x-pack/plugins/file_upload/jest.config.js new file mode 100644 index 0000000000000..6a042a4cc5c1e --- /dev/null +++ b/x-pack/plugins/file_upload/jest.config.js @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/file_upload'], +}; diff --git a/x-pack/plugins/file_upload/kibana.json b/x-pack/plugins/file_upload/kibana.json new file mode 100644 index 0000000000000..7ca024174ec6a --- /dev/null +++ b/x-pack/plugins/file_upload/kibana.json @@ -0,0 +1,8 @@ +{ + "id": "fileUpload", + "version": "8.0.0", + "kibanaVersion": "kibana", + "server": true, + "ui": false, + "requiredPlugins": ["usageCollection"] +} diff --git a/x-pack/plugins/file_upload/server/error_wrapper.ts b/x-pack/plugins/file_upload/server/error_wrapper.ts new file mode 100644 index 0000000000000..fb41d30e34fae --- /dev/null +++ b/x-pack/plugins/file_upload/server/error_wrapper.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { boomify, isBoom } from '@hapi/boom'; +import { ResponseError, CustomHttpResponseOptions } from 'kibana/server'; + +export function wrapError(error: any): CustomHttpResponseOptions { + const boom = isBoom(error) + ? error + : boomify(error, { statusCode: error.status ?? error.statusCode }); + const statusCode = boom.output.statusCode; + return { + body: { + message: boom, + ...(statusCode !== 500 && error.body ? { attributes: { body: error.body } } : {}), + }, + headers: boom.output.headers as { [key: string]: string }, + statusCode, + }; +} diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts b/x-pack/plugins/file_upload/server/import_data.ts similarity index 95% rename from x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts rename to x-pack/plugins/file_upload/server/import_data.ts index 26dba7c2f00c1..1eb495d6570c2 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts +++ b/x-pack/plugins/file_upload/server/import_data.ts @@ -5,19 +5,20 @@ */ import { IScopedClusterClient } from 'kibana/server'; -import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { INDEX_META_DATA_CREATED_BY } from '../common/constants'; import { ImportResponse, ImportFailure, Settings, Mappings, IngestPipelineWrapper, -} from '../../../common/types/file_datavisualizer'; -import { InputData } from './file_data_visualizer'; +} from '../common'; + +export type InputData = any[]; export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { async function importData( - id: string, + id: string | undefined, index: string, settings: Settings, mappings: Mappings, @@ -77,7 +78,7 @@ export function importDataProvider({ asCurrentUser }: IScopedClusterClient) { } catch (error) { return { success: false, - id, + id: id!, index: createdIndex, pipelineId: createdPipelineId, error: error.body !== undefined ? error.body : error, diff --git a/x-pack/plugins/file_upload/server/index.ts b/x-pack/plugins/file_upload/server/index.ts new file mode 100644 index 0000000000000..44a208b7924bc --- /dev/null +++ b/x-pack/plugins/file_upload/server/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FileUploadPlugin } from './plugin'; + +export const plugin = () => new FileUploadPlugin(); diff --git a/x-pack/plugins/file_upload/server/plugin.ts b/x-pack/plugins/file_upload/server/plugin.ts new file mode 100644 index 0000000000000..eea3239e52d1c --- /dev/null +++ b/x-pack/plugins/file_upload/server/plugin.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CoreSetup, CoreStart, Plugin } from 'src/core/server'; +import { fileUploadRoutes } from './routes'; +import { initFileUploadTelemetry } from './telemetry'; +import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; + +interface SetupDeps { + usageCollection: UsageCollectionSetup; +} + +export class FileUploadPlugin implements Plugin { + async setup(coreSetup: CoreSetup, plugins: SetupDeps) { + fileUploadRoutes(coreSetup.http.createRouter()); + + initFileUploadTelemetry(coreSetup, plugins.usageCollection); + } + + start(core: CoreStart) {} +} diff --git a/x-pack/plugins/file_upload/server/routes.ts b/x-pack/plugins/file_upload/server/routes.ts new file mode 100644 index 0000000000000..c98f413caba64 --- /dev/null +++ b/x-pack/plugins/file_upload/server/routes.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter, IScopedClusterClient } from 'kibana/server'; +import { MAX_FILE_SIZE_BYTES, IngestPipelineWrapper, Mappings, Settings } from '../common'; +import { wrapError } from './error_wrapper'; +import { InputData, importDataProvider } from './import_data'; + +import { updateTelemetry } from './telemetry'; +import { importFileBodySchema, importFileQuerySchema } from './schemas'; + +function importData( + client: IScopedClusterClient, + id: string | undefined, + index: string, + settings: Settings, + mappings: Mappings, + ingestPipeline: IngestPipelineWrapper, + data: InputData +) { + const { importData: importDataFunc } = importDataProvider(client); + return importDataFunc(id, index, settings, mappings, ingestPipeline, data); +} + +/** + * Routes for the file upload. + */ +export function fileUploadRoutes(router: IRouter) { + /** + * @apiGroup FileDataVisualizer + * + * @api {post} /api/file_upload/import Import file data + * @apiName ImportFile + * @apiDescription Imports file data into elasticsearch index. + * + * @apiSchema (query) importFileQuerySchema + * @apiSchema (body) importFileBodySchema + */ + router.post( + { + path: '/api/file_upload/import', + validate: { + query: importFileQuerySchema, + body: importFileBodySchema, + }, + options: { + body: { + accepts: ['application/json'], + maxBytes: MAX_FILE_SIZE_BYTES, + }, + tags: ['access:ml:canFindFileStructure'], + }, + }, + async (context, request, response) => { + try { + const { id } = request.query; + const { index, data, settings, mappings, ingestPipeline } = request.body; + + // `id` being `undefined` tells us that this is a new import due to create a new index. + // follow-up import calls to just add additional data will include the `id` of the created + // index, we'll ignore those and don't increment the counter. + if (id === undefined) { + await updateTelemetry(); + } + + const result = await importData( + context.core.elasticsearch.client, + id, + index, + settings, + mappings, + // @ts-expect-error + ingestPipeline, + data + ); + return response.ok({ body: result }); + } catch (e) { + return response.customError(wrapError(e)); + } + } + ); +} diff --git a/x-pack/plugins/file_upload/server/schemas.ts b/x-pack/plugins/file_upload/server/schemas.ts new file mode 100644 index 0000000000000..79db26cdb8c05 --- /dev/null +++ b/x-pack/plugins/file_upload/server/schemas.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; + +export const importFileQuerySchema = schema.object({ + id: schema.maybe(schema.string()), +}); + +export const importFileBodySchema = schema.object({ + index: schema.string(), + data: schema.arrayOf(schema.any()), + settings: schema.maybe(schema.any()), + /** Mappings */ + mappings: schema.any(), + /** Ingest pipeline definition */ + ingestPipeline: schema.object({ + id: schema.maybe(schema.string()), + pipeline: schema.maybe(schema.any()), + }), +}); diff --git a/x-pack/plugins/ml/server/lib/telemetry/index.ts b/x-pack/plugins/file_upload/server/telemetry/index.ts similarity index 82% rename from x-pack/plugins/ml/server/lib/telemetry/index.ts rename to x-pack/plugins/file_upload/server/telemetry/index.ts index b5ec80daf1787..92d8ab425a773 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/index.ts +++ b/x-pack/plugins/file_upload/server/telemetry/index.ts @@ -4,5 +4,5 @@ * you may not use this file except in compliance with the Elastic License. */ -export { initMlTelemetry } from './ml_usage_collector'; +export { initFileUploadTelemetry } from './usage_collector'; export { updateTelemetry } from './telemetry'; diff --git a/x-pack/plugins/ml/server/lib/telemetry/internal_repository.ts b/x-pack/plugins/file_upload/server/telemetry/internal_repository.ts similarity index 100% rename from x-pack/plugins/ml/server/lib/telemetry/internal_repository.ts rename to x-pack/plugins/file_upload/server/telemetry/internal_repository.ts diff --git a/x-pack/plugins/ml/server/lib/telemetry/mappings.ts b/x-pack/plugins/file_upload/server/telemetry/mappings.ts similarity index 86% rename from x-pack/plugins/ml/server/lib/telemetry/mappings.ts rename to x-pack/plugins/file_upload/server/telemetry/mappings.ts index 5aaf9f8c79dc0..3d22bcb4162fd 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/mappings.ts +++ b/x-pack/plugins/file_upload/server/telemetry/mappings.ts @@ -7,13 +7,13 @@ import { SavedObjectsType } from 'src/core/server'; import { TELEMETRY_DOC_ID } from './telemetry'; -export const mlTelemetryMappingsType: SavedObjectsType = { +export const telemetryMappingsType: SavedObjectsType = { name: TELEMETRY_DOC_ID, hidden: false, namespaceType: 'agnostic', mappings: { properties: { - file_data_visualizer: { + file_upload: { properties: { index_creation_count: { type: 'long', diff --git a/x-pack/plugins/ml/server/lib/telemetry/telemetry.test.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts similarity index 97% rename from x-pack/plugins/ml/server/lib/telemetry/telemetry.test.ts rename to x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts index f41c4fda93a54..2ad36338f4928 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/telemetry.test.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts @@ -34,7 +34,7 @@ describe('ml plugin telemetry', () => { it('should update existing telemetry', async () => { const internalRepo = mockInit({ attributes: { - file_data_visualizer: { + file_upload: { index_creation_count: 2, }, }, diff --git a/x-pack/plugins/ml/server/lib/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts similarity index 89% rename from x-pack/plugins/ml/server/lib/telemetry/telemetry.ts rename to x-pack/plugins/file_upload/server/telemetry/telemetry.ts index 06577d6937101..aac45d5d0f871 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -9,10 +9,10 @@ import { ISavedObjectsRepository } from 'kibana/server'; import { getInternalRepository } from './internal_repository'; -export const TELEMETRY_DOC_ID = 'ml-telemetry'; +export const TELEMETRY_DOC_ID = 'file-upload-usage-collection-telemetry'; export interface Telemetry { - file_data_visualizer: { + file_upload: { index_creation_count: number; }; } @@ -23,7 +23,7 @@ export interface TelemetrySavedObject { export function initTelemetry(): Telemetry { return { - file_data_visualizer: { + file_upload: { index_creation_count: 0, }, }; @@ -74,8 +74,8 @@ export async function updateTelemetry(internalRepo?: ISavedObjectsRepository) { function incrementCounts(telemetry: Telemetry) { return { - file_data_visualizer: { - index_creation_count: telemetry.file_data_visualizer.index_creation_count + 1, + file_upload: { + index_creation_count: telemetry.file_upload.index_creation_count + 1, }, }; } diff --git a/x-pack/plugins/ml/server/lib/telemetry/ml_usage_collector.ts b/x-pack/plugins/file_upload/server/telemetry/usage_collector.ts similarity index 62% rename from x-pack/plugins/ml/server/lib/telemetry/ml_usage_collector.ts rename to x-pack/plugins/file_upload/server/telemetry/usage_collector.ts index 35c6936598c40..4a1334ef53d95 100644 --- a/x-pack/plugins/ml/server/lib/telemetry/ml_usage_collector.ts +++ b/x-pack/plugins/file_upload/server/telemetry/usage_collector.ts @@ -8,23 +8,26 @@ import { CoreSetup } from 'kibana/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { getTelemetry, initTelemetry, Telemetry } from './telemetry'; -import { mlTelemetryMappingsType } from './mappings'; +import { telemetryMappingsType } from './mappings'; import { setInternalRepository } from './internal_repository'; -export function initMlTelemetry(coreSetup: CoreSetup, usageCollection: UsageCollectionSetup) { - coreSetup.savedObjects.registerType(mlTelemetryMappingsType); - registerMlUsageCollector(usageCollection); +export function initFileUploadTelemetry( + coreSetup: CoreSetup, + usageCollection: UsageCollectionSetup +) { + coreSetup.savedObjects.registerType(telemetryMappingsType); + registerUsageCollector(usageCollection); coreSetup.getStartServices().then(([core]) => { setInternalRepository(core.savedObjects.createInternalRepository); }); } -function registerMlUsageCollector(usageCollection: UsageCollectionSetup): void { - const mlUsageCollector = usageCollection.makeUsageCollector({ - type: 'mlTelemetry', +function registerUsageCollector(usageCollectionSetup: UsageCollectionSetup): void { + const usageCollector = usageCollectionSetup.makeUsageCollector({ + type: 'fileUpload', isReady: () => true, schema: { - file_data_visualizer: { + file_upload: { index_creation_count: { type: 'long' }, }, }, @@ -38,5 +41,5 @@ function registerMlUsageCollector(usageCollection: UsageCollectionSetup): void { }, }); - usageCollection.registerCollector(mlUsageCollector); + usageCollectionSetup.registerCollector(usageCollector); } diff --git a/x-pack/plugins/file_upload/tsconfig.json b/x-pack/plugins/file_upload/tsconfig.json new file mode 100644 index 0000000000000..f985a4599d5fe --- /dev/null +++ b/x-pack/plugins/file_upload/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": ["common/**/*", "public/**/*", "server/**/*"], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../../../src/plugins/usage_collection/tsconfig.json" } + ] +} diff --git a/x-pack/plugins/maps_file_upload/public/util/indexing_service.js b/x-pack/plugins/maps_file_upload/public/util/indexing_service.js index 28cdb602455b5..14d02ce881cda 100644 --- a/x-pack/plugins/maps_file_upload/public/util/indexing_service.js +++ b/x-pack/plugins/maps_file_upload/public/util/indexing_service.js @@ -119,7 +119,7 @@ async function writeToIndex(indexingDetails) { const { appName, index, data, settings, mappings, ingestPipeline } = indexingDetails; return await httpService({ - url: `/api/fileupload/import`, + url: `/api/maps/fileupload/import`, method: 'POST', ...(query ? { query } : {}), data: { diff --git a/x-pack/plugins/maps_file_upload/server/routes/file_upload.js b/x-pack/plugins/maps_file_upload/server/routes/file_upload.js index 3935d4ca5fe8e..0323f23a51df5 100644 --- a/x-pack/plugins/maps_file_upload/server/routes/file_upload.js +++ b/x-pack/plugins/maps_file_upload/server/routes/file_upload.js @@ -9,7 +9,7 @@ import { updateTelemetry } from '../telemetry/telemetry'; import { MAX_BYTES } from '../../common/constants/file_import'; import { schema } from '@kbn/config-schema'; -export const IMPORT_ROUTE = '/api/fileupload/import'; +export const IMPORT_ROUTE = '/api/maps/fileupload/import'; export const querySchema = schema.maybe( schema.object({ diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts index b1967cfe83f3c..8ba30111c4c8c 100644 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ b/x-pack/plugins/ml/common/types/file_datavisualizer.ts @@ -68,52 +68,3 @@ export interface FindFileStructureResponse { timestamp_field?: string; should_trim_fields?: boolean; } - -export interface ImportResponse { - success: boolean; - id: string; - index?: string; - pipelineId?: string; - docCount: number; - failures: ImportFailure[]; - error?: any; - ingestError?: boolean; -} - -export interface ImportFailure { - item: number; - reason: string; - doc: ImportDoc; -} - -export interface Doc { - message: string; -} - -export type ImportDoc = Doc | string; - -export interface Settings { - pipeline?: string; - index: string; - body: any[]; - [key: string]: any; -} - -export interface Mappings { - _meta?: { - created_by: string; - }; - properties: { - [key: string]: any; - }; -} - -export interface IngestPipelineWrapper { - id: string; - pipeline: IngestPipeline; -} - -export interface IngestPipeline { - description: string; - processors: any[]; -} diff --git a/x-pack/plugins/ml/kibana.json b/x-pack/plugins/ml/kibana.json index 1c47512e0b3de..ede6b8abbd09c 100644 --- a/x-pack/plugins/ml/kibana.json +++ b/x-pack/plugins/ml/kibana.json @@ -10,8 +10,8 @@ "data", "cloud", "features", + "fileUpload", "licensing", - "usageCollection", "share", "embeddable", "uiActions", diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts index 1cc513e778b2f..25d5373b6dc7c 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/combined_fields/utils.ts @@ -8,11 +8,8 @@ import { i18n } from '@kbn/i18n'; import { cloneDeep } from 'lodash'; import uuid from 'uuid/v4'; import { CombinedField } from './types'; -import { - FindFileStructureResponse, - IngestPipeline, - Mappings, -} from '../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; +import { IngestPipeline, Mappings } from '../../../../../../../file_upload/common'; const COMMON_LAT_NAMES = ['latitude', 'lat']; const COMMON_LON_NAMES = ['longitude', 'long', 'lon']; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx index d869676e48827..0c853493293ca 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx @@ -11,7 +11,7 @@ import { EuiCallOut, EuiSpacer, EuiButtonEmpty, EuiHorizontalRule } from '@elast import numeral from '@elastic/numeral'; import { ErrorResponse } from '../../../../../../common/types/errors'; -import { FILE_SIZE_DISPLAY_FORMAT } from '../../../../../../common/constants/file_datavisualizer'; +import { FILE_SIZE_DISPLAY_FORMAT } from '../../../../../../../file_upload/common'; interface FileTooLargeProps { fileSize: number; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts index 718587ad15ad5..ab0e83846661f 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts @@ -15,7 +15,7 @@ import { Mappings, Settings, IngestPipeline, -} from '../../../../../../../common/types/file_datavisualizer'; +} from '../../../../../../../../file_upload/common'; const CHUNK_SIZE = 5000; const MAX_CHUNK_CHAR_COUNT = 1000000; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts index 65be24d9e7be4..a74249ea758a8 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts @@ -5,10 +5,8 @@ */ import { Importer, ImportConfig, CreateDocsResponse } from './importer'; -import { - Doc, - FindFileStructureResponse, -} from '../../../../../../../common/types/file_datavisualizer'; +import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer'; +import { Doc } from '../../../../../../../../file_upload/common'; export class MessageImporter extends Importer { private _excludeLinesRegex: RegExp | null; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts index 781f400180b10..ce15fb9a03fca 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts @@ -14,7 +14,7 @@ import { MAX_FILE_SIZE_BYTES, ABSOLUTE_MAX_FILE_SIZE_BYTES, FILE_SIZE_DISPLAY_FORMAT, -} from '../../../../../../common/constants/file_datavisualizer'; +} from '../../../../../../../file_upload/common'; import { getUiSettings } from '../../../../util/dependency_cache'; import { FILE_DATA_VISUALIZER_MAX_FILE_SIZE } from '../../../../../../common/constants/settings'; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts index 20332546d9cde..27d9b78725bef 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts @@ -7,7 +7,7 @@ import { http } from '../http_service'; import { basePath } from './index'; -import { ImportResponse } from '../../../../common/types/file_datavisualizer'; +import { ImportResponse } from '../../../../../file_upload/common'; export const fileDatavisualizer = { analyzeFile(file: string, params: Record = {}) { @@ -45,7 +45,7 @@ export const fileDatavisualizer = { }); return http({ - path: `${basePath()}/file_data_visualizer/import`, + path: `/api/file_upload/import`, method: 'POST', query, body, diff --git a/x-pack/plugins/ml/public/plugin.ts b/x-pack/plugins/ml/public/plugin.ts index 3ba79e0eb9187..ef3de1a5ce65c 100644 --- a/x-pack/plugins/ml/public/plugin.ts +++ b/x-pack/plugins/ml/public/plugin.ts @@ -21,7 +21,6 @@ import type { SharePluginStart, UrlGeneratorContract, } from 'src/plugins/share/public'; -import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import type { DataPublicPluginStart } from 'src/plugins/data/public'; import type { HomePublicPluginSetup } from 'src/plugins/home/public'; import type { IndexPatternManagementSetup } from 'src/plugins/index_pattern_management/public'; @@ -60,7 +59,6 @@ export interface MlSetupDependencies { security?: SecurityPluginSetup; licensing: LicensingPluginSetup; management?: ManagementSetup; - usageCollection: UsageCollectionSetup; licenseManagement?: LicenseManagementUIPluginSetup; home?: HomePublicPluginSetup; embeddable: EmbeddableSetup; @@ -102,7 +100,6 @@ export class MlPlugin implements Plugin { security: pluginsSetup.security, licensing: pluginsSetup.licensing, management: pluginsSetup.management, - usageCollection: pluginsSetup.usageCollection, licenseManagement: pluginsSetup.licenseManagement, home: pluginsSetup.home, embeddable: { ...pluginsSetup.embeddable, ...pluginsStart.embeddable }, diff --git a/x-pack/plugins/ml/server/lib/register_settings.ts b/x-pack/plugins/ml/server/lib/register_settings.ts index 0cdaaadf7f172..4db2f80633480 100644 --- a/x-pack/plugins/ml/server/lib/register_settings.ts +++ b/x-pack/plugins/ml/server/lib/register_settings.ts @@ -14,7 +14,7 @@ import { DEFAULT_AD_RESULTS_TIME_FILTER, DEFAULT_ENABLE_AD_RESULTS_TIME_FILTER, } from '../../common/constants/settings'; -import { MAX_FILE_SIZE } from '../../common/constants/file_datavisualizer'; +import { MAX_FILE_SIZE } from '../../../file_upload/common'; export function registerKibanaSettings(coreSetup: CoreSetup) { coreSetup.uiSettings.register({ diff --git a/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts b/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts index ea41fb3ae427b..3f9587749d33d 100644 --- a/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts +++ b/x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts @@ -18,7 +18,7 @@ import { DataFrameAnalyticsStats, MapElements, } from '../../../common/types/data_frame_analytics'; -import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { INDEX_META_DATA_CREATED_BY } from '../../../../file_upload/common'; import { getAnalysisType } from '../../../common/util/analytics_utils'; import { ExtendAnalyticsMapArgs, diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts index f8a27fdcd7e1a..aa699694e52a3 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts @@ -5,5 +5,3 @@ */ export { fileDataVisualizerProvider, InputData } from './file_data_visualizer'; - -export { importDataProvider } from './import_data'; diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index e48983c1c5365..3c82f2131e25f 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -23,7 +23,6 @@ import { SpacesPluginSetup } from '../../spaces/server'; import { PLUGIN_ID } from '../common/constants/app'; import { MlCapabilities } from '../common/types/capabilities'; -import { initMlTelemetry } from './lib/telemetry'; import { initMlServerLog } from './lib/log'; import { initSampleDataSets } from './lib/sample_data_sets'; @@ -190,7 +189,6 @@ export class MlServerPlugin trainedModelsRoutes(routeInit); initMlServerLog({ log: this.log }); - initMlTelemetry(coreSetup, plugins.usageCollection); return { ...createSharedServices( diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index 5dc9a3107af86..1b2eb612fda1c 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -43,7 +43,6 @@ "FileDataVisualizer", "AnalyzeFile", - "ImportFile", "ResultsService", "GetAnomaliesTableData", diff --git a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts index c4c449a9e2cb4..9ee19efef13f5 100644 --- a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts @@ -5,28 +5,13 @@ */ import { schema } from '@kbn/config-schema'; -import { IScopedClusterClient } from 'kibana/server'; -import { MAX_FILE_SIZE_BYTES } from '../../common/constants/file_datavisualizer'; -import { - InputOverrides, - Settings, - IngestPipelineWrapper, - Mappings, -} from '../../common/types/file_datavisualizer'; +import { MAX_FILE_SIZE_BYTES } from '../../../file_upload/common'; +import { InputOverrides } from '../../common/types/file_datavisualizer'; import { wrapError } from '../client/error_wrapper'; -import { - InputData, - fileDataVisualizerProvider, - importDataProvider, -} from '../models/file_data_visualizer'; +import { InputData, fileDataVisualizerProvider } from '../models/file_data_visualizer'; import { RouteInitialization } from '../types'; -import { updateTelemetry } from '../lib/telemetry'; -import { - analyzeFileQuerySchema, - importFileBodySchema, - importFileQuerySchema, -} from './schemas/file_data_visualizer_schema'; +import { analyzeFileQuerySchema } from './schemas/file_data_visualizer_schema'; import type { MlClient } from '../lib/ml_client'; function analyzeFiles(mlClient: MlClient, data: InputData, overrides: InputOverrides) { @@ -34,19 +19,6 @@ function analyzeFiles(mlClient: MlClient, data: InputData, overrides: InputOverr return analyzeFile(data, overrides); } -function importData( - client: IScopedClusterClient, - id: string, - index: string, - settings: Settings, - mappings: Mappings, - ingestPipeline: IngestPipelineWrapper, - data: InputData -) { - const { importData: importDataFunc } = importDataProvider(client); - return importDataFunc(id, index, settings, mappings, ingestPipeline, data); -} - /** * Routes for the file data visualizer. */ @@ -84,57 +56,4 @@ export function fileDataVisualizerRoutes({ router, routeGuard }: RouteInitializa } }) ); - - /** - * @apiGroup FileDataVisualizer - * - * @api {post} /api/ml/file_data_visualizer/import Import file data - * @apiName ImportFile - * @apiDescription Imports file data into elasticsearch index. - * - * @apiSchema (query) importFileQuerySchema - * @apiSchema (body) importFileBodySchema - */ - router.post( - { - path: '/api/ml/file_data_visualizer/import', - validate: { - query: importFileQuerySchema, - body: importFileBodySchema, - }, - options: { - body: { - accepts: ['application/json'], - maxBytes: MAX_FILE_SIZE_BYTES, - }, - tags: ['access:ml:canFindFileStructure'], - }, - }, - routeGuard.basicLicenseAPIGuard(async ({ client, request, response }) => { - try { - const { id } = request.query; - const { index, data, settings, mappings, ingestPipeline } = request.body; - - // `id` being `undefined` tells us that this is a new import due to create a new index. - // follow-up import calls to just add additional data will include the `id` of the created - // index, we'll ignore those and don't increment the counter. - if (id === undefined) { - await updateTelemetry(); - } - - const result = await importData( - client, - id, - index, - settings, - mappings, - ingestPipeline, - data - ); - return response.ok({ body: result }); - } catch (e) { - return response.customError(wrapError(e)); - } - }) - ); } diff --git a/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts b/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts index 9a80cf795cabf..685f06f839ee3 100644 --- a/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/file_data_visualizer_schema.ts @@ -24,20 +24,3 @@ export const analyzeFileQuerySchema = schema.maybe( timestamp_format: schema.maybe(schema.string()), }) ); - -export const importFileQuerySchema = schema.object({ - id: schema.maybe(schema.string()), -}); - -export const importFileBodySchema = schema.object({ - index: schema.maybe(schema.string()), - data: schema.arrayOf(schema.any()), - settings: schema.maybe(schema.any()), - /** Mappings */ - mappings: schema.any(), - /** Ingest pipeline definition */ - ingestPipeline: schema.object({ - id: schema.maybe(schema.string()), - pipeline: schema.maybe(schema.any()), - }), -}); diff --git a/x-pack/plugins/ml/server/types.ts b/x-pack/plugins/ml/server/types.ts index 780a4284312e7..f3e3ee22f3817 100644 --- a/x-pack/plugins/ml/server/types.ts +++ b/x-pack/plugins/ml/server/types.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import type { HomeServerPluginSetup } from 'src/plugins/home/server'; import type { IRouter } from 'kibana/server'; import type { CloudSetup } from '../../cloud/server'; @@ -43,7 +42,6 @@ export interface PluginsSetup { licensing: LicensingPluginSetup; security?: SecurityPluginSetup; spaces?: SpacesPluginSetup; - usageCollection: UsageCollectionSetup; } export interface PluginsStart { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 4ca373d9260b7..c1674f8a92669 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -1771,10 +1771,14 @@ } } }, - "fileUploadTelemetry": { + "fileUpload": { "properties": { - "filesUploadedTotalCount": { - "type": "long" + "file_upload": { + "properties": { + "index_creation_count": { + "type": "long" + } + } } } }, @@ -2215,6 +2219,13 @@ } } }, + "fileUploadTelemetry": { + "properties": { + "filesUploadedTotalCount": { + "type": "long" + } + } + }, "maps": { "properties": { "settings": { @@ -2308,17 +2319,6 @@ } } }, - "mlTelemetry": { - "properties": { - "file_data_visualizer": { - "properties": { - "index_creation_count": { - "type": "long" - } - } - } - } - }, "monitoring": { "properties": { "hasMonitoringData": { diff --git a/x-pack/tsconfig.json b/x-pack/tsconfig.json index 7f64a552a5169..64e0b89f88a72 100644 --- a/x-pack/tsconfig.json +++ b/x-pack/tsconfig.json @@ -16,6 +16,7 @@ "plugins/global_search_providers/**/*", "plugins/graph/**/*", "plugins/features/**/*", + "plugins/file_upload/**/*", "plugins/embeddable_enhanced/**/*", "plugins/event_log/**/*", "plugins/enterprise_search/**/*", @@ -100,6 +101,7 @@ { "path": "./plugins/enterprise_search/tsconfig.json" }, { "path": "./plugins/event_log/tsconfig.json" }, { "path": "./plugins/features/tsconfig.json" }, + { "path": "./plugins/file_upload/tsconfig.json" }, { "path": "./plugins/global_search_bar/tsconfig.json" }, { "path": "./plugins/global_search_providers/tsconfig.json" }, { "path": "./plugins/global_search/tsconfig.json" }, diff --git a/x-pack/tsconfig.refs.json b/x-pack/tsconfig.refs.json index 43a488e8727cc..7372c3393689e 100644 --- a/x-pack/tsconfig.refs.json +++ b/x-pack/tsconfig.refs.json @@ -15,6 +15,7 @@ { "path": "./plugins/enterprise_search/tsconfig.json" }, { "path": "./plugins/event_log/tsconfig.json"}, { "path": "./plugins/features/tsconfig.json" }, + { "path": "./plugins/file_upload/tsconfig.json" }, { "path": "./plugins/global_search_bar/tsconfig.json" }, { "path": "./plugins/global_search_providers/tsconfig.json" }, { "path": "./plugins/global_search/tsconfig.json" },