diff --git a/src/platform/packages/private/kbn-index-editor/moon.yml b/src/platform/packages/private/kbn-index-editor/moon.yml index ca88227317438..584df289a9af2 100644 --- a/src/platform/packages/private/kbn-index-editor/moon.yml +++ b/src/platform/packages/private/kbn-index-editor/moon.yml @@ -41,7 +41,6 @@ dependsOn: - '@kbn/restorable-state' - '@kbn/esql-ast' - '@kbn/es-types' - - '@kbn/file-upload-common' - '@kbn/esql-utils' - '@kbn/test-jest-helpers' - '@kbn/esql-types' diff --git a/src/platform/packages/private/kbn-index-editor/src/components/create_flyout.test.tsx b/src/platform/packages/private/kbn-index-editor/src/components/create_flyout.test.tsx index 1d290b4ac812c..df38956c224fc 100644 --- a/src/platform/packages/private/kbn-index-editor/src/components/create_flyout.test.tsx +++ b/src/platform/packages/private/kbn-index-editor/src/components/create_flyout.test.tsx @@ -57,7 +57,7 @@ describe('createFlyout', () => { share: {} as any, fileUpload: {} as any, storage: {} as any, - fileManager: {} as any, + existingIndexName: undefined, }; beforeEach(() => { diff --git a/src/platform/packages/private/kbn-index-editor/src/components/file_clashes.tsx b/src/platform/packages/private/kbn-index-editor/src/components/file_clashes.tsx deleted file mode 100644 index f2691a2580759..0000000000000 --- a/src/platform/packages/private/kbn-index-editor/src/components/file_clashes.tsx +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ -import type { FC } from 'react'; -import React from 'react'; -import { - EuiIcon, - EuiSpacer, - EuiText, - EuiCallOut, - EuiBadge, - useEuiTheme, - EuiToolTip, -} from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n-react'; -import { CLASH_ERROR_TYPE, CLASH_TYPE } from '@kbn/file-upload'; -import type { FileClash } from '@kbn/file-upload/file_upload_manager'; - -interface Props { - fileClash: FileClash; -} - -export const FileClashResult: FC = ({ fileClash }) => { - if (fileClash.clash === CLASH_ERROR_TYPE.ERROR) { - return ; - } - - if (fileClash.clash === CLASH_ERROR_TYPE.WARNING) { - return ( - - {fileClash.newFields?.length || fileClash.missingFields?.length ? ( - - ) : null} - - {fileClash.newFields?.length ? ( - <> - - - - - ) : null} - - {fileClash.missingFields?.length ? ( - <> - - - - - ) : null} - - ); - } -}; - -function getClashText(fileClash: FileClash) { - if (fileClash.clash === CLASH_ERROR_TYPE.ERROR) { - if (fileClash.clashType === CLASH_TYPE.FORMAT) { - return ( - - ); - } - - if (fileClash.clashType === CLASH_TYPE.MAPPING) { - return ( - - ); - } - - if (fileClash.clashType === CLASH_TYPE.EXISTING_INDEX_MAPPING) { - return ( - - ); - } - - if (fileClash.clashType === CLASH_TYPE.UNSUPPORTED) { - return ( - - ); - } - } - - if (fileClash.clash === CLASH_ERROR_TYPE.WARNING) { - return ( - - ); - } -} - -export const FileClashIcon: FC = ({ fileClash }) => { - const { euiTheme } = useEuiTheme(); - switch (fileClash.clash) { - case CLASH_ERROR_TYPE.ERROR: - return ( - - - - - - ); - case CLASH_ERROR_TYPE.WARNING: - return ( - - - - - - ); - default: - return null; - } -}; diff --git a/src/platform/packages/private/kbn-index-editor/src/components/file_drop_zone.test.tsx b/src/platform/packages/private/kbn-index-editor/src/components/file_drop_zone.test.tsx index d5803fac97746..e397f7ee84e4d 100644 --- a/src/platform/packages/private/kbn-index-editor/src/components/file_drop_zone.test.tsx +++ b/src/platform/packages/private/kbn-index-editor/src/components/file_drop_zone.test.tsx @@ -29,9 +29,6 @@ import { IndexEditorErrors } from '../types'; jest.mock('./empty_prompt', () => ({ EmptyPrompt: () =>
EmptyPrompt
, })); -jest.mock('./file_preview', () => ({ - FilesPreview: () =>
FilesPreview
, -})); // Mock hooks and modules jest.mock('@kbn/file-upload'); @@ -113,26 +110,6 @@ describe('FileDropzone', () => { expect(getByText('EmptyPrompt')).toBeInTheDocument(); }); - it('shows analyzing indicator', () => { - fileUploadContext.uploadStatus.analysisStatus = STATUS.STARTED; - const { getByText } = renderWithI18n(); - expect(getByText('Analyzing...')).toBeInTheDocument(); - }); - - it('shows uploading indicator', () => { - fileUploadContext.uploadStatus.overallImportStatus = STATUS.STARTED; - const { getByText } = renderWithI18n(); - expect(getByText('Uploading...')).toBeInTheDocument(); - }); - - it('shows FilesPreview when a file is successfully analyzed', () => { - fileUploadContext.filesStatus = [ - { analysisStatus: STATUS.COMPLETED, importStatus: STATUS.NOT_STARTED }, - ]; - const { getByText } = renderWithI18n(); - expect(getByText('FilesPreview')).toBeInTheDocument(); - }); - it('calls setError when a file is too large', () => { fileUploadContext.filesStatus = [ { diff --git a/src/platform/packages/private/kbn-index-editor/src/components/file_drop_zone.tsx b/src/platform/packages/private/kbn-index-editor/src/components/file_drop_zone.tsx index 7c5aab81c71fc..5c77e286f3aa8 100644 --- a/src/platform/packages/private/kbn-index-editor/src/components/file_drop_zone.tsx +++ b/src/platform/packages/private/kbn-index-editor/src/components/file_drop_zone.tsx @@ -7,19 +7,11 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { - EuiFlexGroup, - EuiFlexItem, - EuiLoadingSpinner, - EuiProgress, - transparentize, - useEuiTheme, -} from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, transparentize, useEuiTheme } from '@elastic/eui'; import { css } from '@emotion/react'; -import { STATUS, useFileUploadContext } from '@kbn/file-upload'; -import { FormattedMessage } from '@kbn/i18n-react'; +import { STATUS, useFileUploadContext, FileUploadLiteLookUpView } from '@kbn/file-upload'; import type { PropsWithChildren } from 'react'; -import React, { type FC, useCallback, useEffect } from 'react'; +import React, { type FC, useCallback, useEffect, useState } from 'react'; import type { FileRejection } from 'react-dropzone'; import { useDropzone } from 'react-dropzone'; import { useKibana } from '@kbn/kibana-react-plugin/public'; @@ -27,7 +19,6 @@ import useObservable from 'react-use/lib/useObservable'; import { i18n } from '@kbn/i18n'; import { getOverrideConfirmation } from './modals/override_warning_modal'; import { EmptyPrompt } from './empty_prompt'; -import { FilesPreview } from './file_preview'; import type { KibanaContextExtra } from '../types'; import { IndexEditorErrors } from '../types'; @@ -55,17 +46,10 @@ export const FileDropzone: FC> = ({ }) => { const { services } = useKibana(); const { indexUpdateService } = services; - const { fileUploadManager, filesStatus, uploadStatus, indexName } = useFileUploadContext(); + const { fileUploadManager, filesStatus, uploadStatus, indexName, reset } = useFileUploadContext(); const isSaving = useObservable(indexUpdateService.isSaving$, false); - - const isAnalyzing = - uploadStatus.analysisStatus === STATUS.STARTED && - uploadStatus.overallImportStatus === STATUS.NOT_STARTED; - - const isUploading = - uploadStatus.overallImportStatus === STATUS.STARTED || - (uploadStatus.overallImportStatus === STATUS.COMPLETED && isSaving); - const overallImportProgress = uploadStatus.overallImportProgress; + const [dropzoneDisabled, setDropzoneDisabled] = useState(false); + const [fileUploadActive, setFileUploadActive] = useState(false); useEffect( function checkForErrors() { @@ -125,7 +109,7 @@ export const FileDropzone: FC> = ({ const onFilesSelected = useCallback( async (files: File[]) => { - if (!files?.length) { + if (!files?.length || dropzoneDisabled) { return; } @@ -138,7 +122,7 @@ export const FileDropzone: FC> = ({ await fileUploadManager.addFiles(files); }, - [services, indexUpdateService, fileUploadManager] + [dropzoneDisabled, services, indexUpdateService, fileUploadManager] ); const onDropRejected = useCallback( @@ -161,6 +145,7 @@ export const FileDropzone: FC> = ({ multiple: true, noClick: true, // we'll trigger open manually noKeyboard: true, + disabled: dropzoneDisabled, }); const onFileSelectorClick = useCallback(() => { @@ -192,46 +177,6 @@ export const FileDropzone: FC> = ({ }, ]); - const loadingIndicator = ( -
- {overallImportProgress ? ( - - ) : null} -
- -
- {isAnalyzing ? ( - - ) : null} - {isUploading ? ( - - ) : null} -
-
-
- ); - const successfulPreviews = filesStatus.filter( (f) => f.analysisStatus === STATUS.COMPLETED && f.importStatus !== STATUS.COMPLETED ); @@ -239,7 +184,7 @@ export const FileDropzone: FC> = ({ let content: React.ReactNode = children; - if (noResults && !showFilePreview && !isSaving) { + if (noResults && !showFilePreview && !isSaving && !fileUploadActive) { content = ( {content} @@ -248,19 +193,26 @@ export const FileDropzone: FC> = ({ ); - } else if (showFilePreview) { - content = ; + } else if (showFilePreview || fileUploadActive) { + content = ( + { + await indexUpdateService.onFileUploadFinished(indexName); + setDropzoneDisabled(false); + reset?.(indexName); + }} + /> + ); } - const showLoadingOverlay = isUploading || isAnalyzing; - if (indexName) { return ( {indexUpdateService.canEditIndex ? (
{isDragActive ?
: null} - {showLoadingOverlay ? loadingIndicator : null} {content}
diff --git a/src/platform/packages/private/kbn-index-editor/src/components/file_preview.tsx b/src/platform/packages/private/kbn-index-editor/src/components/file_preview.tsx deleted file mode 100644 index f23d06ba7083d..0000000000000 --- a/src/platform/packages/private/kbn-index-editor/src/components/file_preview.tsx +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -import type { EuiBasicTableColumn } from '@elastic/eui'; -import { - EuiAccordion, - EuiBasicTable, - EuiButton, - EuiCodeBlock, - EuiSpacer, - EuiTabbedContent, - type EuiTabbedContentTab, - EuiTitle, -} from '@elastic/eui'; -import { CLASH_ERROR_TYPE, STATUS, useFileUploadContext } from '@kbn/file-upload'; -import type { FC } from 'react'; -import React, { Fragment, useEffect, useMemo, useState } from 'react'; -import type { DataTableRecord } from '@kbn/discover-utils'; -import type { FindFileStructureResponse } from '@kbn/file-upload-common'; -import { FormattedMessage } from '@kbn/i18n-react'; -import type { FileClash } from '@kbn/file-upload/file_upload_manager'; -import { FileClashResult } from './file_clashes'; - -interface FilePreviewItem { - fileName: string; - fileContents?: string; - // Sample documents created by the inference pipeline simulate - sampleDocs?: DataTableRecord[]; - columnNames: Exclude; - fileClash?: FileClash; -} - -const FILE_PREVIEW_ROWS_LIMIT = 10; - -export const FilesPreview: FC = () => { - const { filesStatus, uploadStatus, deleteFile } = useFileUploadContext(); - - const [filePreviewItems, setFilePreviewItems] = useState([]); - - useEffect( - function updateFilePreviewItems() { - // don't fetch preview if importing is in progress - if (uploadStatus.overallImportStatus === STATUS.STARTED) { - return; - } - - // wait for all files to be analyzed before fetching previews - if ( - filesStatus.length > 0 && - filesStatus.every((f) => f.analysisStatus === STATUS.COMPLETED) - ) { - setFilePreviewItems( - filesStatus.map((status, index) => { - const columnNames = status.results?.column_names || []; - - let fileClash: FileClash | undefined = uploadStatus.fileClashes[index]; - if (fileClash.clash === CLASH_ERROR_TYPE.NONE) { - fileClash = undefined; - } - - const item: FilePreviewItem = { - fileName: status.fileName, - columnNames, - fileContents: status.fileContents - .split('\n') - .slice(0, FILE_PREVIEW_ROWS_LIMIT) - .join('\n'), - sampleDocs: status.sampleDocs, - ...(fileClash ? { fileClash } : {}), - }; - - return item; - }) - ); - } - }, - [filesStatus, uploadStatus.fileClashes, uploadStatus.overallImportStatus] - ); - - if (!filePreviewItems.length) return null; - - return filesStatus.length > 0 ? ( -
- {filePreviewItems.map((filePreviewItem, i) => { - const tabs: EuiTabbedContentTab[] = []; - - if (filePreviewItem.sampleDocs?.length) { - // Hide the File Preview tab if ingest pipeline didn't produce any results - tabs.push({ - id: 'previewDoc', - name: ( - - ), - content: ( - <> - - - - ), - }); - } - - if (filePreviewItem.fileContents) { - tabs.push({ - id: 'fileContent', - name: ( - - ), - content: ( - <> - - - {filePreviewItem.fileContents} - - - ), - }); - } - - if (filePreviewItem.fileClash) { - tabs.push({ - id: 'fileClash', - name: ( - - ), - content: ( - <> - - - - ), - }); - } - - return ( - - -

{filePreviewItem.fileName}

- - } - initialIsOpen={i === 0} - extraAction={ - { - await deleteFile(i); - }} - > - - - } - paddingSize="s" - > - - - - - ); - })} - -
- ) : null; -}; - -export type ResultsPreviewProps = Omit; - -const ResultsPreview: FC = ({ sampleDocs, columnNames }) => { - const columns = useMemo>>(() => { - return columnNames.map((name: string) => { - return { - field: name, - name, - dataType: 'auto', - truncateText: { lines: 2 }, - }; - }); - }, [columnNames]); - - const items = useMemo(() => { - return ( - sampleDocs?.map((doc) => { - return doc.flattened; - }) || [] - ); - }, [sampleDocs]); - - return ( - <> - {sampleDocs?.length ? ( - - ) : null} - - ); -}; diff --git a/src/platform/packages/private/kbn-index-editor/src/components/flyout_content.tsx b/src/platform/packages/private/kbn-index-editor/src/components/flyout_content.tsx index d770558cc6812..4a2dc0e12594c 100644 --- a/src/platform/packages/private/kbn-index-editor/src/components/flyout_content.tsx +++ b/src/platform/packages/private/kbn-index-editor/src/components/flyout_content.tsx @@ -20,15 +20,14 @@ import { useEuiTheme, } from '@elastic/eui'; import { CellActionsProvider } from '@kbn/cell-actions'; -import { FileUploadContext, useFileUpload } from '@kbn/file-upload'; +import { FileUploadContext, FileUploadManager, useFileUpload } from '@kbn/file-upload'; import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public'; import { withSuspense } from '@kbn/shared-ux-utility'; import type { FC } from 'react'; -import React, { lazy, useMemo } from 'react'; +import React, { lazy, useCallback, useState, useMemo } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { FormattedMessage } from '@kbn/i18n-react'; import { css } from '@emotion/react'; -import type { FileUploadResults } from '@kbn/file-upload-common'; import { i18n } from '@kbn/i18n'; import { ErrorCallout } from './error_callout'; import { UnsavedChangesModal } from './modals/unsaved_changes_modal'; @@ -57,22 +56,47 @@ export const FlyoutContent: FC = ({ deps, props }) => { const rows = useObservable(deps.indexUpdateService.rows$, []); const isLoading = useObservable(deps.indexUpdateService.isFetching$, false); + const createFileUploadManager = useCallback( + (existingIndex?: string | null) => { + return new FileUploadManager( + { + analytics: coreStart.analytics, + data: deps.data, + fileUpload: deps.fileUpload, + http: coreStart.http, + notifications: coreStart.notifications, + }, + null, + false, + true, + existingIndex, + { index: { mode: 'lookup' } }, + 'lookup-index-editor' + ); + }, + [coreStart.analytics, coreStart.http, coreStart.notifications, deps.data, deps.fileUpload] + ); + + const [fileUploadManager, setFileUploadManager] = useState(() => + createFileUploadManager(deps.existingIndexName) + ); + + const reset = useCallback( + (existingIndex?: string) => { + setFileUploadManager(createFileUploadManager(existingIndex)); + }, + [createFileUploadManager] + ); + const fileUploadContextValue = useFileUpload( - deps.fileManager, + fileUploadManager, deps.data, coreStart.application, coreStart.http, coreStart.notifications, undefined, - // onUploadComplete - (results: FileUploadResults | null) => { - if (results) { - fileUploadContextValue.setExistingIndexName(results.index); - results.files.forEach((_, index) => { - deps.fileManager.removeFile(index); - }); - } - } + undefined, + reset ); const noResults = useMemo(() => { diff --git a/src/platform/packages/private/kbn-index-editor/src/components/flyout_footer.test.tsx b/src/platform/packages/private/kbn-index-editor/src/components/flyout_footer.test.tsx index 3893ae24129fd..9e8e528032af4 100644 --- a/src/platform/packages/private/kbn-index-editor/src/components/flyout_footer.test.tsx +++ b/src/platform/packages/private/kbn-index-editor/src/components/flyout_footer.test.tsx @@ -124,31 +124,6 @@ describe('FlyoutFooter', () => { }); }); - it('shows import button when canImport is true', async () => { - mockFileUploadContext.canImport = true; - renderWithI18n(); - await waitFor(() => { - expect(screen.getByTestId('indexEditorImportButton')).toBeInTheDocument(); - }); - }); - - it('calls onImportClick when import button is clicked', async () => { - mockFileUploadContext.canImport = true; - renderWithI18n(); - fireEvent.click(screen.getByTestId('indexEditorImportButton')); - await waitFor(() => { - expect(mockFileUploadContext.onImportClick).toHaveBeenCalled(); - }); - }); - - it('shows importing status when upload is in progress', async () => { - mockFileUploadContext.uploadStatus.overallImportStatus = STATUS.STARTED; - renderWithI18n(); - await waitFor(() => { - expect(screen.getByText(/Importing.../)).toBeInTheDocument(); - }); - }); - it('shows saving spinner when isSaving is true', async () => { isSaving$.next(true); renderWithI18n(); diff --git a/src/platform/packages/private/kbn-index-editor/src/components/flyout_footer.tsx b/src/platform/packages/private/kbn-index-editor/src/components/flyout_footer.tsx index 5f6e1ff3521eb..48b0eaa2b3be0 100644 --- a/src/platform/packages/private/kbn-index-editor/src/components/flyout_footer.tsx +++ b/src/platform/packages/private/kbn-index-editor/src/components/flyout_footer.tsx @@ -15,9 +15,9 @@ import { EuiFlyoutFooter, EuiLoadingSpinner, } from '@elastic/eui'; -import { STATUS, useFileUploadContext } from '@kbn/file-upload'; +import { useFileUploadContext } from '@kbn/file-upload'; import { FormattedMessage } from '@kbn/i18n-react'; -import React, { useCallback, type FC } from 'react'; +import React, { type FC } from 'react'; import useObservable from 'react-use/lib/useObservable'; import { i18n } from '@kbn/i18n'; import { useKibana } from '@kbn/kibana-react-plugin/public'; @@ -41,12 +41,7 @@ export const FlyoutFooter: FC = ({ onClose }) => { const indexName = useObservable(indexUpdateService.indexName$, indexUpdateService.getIndexName()); - const { uploadStatus, onImportClick, canImport, setExistingIndexName } = useFileUploadContext(); - - const onImport = useCallback(async () => { - indexUpdateService.setIsSaving(true); - await onImportClick(); - }, [indexUpdateService, onImportClick]); + const { setExistingIndexName } = useFileUploadContext(); const onSave = async ({ exitAfterFlush = false }) => { if (isIndexCreated) { @@ -117,33 +112,6 @@ export const FlyoutFooter: FC = ({ onClose }) => { ) : null} - {uploadStatus.overallImportStatus !== STATUS.STARTED && canImport ? ( - - - - - - ) : null} - - {uploadStatus.overallImportStatus === STATUS.STARTED ? ( - - - - - - - - - - - ) : null} - {isSaving ? ( diff --git a/src/platform/packages/private/kbn-index-editor/src/types.ts b/src/platform/packages/private/kbn-index-editor/src/types.ts index ccb6548b4ae14..f3077037e0eb1 100644 --- a/src/platform/packages/private/kbn-index-editor/src/types.ts +++ b/src/platform/packages/private/kbn-index-editor/src/types.ts @@ -48,7 +48,7 @@ export type FlyoutDeps = EditLookupIndexFlyoutDeps & { storage: Storage; indexUpdateService: IndexUpdateService; indexEditorTelemetryService: IndexEditorTelemetryService; - fileManager: FileUploadManager; + existingIndexName: string | undefined | null; }; /** Extended kibana context */ diff --git a/src/platform/packages/private/kbn-index-editor/src/ui_action/create_edit_index_action.ts b/src/platform/packages/private/kbn-index-editor/src/ui_action/create_edit_index_action.ts index 08816fe946c45..ed1e445753f68 100644 --- a/src/platform/packages/private/kbn-index-editor/src/ui_action/create_edit_index_action.ts +++ b/src/platform/packages/private/kbn-index-editor/src/ui_action/create_edit_index_action.ts @@ -9,7 +9,6 @@ import { i18n } from '@kbn/i18n'; import type { UiActionsActionDefinition } from '@kbn/ui-actions-plugin/public'; -import { FileUploadManager } from '@kbn/file-upload'; import { Storage } from '@kbn/kibana-utils-plugin/public'; import { createFlyout } from '../components/create_flyout'; import { IndexUpdateService } from '../index_update_service'; @@ -31,7 +30,7 @@ export function createEditLookupIndexContentAction( defaultMessage: 'Open lookup index editor UI', }), async execute(context: EditLookupIndexContentContext) { - const { coreStart, data, fileUpload } = dependencies; + const { coreStart, data } = dependencies; const indexEditorTelemetryService = new IndexEditorTelemetryService( coreStart.analytics, @@ -66,28 +65,6 @@ export function createEditLookupIndexContentAction( const existingIndexName = doesIndexExist ? indexName : null; - const fileManager = new FileUploadManager( - { - analytics: coreStart.analytics, - data, - fileUpload, - http: coreStart.http, - notifications: coreStart.notifications, - }, - null, - false, - true, - existingIndexName, - { index: { mode: 'lookup' } }, - 'lookup-index-editor', - // On index searchable - undefined, - // On all docs searchable - (index) => { - indexUpdateService.onFileUploadFinished(index); - } - ); - const storage = new Storage(localStorage); try { @@ -96,8 +73,8 @@ export function createEditLookupIndexContentAction( ...dependencies, indexUpdateService, indexEditorTelemetryService, - fileManager, storage, + existingIndexName, }, context ); diff --git a/src/platform/packages/private/kbn-index-editor/tsconfig.json b/src/platform/packages/private/kbn-index-editor/tsconfig.json index 2abdfe727c23d..065d156f2a45d 100644 --- a/src/platform/packages/private/kbn-index-editor/tsconfig.json +++ b/src/platform/packages/private/kbn-index-editor/tsconfig.json @@ -37,7 +37,6 @@ "@kbn/restorable-state", "@kbn/esql-ast", "@kbn/es-types", - "@kbn/file-upload-common", "@kbn/esql-utils", "@kbn/test-jest-helpers", "@kbn/esql-types", diff --git a/src/platform/test/functional/apps/discover/esql/_index_editor.ts b/src/platform/test/functional/apps/discover/esql/_index_editor.ts index 1884ded094d45..bad753ff248a3 100644 --- a/src/platform/test/functional/apps/discover/esql/_index_editor.ts +++ b/src/platform/test/functional/apps/discover/esql/_index_editor.ts @@ -87,8 +87,18 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Import a file await indexEditor.uploadFile(IMPORT_FILE_PATH); - await testSubjects.isDisplayed('indexEditorPreviewFile'); - await testSubjects.click('indexEditorImportButton'); + await testSubjects.isDisplayed('fileUploadLiteLookupSteps'); + await testSubjects.click('fileUploadLiteLookupReviewButton', 6000); + + await testSubjects.click('fileUploadLiteLookupImportButton', 6000); + + // Wait for finish button to be enabled and click it + // wait for file upload component to disappear + await retry.tryForTime(20000, async () => { + await testSubjects.waitForEnabled('fileUploadLiteLookupFinishButton', 6000); + await testSubjects.click('fileUploadLiteLookupFinishButton', 6000); + await testSubjects.missingOrFail('fileUploadLiteLookupSteps'); + }); // Check data grid has been populated correctly await retry.tryForTime(6000, async () => { diff --git a/x-pack/platform/packages/shared/file-upload-common/src/types.ts b/x-pack/platform/packages/shared/file-upload-common/src/types.ts index 4fdbd9f10f5c6..4d0816bbd7631 100644 --- a/x-pack/platform/packages/shared/file-upload-common/src/types.ts +++ b/x-pack/platform/packages/shared/file-upload-common/src/types.ts @@ -178,8 +178,8 @@ export interface IngestPipelineWrapper { } export interface IngestPipeline { - description: string; - processors: any[]; + description?: string; + processors?: any[]; isManaged?: boolean; name?: string; } diff --git a/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_manager.ts b/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_manager.ts index 5cc87a404f27b..2f32f39c38564 100644 --- a/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_manager.ts +++ b/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_manager.ts @@ -90,6 +90,8 @@ export interface UploadStatus { indexSearchable: boolean; allDocsSearchable: boolean; errors: Array<{ title: string; error: any }>; + totalDocs: number; + totalFailedDocs: number; } export class FileUploadManager { @@ -157,6 +159,8 @@ export class FileUploadManager { allDocsSearchable: false, errors: [], overallImportProgress: 0, + totalDocs: 0, + totalFailedDocs: 0, }); public readonly uploadStatus$ = this._uploadStatus$.asObservable(); @@ -352,6 +356,10 @@ export class FileUploadManager { } } + /** + * Removes files that have clashing mappings and cannot be imported together. + * Files marked with ERROR clash type will be removed from the file list. + */ public async removeClashingFiles() { const fileClashes = this._uploadStatus$.getValue().fileClashes; const filesToDestroy: FileWrapper[] = []; @@ -371,6 +379,11 @@ export class FileUploadManager { }); } + /** + * Creates a function to analyze a file at the specified index with custom overrides. + * @param index - The index of the file to analyze + * @returns A function that accepts overrides and performs the file analysis + */ public analyzeFileWithOverrides(index: number) { return async (overrides: InputOverrides) => { const files = this.getFiles(); @@ -382,17 +395,34 @@ export class FileUploadManager { }; } + /** + * Gets the current upload status including file clashes, analysis status, and import progress. + * @returns The current upload status + */ public getUploadStatus() { return this._uploadStatus$.getValue(); } + /** + * Checks if the file upload manager was initialized with an existing index. + * @returns True if initialized with an existing index, false otherwise + */ public getInitializedWithExistingIndex() { return this.initializedWithExistingIndex; } + /** + * Gets the name of the existing index being used for import. + * @returns The existing index name or null if none is set + */ public getExistingIndexName() { return this._existingIndexName$.getValue(); } + + /** + * Sets the existing index name and resets the analysis status. + * @param name - The index name to set, or null to clear + */ public setExistingIndexName(name: string | null) { this.setStatus({ analysisStatus: STATUS.NOT_STARTED, @@ -407,10 +437,18 @@ export class FileUploadManager { } } + /** + * Checks if this upload is targeting an existing index. + * @returns True if uploading to an existing index, false otherwise + */ public isExistingIndexUpload() { return this.getExistingIndexName() !== null; } + /** + * Gets the current array of file wrappers being managed. + * @returns Array of FileWrapper instances + */ public getFiles() { return this.files$.getValue(); } @@ -466,6 +504,10 @@ export class FileUploadManager { }; } + /** + * Updates the pipelines for all files with the provided array. + * @param pipelines - Array of pipelines corresponding to each file + */ public updatePipelines(pipelines: Array) { const files = this.getFiles(); files.forEach((file, i) => { @@ -473,18 +515,34 @@ export class FileUploadManager { }); } + /** + * Gets the current index mappings. + * @returns The current mappings configuration + */ public getMappings() { return this._mappings$.getValue(); } + /** + * Updates the index mappings configuration. + * @param mappings - New mappings as object or JSON string + */ public updateMappings(mappings: MappingTypeMapping | string) { this.updateSettingsOrMappings('mappings', mappings); } + /** + * Gets the current index settings. + * @returns The current index settings configuration + */ public getSettings() { return this._settings$.getValue(); } + /** + * Updates the index settings configuration. + * @param settings - New settings as object or JSON string + */ public updateSettings(settings: IndicesIndexSettings | string) { this.updateSettingsOrMappings('settings', settings); } @@ -526,10 +584,20 @@ export class FileUploadManager { } } + /** + * Gets whether a data view should be automatically created after import. + * @returns True if auto-creating data view, false otherwise + */ public getAutoCreateDataView() { return this.autoCreateDataView; } + /** + * Imports all files into the specified index with optional data view creation. + * @param indexName - Name of the target index + * @param dataViewName - Optional name for the data view to create + * @returns Promise resolving to import results or null if cancelled + */ public async import( indexName: string, dataViewName?: string | null @@ -690,16 +758,29 @@ export class FileUploadManager { checkImportAborted(); - const totalDocCount = files.reduce((acc, file) => { - const { docCount, failures } = file.getStatus(); - const count = docCount - failures.length; - return acc + count; - }, 0); + // Calculate document counts across all imported files + const documentCounts = files.reduce( + (totals, file) => { + const { docCount, failures } = file.getStatus(); + totals.totalDocs += docCount; + totals.totalDocFailures += failures.length; + return totals; + }, + { + totalDocs: 0, + totalDocFailures: 0, + } + ); - this.docCountService.startAllDocsSearchableCheck(indexName, totalDocCount); + this.docCountService.startAllDocsSearchableCheck( + indexName, + documentCounts.totalDocs - documentCounts.totalDocFailures + ); this.setStatus({ fileImport: STATUS.COMPLETED, + totalDocs: documentCounts.totalDocs, + totalFailedDocs: documentCounts.totalDocFailures, }); if (this.removePipelinesAfterImport) { @@ -829,7 +910,7 @@ export class FileUploadManager { }; pipelines.forEach((pipeline) => { - if (pipeline === undefined) { + if (pipeline === undefined || pipeline.processors === undefined) { return; } pipeline.processors.push({ @@ -869,6 +950,52 @@ export class FileUploadManager { }); } } + + /** + * Renames target fields in CSV processors across all file pipelines. + * @param changes - Array of field name changes to apply + */ + public renamePipelineTargetFields( + changes: { + oldName: string; + newName: string; + }[] + ) { + // Filter out changes where oldName equals newName (no actual difference) + const actualChanges = changes.filter((change) => change.oldName !== change.newName); + + if (actualChanges.length === 0) { + return; + } + + // Update pipeline configurations for all files + const files = this.getFiles(); + for (const file of files) { + file.renameTargetFields(actualChanges); + } + } + + /** + * Removes all convert processors from all file pipelines. + */ + public removeConvertProcessors() { + const files = this.getFiles(); + for (const file of files) { + file.removeConvertProcessors(); + } + } + + /** + * Updates date field processors in all file pipelines based on current mappings. + * @param mappings - Current index mappings to validate against + */ + public updateDateFields(mappings: MappingTypeMapping) { + const files = this.getFiles(); + for (const file of files) { + file.updateDateField(mappings); + } + } + private sendTelemetryProvider( files: FileWrapper[], startTime: number, diff --git a/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_wrapper.test.ts b/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_wrapper.test.ts new file mode 100644 index 0000000000000..4ff7ea5cb7ae1 --- /dev/null +++ b/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_wrapper.test.ts @@ -0,0 +1,549 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FileWrapper } from './file_wrapper'; +import type { MappingTypeMapping, IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; + +// Mock the dependencies +jest.mock('@kbn/file-upload-plugin/public/api'); +jest.mock('@kbn/data-plugin/public'); +jest.mock('./tika_utils'); +jest.mock('./tika_analyzer'); +jest.mock('./file_size_check'); +jest.mock('../src/utils'); +jest.mock('./doc_count_service'); + +describe('FileWrapper', () => { + let fileWrapper: FileWrapper; + let mockFile: File; + let mockFileUpload: any; + let mockData: any; + let mockTelemetryService: any; + + beforeEach(() => { + mockFile = new File(['test content'], 'test.csv', { type: 'text/csv' }); + mockFileUpload = {}; + mockData = {}; + mockTelemetryService = {}; + + fileWrapper = new FileWrapper( + mockFile, + mockFileUpload, + mockData, + mockTelemetryService, + 'test-session-id' + ); + }); + + afterEach(() => { + fileWrapper.destroy(); + }); + + describe('updateDateField', () => { + it('should remove date processors when field is no longer of type date', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + csv: { + field: 'message', + target_fields: ['time', 'name'], + }, + }, + { + date: { + field: 'time', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + { + remove: { + field: 'message', + }, + }, + ], + }; + + const mappings: MappingTypeMapping = { + properties: { + time: { type: 'text' }, // Changed from date to text + name: { type: 'keyword' }, + }, + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.updateDateField(mappings); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(2); + expect(updatedPipeline?.processors!.find((p) => p.date)).toBeUndefined(); + }); + + it('should replace date processors when field format does not match', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + date: { + field: 'timestamp', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + ], + }; + + const mappings: MappingTypeMapping = { + properties: { + timestamp: { + type: 'date', + format: 'yyyy/MM/dd', // Different format + }, + }, + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.updateDateField(mappings); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toEqual([ + { date: { field: 'timestamp', formats: ['yyyy/MM/dd'] } }, + ]); + }); + + it('should keep date processors when field type and format match', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + date: { + field: 'timestamp', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + ], + }; + + const mappings: MappingTypeMapping = { + properties: { + timestamp: { + type: 'date', + format: 'yyyy-MM-dd HH:mm:ss', + }, + }, + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.updateDateField(mappings); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(1); + expect(updatedPipeline?.processors![0].date?.field).toBe('timestamp'); + }); + + it('should add new date processors for date fields with formats', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + csv: { + field: 'message', + target_fields: ['time', 'name'], + }, + }, + { + remove: { + field: 'message', + }, + }, + ], + }; + + const mappings: MappingTypeMapping = { + properties: { + time: { + type: 'date', + format: 'yyyy-MM-dd HH:mm:ss', + }, + created_at: { + type: 'date', + format: 'ISO8601', + }, + name: { type: 'keyword' }, + }, + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.updateDateField(mappings); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(4); // csv + 2 new date + remove + + const dateProcessors = updatedPipeline?.processors!.filter((p) => p.date); + expect(dateProcessors).toHaveLength(2); + + const timeProcessor = dateProcessors?.find((p) => p.date?.field === 'time'); + const createdAtProcessor = dateProcessors?.find((p) => p.date?.field === 'created_at'); + + expect(timeProcessor?.date?.formats).toEqual(['yyyy-MM-dd HH:mm:ss']); + expect(createdAtProcessor?.date?.formats).toEqual(['ISO8601']); + + // Check that remove processor is still last + const lastProcessor = updatedPipeline?.processors![updatedPipeline.processors!.length - 1]; + expect(lastProcessor?.remove).toBeDefined(); + }); + + it('should handle array formats in mappings', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [], + }; + + const mappings: MappingTypeMapping = { + properties: { + timestamp: { + type: 'date', + format: 'yyyy-MM-dd HH:mm:ss', + }, + }, + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.updateDateField(mappings); + + const updatedPipeline = fileWrapper.getPipeline(); + const dateProcessor = updatedPipeline?.processors!.find((p) => p.date); + + expect(dateProcessor?.date?.formats).toEqual(['yyyy-MM-dd HH:mm:ss']); + }); + + it('should not add duplicate date processors', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + date: { + field: 'timestamp', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + ], + }; + + const mappings: MappingTypeMapping = { + properties: { + timestamp: { + type: 'date', + format: 'yyyy-MM-dd HH:mm:ss', + }, + }, + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.updateDateField(mappings); + + const updatedPipeline = fileWrapper.getPipeline(); + const dateProcessors = updatedPipeline?.processors!.filter((p) => p.date); + expect(dateProcessors).toHaveLength(1); + }); + + it('should handle empty pipeline processors', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [], + }; + + const mappings: MappingTypeMapping = { + properties: { + timestamp: { + type: 'date', + format: 'yyyy-MM-dd HH:mm:ss', + }, + }, + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.updateDateField(mappings); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(1); + expect(updatedPipeline?.processors![0].date?.field).toBe('timestamp'); + }); + + it('should handle undefined pipeline', () => { + const mappings: MappingTypeMapping = { + properties: { + timestamp: { + type: 'date', + format: 'yyyy-MM-dd HH:mm:ss', + }, + }, + }; + + fileWrapper.setPipeline(undefined); + + expect(() => { + fileWrapper.updateDateField(mappings); + }).not.toThrow(); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline).toBeUndefined(); + }); + + it('should handle empty mappings properties', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + date: { + field: 'timestamp', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + ], + }; + + const mappings: MappingTypeMapping = { + properties: {}, + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.updateDateField(mappings); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(0); + }); + }); + + describe('renameTargetFields', () => { + it('should rename target_fields in CSV processors', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + csv: { + field: 'message', + target_fields: ['old_field1', 'old_field2', 'unchanged_field'], + }, + }, + { + date: { + field: 'old_field1', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + ], + }; + + const changes = [ + { oldName: 'old_field1', newName: 'new_field1' }, + { oldName: 'old_field2', newName: 'new_field2' }, + ]; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.renameTargetFields(changes); + + const updatedPipeline = fileWrapper.getPipeline(); + const csvProcessor = updatedPipeline?.processors!.find((p) => p.csv); + + expect(csvProcessor?.csv?.target_fields).toEqual([ + 'new_field1', + 'new_field2', + 'unchanged_field', + ]); + }); + + it('should handle changes where oldName equals newName', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + csv: { + field: 'message', + target_fields: ['field1', 'field2'], + }, + }, + ], + }; + + const changes = [ + { oldName: 'field1', newName: 'field1' }, // Same name + { oldName: 'field2', newName: 'new_field2' }, + ]; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.renameTargetFields(changes); + + const updatedPipeline = fileWrapper.getPipeline(); + const csvProcessor = updatedPipeline?.processors!.find((p) => p.csv); + + expect(csvProcessor?.csv?.target_fields).toEqual(['field1', 'new_field2']); + }); + + it('should handle empty changes array', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + csv: { + field: 'message', + target_fields: ['field1', 'field2'], + }, + }, + ], + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.renameTargetFields([]); + + const updatedPipeline = fileWrapper.getPipeline(); + const csvProcessor = updatedPipeline?.processors!.find((p) => p.csv); + + expect(csvProcessor?.csv?.target_fields).toEqual(['field1', 'field2']); + }); + + it('should handle pipeline without CSV processors', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + date: { + field: 'timestamp', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + ], + }; + + const changes = [{ oldName: 'old_field', newName: 'new_field' }]; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.renameTargetFields(changes); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(1); + expect(updatedPipeline?.processors![0].date).toBeDefined(); + }); + + it('should handle undefined pipeline', () => { + const changes = [{ oldName: 'old_field', newName: 'new_field' }]; + + fileWrapper.setPipeline(undefined); + + expect(() => { + fileWrapper.renameTargetFields(changes); + }).not.toThrow(); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline).toBeUndefined(); + }); + }); + + describe('removeConvertProcessors', () => { + it('should remove all convert processors', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + csv: { + field: 'message', + target_fields: ['field1', 'field2'], + }, + }, + { + convert: { + field: 'field1', + type: 'integer', + }, + }, + { + convert: { + field: 'field2', + type: 'boolean', + }, + }, + { + date: { + field: 'timestamp', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + { + remove: { + field: 'message', + }, + }, + ], + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.removeConvertProcessors(); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(3); // csv + date + remove + + const convertProcessors = updatedPipeline?.processors!.filter((p) => p.convert); + expect(convertProcessors).toHaveLength(0); + + // Verify other processors remain + expect(updatedPipeline?.processors!.find((p) => p.csv)).toBeDefined(); + expect(updatedPipeline?.processors!.find((p) => p.date)).toBeDefined(); + expect(updatedPipeline?.processors!.find((p) => p.remove)).toBeDefined(); + }); + + it('should handle pipeline without convert processors', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [ + { + csv: { + field: 'message', + target_fields: ['field1', 'field2'], + }, + }, + { + date: { + field: 'timestamp', + formats: ['yyyy-MM-dd HH:mm:ss'], + }, + }, + ], + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.removeConvertProcessors(); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(2); + expect(updatedPipeline?.processors!.find((p) => p.csv)).toBeDefined(); + expect(updatedPipeline?.processors!.find((p) => p.date)).toBeDefined(); + }); + + it('should handle empty processors array', () => { + const initialPipeline: IngestPipeline = { + description: 'Test pipeline', + processors: [], + }; + + fileWrapper.setPipeline(initialPipeline); + fileWrapper.removeConvertProcessors(); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline?.processors).toHaveLength(0); + }); + + it('should handle undefined pipeline', () => { + fileWrapper.setPipeline(undefined); + + expect(() => { + fileWrapper.removeConvertProcessors(); + }).not.toThrow(); + + const updatedPipeline = fileWrapper.getPipeline(); + expect(updatedPipeline).toBeUndefined(); + }); + }); +}); diff --git a/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_wrapper.ts b/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_wrapper.ts index 76b055ce61540..11bf092cf53e3 100644 --- a/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_wrapper.ts +++ b/x-pack/platform/packages/shared/file-upload/file_upload_manager/file_wrapper.ts @@ -128,6 +128,10 @@ export class FileWrapper { }); } + /** + * Cleans up resources and aborts any ongoing analysis. + * Should be called when the file wrapper is no longer needed. + */ public destroy() { this.analysisAbortController?.abort(); @@ -136,6 +140,10 @@ export class FileWrapper { this.pipelineJsonValid$.complete(); } + /** + * Analyzes the file to determine its structure and create mappings/pipelines. + * @param overrides - Optional analysis overrides to customize the analysis + */ public async analyzeFile(overrides: InputOverrides = {}) { const startTime = new Date().getTime(); this.setStatus({ analysisStatus: STATUS.STARTED }); @@ -162,6 +170,7 @@ export class FileWrapper { data, supportedFormat, }); + this.setPipeline(analysisResults.results?.ingest_pipeline); this.analyzeFileTelemetry(analysisResults, overrides, new Date().getTime() - startTime); @@ -252,6 +261,9 @@ export class FileWrapper { } } + /** + * Aborts the current file analysis operation. + */ public abortAnalysis() { this.analysisAbortController?.abort(); this.setStatus({ @@ -266,28 +278,67 @@ export class FileWrapper { }); } + /** + * Gets the current analysis status and file information. + * @returns Current file analysis status + */ public getStatus() { return this.analyzedFile$.getValue(); } + /** + * Gets the name of the file being processed. + * @returns The file name + */ public getFileName() { return this.analyzedFile$.getValue().fileName; } + + /** + * Gets the Elasticsearch mappings generated from file analysis. + * @returns The mappings or undefined if analysis hasn't completed + */ public getMappings() { return this.analyzedFile$.getValue().results?.mappings; } + + /** + * Gets the current ingest pipeline for processing the file. + * @returns The ingest pipeline or undefined if none is set + */ public getPipeline(): IngestPipeline | undefined { return this.pipeline$.getValue(); } + + /** + * Checks if the current pipeline JSON is valid. + * @returns True if the pipeline is valid, false otherwise + */ public isPipelineValid() { return this.pipelineJsonValid$.getValue(); } + + /** + * Sets the ingest pipeline for processing the file. + * @param pipeline - The ingest pipeline to set + */ public setPipeline(pipeline: IngestPipeline | undefined) { this.pipeline$.next(pipeline); } + + /** + * Sets whether the pipeline JSON is valid. + * @param valid - True if the pipeline is valid, false otherwise + */ public setPipelineValid(valid: boolean) { this.pipelineJsonValid$.next(valid); } + + /** + * Updates the pipeline with a new pipeline object or JSON string. + * Validates JSON and sets pipeline validity accordingly. + * @param pipeline - New pipeline as object or JSON string + */ public updatePipeline(pipeline: IngestPipeline | string) { if (typeof pipeline === 'string') { try { @@ -311,15 +362,35 @@ export class FileWrapper { this.setPipeline(pipeline); } } + + /** + * Gets the detected file format from analysis. + * @returns The file format or undefined if analysis hasn't completed + */ public getFormat() { return this.analyzedFile$.getValue().results?.format; } + + /** + * Gets the raw file data as ArrayBuffer. + * @returns The file data or null if not loaded + */ public getData() { return this.analyzedFile$.getValue().data; } + + /** + * Gets the file size in bytes. + * @returns The file size in bytes + */ public getSizeInBytes() { return this.file.size; } + + /** + * Gets formatted file size information including max allowed size. + * @returns Object containing formatted size strings + */ public getFileSizeInfo() { return { fileSizeFormatted: this.fileSizeChecker.fileSizeFormatted(), @@ -328,6 +399,15 @@ export class FileWrapper { }; } + /** + * Imports the file data into the specified Elasticsearch index. + * @param index - Target index name + * @param mappings - Index mappings configuration + * @param pipelineId - Optional ingest pipeline ID + * @param getFileClashes - Function to get file clash information + * @param signal - Optional abort signal to cancel import + * @returns Promise resolving to import results + */ public async import( index: string, mappings: MappingTypeMapping, @@ -376,6 +456,125 @@ export class FileWrapper { } } + /** + * Removes all convert processors from the current pipeline. + * Convert processors are typically used for type conversion during ingestion. + */ + public removeConvertProcessors() { + const pipeline = this.getPipeline(); + if (pipeline?.processors === undefined) { + return; + } + const tempPipeline = { + ...pipeline, + processors: pipeline.processors.filter((processor) => processor.convert === undefined), + }; + this.setPipeline(tempPipeline); + } + + /** + * Renames target fields in CSV processors within the pipeline. + * @param changes - Array of field name changes to apply + */ + public renameTargetFields( + changes: { + oldName: string; + newName: string; + }[] + ) { + const pipeline = this.getPipeline(); + if (pipeline?.processors === undefined) { + return; + } + + const renameMap = new Map(); + changes.forEach((change) => { + if (change.oldName !== change.newName) { + renameMap.set(change.oldName, change.newName); + } + }); + + pipeline.processors.forEach((processor) => { + if (Array.isArray(processor?.csv?.target_fields)) { + processor.csv.target_fields = (processor.csv.target_fields as string[]).map((fieldName) => { + const newName = renameMap.get(fieldName); + return newName !== undefined ? newName : fieldName; + }); + } + }); + + this.setPipeline(pipeline); + } + + /** + * Updates date processors in the pipeline based on current field mappings. + * Removes invalid date processors and adds new ones for date fields with formats. + * @param mappings - Current index mappings to validate against + */ + public updateDateField(mappings: MappingTypeMapping) { + const pipeline = this.getPipeline(); + if (pipeline?.processors === undefined || !mappings.properties) { + return; + } + + pipeline.processors = pipeline.processors.filter((processor) => { + if (processor.date) { + const fieldName = processor.date.field; + const fieldMapping = mappings.properties![fieldName]; + + if (!fieldMapping || fieldMapping.type !== 'date') { + return false; + } + + const processorFormats: string[] = processor.date.formats; + const mappingFormat = fieldMapping.format; + + if (processorFormats && mappingFormat) { + const hasMatchingFormat = processorFormats.some( + (format) => mappingFormat === format || mappingFormat.includes(format) + ); + if (!hasMatchingFormat) { + return false; + } + } + } + + return true; + }); + + const existingDateFields = new Set( + pipeline.processors.filter((p) => p.date).map((p) => p.date!.field) + ); + + const newDateProcessors = []; + for (const [fieldName, fieldMapping] of Object.entries(mappings.properties)) { + if ( + fieldMapping.type === 'date' && + fieldMapping.format && + !existingDateFields.has(fieldName) + ) { + newDateProcessors.push({ + date: { + field: fieldName, + formats: [fieldMapping.format], + }, + }); + } + } + + if (newDateProcessors.length > 0 && pipeline.processors.length > 0) { + const lastProcessor = pipeline.processors.pop(); + pipeline.processors.push(...newDateProcessors); + if (lastProcessor) { + pipeline.processors.push(lastProcessor); + } + } else if (newDateProcessors.length > 0) { + pipeline.processors.push(...newDateProcessors); + } + + this.setPipeline(pipeline); + } + private analyzeFileTelemetry( analysisResults: AnalysisResults | undefined, overrides: InputOverrides, diff --git a/x-pack/platform/packages/shared/file-upload/index.ts b/x-pack/platform/packages/shared/file-upload/index.ts index 97e3a38f13bb0..4443cfc6a840e 100644 --- a/x-pack/platform/packages/shared/file-upload/index.ts +++ b/x-pack/platform/packages/shared/file-upload/index.ts @@ -21,4 +21,6 @@ export { createOpenFileUploadLiteTrigger, } from './src/file_upload_component/new/file_upload_lite_action'; +export { FileUploadLiteLookUpView } from './src/file_upload_component/new/file_upload_lite_lookup_view'; + export type { FileUploadStartDependencies } from './src/file_upload_component/kibana_context'; diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/file_data_visualizer.tsx b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/file_data_visualizer.tsx index dbcfd2a23e597..e4a7ef9bb3bc7 100644 --- a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/file_data_visualizer.tsx +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/file_data_visualizer.tsx @@ -70,20 +70,22 @@ export const FileDataVisualizer: FC = ({ createFileUploadManager() ); + const reset = useCallback( + (existingIndex?: string) => { + setFileUploadManager(createFileUploadManager(existingIndex)); + }, + [createFileUploadManager] + ); + const fileUploadContextValue = useFileUpload( fileUploadManager, data, application, http, notifications, - getFieldsStatsGrid - ); - - const reset = useCallback( - (existingIndex?: string) => { - setFileUploadManager(createFileUploadManager(existingIndex)); - }, - [createFileUploadManager] + getFieldsStatsGrid, + undefined, + reset ); return ( @@ -94,9 +96,6 @@ export const FileDataVisualizer: FC = ({ getAdditionalLinks={getAdditionalLinks} resultLinks={resultLinks} setUploadResults={setUploadResults} - reset={(existingIndex?: string) => { - reset(existingIndex); - }} /> diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/combined_fields/semantic_text.tsx b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/combined_fields/semantic_text.tsx index 405e366784ea2..9547b774c20c1 100644 --- a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/combined_fields/semantic_text.tsx +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/combined_fields/semantic_text.tsx @@ -111,7 +111,7 @@ export const SemanticTextForm: FC = ({ addCombinedField, hasNameCollision (pipelines: IngestPipeline[]) => { return cloneDeep(pipelines).map((p) => { if (renameToFieldOption !== null) { - p.processors.push({ + p.processors!.push({ set: { field: renameToFieldOption, copy_from: selectedFieldOption, diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/combined_fields/utils.ts b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/combined_fields/utils.ts index 8f54e13707ee6..f6343f641b0e7 100644 --- a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/combined_fields/utils.ts +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/combined_fields/utils.ts @@ -54,7 +54,7 @@ export function addCombinedFieldsToPipeline( ) { const updatedPipeline = cloneDeep(pipeline); combinedFields.forEach((combinedField) => { - updatedPipeline.processors.push({ + updatedPipeline.processors!.push({ set: { field: combinedField.combinedFieldName, value: combinedField.fieldNames @@ -75,7 +75,7 @@ export function removeCombinedFieldsFromPipeline( const updatedPipeline = cloneDeep(pipeline); return { ...updatedPipeline, - processors: updatedPipeline.processors.filter((processor) => { + processors: updatedPipeline.processors!.filter((processor) => { return 'set' in processor ? !combinedFields.some((combinedField) => { return processor.set.field === combinedField.combinedFieldName; diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_status/file_status.tsx b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_status/file_status.tsx index af0f07ec51a97..d8a1c4bd67309 100644 --- a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_status/file_status.tsx +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_status/file_status.tsx @@ -50,15 +50,21 @@ enum TAB { interface Props { index: number; showFileContentPreview?: boolean; + showFileContents?: boolean; lite: boolean; showOverrideButton?: boolean; + showExplanationButton?: boolean; + showSettingsButton?: boolean; } export const FileStatus: FC = ({ lite, index, - showFileContentPreview, + showFileContentPreview = true, + showFileContents = false, showOverrideButton = false, + showExplanationButton = true, + showSettingsButton = true, }) => { const { deleteFile, @@ -153,29 +159,33 @@ export const FileStatus: FC = ({ {fileStatus.results !== null ? ( <> - - - } - > - - - + {showExplanationButton ? ( + + + } + > + + + + ) : null} - - - + {showSettingsButton ? ( + + + + ) : null} ) : null} @@ -224,6 +234,19 @@ export const FileStatus: FC = ({ ) : null} + {showFileContents ? ( + setSelectedTab(TAB.CONTENT)} + data-test-subj={`mlFileUploadFileStatusContentsTab-${index}`} + > + + + ) : null} + {lite === false && FieldsStatsGrid !== undefined ? ( = ({ ) : null} + {selectedTab === TAB.CONTENT ? ( + + ) : null} + {selectedTab === TAB.STATS ? (
{FieldsStatsGrid !== undefined ? ( diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_upload_lite_lookup_view.tsx b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_upload_lite_lookup_view.tsx new file mode 100644 index 0000000000000..1a1f53356e18b --- /dev/null +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_upload_lite_lookup_view.tsx @@ -0,0 +1,225 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EuiStepStatus } from '@elastic/eui'; +import { EuiButton, EuiSpacer, EuiSteps, EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n-react'; +import type { FC } from 'react'; +import React, { useState, useCallback, useEffect, useMemo } from 'react'; +import type { GetAdditionalLinks, ResultLinks } from '@kbn/file-upload-common'; +import type { EuiContainedStepProps } from '@elastic/eui/src/components/steps/steps'; +import { i18n } from '@kbn/i18n'; +import { STATUS, useFileUploadContext } from '../../..'; +import { FileClashWarning } from './file_clash_warning'; +import { FileStatus } from './file_status'; +import { MappingEditor } from './mapping_editor'; + +interface Props { + resultLinks?: ResultLinks; + getAdditionalLinks?: GetAdditionalLinks; + onClose?: () => void; + setFileUploadActive: (active: boolean) => void; + setDropzoneDisabled?: (disabled: boolean) => void; +} + +interface StepsStatus { + analysis: STATUS; + mapping: STATUS; + upload: STATUS; + finish: STATUS; +} + +export const FileUploadLiteLookUpView: FC = ({ + onClose, + setDropzoneDisabled, + setFileUploadActive, +}) => { + const { filesStatus, uploadStatus, fileClashes, onImportClick, indexName } = + useFileUploadContext(); + + const [stepsStatus, setStepsStatus] = useState({ + analysis: STATUS.STARTED, + mapping: STATUS.NOT_STARTED, + upload: STATUS.NOT_STARTED, + finish: STATUS.NOT_STARTED, + }); + + const setStep = useCallback((step: keyof StepsStatus, status: STATUS) => { + setStepsStatus((prev) => ({ + ...prev, + [step]: status, + })); + }, []); + + useEffect(() => { + if (uploadStatus.overallImportStatus === STATUS.COMPLETED) { + setStep('finish', STATUS.COMPLETED); + } + }, [uploadStatus.overallImportStatus, setStep]); + + useEffect(() => { + setDropzoneDisabled?.(stepsStatus.analysis !== STATUS.STARTED); + }, [stepsStatus.analysis, setDropzoneDisabled]); + + useEffect(() => { + setFileUploadActive(filesStatus.length > 0); + return () => { + setFileUploadActive(false); + }; + }, [filesStatus, setFileUploadActive]); + + const importClick = useCallback(() => { + onImportClick(); + setStep('mapping', STATUS.COMPLETED); + setStep('upload', STATUS.STARTED); + }, [onImportClick, setStep]); + + const finalStatement = useMemo(() => { + const { totalDocs, totalFailedDocs } = uploadStatus; + if (totalFailedDocs === totalDocs) { + return i18n.translate('xpack.fileUpload.lookupJoinUpload.allDocumentsFailed', { + defaultMessage: 'Index created, but all documents failed to upload.', + }); + } else if (totalFailedDocs > 0) { + return i18n.translate('xpack.fileUpload.lookupJoinUpload.someDocumentsFailed', { + defaultMessage: + 'Index created, but {totalFailedDocs} out of {totalDocs} documents failed to upload.', + values: { totalFailedDocs, totalDocs }, + }); + } else { + return i18n.translate('xpack.fileUpload.lookupJoinUpload.allDocumentsUploaded', { + defaultMessage: 'All files uploaded successfully.', + }); + } + }, [uploadStatus]); + + if (indexName === null) { + return null; + } + + const css = { + '.euiStep__content': { paddingBlockEnd: '0px' }, + }; + + const steps: EuiContainedStepProps[] = [ + { + title: i18n.translate('xpack.fileUpload.lookupJoinUpload.analysis', { + defaultMessage: 'Review data', + }), + children: + stepsStatus.analysis === STATUS.STARTED ? ( + <> + {filesStatus.map((status, i) => ( + + ))} + + {fileClashes ? : null} + + + + { + setStep('analysis', STATUS.COMPLETED); + setStep('mapping', STATUS.STARTED); + }} + > + + + + + + ) : null, + status: generateStatus(stepsStatus.analysis), + }, + { + title: i18n.translate('xpack.fileUpload.lookupJoinUpload.reviewMapping', { + defaultMessage: 'Review mapping', + }), + children: + stepsStatus.mapping === STATUS.STARTED ? ( + <> + + + + + ) : null, + status: generateStatus(stepsStatus.mapping), + }, + { + title: i18n.translate('xpack.fileUpload.lookupJoinUpload.uploadingFilesToIndex', { + defaultMessage: 'Upload {count, plural, one {# file} other {# files}} to {indexName}', + values: { count: filesStatus.length, indexName }, + }), + children: + stepsStatus.upload === STATUS.STARTED ? ( + <> + {filesStatus.map((status, i) => ( + + ))} + + + ) : null, + status: generateStatus(uploadStatus.overallImportStatus), + }, + { + title: i18n.translate('xpack.fileUpload.lookupJoinUpload.finish', { + defaultMessage: 'Finalizing', + }), + children: + stepsStatus.finish === STATUS.COMPLETED ? ( + <> + {finalStatement} + + + + { + onClose?.(); + }} + > + + + + ) : null, + status: generateStatus( + stepsStatus.finish === STATUS.COMPLETED + ? uploadStatus.allDocsSearchable === true + ? STATUS.COMPLETED + : STATUS.STARTED + : stepsStatus.finish + ), + }, + ]; + + return ( + + ); +}; + +function generateStatus(status: STATUS): EuiStepStatus { + if (status === STATUS.STARTED) { + return 'current'; + } else if (status === STATUS.FAILED || status === STATUS.ABORTED) { + return 'danger'; + } else if (status === STATUS.COMPLETED) { + return 'complete'; + } else { + return 'incomplete'; + } +} diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_upload_view.tsx b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_upload_view.tsx index 4dddedfa939a4..9dbf16b05bcb6 100644 --- a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_upload_view.tsx +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/file_upload_view.tsx @@ -27,10 +27,9 @@ interface Props { resultLinks?: ResultLinks; getAdditionalLinks?: GetAdditionalLinks; setUploadResults?: (results: FileUploadResults) => void; - reset?: (existingIndex?: string) => void; } -export const FileUploadView: FC = ({ reset, getAdditionalLinks }) => { +export const FileUploadView: FC = ({ getAdditionalLinks }) => { const { fileUploadManager, filesStatus, @@ -42,6 +41,7 @@ export const FileUploadView: FC = ({ reset, getAdditionalLinks }) => { importResults, indexName, abortImport, + reset, } = useFileUploadContext(); const showImportControls = diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/index.ts b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/index.ts new file mode 100644 index 0000000000000..7c5ca0001e3ca --- /dev/null +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { MappingEditor } from './mapping_editor'; diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor.test.tsx b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor.test.tsx new file mode 100644 index 0000000000000..e41bfcc427c28 --- /dev/null +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor.test.tsx @@ -0,0 +1,320 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { screen, fireEvent, waitFor, act } from '@testing-library/react'; +import { renderWithI18n } from '@kbn/test-jest-helpers'; +import { BehaviorSubject } from 'rxjs'; +import { MappingEditor } from './mapping_editor'; +import { MappingEditorService } from './mapping_editor_service'; + +jest.mock('../../../use_file_upload', () => ({ + useFileUploadContext: () => ({ + fileUploadManager: { + getMappings: jest.fn(() => ({ + json: { + properties: { + field1: { type: 'text' }, + field2: { type: 'keyword' }, + }, + }, + })), + updateMappings: jest.fn(), + renamePipelineTargetFields: jest.fn(), + }, + }), +})); + +jest.mock('@kbn/field-utils/src/components/field_select/field_select', () => ({ + FieldSelect: ({ + selectedType, + onTypeChange, + }: { + selectedType: string | null; + onTypeChange: (type: string) => void; + }) => { + return ( + // eslint-disable-next-line jsx-a11y/no-onchange + + ); + }, +})); + +jest.mock('./mapping_editor_service'); + +const MockMappingEditorService = MappingEditorService as jest.MockedClass< + typeof MappingEditorService +>; + +describe('MappingEditor', () => { + let mockService: jest.Mocked; + let onImportClick: jest.MockedFunction<() => void>; + + beforeEach(() => { + onImportClick = jest.fn(); + + const mockMappings = [ + { + name: 'field1', + originalName: 'field1', + mappingProperty: { type: 'text' }, + originalMappingProperty: { type: 'text' }, + }, + { + name: 'field2', + originalName: 'field2', + mappingProperty: { type: 'keyword' }, + originalMappingProperty: { type: 'keyword' }, + }, + ]; + + mockService = { + mappings$: new BehaviorSubject(mockMappings).asObservable(), + mappingsError$: new BehaviorSubject(null).asObservable(), + mappingsEdited$: new BehaviorSubject(false).asObservable(), + getMappings: jest.fn(() => mockMappings), + getMappingsError: jest.fn(() => null), + getMappingsEdited: jest.fn(() => false), + updateMapping: jest.fn(), + reset: jest.fn(), + destroy: jest.fn(), + } as Partial as jest.Mocked; + + MockMappingEditorService.mockImplementation(() => mockService); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('renders field count and mappings correctly', () => { + renderWithI18n(); + + expect(screen.getByText(/2 fields found/)).toBeInTheDocument(); + expect(screen.getByText('Field name')).toBeInTheDocument(); + expect(screen.getByText('Field type')).toBeInTheDocument(); + + expect(screen.getByDisplayValue('field1')).toBeInTheDocument(); + expect(screen.getByDisplayValue('field2')).toBeInTheDocument(); + }); + + it('updates field name when input changes', () => { + renderWithI18n(); + + const fieldInput = screen.getByDisplayValue('field1'); + fireEvent.change(fieldInput, { target: { value: 'newFieldName' } }); + + expect(mockService.updateMapping).toHaveBeenCalledWith(0, 'newFieldName', 'text'); + }); + + it('updates field type when select changes', () => { + renderWithI18n(); + + const fieldSelects = screen.getAllByTestId('field-type-select'); + fireEvent.change(fieldSelects[0], { target: { value: 'keyword' } }); + + expect(mockService.updateMapping).toHaveBeenCalledWith(0, 'field1', 'keyword'); + }); + + it('displays error message when mappingsError is present', () => { + const errorObj = { + message: 'Mapping name and type cannot be blank', + errors: [{ index: 1, nameError: false, typeError: true }], + }; + (mockService.mappingsError$ as any) = new BehaviorSubject(errorObj).asObservable(); + (mockService.getMappingsError as jest.Mock).mockReturnValue(errorObj); + + renderWithI18n(); + + const errorElement = screen.getByText('Mapping name and type cannot be blank'); + expect(errorElement).toBeInTheDocument(); + expect(errorElement).toHaveClass('euiText'); + }); + + it('does not display error message when mappingsError is null', () => { + renderWithI18n(); + + expect(screen.queryByText(/error/i)).not.toBeInTheDocument(); + }); + + it('renders field name placeholders correctly', () => { + renderWithI18n(); + + const fieldInputs = screen.getAllByPlaceholderText('Enter field name'); + expect(fieldInputs).toHaveLength(2); + }); + + it('handles empty mappings array', () => { + mockService.getMappings = jest.fn(() => []); + mockService.mappings$ = new BehaviorSubject([]).asObservable(); + + renderWithI18n(); + + expect(screen.getByText(/0 fields found/)).toBeInTheDocument(); + expect(screen.queryByDisplayValue('field1')).not.toBeInTheDocument(); + }); + + it('updates mappings when observable changes', async () => { + const mappingsSubject = new BehaviorSubject([ + { + name: 'initialField', + originalName: 'initialField', + mappingProperty: { type: 'text' }, + originalMappingProperty: { type: 'text' }, + }, + ]); + + (mockService.mappings$ as any) = mappingsSubject.asObservable(); + (mockService.getMappings as jest.Mock).mockImplementation(() => mappingsSubject.getValue()); + + renderWithI18n(); + + expect(screen.getByDisplayValue('initialField')).toBeInTheDocument(); + + const newMappings = [ + { + name: 'updatedField', + originalName: 'updatedField', + mappingProperty: { type: 'keyword' }, + originalMappingProperty: { type: 'keyword' }, + }, + ]; + + act(() => { + mappingsSubject.next(newMappings); + }); + + await waitFor(() => { + expect(screen.getByDisplayValue('updatedField')).toBeInTheDocument(); + }); + }); + + it('renders reset button and calls reset method when clicked', async () => { + const mappingsEditedSubject = new BehaviorSubject(true); + (mockService.mappingsEdited$ as any) = mappingsEditedSubject.asObservable(); + (mockService.getMappingsEdited as jest.Mock).mockReturnValue(true); + + renderWithI18n(); + + const resetButton = screen.getByRole('button', { name: 'Reset to default' }); + expect(resetButton).toBeInTheDocument(); + + expect(resetButton).toBeEnabled(); + + fireEvent.click(resetButton); + + expect(mockService.reset).toHaveBeenCalledTimes(1); + }); + + it('disables reset button when mappings are not edited', () => { + const mappingsEditedSubject = new BehaviorSubject(false); + (mockService.mappingsEdited$ as any) = mappingsEditedSubject.asObservable(); + (mockService.getMappingsEdited as jest.Mock).mockReturnValue(false); + + renderWithI18n(); + + const resetButton = screen.getByRole('button', { name: 'Reset to default' }); + expect(resetButton).toBeInTheDocument(); + + expect(resetButton).toBeDisabled(); + + fireEvent.click(resetButton); + expect(mockService.reset).not.toHaveBeenCalled(); + }); + + it('resets fields to initial values when reset button is clicked after editing', async () => { + const initialMappings = [ + { + name: 'originalField1', + originalName: 'originalField1', + mappingProperty: { type: 'text' }, + originalMappingProperty: { type: 'text' }, + }, + { + name: 'originalField2', + originalName: 'originalField2', + mappingProperty: { type: 'keyword' }, + originalMappingProperty: { type: 'keyword' }, + }, + ]; + + const mappingsSubject = new BehaviorSubject(initialMappings); + const mappingsEditedSubject = new BehaviorSubject(false); + + (mockService.mappings$ as any) = mappingsSubject.asObservable(); + (mockService.mappingsEdited$ as any) = mappingsEditedSubject.asObservable(); + (mockService.getMappings as jest.Mock).mockImplementation(() => mappingsSubject.getValue()); + (mockService.getMappingsEdited as jest.Mock).mockImplementation(() => + mappingsEditedSubject.getValue() + ); + + renderWithI18n(); + + expect(screen.getByDisplayValue('originalField1')).toBeInTheDocument(); + expect(screen.getByDisplayValue('originalField2')).toBeInTheDocument(); + + const firstFieldInput = screen.getByDisplayValue('originalField1'); + fireEvent.change(firstFieldInput, { target: { value: 'editedField1' } }); + + const editedMappings = [ + { + name: 'editedField1', + originalName: 'originalField1', + mappingProperty: { type: 'text' }, + originalMappingProperty: { type: 'text' }, + }, + { + name: 'originalField2', + originalName: 'originalField2', + mappingProperty: { type: 'keyword' }, + originalMappingProperty: { type: 'keyword' }, + }, + ]; + + act(() => { + mappingsSubject.next(editedMappings); + mappingsEditedSubject.next(true); + }); + + await waitFor(() => { + expect(screen.getByDisplayValue('editedField1')).toBeInTheDocument(); + }); + + const resetButton = screen.getByRole('button', { name: 'Reset to default' }); + expect(resetButton).toBeEnabled(); + + (mockService.reset as jest.Mock).mockImplementation(() => { + mappingsSubject.next(initialMappings); + mappingsEditedSubject.next(false); + }); + + fireEvent.click(resetButton); + + expect(mockService.reset).toHaveBeenCalledTimes(1); + + await waitFor(() => { + expect(screen.getByDisplayValue('originalField1')).toBeInTheDocument(); + expect(screen.getByDisplayValue('originalField2')).toBeInTheDocument(); + }); + + expect(screen.queryByDisplayValue('editedField1')).not.toBeInTheDocument(); + + expect(resetButton).toBeDisabled(); + }); +}); diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor.tsx b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor.tsx new file mode 100644 index 0000000000000..91e6c86cc3c28 --- /dev/null +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor.tsx @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FC } from 'react'; +import React, { useMemo } from 'react'; +import { + EuiSpacer, + EuiFlexGroup, + EuiFlexItem, + EuiFieldText, + EuiText, + EuiButton, + EuiButtonEmpty, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { FieldSelect } from '@kbn/field-utils/src/components/field_select/field_select'; +import useObservable from 'react-use/lib/useObservable'; +import { useFileUploadContext } from '../../../use_file_upload'; +import { MappingEditorService } from './mapping_editor_service'; + +interface Props { + onImportClick: () => void; +} + +export const MappingEditor: FC = ({ onImportClick }) => { + const { fileUploadManager } = useFileUploadContext(); + const mappingEditorService = useMemo( + () => new MappingEditorService(fileUploadManager), + [fileUploadManager] + ); + + const mappingsError = useObservable( + mappingEditorService.mappingsError$, + mappingEditorService.getMappingsError() + ); + + const mappingsEdited = useObservable( + mappingEditorService.mappingsEdited$, + mappingEditorService.getMappingsEdited() + ); + + const mappings = useObservable( + mappingEditorService.mappings$, + mappingEditorService.getMappings() + ); + + const fieldCount = useMemo(() => mappings.length, [mappings]); + + return ( + <> + + + + + + + + + + + + + + + + + + + + + + {mappings.map((mapping, index) => { + const { name, mappingProperty } = mapping; + const nameInvalid = + mappingsError?.errors[index]?.nameError || mappingsError?.errors[index]?.duplicateError; + + return ( + + + + + mappingEditorService.updateMapping( + index, + e.target.value, + mappingProperty.type! + ) + } + /> + + + { + mappingEditorService.updateMapping(index, name, newType); + }} + selectedType={mappingProperty.type || null} + css={{ maxWidth: '250px' }} + /> + + + + ); + })} + + + {mappingsError ? ( + <> + + + {mappingsError.message} + + + ) : null} + + + + + + + + { + mappingEditorService.applyChanges(); + onImportClick(); + }} + fullWidth={false} + > + + + + + + + +
+ { + mappingEditorService.reset(); + }} + > + + +
+ + + + ); +}; diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor_service.ts b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor_service.ts new file mode 100644 index 0000000000000..eb06e2b55456a --- /dev/null +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/mapping_editor/mapping_editor_service.ts @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { BehaviorSubject } from 'rxjs'; +import type { MappingProperty, MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import { i18n } from '@kbn/i18n'; +import { cloneDeep } from 'lodash'; +import type { FileUploadManager } from '../../../../file_upload_manager/file_manager'; + +interface MappingEdits { + name: string; + originalName: string; + mappingProperty: MappingProperty; + originalMappingProperty: MappingProperty; +} + +interface MappingError { + message: string; + errors: Array<{ + nameError: boolean; + typeError: boolean; + duplicateError: boolean; + }>; +} + +const blankMappingErrorText = i18n.translate('xpack.fileUpload.mappingEditor.blankMappingError', { + defaultMessage: 'Mapping name and type cannot be blank', +}); +const duplicateMappingErrorText = i18n.translate( + 'xpack.fileUpload.mappingEditor.duplicateMappingError', + { + defaultMessage: 'Duplicate field names are not allowed', + } +); + +export class MappingEditorService { + // private mappingsSubscription: Subscription; + private _mappings$ = new BehaviorSubject>([]); + + /** Observable stream of current mapping edits */ + public mappings$ = this._mappings$.asObservable(); + + private _mappingsError$ = new BehaviorSubject(null); + + /** Observable stream of mapping validation errors */ + public mappingsError$ = this._mappingsError$.asObservable(); + + private originalMappingJSON: MappingTypeMapping; + + private _mappingsEdited$ = new BehaviorSubject(false); + + /** Observable stream indicating if mappings have been edited */ + public mappingsEdited$ = this._mappingsEdited$.asObservable(); + + /** + * Creates a new MappingEditorService instance. + * @param fileUploadManager - The file upload manager to manage mappings for + */ + constructor(private readonly fileUploadManager: FileUploadManager) { + const originalMappings = this.fileUploadManager.getMappings().json; + this.originalMappingJSON = cloneDeep(originalMappings); + this.initializeMappings(originalMappings); + } + + private initializeMappings(mappings: MappingTypeMapping) { + if (mappings.properties) { + const mappingsArray = Object.entries(mappings.properties).map(([fieldName, fieldConfig]) => ({ + name: fieldName, + originalName: fieldName, + mappingProperty: fieldConfig as MappingProperty, + originalMappingProperty: fieldConfig as MappingProperty, + })); + + this._mappings$.next(mappingsArray); + this.checkMappingsValid(mappingsArray); + } + } + + /** + * Applies all mapping changes to the file upload manager. + * Updates mappings, renames pipeline fields, updates date processors, and removes convert processors. + */ + public applyChanges() { + const mappings = this._mappings$.getValue(); + const nameChanges = mappings + .filter((mapping) => mapping.name !== mapping.originalName) + .map((mapping) => ({ + oldName: mapping.originalName, + newName: mapping.name, + })); + + const hasTypeChanges = mappings.some( + (mapping) => + mapping.mappingProperty.type !== mapping.originalMappingProperty.type || + JSON.stringify(mapping.mappingProperty) !== JSON.stringify(mapping.originalMappingProperty) + ); + + if (nameChanges.length > 0 || hasTypeChanges) { + const mappingTypeMapping: MappingTypeMapping = { + properties: mappings.reduce>((acc, mapping) => { + acc[mapping.name] = mapping.mappingProperty; + return acc; + }, {}), + }; + + this.fileUploadManager.updateMappings(mappingTypeMapping); + if (nameChanges.length > 0) { + this.fileUploadManager.renamePipelineTargetFields(nameChanges); + } + this.fileUploadManager.updateDateFields(mappingTypeMapping); + } + this.fileUploadManager.removeConvertProcessors(); + } + + /** + * Gets the current array of mapping edits. + * @returns Array of mapping edit objects + */ + getMappings() { + return this._mappings$.getValue(); + } + + /** + * Gets the current mapping validation error, if any. + * @returns Mapping error object or null if no errors + */ + getMappingsError() { + return this._mappingsError$.getValue(); + } + + /** + * Checks if any mappings have been edited from their original state. + * @returns True if mappings have been edited, false otherwise + */ + getMappingsEdited() { + return this._mappingsEdited$.getValue(); + } + + /** + * Updates a mapping at the specified index with new field name and/or type. + * @param index - Index of the mapping to update + * @param fieldName - New field name + * @param fieldType - New field type or null + */ + updateMapping(index: number, fieldName: string, fieldType: string | null) { + const mappings = [...this._mappings$.getValue()]; + if (mappings[index]) { + const currentMapping = mappings[index]; + const updatedMapping = { + ...currentMapping, + name: fieldName, + }; + + // Only update mappingProperty if the type has changed + if (currentMapping.mappingProperty.type !== fieldType) { + updatedMapping.mappingProperty = { type: fieldType } as MappingProperty; + } + + mappings[index] = updatedMapping; + this._mappings$.next(mappings); + this.checkMappingsValid(mappings); + this._mappingsEdited$.next(true); + } + } + + private checkMappingsValid(mappingsArray: Array) { + const errors: MappingError['errors'] = []; + const tempMappingsArray = mappingsArray.map((mapping) => ({ + ...mapping, + name: mapping.name.trim(), + })); + + let hasBlankError = false; + let hasDuplicateError = false; + + // Check for blank names and null types + tempMappingsArray.forEach((mapping, index) => { + const nameError = !mapping.name || mapping.name === ''; + const typeError = mapping.mappingProperty.type === null; + + errors.push({ nameError, typeError, duplicateError: false }); + hasBlankError = hasBlankError || nameError || typeError; + }); + + if (hasBlankError) { + this._mappingsError$.next({ + message: blankMappingErrorText, + errors, + }); + return; + } + + errors.length = 0; + + const nameCounts: Record = {}; + tempMappingsArray.forEach((mapping) => { + const name = mapping.name; + nameCounts[name] = (nameCounts[name] ?? 0) + 1; + }); + + // Check for duplicate names + tempMappingsArray.forEach((mapping, index) => { + const name = mapping.name; + const duplicateError = nameCounts[name] > 1; + errors.push({ + nameError: false, + typeError: false, + duplicateError, + }); + hasDuplicateError = hasDuplicateError || duplicateError; + }); + + if (hasDuplicateError) { + this._mappingsError$.next({ + message: duplicateMappingErrorText, + errors, + }); + return; + } + + // No errors + this._mappingsError$.next(null); + } + + /** + * Resets the mapping editor to its original state. + * Clears all edits and restores original mappings. + */ + public reset() { + const mappings = this.originalMappingJSON; + this.initializeMappings(mappings); + this._mappingsEdited$.next(false); + } +} diff --git a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/overall_upload_status.tsx b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/overall_upload_status.tsx index eaf5edce8b2cb..61b8d7fbe1ec3 100644 --- a/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/overall_upload_status.tsx +++ b/x-pack/platform/packages/shared/file-upload/src/file_upload_component/new/overall_upload_status.tsx @@ -17,17 +17,6 @@ import { useFileUploadContext } from '../../use_file_upload'; export const OverallUploadStatus: FC = () => { const { filesStatus, uploadStatus, existingIndexName, indexName } = useFileUploadContext(); - const generateStatus = (statuses: STATUS[]): EuiStepStatus => { - if (statuses.includes(STATUS.STARTED)) { - return 'current'; - } else if (statuses.includes(STATUS.FAILED) || statuses.includes(STATUS.ABORTED)) { - return 'danger'; - } else if (statuses.every((status) => status === STATUS.COMPLETED)) { - return 'complete'; - } else { - return 'incomplete'; - } - }; const css = { '.euiStep__content': { paddingBlockEnd: '0px' }, @@ -134,3 +123,15 @@ export const OverallUploadStatus: FC = () => { return ; }; + +function generateStatus(statuses: STATUS[]): EuiStepStatus { + if (statuses.includes(STATUS.STARTED)) { + return 'current'; + } else if (statuses.includes(STATUS.FAILED) || statuses.includes(STATUS.ABORTED)) { + return 'danger'; + } else if (statuses.every((status) => status === STATUS.COMPLETED)) { + return 'complete'; + } else { + return 'incomplete'; + } +} diff --git a/x-pack/platform/packages/shared/file-upload/src/use_file_upload.ts b/x-pack/platform/packages/shared/file-upload/src/use_file_upload.ts index bd9b13f817fb4..38f56144e19f2 100644 --- a/x-pack/platform/packages/shared/file-upload/src/use_file_upload.ts +++ b/x-pack/platform/packages/shared/file-upload/src/use_file_upload.ts @@ -29,7 +29,8 @@ export function useFileUpload( http: HttpSetup, notifications: NotificationsStart, getFieldsStatsGrid?: () => React.FC<{ results: FindFileStructureResponse | null }>, - onUploadComplete?: (results: FileUploadResults | null) => void + onUploadComplete?: (results: FileUploadResults | null) => void, + reset?: (existingIndex?: string) => void ) { const isMounted = useMountedState(); const { dataViews } = data; @@ -276,6 +277,7 @@ export function useFileUpload( abortAllAnalysis, abortImport, getFieldsStatsGrid, + reset, }; } diff --git a/x-pack/platform/packages/shared/file-upload/tsconfig.json b/x-pack/platform/packages/shared/file-upload/tsconfig.json index a20a2c172e005..2db3072034ebb 100644 --- a/x-pack/platform/packages/shared/file-upload/tsconfig.json +++ b/x-pack/platform/packages/shared/file-upload/tsconfig.json @@ -6,6 +6,7 @@ "jest", "node", "react", + "@testing-library/jest-dom", "@emotion/react/types/css-prop", "../../../../../typings/emotion.d.ts" ]