From 109b7ab6e35328a80a5f2315f545234301e3dafb Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 30 Sep 2025 00:04:34 +0300 Subject: [PATCH 1/7] feat(web-console): import Parquet files --- packages/web-console/assets/csv-file.svg | 5 + packages/web-console/assets/parquet-file.svg | 5 + packages/web-console/serve-dist.js | 3 +- .../src/components/Toast/index.tsx | 2 +- packages/web-console/src/consts/index.ts | 2 + .../src/modules/ZeroState/start.tsx | 24 +- .../web-console/src/scenes/Console/import.tsx | 26 -- .../web-console/src/scenes/Console/index.tsx | 85 ++-- .../dropbox.tsx => Dropbox.tsx} | 45 +- .../src/scenes/Import/DropboxUploadArea.tsx | 97 +++++ .../src/scenes/Import/FileStatus.tsx | 196 +++++++++ .../Import/ImportCSVFiles/CSVUploadList.tsx | 316 ++++++++++++++ .../Import/ImportCSVFiles/file-status.tsx | 86 ---- .../Import/ImportCSVFiles/files-to-upload.tsx | 400 ++---------------- .../scenes/Import/ImportCSVFiles/index.tsx | 48 ++- .../src/scenes/Import/ImportCSVFiles/types.ts | 20 +- .../Import/ImportCSVFiles/upload-actions.tsx | 2 +- .../scenes/Import/ImportCSVFiles/upload.tsx | 69 +-- .../Import/ImportParquet/ParquetFileList.tsx | 194 +++++++++ .../src/scenes/Import/ImportParquet/index.tsx | 344 +++++++++++++++ .../ImportParquet/rename-file-dialog.tsx | 119 ++++++ .../src/scenes/Import/ImportParquet/types.ts | 25 ++ .../web-console/src/scenes/Import/index.tsx | 43 ++ .../web-console/src/store/Console/actions.ts | 7 + .../web-console/src/store/Console/reducers.ts | 8 + .../src/store/Console/selectors.ts | 6 +- .../web-console/src/store/Console/types.ts | 10 + .../web-console/src/utils/questdb/client.ts | 63 ++- .../web-console/src/utils/questdb/types.ts | 12 +- packages/web-console/webpack.config.js | 2 +- 30 files changed, 1632 insertions(+), 632 deletions(-) create mode 100644 packages/web-console/assets/csv-file.svg create mode 100644 packages/web-console/assets/parquet-file.svg delete mode 100644 packages/web-console/src/scenes/Console/import.tsx rename packages/web-console/src/scenes/Import/{ImportCSVFiles/dropbox.tsx => Dropbox.tsx} (73%) create mode 100644 packages/web-console/src/scenes/Import/DropboxUploadArea.tsx create mode 100644 packages/web-console/src/scenes/Import/FileStatus.tsx create mode 100644 packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx delete mode 100644 packages/web-console/src/scenes/Import/ImportCSVFiles/file-status.tsx create mode 100644 packages/web-console/src/scenes/Import/ImportParquet/ParquetFileList.tsx create mode 100644 packages/web-console/src/scenes/Import/ImportParquet/index.tsx create mode 100644 packages/web-console/src/scenes/Import/ImportParquet/rename-file-dialog.tsx create mode 100644 packages/web-console/src/scenes/Import/ImportParquet/types.ts create mode 100644 packages/web-console/src/scenes/Import/index.tsx diff --git a/packages/web-console/assets/csv-file.svg b/packages/web-console/assets/csv-file.svg new file mode 100644 index 000000000..7713717a7 --- /dev/null +++ b/packages/web-console/assets/csv-file.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/packages/web-console/assets/parquet-file.svg b/packages/web-console/assets/parquet-file.svg new file mode 100644 index 000000000..3c62a6676 --- /dev/null +++ b/packages/web-console/assets/parquet-file.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/packages/web-console/serve-dist.js b/packages/web-console/serve-dist.js index a1a5fe2c9..1e35140a4 100644 --- a/packages/web-console/serve-dist.js +++ b/packages/web-console/serve-dist.js @@ -22,7 +22,8 @@ const server = http.createServer((req, res) => { reqPathName.startsWith("/settings") || reqPathName.startsWith("/warnings") || reqPathName.startsWith("/chk") || - reqPathName.startsWith("/imp") + reqPathName.startsWith("/imp") || + reqPathName.startsWith("/api/") ) { // proxy /exec requests to localhost:9000 const options = { diff --git a/packages/web-console/src/components/Toast/index.tsx b/packages/web-console/src/components/Toast/index.tsx index 4468e7ad0..9af7c356b 100644 --- a/packages/web-console/src/components/Toast/index.tsx +++ b/packages/web-console/src/components/Toast/index.tsx @@ -9,7 +9,7 @@ import { } from "react-toastify" import { useNotificationCenter as RTNotificationCenter } from "react-toastify/addons/use-notification-center" import { NotificationCenterItem as RNotificationCenterItem } from "react-toastify/addons/use-notification-center/useNotificationCenter" -import { BadgeType } from "../../scenes/Import/ImportCSVFiles/types" +import { BadgeType } from "../../scenes/Import/FileStatus" import { CloseCircle, ErrorWarning, diff --git a/packages/web-console/src/consts/index.ts b/packages/web-console/src/consts/index.ts index 481421172..b06b19892 100644 --- a/packages/web-console/src/consts/index.ts +++ b/packages/web-console/src/consts/index.ts @@ -38,4 +38,6 @@ export const API = `https://${BASE}.questdb.io` // the console will understand export const API_VERSION = "2" +export const API_ROUTE_V1 = "/api/v1" + export const BUTTON_ICON_SIZE = "26px" diff --git a/packages/web-console/src/modules/ZeroState/start.tsx b/packages/web-console/src/modules/ZeroState/start.tsx index 3a3a48987..4939bdbb0 100644 --- a/packages/web-console/src/modules/ZeroState/start.tsx +++ b/packages/web-console/src/modules/ZeroState/start.tsx @@ -66,28 +66,32 @@ export const Start = () => { + onClick={() => { + dispatch(actions.console.setImportType("parquet")) dispatch(actions.console.setActiveBottomPanel("import")) - } + }} > File upload icon - Import CSV + Import Parquet dispatch(actions.console.setActiveSidebar("create"))} + onClick={() => { + dispatch(actions.console.setImportType("csv")) + dispatch(actions.console.setActiveBottomPanel("import")) + }} > Create table icon - Create table + Import CSV diff --git a/packages/web-console/src/scenes/Console/import.tsx b/packages/web-console/src/scenes/Console/import.tsx deleted file mode 100644 index 21cf4a0e9..000000000 --- a/packages/web-console/src/scenes/Console/import.tsx +++ /dev/null @@ -1,26 +0,0 @@ -import React from "react" -import { PaneContent, PaneWrapper } from "../../components" -import { ImportCSVFiles } from "../Import/ImportCSVFiles" -import { eventBus } from "../../modules/EventBus" -import { EventType } from "../../modules/EventBus/types" - -export const Import = () => ( - - - { - eventBus.publish(EventType.MSG_QUERY_SCHEMA) - }} - onViewData={(result) => { - if (result.status === "OK") { - eventBus.publish(EventType.MSG_QUERY_SCHEMA) - eventBus.publish(EventType.MSG_QUERY_FIND_N_EXEC, { - query: `"${result.location}"`, - options: { appendAt: "end" }, - }) - } - }} - /> - - -) diff --git a/packages/web-console/src/scenes/Console/index.tsx b/packages/web-console/src/scenes/Console/index.tsx index 31b499d86..2a7775306 100644 --- a/packages/web-console/src/scenes/Console/index.tsx +++ b/packages/web-console/src/scenes/Console/index.tsx @@ -13,11 +13,12 @@ import { actions, selectors } from "../../store" import { Tooltip } from "../../components" import { Sidebar } from "../../components/Sidebar" import { Navigation } from "../../components/Sidebar/navigation" -import { Database2, Grid, PieChart, Search, FileSearch } from "@styled-icons/remix-line" +import { Database2, Grid, PieChart, FileSearch } from "@styled-icons/remix-line" import { ResultViewMode } from "./types" import { BUTTON_ICON_SIZE } from "../../consts" import { PrimaryToggleButton } from "../../components" -import { Import } from "./import" +import { Import } from "../Import" +import { DropdownMenu } from "../../components/DropdownMenu" import { BottomPanel } from "../../store/Console/types" import { Allotment, AllotmentHandle } from "allotment" import { Import as ImportIcon } from "../../components/icons/import" @@ -78,6 +79,7 @@ const Console = () => { useLocalStorage() const result = useSelector(selectors.query.getResult) const activeBottomPanel = useSelector(selectors.console.getActiveBottomPanel) + const importType = useSelector(selectors.console.getImportType) const { consoleConfig } = useSettings() const { isSearchPanelOpen, setSearchPanelOpen, searchPanelRef } = useSearch() @@ -231,29 +233,62 @@ const Console = () => { {tooltipText} ))} - { - dispatch(actions.console.setActiveBottomPanel("import")) - }, - })} - selected={activeBottomPanel === "import"} - data-hook="import-panel-button" + {consoleConfig.readOnly ? ( + + + + } + > + + To use this feature, turn off read-only mode in the configuration file + + + ) : ( + + + + + + + } > - - - } - > - - {consoleConfig.readOnly - ? "To use this feature, turn off read-only mode in the configuration file" - : "Import files from CSV"} - - + Import data + + + + { + dispatch(actions.console.setImportType("parquet")) + dispatch(actions.console.setActiveBottomPanel("import")) + }} + > + Import Parquet + + { + dispatch(actions.console.setImportType("csv")) + dispatch(actions.console.setActiveBottomPanel("import")) + }} + > + Import CSV + + + + + )} {result && } @@ -262,7 +297,7 @@ const Console = () => { - + diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx b/packages/web-console/src/scenes/Import/Dropbox.tsx similarity index 73% rename from packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx rename to packages/web-console/src/scenes/Import/Dropbox.tsx index a1f3596c2..8a12c20c1 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/dropbox.tsx +++ b/packages/web-console/src/scenes/Import/Dropbox.tsx @@ -1,7 +1,6 @@ import React, { useRef, useState, useEffect } from "react" import styled from "styled-components" -import { Box } from "../../../components/Box" -import { ProcessedFile } from "./types" +import { Box } from "../../components/Box" const getFileDuplicates = ( inputFiles: FileList, @@ -13,39 +12,37 @@ const getFileDuplicates = ( return duplicates } -const Root = styled(Box).attrs({ flexDirection: "column" })<{ - isDragging: boolean -}>` +const Root = styled(Box).attrs({ flexDirection: "column" })<{ $isDragging: boolean }>` flex: 1; width: 100%; - padding: 4rem 0 0; gap: 2rem; - background: ${({ theme }) => theme.color.backgroundLighter}; - border: 3px dashed ${({ isDragging }) => (isDragging ? "#7f839b" : "#333543")}; + background: ${({ theme, $isDragging }) => $isDragging ? theme.color.selectionDarker : theme.color.backgroundLighter}; box-shadow: inset 0 0 10px 0 #1b1c23; transition: all 0.15s ease-in-out; ` type Props = { - files: ProcessedFile[] + existingFileNames: string[] onFilesDropped: (files: File[]) => void - dialogOpen: boolean + dialogOpen?: boolean + enablePaste?: boolean render: (props: { duplicates: File[] addToQueue: (inputFiles: FileList) => void + uploadInputRef: React.RefObject }) => React.ReactNode } -export const DropBox = ({ - files, +export const Dropbox = ({ + existingFileNames, onFilesDropped, - dialogOpen, + dialogOpen = false, + enablePaste = true, render, }: Props) => { const [isDragging, setIsDragging] = useState(false) const [duplicates, setDuplicates] = useState([]) const uploadInputRef = useRef(null) - const filenames = useRef(files.map((f) => f.table_name)) const handleDrag = (e: React.DragEvent) => { e.preventDefault() @@ -61,7 +58,7 @@ export const DropBox = ({ } const addToQueue = (inputFiles: FileList) => { - const duplicates = getFileDuplicates(inputFiles, filenames.current) + const duplicates = getFileDuplicates(inputFiles, existingFileNames) setDuplicates(duplicates) onFilesDropped( Array.from(inputFiles).filter((f) => !duplicates.includes(f)), @@ -78,22 +75,22 @@ export const DropBox = ({ } useEffect(() => { + if (!enablePaste) return + return () => { window.removeEventListener("paste", handlePaste) } - }, []) + }, [enablePaste]) useEffect(() => { + if (!enablePaste) return + if (dialogOpen) { window.removeEventListener("paste", handlePaste) } else { window.addEventListener("paste", handlePaste) } - }, [dialogOpen]) - - useEffect(() => { - filenames.current = files.map((f) => f.table_name) - }, [files]) + }, [dialogOpen, enablePaste]) return ( - {render({ duplicates, addToQueue })} + {render({ duplicates, addToQueue, uploadInputRef })} ) -} +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/DropboxUploadArea.tsx b/packages/web-console/src/scenes/Import/DropboxUploadArea.tsx new file mode 100644 index 000000000..d264e5c7a --- /dev/null +++ b/packages/web-console/src/scenes/Import/DropboxUploadArea.tsx @@ -0,0 +1,97 @@ +import React from "react" +import { Button, Heading, Text } from "@questdb/react-components" +import { Box } from "../../components/Box" +import { Upload2 } from "@styled-icons/remix-line" +import styled from "styled-components" + +const BrowseTextLink = styled.span` + text-decoration: underline; + cursor: pointer; + + &:hover { + text-decoration: none; + } +` + +type Props = { + title: string + accept: string + uploadInputRef: React.RefObject + addToQueue: (inputFiles: FileList) => void + duplicates: File[] + mode?: "initial" | "list" + children?: React.ReactNode +} + +export const DropboxUploadArea = ({ + title, + accept, + uploadInputRef, + addToQueue, + duplicates, + mode = "initial", + children +}: Props) => { + const handleFileChange = (e: React.ChangeEvent) => { + if (e.target.files) { + addToQueue(e.target.files) + e.target.value = "" + } + } + + return ( + <> + + + {mode === "initial" ? ( + + + + + {title} + + + + {duplicates.length > 0 && ( + + File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} + {duplicates.map((f) => f.name).join(", ")}. Change target table + name and try again. + + )} + {children} + + ) : ( + <> + + You can drag and drop more files or{" "} + uploadInputRef.current?.click()}> + browse from disk + + + {duplicates.length > 0 && ( + + File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} + {duplicates.map((f) => f.name).join(", ")}. Change import name + and try again. + + )} + + )} + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/FileStatus.tsx b/packages/web-console/src/scenes/Import/FileStatus.tsx new file mode 100644 index 000000000..b38e0b74b --- /dev/null +++ b/packages/web-console/src/scenes/Import/FileStatus.tsx @@ -0,0 +1,196 @@ +import React, { useState } from "react" +import { FileCheckStatus as FileStatusType, CSVUploadResult } from "../../utils" +import { Badge } from "@questdb/react-components" +import { Box } from "../../components/Box" +import styled from "styled-components" +import { ChevronDown } from "@styled-icons/boxicons-solid" +import { Error as ErrorIcon } from "@styled-icons/boxicons-regular" +import { CheckboxCircle } from "@styled-icons/remix-fill" +import { Text } from "../../components/Text" +import { ColorShape } from "../../types/styled" +import { ProcessedFile } from "./ImportCSVFiles/types" +import { ProcessedParquet } from "./ImportParquet/types" + +export enum BadgeType { + SUCCESS = "success", + INFO = "info", + WARNING = "warning", + ERROR = "error", +} + +const CheckboxCircleIcon = styled(CheckboxCircle)` + color: ${({ theme }) => theme.color.green}; +` + +const ChevronIcon = styled(ChevronDown)<{ $expanded?: boolean; $color?: keyof ColorShape }>` + transform: rotate(${({ $expanded }) => $expanded ? "180deg" : "0deg"}); + cursor: pointer; + position: relative; + z-index: 1; + color: ${({ $color, theme }) => theme.color[$color ?? "gray2"]}; +` + +const ExclamationCircleIcon = styled(ErrorIcon)` + color: ${({ theme }) => theme.color.red}; +` + +const StyledBadge = styled(Badge)` + display: flex; + align-items: flex-start; + flex-direction: column; + gap: 0.5rem; + min-height: 3rem; + height: unset; +` + +const StyledBox = styled(Box)` + gap: 0.5rem; + white-space: nowrap; + height: 3rem; +` + +const FileTextBox = styled(Box)` + padding: 0.5rem 0; + text-align: left; + width: 350px; +` + +const mapStatusToLabel = ( + file: ProcessedFile | ProcessedParquet, +): + | { + label: string + type: BadgeType + icon?: React.ReactNode + } + | undefined => { + // For Parquet files + if ('file_name' in file) { + if (file.isUploading) { + return { + label: "Uploading...", + type: BadgeType.WARNING, + } + } + if (file.uploaded) { + return { + label: "Imported", + type: BadgeType.SUCCESS, + icon: , + } + } + if (file.error) { + return { + label: "Upload error", + type: BadgeType.ERROR, + icon: , + } + } + if (file.cancelled) { + return { + label: "Cancelled", + type: BadgeType.ERROR, + icon: , + } + } + return { + label: "Ready to upload", + type: BadgeType.SUCCESS, + icon: , + } + } + // For CSV files + else if (!file.isUploading && file.uploaded && file.uploadResult) { + let label = "Imported" + const csvResult = file.uploadResult as CSVUploadResult + label += ` ${csvResult.rowsImported.toLocaleString()} row${ + csvResult.rowsImported > 1 || + csvResult.rowsImported === 0 + ? "s" + : "" + }` + return { + label, + type: BadgeType.SUCCESS, + icon: , + } + } + + if (file.error) { + return { + label: "Upload error", + type: BadgeType.ERROR, + icon: , + } + } + + if (file.isUploading) { + return { + label: `Uploading: ${(file.uploadProgress || 0).toFixed(2)}%`, + type: BadgeType.WARNING, + } + } + + if ('status' in file) { + switch (file.status) { + case FileStatusType.EXISTS: + return { + label: "Table already exists", + type: BadgeType.WARNING, + } + case FileStatusType.RESERVED_NAME: + return { + label: "Reserved table name", + type: BadgeType.ERROR, + } + case FileStatusType.DOES_NOT_EXIST: + return { + label: "Ready to upload", + type: BadgeType.SUCCESS, + } + } + } +} + +const mapStatusToColor = (type: BadgeType): keyof ColorShape => { + switch (type) { + case BadgeType.SUCCESS: + return "green" + case BadgeType.ERROR: + return "red" + case BadgeType.WARNING: + return "orange" + case BadgeType.INFO: + return "cyan" + default: + return "gray2" + } +} + +export const FileStatus = ({ file }: { file: ProcessedFile | ProcessedParquet }) => { + const [expanded, setExpanded] = useState(false) + const mappedStatus = mapStatusToLabel(file) + const statusDetails = file.error + + if (!mappedStatus) { + return null + } + + return ( + + + + {mappedStatus.icon} {mappedStatus.label} + {statusDetails && setExpanded(!expanded)} $color={mapStatusToColor(mappedStatus.type)} />} + + {expanded && statusDetails && ( + + + {statusDetails} + + + )} + + + ) +} diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx new file mode 100644 index 000000000..2e8bec5fc --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx @@ -0,0 +1,316 @@ +import React, { useState, useEffect } from "react" +import styled from "styled-components" +import { Table, Button, Select } from "@questdb/react-components" +import { Column } from "@questdb/react-components/dist/components/Table" +import { Box } from "../../../components/Box" +import { Text } from "../../../components/Text" +import { Grid, Information } from "@styled-icons/remix-line" +import { ProcessedFile } from "./types" +import { CSVUploadResult, UploadModeSettings } from "../../../utils" +import { RenameTableDialog } from "./rename-table-dialog" +import { FileStatus } from "../FileStatus" +import { UploadActions } from "./upload-actions" +import { UploadResultDialog } from "./upload-result-dialog" +import { shortenText } from "../../../utils" +import { bytesWithSuffix } from "../../../utils/bytesWithSuffix" +import { PopperHover, Tooltip } from "../../../components" +import { Dialog as TableSchemaDialog } from "../../../components/TableSchemaDialog/dialog" + +const StyledTable = styled(Table)` + width: 100%; + table-layout: fixed; + border-collapse: separate; + border-spacing: 0 2rem; + + th { + padding: 0 1.5rem; + } + + td { + padding: 1.5rem; + } + + tbody td { + background: #242531; + + &:first-child { + border-top-left-radius: ${({ theme }) => theme.borderRadius}; + border-bottom-left-radius: ${({ theme }) => theme.borderRadius}; + } + + &:last-child { + border-top-right-radius: ${({ theme }) => theme.borderRadius}; + border-bottom-right-radius: ${({ theme }) => theme.borderRadius}; + } + } +` + +const FileTextBox = styled(Box)` + padding: 0 1.1rem; +` + +interface Props { + files: ProcessedFile[] + ownedByList: string[] + onFileRemove: (id: string) => void + onFileUpload: (id: string) => void + onFilePropertyChange: (id: string, file: Partial) => void + onViewData: (query: string) => void + onDialogToggle: (open: boolean) => void +} + +export const CSVUploadList = ({ + files, + ownedByList, + onFileRemove, + onFileUpload, + onFilePropertyChange, + onViewData, + onDialogToggle, +}: Props) => { + const [renameDialogOpen, setRenameDialogOpen] = useState() + const [schemaDialogOpen, setSchemaDialogOpen] = useState() + + useEffect(() => { + onDialogToggle(renameDialogOpen !== undefined || schemaDialogOpen !== undefined) + }, [renameDialogOpen, schemaDialogOpen, onDialogToggle]) + + if (files.length === 0) { + return null + } + + const columns: Column[] = [ + { + header: "CSV File", + align: "flex-start", + width: "400px", + render: ({ data }) => { + const needsTooltip = data.fileObject.name.length > 20 + const fileInfo = ( + + + {shortenText(data.fileObject.name, 20)} + + + {bytesWithSuffix(data.fileObject.size)} + + + ) + + return ( + + + CSV file icon + + {needsTooltip ? ( + + {data.fileObject.name} + + ) : ( + fileInfo + )} + + + {!data.isUploading && data.uploadResult && ( + <> + + + + )} + + + + {data.uploadResult && data.uploadResult.rowsRejected > 0 && ( + + + {data.uploadResult.rowsRejected.toLocaleString()} row + {data.uploadResult.rowsRejected > 1 ? "s" : ""} rejected + + + )} + + ) + }, + }, + { + header: "Table name", + align: "flex-end", + width: "180px", + render: ({ data }) => ( + setRenameDialogOpen(f?.id)} + onNameChange={(name) => { + onFilePropertyChange(data.id, { table_name: name }) + }} + file={data} + /> + ), + }, + ] + + if (ownedByList.length > 0) { + columns.push({ + header: ( + + Table owner + + + } + > + Required for external (non-database) users. + + ), + align: "center", + width: "150px", + render: ({ data }) => ( + ) => + onFilePropertyChange(data.id, { + settings: { + ...data.settings, + overwrite: e.target.value === "true", + }, + }) + } + options={[ + { + label: "Append", + value: "false", + }, + { + label: "Overwrite", + value: "true", + }, + ]} + /> + ), + }) + columns.push({ + align: "flex-end", + width: "300px", + render: ({ data }) => ( + onFileUpload(data.id)} + onRemove={() => onFileRemove(data.id)} + onSettingsChange={(settings: UploadModeSettings) => { + onFilePropertyChange(data.id, { settings }) + }} + /> + ), + }) + + return ( + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/file-status.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/file-status.tsx deleted file mode 100644 index ec9116dbe..000000000 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/file-status.tsx +++ /dev/null @@ -1,86 +0,0 @@ -import React from "react" -import { FileCheckStatus as FileStatusType } from "../../../utils" -import { Badge } from "@questdb/react-components" -import { BadgeType, ProcessedFile } from "./types" -import { Box } from "../../../components/Box" -import styled from "styled-components" -import { CheckboxCircle } from "@styled-icons/remix-fill" - -const CheckboxCircleIcon = styled(CheckboxCircle)` - color: ${({ theme }) => theme.color.green}; -` - -const StyledBox = styled(Box)` - gap: 0.5rem; - white-space: nowrap; -` - -const mapStatusToLabel = ( - file: ProcessedFile, -): - | { - label: string - type: BadgeType - icon?: React.ReactNode - } - | undefined => { - if (!file.isUploading && file.uploaded && file.uploadResult) { - return { - label: `Imported ${file.uploadResult.rowsImported.toLocaleString()} row${ - file.uploadResult.rowsImported > 1 || - file.uploadResult.rowsImported === 0 - ? "s" - : "" - }`, - type: BadgeType.SUCCESS, - icon: , - } - } - - if (file.error) { - return { - label: "Upload error", - type: BadgeType.ERROR, - } - } - - if (file.isUploading) { - return { - label: `Uploading: ${file.uploadProgress.toFixed(2)}%`, - type: BadgeType.WARNING, - } - } - - switch (file.status) { - case FileStatusType.EXISTS: - return { - label: "Table already exists", - type: BadgeType.WARNING, - } - case FileStatusType.RESERVED_NAME: - return { - label: "Reserved table name", - type: BadgeType.ERROR, - } - case FileStatusType.DOES_NOT_EXIST: - return { - label: "Ready to upload", - type: BadgeType.SUCCESS, - } - } -} - -export const FileStatus = ({ file }: { file: ProcessedFile }) => { - const mappedStatus = mapStatusToLabel(file) - return mappedStatus ? ( - - - - {mappedStatus.icon} {mappedStatus.label} - - - - ) : ( - <> - ) -} diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx index 120cb1654..4a83429a6 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/files-to-upload.tsx @@ -1,54 +1,17 @@ -import React, { useEffect, useRef } from "react" +import React from "react" import styled from "styled-components" -import { Heading, Table, Select, Button } from "@questdb/react-components" -import { Column, Props as TableProps } from "@questdb/react-components/dist/components/Table" -import { PopperHover, Text, Tooltip } from "../../../components" +import { Heading } from "@questdb/react-components" +import { Text } from "../../../components" import { Box } from "../../../components/Box" -import { bytesWithSuffix } from "../../../utils/bytesWithSuffix" -import { FileStatus } from "./file-status" -import { Grid, Information } from "@styled-icons/remix-line" -import { FiletypeCsv } from "@styled-icons/bootstrap/FiletypeCsv" import { ProcessedFile } from "./types" -import { UploadActions } from "./upload-actions" -import { RenameTableDialog } from "./rename-table-dialog" -import { Dialog as TableSchemaDialog } from "../../../components/TableSchemaDialog/dialog" -import { UploadResultDialog } from "./upload-result-dialog" -import { shortenText, UploadResult } from "../../../utils" -import { DropBox } from "./dropbox" +import { Dropbox } from "../Dropbox" +import { DropboxUploadArea } from "../DropboxUploadArea" +import { CSVUploadList } from "./CSVUploadList" const Root = styled(Box).attrs({ flexDirection: "column", gap: "2rem" })` padding: 2rem; ` -const StyledTable = styled(Table)` - width: 100%; - table-layout: fixed; - border-collapse: separate; - border-spacing: 0 2rem; - - th { - padding: 0 1.5rem; - } - - td { - padding: 1.5rem; - } - - tbody td { - background: #242531; - - &:first-child { - border-top-left-radius: ${({ theme }) => theme.borderRadius}; - border-bottom-left-radius: ${({ theme }) => theme.borderRadius}; - } - - &:last-child { - border-top-right-radius: ${({ theme }) => theme.borderRadius}; - border-bottom-right-radius: ${({ theme }) => theme.borderRadius}; - } - } -` - const EmptyState = styled(Box).attrs({ justifyContent: "center" })` width: 100%; background: #242531; @@ -56,27 +19,14 @@ const EmptyState = styled(Box).attrs({ justifyContent: "center" })` padding: 1rem; ` -const FileTextBox = styled(Box)` - padding: 0 1.1rem; -` - -const BrowseTextLink = styled.span` - text-decoration: underline; - cursor: pointer; - - &:hover { - text-decoration: none; - } -` - type Props = { files: ProcessedFile[] onDialogToggle: (open: boolean) => void - onFileRemove: (id: string) => void - onFileUpload: (id: string) => void + onFileRemove: (filename: string) => void + onFileUpload: (filename: string) => void onFilePropertyChange: (id: string, file: Partial) => void onFilesDropped: (files: File[]) => void - onViewData: (result: UploadResult) => void + onViewData: (query: string) => void dialogOpen: boolean ownedByList: string[] } @@ -92,324 +42,42 @@ export const FilesToUpload = ({ dialogOpen, ownedByList, }: Props) => { - const uploadInputRef = useRef(null) - const [renameDialogOpen, setRenameDialogOpen] = React.useState< - string | undefined - >() + const existingFileNames = files.map((f) => f.table_name) - const [schemaDialogOpen, setSchemaDialogOpen] = React.useState< - string | undefined - >() - - useEffect(() => { - onDialogToggle( - renameDialogOpen !== undefined || schemaDialogOpen !== undefined, - ) - }, [renameDialogOpen, schemaDialogOpen]) - - const columns: Column[] = [] - columns.push( - { - header: "File", - align: "flex-start", - ...(files.length > 0 && { width: "400px" }), - render: ({ data }) => { - const file = ( - - - {shortenText(data.fileObject.name, 20)} - - - - {bytesWithSuffix(data.fileObject.size)} - - - ) - return ( - - - - {data.fileObject.name.length > 20 && ( - - {data.fileObject.name} - - )} - {data.fileObject.name.length <= 20 && file} - - - {!data.isUploading && - data.uploadResult !== undefined && ( - - - - - )} - - {(data.uploadResult && - data.uploadResult.rowsRejected > 0) || - (data.error && ( - - {data.uploadResult && - data.uploadResult.rowsRejected > 0 && ( - - {data.uploadResult.rowsRejected.toLocaleString()}{" "} - row - {data.uploadResult.rowsRejected > 1 - ? "s" - : ""}{" "} - rejected - - )} - {data.error && ( - - {data.error} - - )} - - ))} - - - ) - }, - }, - { - header: "Table name", - align: "flex-end", - width: "180px", - render: ({ data }) => { - return ( - setRenameDialogOpen(f?.id)} - onNameChange={(name) => { - onFilePropertyChange(data.id, { - table_name: name, - }) - }} - file={data} - /> - ) - }, - }, - ) - - if (ownedByList && ownedByList.length > 0) { - columns.push( - { - header: ( - - Table owner - - - } - > - - Required for external (non-database) users. - - - ), - align: "center", - width: "150px", - render: ({ data }) => ( - ) => - onFilePropertyChange(data.id, { - settings: { - ...data.settings, - overwrite: e.target.value === "true", - }, - }) - } - options={[ - { - label: "Append", - value: "false", - }, - { - label: "Overwrite", - value: "true", - }, - ]} - /> - ), - }, - { - align: "flex-end", - width: "300px", - render: ({ data }) => ( - { - onFilePropertyChange(data.id, { - settings, - }) - }} - /> - ), - }, - ) - return ( - ( + enablePaste={true} + render={({ duplicates, addToQueue, uploadInputRef }) => ( - Upload queue - { - if (e.target.files === null) return - addToQueue(e.target.files) - }} - multiple={true} - ref={uploadInputRef} - style={{ display: "none" }} - value="" - /> - - You can drag and drop more files or{" "} - { - uploadInputRef.current?.click() - }} - > - browse from disk - - - {duplicates.length > 0 && ( - - File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} - {duplicates.map((f) => f.name).join(", ")}. Change target table - name and try again. - - )} - >> - columns={columns} - rows={files} + Import queue + - {files.length === 0 && ( + + {files.length === 0 ? ( No files in queue + ) : ( + onFilePropertyChange(id, file as Partial)} + onViewData={onViewData} + onDialogToggle={onDialogToggle} + /> )} )} diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx index cf0353049..168fea375 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/index.tsx @@ -6,7 +6,7 @@ import { ProcessedFile } from "./types" import { SchemaColumn } from "components/TableSchemaDialog/types" import { useContext } from "react" import { QuestContext } from "../../../providers" -import { pick, UploadResult, FileCheckStatus, Parameter } from "../../../utils" +import { pick, FileCheckStatus, Parameter, FileUploadResult } from "../../../utils" import * as QuestDB from "../../../utils/questdb" import { useSelector } from "react-redux" import { selectors } from "../../../store" @@ -28,8 +28,8 @@ import { ssoAuthState } from "../../../modules/OAuth2/ssoAuthState" type State = "upload" | "list" type Props = { - onViewData: (result: UploadResult) => void - onUpload: (result: UploadResult) => void + onViewData: (query: string) => void + onUpload: (result: FileUploadResult) => void } const Root = styled(Box).attrs({ gap: "4rem", flexDirection: "column" })` @@ -96,7 +96,7 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { return { ...f, ...file, - } + } as ProcessedFile } return f }), @@ -108,8 +108,9 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { } const getFileConfigs = async (files: File[]): Promise => { + const csvFiles = files.filter(file => file.type === "text/csv") return await Promise.all( - files.map(async (file) => { + csvFiles.map(async (file) => { const result = await quest.checkCSVFile(file.name) const schema = @@ -168,6 +169,7 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { return { id: uuid(), + type: "csv" as const, fileObject: file, table_name: file.name, table_owner: ownedByList[0], @@ -190,7 +192,7 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { uploaded: false, uploadResult: undefined, uploadProgress: 0, - } + } as ProcessedFile }), ) } @@ -220,6 +222,12 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { } }, [isVisible]) + useEffect(() => { + if (filesDropped.length === 0) { + setState("upload") + } + }, [filesDropped]) + return ( {state === "upload" && ( @@ -240,7 +248,6 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { ownedByList={ownedByList} onFileUpload={async (id) => { const file = filesDropped.find((f) => f.id === id) as ProcessedFile - if (file.isUploading) { return } @@ -290,13 +297,13 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { }) setIsUploading(file, false) onUpload(response) - } catch (err) { + } catch (err: any) { setIsUploading(file, false) setFileProperties(file.id, { uploaded: false, uploadResult: undefined, uploadProgress: 0, - error: "Upload error", + error: err.statusText || "Upload error", }) } }} @@ -312,15 +319,20 @@ export const ImportCSVFiles = ({ onViewData, onUpload }: Props) => { const processedFiles = await Promise.all( filesDropped.map(async (file) => { if (file.id === id) { - // Only check for file existence if table name is changed - const result = partialFile.table_name - ? await quest.checkCSVFile(partialFile.table_name) - : await Promise.resolve({ status: file.status }) - return { - ...file, - ...partialFile, - status: result.status, - error: partialFile.table_name ? undefined : file.error, // reset prior error if table name is changed + if ('table_name' in partialFile && partialFile.table_name) { + // Only check for file existence if table name is changed + const result = await quest.checkCSVFile(partialFile.table_name) + return { + ...file, + ...partialFile, + status: result.status, + error: undefined, // reset prior error if table name is changed + } as ProcessedFile + } else { + return { + ...file, + ...partialFile, + } as ProcessedFile } } else { return file diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/types.ts b/packages/web-console/src/scenes/Import/ImportCSVFiles/types.ts index dd68e06af..417940b7b 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/types.ts +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/types.ts @@ -1,10 +1,14 @@ -import { UploadResult, UploadModeSettings } from "utils" +import { CSVUploadResult, UploadModeSettings } from "utils" import { SchemaColumn } from "../../../components/TableSchemaDialog/types" export type ProcessedFile = { id: string fileObject: File status: string + isUploading: boolean + uploaded: boolean + uploadProgress: number + error?: string table_name: string table_owner: string settings: UploadModeSettings @@ -13,18 +17,6 @@ export type ProcessedFile = { timestamp: string ttlValue: number ttlUnit: string - isUploading: boolean - uploaded: boolean - uploadResult?: UploadResult - uploadProgress: number - error?: string + uploadResult?: CSVUploadResult exists: boolean } - -// TODO: Refactor @questdb/react-components/Badge to ditch enum as prop value -export enum BadgeType { - SUCCESS = "success", - INFO = "info", - WARNING = "warning", - ERROR = "error", -} diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-actions.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-actions.tsx index 350217e30..c48306856 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-actions.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/upload-actions.tsx @@ -27,7 +27,7 @@ export const UploadActions = ({ open={settingsOpen} onOpenChange={setSettingsOpen} onSubmit={onSettingsChange} - file={file} + file={file as ProcessedFile} /> - {duplicates.length > 0 && ( - - File{duplicates.length > 1 ? "s" : ""} already added to queue:{" "} - {duplicates.map((f) => f.name).join(", ")}. Change target table - name and try again. - - )} + render={({ duplicates, addToQueue, uploadInputRef }) => ( + {copyEnabled ? ( @@ -130,8 +92,7 @@ export const Upload = ({ files, onFilesDropped, dialogOpen }: Props) => { )} - - + )} /> ) diff --git a/packages/web-console/src/scenes/Import/ImportParquet/ParquetFileList.tsx b/packages/web-console/src/scenes/Import/ImportParquet/ParquetFileList.tsx new file mode 100644 index 000000000..874a36fca --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportParquet/ParquetFileList.tsx @@ -0,0 +1,194 @@ +import React, { useCallback, useRef, useState } from "react" +import styled from "styled-components" +import { Table, Button } from "@questdb/react-components" +import { Column } from "@questdb/react-components/dist/components/Table" +import { Box } from "../../../components/Box" +import { Text } from "../../../components/Text" +import { Close } from "@styled-icons/remix-line" +import { Eye } from "@styled-icons/remix-line" +import { ProcessedParquet } from "./types" +import { bytesWithSuffix } from "../../../utils/bytesWithSuffix" +import { PopperHover, Tooltip } from "../../../components" +import { RenameFileDialog } from "./rename-file-dialog" +import { FileStatus } from "../FileStatus" +import { shortenText } from "../../../utils" + +const StyledTable = styled(Table)` + width: 100%; + table-layout: fixed; + border-collapse: separate; + border-spacing: 0 2rem; + + th { + padding: 0 1.5rem; + } + + td { + padding: 1.5rem; + } + + tbody td { + background: #242531; + + &:first-child { + border-top-left-radius: ${({ theme }) => theme.borderRadius}; + border-bottom-left-radius: ${({ theme }) => theme.borderRadius}; + } + + &:last-child { + border-top-right-radius: ${({ theme }) => theme.borderRadius}; + border-bottom-right-radius: ${({ theme }) => theme.borderRadius}; + } + } +` + +const FileTextBox = styled(Box)` + padding: 0 1.1rem; +` + +const FileSize = styled(Text)` + font-size: 13px; + line-height: 2; +` + +const HiddenInput = styled.input` + display: none; +` + +interface Props { + files: ProcessedParquet[] + onRemoveFile: (id: string) => void + onFileNameChange: (id: string, name: string) => void + onAddMoreFiles: (files: File[]) => void + onViewData: (query: string) => void + onSingleFileUpload: (id: string) => void + isUploading: boolean +} + +export const ParquetFileList = ({ + files, + onRemoveFile, + onFileNameChange, + onAddMoreFiles, + onViewData, + onSingleFileUpload, + isUploading, +}: Props) => { + const fileInputRef = useRef(null) + const [renameDialogOpen, setRenameDialogOpen] = useState() + + const handleFileInputChange = useCallback((e: React.ChangeEvent) => { + if (e.target.files) { + onAddMoreFiles(Array.from(e.target.files)) + e.target.value = "" + } + }, [onAddMoreFiles]) + + const columns: Column[] = [ + { + header: "Parquet File", + align: "flex-start", + width: "400px", + render: ({ data }) => { + const fileInfo = ( + + + {shortenText(data.fileObject.name, 40)} + + + {bytesWithSuffix(data.fileObject.size)} + + + ) + + return ( + + + Parquet icon + + {fileInfo} + + + {data.uploaded && ( + + )} + + + + + ) + }, + }, + { + header: "Import Path", + align: "flex-end", + width: "400px", + render: ({ data }) => ( + setRenameDialogOpen(f?.id)} + onNameChange={(name) => { + onFileNameChange(data.id, name) + }} + file={data} + /> + ), + }, + { + header: "", + align: "flex-end", + width: "300px", + render: ({ data }) => ( + + {data.error && ( + + )} + onRemoveFile(data.id)} + > + + + } + > + Remove file from queue + + + ), + }, + ] + + return ( + <> + + + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/ImportParquet/index.tsx b/packages/web-console/src/scenes/Import/ImportParquet/index.tsx new file mode 100644 index 000000000..2311add9c --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportParquet/index.tsx @@ -0,0 +1,344 @@ +import React, { useCallback, useState } from "react" +import styled from "styled-components" +import { Heading, Button, Switch, Loader } from "@questdb/react-components" +import { Box } from "../../../components/Box" +import { Text } from "../../../components/Text" +import { Dropbox } from "../Dropbox" +import { DropboxUploadArea } from "../DropboxUploadArea" +import { ParquetFileList } from "./ParquetFileList" +import { ParquetUploadError, ProcessedParquet, UploadError } from "./types" +import { useContext } from "react" +import { QuestContext } from "../../../providers" +import { CheckmarkOutline, CloseOutline } from "@styled-icons/evaicons-outline" +import { theme } from "../../../theme" + +const Root = styled(Box).attrs({ gap: "3rem", flexDirection: "column" })` + flex: 1; +` + +const ControlPanel = styled(Box).attrs({ gap: "2rem", justifyContent: "space-between" })` + align-self: flex-end; +` + +const UploadButton = styled(Button)` + min-width: 150px; +` + +const CheckmarkIcon = styled(CheckmarkOutline)` + color: ${({ theme }) => theme.color.green}; + flex-shrink: 0; +` + +const CloseIcon = styled(CloseOutline)` + color: ${({ theme }) => theme.color.red}; + flex-shrink: 0; +` + +type State = "upload" | "list" + +type Props = { + onViewData: (query: string) => void +} + +export const ImportParquet = ({ onViewData }: Props) => { + const { quest } = useContext(QuestContext) + const [files, setFiles] = useState([]) + const [state, setState] = useState("upload") + const [overwrite, setOverwrite] = useState(false) + const [isUploading, setIsUploading] = useState(false) + const [status, setStatus] = useState<{ type: "error" | "success" | "warning", message: string } | undefined>(undefined) + + const handleFilesDropped = useCallback((droppedFiles: File[]) => { + const newFiles: ProcessedParquet[] = droppedFiles.map((file) => ({ + id: Math.random().toString(36).substr(2, 9), + fileObject: file, + file_name: file.name, + })) + + setFiles((prevFiles) => [...prevFiles, ...newFiles]) + setState("list") + }, []) + + const handleRemoveFile = useCallback((id: string) => { + setFiles((prevFiles) => prevFiles.filter((f) => f.id !== id)) + if (files.length === 1) { + setState("upload") + } + }, [files]) + + const handleFileNameChange = useCallback((id: string, name: string) => { + setFiles((prevFiles) => + prevFiles.map((f) => (f.id === id ? { ...f, file_name: name } : f)) + ) + }, []) + + const handleUploadAll = useCallback(async () => { + if (files.length === 0 || isUploading) return + + setIsUploading(true) + setStatus({ type: "warning", message: `Uploading...` }) + setFiles(prevFiles => + prevFiles.map(f => ({ ...f, isUploading: true, cancelled: false, error: undefined, uploaded: false })) + ) + + let remainingFiles = files.map((file, index) => ({ + file: file.fileObject, + name: file.file_name, + originalIndex: index + })) + + const failedFiles: number[] = [] + let successCount = 0 + let processedCount = 0 + + while (remainingFiles.length > 0) { + try { + const filesToUpload = remainingFiles.map(f => ({ + file: f.file, + name: f.name + })) + + await quest.uploadParquetFiles( + filesToUpload, + overwrite + ) + + const successfulIndices = remainingFiles.map(f => f.originalIndex) + successCount += remainingFiles.length + processedCount += remainingFiles.length + + setFiles(prevFiles => + prevFiles.map((f, i) => { + if (successfulIndices.includes(i)) { + return { ...f, isUploading: false, uploaded: true, error: undefined } + } + return f + }) + ) + break + + } catch (error) { + const uploadError = error as UploadError + + if (!uploadError.response) { + setStatus({ type: "error", message: `Upload error: ${uploadError.statusText || 'Upload failed'}` }) + setFiles(prevFiles => + prevFiles.map(f => ({ ...f, isUploading: false, uploaded: false })) + ) + break + } + + try { + const errorData = JSON.parse(uploadError.response) as ParquetUploadError + if (errorData.errors && Array.isArray(errorData.errors)) { + const uploadError = errorData.errors[0] + const errorFileName = uploadError.meta.name + + const failedFileIndex = remainingFiles.findIndex(f => f.name === errorFileName) + + if (failedFileIndex !== -1) { + const failedFile = remainingFiles[failedFileIndex] + failedFiles.push(failedFile.originalIndex) + + const successfulFiles = remainingFiles.slice(0, failedFileIndex) + successCount += successfulFiles.length + processedCount += failedFileIndex + 1 + + setStatus({ + type: "warning", + message: `Uploading...${processedCount > 0 ? ` (Processed ${processedCount}/${files.length})` : ''}` + }) + + setFiles(prevFiles => + prevFiles.map((f, i) => { + if (i === failedFile.originalIndex) { + return { ...f, error: uploadError.detail, isUploading: false, uploaded: false } + } + if (successfulFiles.some(sf => sf.originalIndex === i)) { + return { ...f, isUploading: false, uploaded: true } + } + return f + }) + ) + + remainingFiles = remainingFiles.slice(failedFileIndex + 1) + + if (remainingFiles.length > 0) { + continue + } + } else { + setStatus({ type: "error", message: "Failed to identify the problematic file" }) + break + } + } else { + setStatus({ type: "error", message: `Server error: ${uploadError.statusText || 'Unknown error'}` }) + break + } + } catch (parseError) { + setStatus({ type: "error", message: `Failed to parse error response: ${uploadError.statusText || 'Unknown error'}` }) + break + } + } + } + + setFiles(prevFiles => + prevFiles.map((f, i) => { + if (!failedFiles.includes(i) && !f.uploaded && !f.error) { + return { ...f, isUploading: false, cancelled: true } + } + return { ...f, isUploading: false } + }) + ) + + if (successCount === files.length) { + setStatus({ type: "success", message: `Uploaded ${files.length} files successfully` }) + } else if (failedFiles.length > 0) { + setStatus({ type: "error", message: `Failed after uploading ${successCount}/${files.length} files` }) + } + + setIsUploading(false) + }, [files, overwrite, quest, isUploading]) + + const handleSingleFileUpload = useCallback(async (id: string) => { + const file = files.find(f => f.id === id) + if (!file || isUploading) return + + setIsUploading(true) + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: true, uploaded: false, error: undefined } + : f + ) + ) + + try { + await quest.uploadParquetFiles( + [{ file: file.fileObject, name: file.file_name }], + overwrite + ) + + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: false, uploaded: true, error: undefined } + : f + ) + ) + } catch (error) { + const uploadError = error as UploadError + + if (!uploadError.response) { + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: false, uploaded: false, error: uploadError.statusText || 'Network error: Unable to connect to server' } + : f + ) + ) + } else { + try { + const errorData = JSON.parse(uploadError.response) as ParquetUploadError + const errorMessage = errorData.errors?.[0]?.detail || 'Upload failed' + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: false, uploaded: false, error: errorMessage } + : f + ) + ) + } catch (parseError) { + setFiles(prevFiles => + prevFiles.map(f => f.id === id + ? { ...f, isUploading: false, uploaded: false, error: uploadError.statusText || 'Failed to parse the error response from the server' } + : f + ) + ) + } + } + } finally { + setIsUploading(false) + } + }, [files, overwrite, quest, isUploading]) + + return ( + + {state === "upload" && ( + f.file_name)} + onFilesDropped={handleFilesDropped} + render={({ duplicates, addToQueue, uploadInputRef }) => ( + + )} + /> + )} + + {state === "list" && ( + f.file_name)} + onFilesDropped={handleFilesDropped} + render={({ duplicates, addToQueue, uploadInputRef }) => ( + + + Import queue + + + + + {status && ( + + {status.type === "success" && } + {status.type === "error" && } + {status.type === "warning" && } + + {status.message} + + + )} + + setOverwrite(checked)} + /> + + Overwrite existing files + + + + + + {isUploading ? "Uploading..." : "Import all files"} + + + + + + )} + /> + )} + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/ImportParquet/rename-file-dialog.tsx b/packages/web-console/src/scenes/Import/ImportParquet/rename-file-dialog.tsx new file mode 100644 index 000000000..9786ebd33 --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportParquet/rename-file-dialog.tsx @@ -0,0 +1,119 @@ +import React from "react" +import { ProcessedParquet } from "./types" +import { Dialog, ForwardRef, Button, Overlay } from "@questdb/react-components" +import { Edit } from "@styled-icons/remix-line" +import { Undo } from "@styled-icons/boxicons-regular" +import { Text } from "../../../components/Text" +import { Form } from "../../../components/Form" +import { Box } from "../../../components/Box" +import Joi from "joi" +import styled from "styled-components" +import { shortenText } from "../../../utils" + +const StyledDescription = styled(Dialog.Description)` + display: grid; + gap: 2rem; +` + +type Props = { + open: boolean + onOpenChange: (file?: ProcessedParquet) => void + onNameChange: (name: string) => void + file: ProcessedParquet +} + +const schema = Joi.object({ + name: Joi.string() + .required() + .messages({ + "string.empty": "Please enter a name", + }), +}) + +export const RenameFileDialog = ({ + open, + onOpenChange, + onNameChange, + file, +}: Props) => { + const name = file.file_name + return ( + + + + + + + + + + + + + onOpenChange(undefined)} + onInteractOutside={() => onOpenChange(undefined)} + > + + name="rename-file" + defaultValues={{ name }} + onSubmit={(values) => { + onNameChange(values.name) + onOpenChange(undefined) + }} + validationSchema={schema} + > + + + + Change import path + + + + + + + + + This path is a relative path to the sql.copy.input.root directory. +
+
+ Example: subdir/test.parquet + {' '}will import the data into {"{"}sql.copy.input.root{"}"}/subdir/test.parquet + + + + + + + + + + + } + variant="success" + > + Change + + + + + + + + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/ImportParquet/types.ts b/packages/web-console/src/scenes/Import/ImportParquet/types.ts new file mode 100644 index 000000000..6cd0bdb0c --- /dev/null +++ b/packages/web-console/src/scenes/Import/ImportParquet/types.ts @@ -0,0 +1,25 @@ +export type ProcessedParquet = { + id: string + fileObject: File + file_name: string + isUploading?: boolean + uploaded?: boolean + error?: string + cancelled?: boolean +} + +export type ParquetUploadError = { + errors: { + meta: { + name: string + } + detail: string + status: string + }[] +} + +export type UploadError = { + status: number + statusText: string + response?: string +} \ No newline at end of file diff --git a/packages/web-console/src/scenes/Import/index.tsx b/packages/web-console/src/scenes/Import/index.tsx new file mode 100644 index 000000000..ac180f071 --- /dev/null +++ b/packages/web-console/src/scenes/Import/index.tsx @@ -0,0 +1,43 @@ +import React from "react" +import { PaneContent, PaneWrapper } from "../../components" +import { ImportCSVFiles } from "./ImportCSVFiles" +import { ImportParquet } from "./ImportParquet" +import { eventBus } from "../../modules/EventBus" +import { EventType } from "../../modules/EventBus/types" + +export type ImportType = "csv" | "parquet" + +interface Props { + type: ImportType +} + +export const Import = ({ type }: Props) => { + const handleViewData = (query: string) => { + if (query) { + eventBus.publish(EventType.MSG_QUERY_SCHEMA) + eventBus.publish(EventType.MSG_QUERY_FIND_N_EXEC, { + query, + options: { appendAt: "end" }, + }) + } + } + + const handleUpload = () => { + eventBus.publish(EventType.MSG_QUERY_SCHEMA) + } + + return ( + + + {type === "csv" ? ( + + ) : ( + + )} + + + ) +} \ No newline at end of file diff --git a/packages/web-console/src/store/Console/actions.ts b/packages/web-console/src/store/Console/actions.ts index 64f882e1b..14e2124a7 100644 --- a/packages/web-console/src/store/Console/actions.ts +++ b/packages/web-console/src/store/Console/actions.ts @@ -27,6 +27,7 @@ import { ImageToZoom, Sidebar, BottomPanel, + ImportType, } from "./types" const setActiveSidebar = (panel: Sidebar): ConsoleAction => ({ @@ -39,6 +40,11 @@ const setActiveBottomPanel = (panel: BottomPanel): ConsoleAction => ({ type: ConsoleAT.SET_ACTIVE_BOTTOM_PANEL, }) +const setImportType = (type: ImportType): ConsoleAction => ({ + payload: type, + type: ConsoleAT.SET_IMPORT_TYPE, +}) + const setImageToZoom = (image?: ImageToZoom): ConsoleAction => ({ payload: image, type: ConsoleAT.SET_IMAGE_TO_ZOOM, @@ -52,5 +58,6 @@ export default { toggleSideMenu, setActiveSidebar, setActiveBottomPanel, + setImportType, setImageToZoom, } diff --git a/packages/web-console/src/store/Console/reducers.ts b/packages/web-console/src/store/Console/reducers.ts index fcb626e3c..e23251a75 100644 --- a/packages/web-console/src/store/Console/reducers.ts +++ b/packages/web-console/src/store/Console/reducers.ts @@ -28,6 +28,7 @@ export const initialState: ConsoleStateShape = { sideMenuOpened: false, activeSidebar: undefined, activeBottomPanel: "zeroState", + importType: "csv", imageToZoom: undefined, } @@ -57,6 +58,13 @@ const _console = ( } } + case ConsoleAT.SET_IMPORT_TYPE: { + return { + ...state, + importType: action.payload, + } + } + case ConsoleAT.SET_IMAGE_TO_ZOOM: { return { ...state, diff --git a/packages/web-console/src/store/Console/selectors.ts b/packages/web-console/src/store/Console/selectors.ts index 7e972c46d..256056a1c 100644 --- a/packages/web-console/src/store/Console/selectors.ts +++ b/packages/web-console/src/store/Console/selectors.ts @@ -21,7 +21,7 @@ * limitations under the License. * ******************************************************************************/ -import { StoreShape, Sidebar, BottomPanel, ImageToZoom } from "types" +import { StoreShape, Sidebar, BottomPanel, ImageToZoom, ImportType } from "types" const getSideMenuOpened: (store: StoreShape) => boolean = (store) => store.console.sideMenuOpened @@ -32,6 +32,9 @@ const getActiveSidebar: (store: StoreShape) => Sidebar = (store) => const getActiveBottomPanel: (store: StoreShape) => BottomPanel = (store) => store.console.activeBottomPanel +const getImportType: (store: StoreShape) => ImportType = (store) => + store.console.importType + const getImageToZoom: (store: StoreShape) => ImageToZoom | undefined = ( store, ) => store.console.imageToZoom @@ -40,5 +43,6 @@ export default { getSideMenuOpened, getActiveSidebar, getActiveBottomPanel, + getImportType, getImageToZoom, } diff --git a/packages/web-console/src/store/Console/types.ts b/packages/web-console/src/store/Console/types.ts index e9c52f75b..218a69766 100644 --- a/packages/web-console/src/store/Console/types.ts +++ b/packages/web-console/src/store/Console/types.ts @@ -26,6 +26,8 @@ export type Sidebar = "news" | "create" | undefined export type BottomPanel = "result" | "zeroState" | "import" +export type ImportType = "csv" | "parquet" + export type ImageToZoom = { src: string alt: string @@ -37,6 +39,7 @@ export type ConsoleStateShape = Readonly<{ sideMenuOpened: boolean activeSidebar: Sidebar activeBottomPanel: BottomPanel + importType: ImportType imageToZoom: ImageToZoom | undefined }> @@ -44,6 +47,7 @@ export enum ConsoleAT { TOGGLE_SIDE_MENU = "CONSOLE/TOGGLE_SIDE_MENU", SET_ACTIVE_SIDEBAR = "CONSOLE/SET_ACTIVE_SIDEBAR", SET_ACTIVE_BOTTOM_PANEL = "CONSOLE/SET_ACTIVE_BOTTOM_PANEL", + SET_IMPORT_TYPE = "CONSOLE/SET_IMPORT_TYPE", SET_IMAGE_TO_ZOOM = "CONSOLE/SET_IMAGE_TO_ZOOM", } @@ -61,6 +65,11 @@ type setActiveBottomPanelAction = Readonly<{ type: ConsoleAT.SET_ACTIVE_BOTTOM_PANEL }> +type setImportTypeAction = Readonly<{ + payload: ImportType + type: ConsoleAT.SET_IMPORT_TYPE +}> + type setImageToZoomAction = Readonly<{ payload?: ImageToZoom type: ConsoleAT.SET_IMAGE_TO_ZOOM @@ -70,4 +79,5 @@ export type ConsoleAction = | ToggleSideMenuAction | setActiveSidebarAction | setActiveBottomPanelAction + | setImportTypeAction | setImageToZoomAction diff --git a/packages/web-console/src/utils/questdb/client.ts b/packages/web-console/src/utils/questdb/client.ts index 8724c9ad7..cf7634ec6 100644 --- a/packages/web-console/src/utils/questdb/client.ts +++ b/packages/web-console/src/utils/questdb/client.ts @@ -3,7 +3,7 @@ import { TelemetryConfigShape } from "../../store/Telemetry/types" import { eventBus } from "../../modules/EventBus" import { EventType } from "../../modules/EventBus/types" import { AuthPayload } from "../../modules/OAuth2/types" -import { API_VERSION } from "../../consts" +import { API_ROUTE_V1, API_VERSION } from "../../consts" import { Type, ErrorResult, @@ -19,7 +19,7 @@ import { FileCheckResponse, UploadModeSettings, UploadOptions, - UploadResult, + CSVUploadResult, Value, Preferences, Permission, @@ -372,7 +372,7 @@ export class Client { partitionBy, timestamp, onProgress, - }: UploadOptions): Promise { + }: UploadOptions): Promise { const formData = new FormData() if (schema) { formData.append("schema", JSON.stringify(schema)) @@ -530,6 +530,63 @@ export class Client { return Promise.reject(error) } } + + async uploadParquetFiles( + files: { file: File; name: string }[], + overwrite: boolean, + onProgress?: (progress: number) => void + ): Promise { + const formData = new FormData() + files.forEach(({ file, name }) => { + formData.append(name, file) + }) + + return new Promise((resolve, reject) => { + const request = new XMLHttpRequest() + request.open("POST", `${API_ROUTE_V1}/imports?overwrite=${overwrite}`) + request.withCredentials = true + + Object.keys(this.commonHeaders).forEach((key) => { + request.setRequestHeader(key, this.commonHeaders[key]) + }) + + if (onProgress) { + request.upload.addEventListener("progress", (e) => { + const percent_completed = (e.loaded / e.total) * 100 + onProgress(percent_completed) + }) + } + + request.onload = () => { + if (request.status >= 200 && request.status < 300) { + try { + const response = JSON.parse(request.responseText) + resolve(response) + } catch (error) { + reject({ + status: request.status, + statusText: "Invalid JSON response", + }) + } + } else { + reject({ + status: request.status, + statusText: request.statusText, + response: request.responseText, + }) + } + } + + request.onerror = () => { + reject({ + status: request.status, + statusText: request.statusText || "Upload error", + }) + } + + request.send(formData) + }) + } } async function extractErrorMessage(response: Response) { diff --git a/packages/web-console/src/utils/questdb/types.ts b/packages/web-console/src/utils/questdb/types.ts index 6f8054a94..2a715a080 100644 --- a/packages/web-console/src/utils/questdb/types.ts +++ b/packages/web-console/src/utils/questdb/types.ts @@ -298,7 +298,7 @@ export type UploadResultColumn = { errors: number } -export type UploadResult = { +export type CSVUploadResult = { columns: UploadResultColumn[] header: boolean location: string @@ -307,6 +307,16 @@ export type UploadResult = { status: string } +export type ParquetUploadResult = { + operation: "parquet_import" + status: string + file: string + size: number + location: string +} + +export type FileUploadResult = CSVUploadResult | ParquetUploadResult + export type Parameter = { property_path: string env_var_name: string diff --git a/packages/web-console/webpack.config.js b/packages/web-console/webpack.config.js index 1923b8e9d..a4afa02be 100644 --- a/packages/web-console/webpack.config.js +++ b/packages/web-console/webpack.config.js @@ -86,7 +86,7 @@ module.exports = { context: [ config.contextPath + "/imp", config.contextPath + "/exp", config.contextPath + "/exec", config.contextPath + "/chk", config.contextPath + "/settings", config.contextPath + "/warnings", - config.contextPath + "/preferences" + config.contextPath + "/preferences", config.contextPath + "/api" ], target: config.backendUrl, }, From d56ac625c7673692b51687a95dfaed8206c88a2c Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 30 Sep 2025 00:05:35 +0300 Subject: [PATCH 2/7] submodule --- packages/browser-tests/questdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/browser-tests/questdb b/packages/browser-tests/questdb index c113299ed..dea713f02 160000 --- a/packages/browser-tests/questdb +++ b/packages/browser-tests/questdb @@ -1 +1 @@ -Subproject commit c113299ed19ad020bc57415ef00658a2a17af147 +Subproject commit dea713f02c5467e99263747ebbf7c421cdd43025 From fd6b40e5fee63907d8ce2a80b9e76ccb82ead8b8 Mon Sep 17 00:00:00 2001 From: emrberk Date: Tue, 30 Sep 2025 01:19:34 +0300 Subject: [PATCH 3/7] improve types --- .../web-console/src/scenes/Console/index.tsx | 2 ++ .../src/scenes/Import/FileStatus.tsx | 8 +++--- .../Import/ImportCSVFiles/CSVUploadList.tsx | 14 +++++----- .../Import/ImportCSVFiles/files-to-upload.tsx | 8 +++--- .../scenes/Import/ImportCSVFiles/index.tsx | 27 +++++++++---------- .../ImportCSVFiles/rename-table-dialog.tsx | 6 ++--- .../src/scenes/Import/ImportCSVFiles/types.ts | 2 +- .../Import/ImportCSVFiles/upload-actions.tsx | 6 ++--- .../ImportCSVFiles/upload-result-dialog.tsx | 4 +-- .../ImportCSVFiles/upload-settings-dialog.tsx | 4 +-- .../scenes/Import/ImportCSVFiles/upload.tsx | 4 +-- .../src/scenes/Import/ImportParquet/index.tsx | 16 ++++++----- .../web-console/src/scenes/Import/index.tsx | 3 +-- .../web-console/src/utils/questdb/client.ts | 5 ++-- .../web-console/src/utils/questdb/types.ts | 6 +---- 15 files changed, 58 insertions(+), 57 deletions(-) diff --git a/packages/web-console/src/scenes/Console/index.tsx b/packages/web-console/src/scenes/Console/index.tsx index 2a7775306..8cf1182c9 100644 --- a/packages/web-console/src/scenes/Console/index.tsx +++ b/packages/web-console/src/scenes/Console/index.tsx @@ -270,6 +270,7 @@ const Console = () => { { dispatch(actions.console.setImportType("parquet")) dispatch(actions.console.setActiveBottomPanel("import")) @@ -278,6 +279,7 @@ const Console = () => { Import Parquet { dispatch(actions.console.setImportType("csv")) dispatch(actions.console.setActiveBottomPanel("import")) diff --git a/packages/web-console/src/scenes/Import/FileStatus.tsx b/packages/web-console/src/scenes/Import/FileStatus.tsx index b38e0b74b..66859723d 100644 --- a/packages/web-console/src/scenes/Import/FileStatus.tsx +++ b/packages/web-console/src/scenes/Import/FileStatus.tsx @@ -8,7 +8,7 @@ import { Error as ErrorIcon } from "@styled-icons/boxicons-regular" import { CheckboxCircle } from "@styled-icons/remix-fill" import { Text } from "../../components/Text" import { ColorShape } from "../../types/styled" -import { ProcessedFile } from "./ImportCSVFiles/types" +import { ProcessedCSV } from "./ImportCSVFiles/types" import { ProcessedParquet } from "./ImportParquet/types" export enum BadgeType { @@ -56,7 +56,7 @@ const FileTextBox = styled(Box)` ` const mapStatusToLabel = ( - file: ProcessedFile | ProcessedParquet, + file: ProcessedCSV | ProcessedParquet, ): | { label: string @@ -102,7 +102,7 @@ const mapStatusToLabel = ( // For CSV files else if (!file.isUploading && file.uploaded && file.uploadResult) { let label = "Imported" - const csvResult = file.uploadResult as CSVUploadResult + const csvResult = file.uploadResult label += ` ${csvResult.rowsImported.toLocaleString()} row${ csvResult.rowsImported > 1 || csvResult.rowsImported === 0 @@ -167,7 +167,7 @@ const mapStatusToColor = (type: BadgeType): keyof ColorShape => { } } -export const FileStatus = ({ file }: { file: ProcessedFile | ProcessedParquet }) => { +export const FileStatus = ({ file }: { file: ProcessedCSV | ProcessedParquet }) => { const [expanded, setExpanded] = useState(false) const mappedStatus = mapStatusToLabel(file) const statusDetails = file.error diff --git a/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx b/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx index 2e8bec5fc..6d6dba6f5 100644 --- a/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx +++ b/packages/web-console/src/scenes/Import/ImportCSVFiles/CSVUploadList.tsx @@ -5,7 +5,7 @@ import { Column } from "@questdb/react-components/dist/components/Table" import { Box } from "../../../components/Box" import { Text } from "../../../components/Text" import { Grid, Information } from "@styled-icons/remix-line" -import { ProcessedFile } from "./types" +import { ProcessedCSV } from "./types" import { CSVUploadResult, UploadModeSettings } from "../../../utils" import { RenameTableDialog } from "./rename-table-dialog" import { FileStatus } from "../FileStatus" @@ -50,11 +50,11 @@ const FileTextBox = styled(Box)` ` interface Props { - files: ProcessedFile[] + files: ProcessedCSV[] ownedByList: string[] onFileRemove: (id: string) => void onFileUpload: (id: string) => void - onFilePropertyChange: (id: string, file: Partial) => void + onFilePropertyChange: (id: string, file: Partial) => void onViewData: (query: string) => void onDialogToggle: (open: boolean) => void } @@ -79,7 +79,7 @@ export const CSVUploadList = ({ return null } - const columns: Column[] = [ + const columns: Column[] = [ { header: "CSV File", align: "flex-start", @@ -122,11 +122,11 @@ export const CSVUploadList = ({