From f3ab2dff816430c83f16327e5ba648095aaf113f Mon Sep 17 00:00:00 2001 From: Marco Date: Sun, 7 Sep 2025 17:39:06 +0000 Subject: [PATCH 1/6] latest --- .../client/src/api/presto-search/index.ts | 34 ++-- .../webui/client/src/api/search/index.ts | 42 ++-- .../src/api/socket/MongoSocketCollection.ts | 2 +- .../src/api/socket/MongoSocketCursor.ts | 4 +- .../client/src/api/socket/SocketSingleton.ts | 4 +- components/webui/client/src/api/sql/index.ts | 12 +- .../client/src/api/stream-files/index.ts | 4 +- components/webui/client/src/config/index.ts | 2 +- .../src/pages/IngestPage/Details/sql.ts | 2 +- .../client/src/pages/IngestPage/Jobs/sql.ts | 4 +- .../usePrestoSearchResults.ts | 2 +- .../Presto/PrestoResultsVirtualTable/utils.ts | 2 +- .../SearchState/usePseudoProgress.ts | 2 +- .../SearchState/useResultsMetadata.ts | 2 +- .../SearchState/useUpdateStateWithMetadata.ts | 2 +- components/webui/client/src/typings/query.ts | 13 +- components/webui/common/package.json | 7 +- components/webui/common/src/config.ts | 12 ++ components/webui/common/src/index.ts | 185 +----------------- components/webui/common/src/metadata.ts | 61 ++++++ components/webui/common/src/presto.ts | 19 ++ components/webui/common/src/query.ts | 22 +++ .../src/schemas/archive-metadata.ts | 0 .../{server => common}/src/schemas/common.ts | 0 .../{server => common}/src/schemas/error.ts | 0 components/webui/common/src/schemas/index.ts | 6 + .../src/schemas/presto-search.ts | 0 .../{server => common}/src/schemas/search.ts | 0 .../src/schemas/stream-files.ts | 2 +- components/webui/common/src/socket.ts | 83 ++++++++ .../common.ts => common/src/utility-types.ts} | 2 +- .../webui/server/src/plugins/app/Presto.ts | 2 +- .../plugins/app/QueryJobDbManager/index.ts | 4 +- .../server/src/plugins/app/S3Manager/index.ts | 2 +- .../src/plugins/app/StreamFileManager.ts | 4 +- .../MongoWatcherCollection.ts | 2 +- .../app/socket/MongoSocketIoServer/index.ts | 2 +- .../app/socket/MongoSocketIoServer/typings.ts | 2 +- .../src/routes/api/archive-metadata/index.ts | 2 +- .../src/routes/api/presto-search/index.ts | 11 +- .../src/routes/api/presto-search/utils.ts | 2 +- .../server/src/routes/api/search/index.ts | 12 +- .../server/src/routes/api/search/typings.ts | 2 +- .../server/src/routes/api/search/utils.ts | 2 +- .../src/routes/api/stream-files/index.ts | 6 +- components/webui/server/src/typings/common.ts | 4 - components/webui/server/src/typings/query.ts | 21 +- 47 files changed, 298 insertions(+), 314 deletions(-) create mode 100644 components/webui/common/src/config.ts create mode 100644 components/webui/common/src/metadata.ts create mode 100644 components/webui/common/src/presto.ts create mode 100644 components/webui/common/src/query.ts rename components/webui/{server => common}/src/schemas/archive-metadata.ts (100%) rename components/webui/{server => common}/src/schemas/common.ts (100%) rename components/webui/{server => common}/src/schemas/error.ts (100%) create mode 100644 components/webui/common/src/schemas/index.ts rename components/webui/{server => common}/src/schemas/presto-search.ts (100%) rename components/webui/{server => common}/src/schemas/search.ts (100%) rename components/webui/{server => common}/src/schemas/stream-files.ts (91%) create mode 100644 components/webui/common/src/socket.ts rename components/webui/{client/src/typings/common.ts => common/src/utility-types.ts} (55%) delete mode 100644 components/webui/server/src/typings/common.ts diff --git a/components/webui/client/src/api/presto-search/index.ts b/components/webui/client/src/api/presto-search/index.ts index 25b093fca6..b85cfdfa86 100644 --- a/components/webui/client/src/api/presto-search/index.ts +++ b/components/webui/client/src/api/presto-search/index.ts @@ -1,18 +1,12 @@ import axios, {AxiosResponse} from "axios"; +import { Static } from '@sinclair/typebox' +import { + PrestoQueryJobCreationSchema, + PrestoQueryJobSchema, +} from "@webui/common/schemas/presto-search" - -// eslint-disable-next-line no-warning-comments -// TODO: Replace with shared type from the `@common` directory once refactoring is completed. -// Currently, server schema types require typebox dependency so they cannot be moved to the -// `@common` directory with current implementation. -type PrestoQueryJobCreationSchema = { - queryString: string; -}; - -type PrestoQueryJobSchema = { - searchJobId: string; -}; - +type PrestoQueryJobCreation = Static; +type PrestoQueryJob = Static; /** * Sends post request to server to submit presto query. @@ -21,11 +15,11 @@ type PrestoQueryJobSchema = { * @return */ const submitQuery = async ( - payload: PrestoQueryJobCreationSchema -): Promise> => { + payload: PrestoQueryJobCreation +): Promise> => { console.log("Submitting query:", JSON.stringify(payload)); - return axios.post("/api/presto-search/query", payload); + return axios.post("/api/presto-search/query", payload); }; @@ -36,7 +30,7 @@ const submitQuery = async ( * @return */ const cancelQuery = async ( - payload: PrestoQueryJobSchema + payload: PrestoQueryJob ): Promise> => { console.log("Cancelling query:", JSON.stringify(payload)); @@ -50,16 +44,12 @@ const cancelQuery = async ( * @param payload * @return */ -const clearQueryResults = (payload: PrestoQueryJobSchema): Promise> => { +const clearQueryResults = (payload: PrestoQueryJob): Promise> => { console.log("Clearing query:", JSON.stringify(payload)); return axios.delete("/api/presto-search/results", {data: payload}); }; -export type { - PrestoQueryJobCreationSchema, - PrestoQueryJobSchema, -}; export { cancelQuery, diff --git a/components/webui/client/src/api/search/index.ts b/components/webui/client/src/api/search/index.ts index 3f6e42973c..f4907633bc 100644 --- a/components/webui/client/src/api/search/index.ts +++ b/components/webui/client/src/api/search/index.ts @@ -1,29 +1,14 @@ import axios, {AxiosResponse} from "axios"; -import {Nullable} from "../../typings/common"; +import { Static } from '@sinclair/typebox' +import { + QueryJobCreationSchema, + QueryJobSchema, +} from "@webui/common/schemas/search" +type QueryJobCreation = Static; +type QueryJob = Static; -// eslint-disable-next-line no-warning-comments -// TODO: Replace with shared type from the `@common` directory once refactoring is completed. -// Currently, server schema types require typebox dependency so they cannot be moved to the -// `@common` directory with current implementation. -type QueryJobSchema = { - searchJobId: string; - aggregationJobId: string; -}; - -// eslint-disable-next-line no-warning-comments -// TODO: Replace with shared type from the `@common` directory once refactoring is completed. -// Currently, server schema types require typebox dependency so they cannot be moved to the -// `@common` directory with current implementation. -type QueryJobCreationSchema = { - dataset: Nullable; - ignoreCase: boolean; - queryString: string; - timeRangeBucketSizeMillis: number; - timestampBegin: number; - timestampEnd: number; -}; /** * Sends post request to server to submit query. @@ -31,10 +16,10 @@ type QueryJobCreationSchema = { * @param payload * @return */ -const submitQuery = (payload: QueryJobCreationSchema): Promise> => { +const submitQuery = (payload: QueryJobCreation): Promise> => { console.log("Submitting query:", JSON.stringify(payload)); - return axios.post("/api/search/query", payload); + return axios.post("/api/search/query", payload); }; /** @@ -43,7 +28,7 @@ const submitQuery = (payload: QueryJobCreationSchema): Promise> => { +const cancelQuery = (payload: QueryJob): Promise> => { console.log("Cancelling query:", JSON.stringify(payload)); return axios.post("/api/search/cancel", payload); @@ -55,15 +40,12 @@ const cancelQuery = (payload: QueryJobSchema): Promise> => { * @param payload * @return */ -const clearQueryResults = (payload: QueryJobSchema): Promise> => { +const clearQueryResults = (payload: QueryJob): Promise> => { console.log("Clearing query:", JSON.stringify(payload)); return axios.delete("/api/search/results", {data: payload}); }; -export type { - QueryJobCreationSchema, - QueryJobSchema, -}; + export { cancelQuery, clearQueryResults, diff --git a/components/webui/client/src/api/socket/MongoSocketCollection.ts b/components/webui/client/src/api/socket/MongoSocketCollection.ts index e1d1d1b092..a79583f91e 100644 --- a/components/webui/client/src/api/socket/MongoSocketCollection.ts +++ b/components/webui/client/src/api/socket/MongoSocketCollection.ts @@ -1,7 +1,7 @@ import { ClientToServerEvents, ServerToClientEvents, -} from "@webui/common"; +} from "@webui/common/socket"; import {Socket} from "socket.io-client"; import {MongoSocketCursor} from "./MongoSocketCursor.js"; diff --git a/components/webui/client/src/api/socket/MongoSocketCursor.ts b/components/webui/client/src/api/socket/MongoSocketCursor.ts index 363f0b185e..c66ab04b4c 100644 --- a/components/webui/client/src/api/socket/MongoSocketCursor.ts +++ b/components/webui/client/src/api/socket/MongoSocketCursor.ts @@ -3,10 +3,10 @@ import { QueryId, Response, ServerToClientEvents, -} from "@webui/common"; +} from "@webui/common/socket" import {Socket} from "socket.io-client"; +import {Nullable} from "@webui/common/utility-types" -import {Nullable} from "../../typings/common"; /** diff --git a/components/webui/client/src/api/socket/SocketSingleton.ts b/components/webui/client/src/api/socket/SocketSingleton.ts index dee90283d6..a87a7f9147 100644 --- a/components/webui/client/src/api/socket/SocketSingleton.ts +++ b/components/webui/client/src/api/socket/SocketSingleton.ts @@ -1,13 +1,13 @@ import { ClientToServerEvents, ServerToClientEvents, -} from "@webui/common"; +} from "@webui/common/socket"; import { io, Socket, } from "socket.io-client"; -import {Nullable} from "../../typings/common"; +import {Nullable} from "@webui/common/utility-types" let sharedSocket: Nullable> = null; diff --git a/components/webui/client/src/api/sql/index.ts b/components/webui/client/src/api/sql/index.ts index 3fd2b39928..2a65705360 100644 --- a/components/webui/client/src/api/sql/index.ts +++ b/components/webui/client/src/api/sql/index.ts @@ -1,14 +1,20 @@ import axios from "axios"; +import { Static } from '@sinclair/typebox' +import { + SqlSchema, +} from "@webui/common/schemas/archive-metadata" + +type Sql = Static; /** * Query the SQL server with the queryString. * - * @param queryString + * @param payload * @return */ -const querySql = async (queryString: string) => { - return axios.post("/api/archive-metadata/sql", {queryString}); +const querySql = async (payload: Sql) => { + return axios.post("/api/archive-metadata/sql", payload); }; export {querySql}; diff --git a/components/webui/client/src/api/stream-files/index.ts b/components/webui/client/src/api/stream-files/index.ts index 2552263b14..933c3c20a4 100644 --- a/components/webui/client/src/api/stream-files/index.ts +++ b/components/webui/client/src/api/stream-files/index.ts @@ -2,13 +2,13 @@ import axios, { AxiosProgressEvent, AxiosResponse, } from "axios"; -import {Nullable} from "src/typings/common"; +import {Nullable} from "@webui/common/utility-types" import { ExtractStreamResp, - QUERY_JOB_TYPE, } from "../../typings/query"; +import {QUERY_JOB_TYPE} from "@webui/common/query" interface SubmitExtractStreamJobProps { dataset: Nullable; diff --git a/components/webui/client/src/config/index.ts b/components/webui/client/src/config/index.ts index cc77f022ba..08a29413ea 100644 --- a/components/webui/client/src/config/index.ts +++ b/components/webui/client/src/config/index.ts @@ -1,4 +1,4 @@ -import {CLP_QUERY_ENGINES} from "@webui/common"; +import {CLP_QUERY_ENGINES} from "@webui/common/config"; import {settings} from "../settings"; diff --git a/components/webui/client/src/pages/IngestPage/Details/sql.ts b/components/webui/client/src/pages/IngestPage/Details/sql.ts index bf74da7272..99d15348d6 100644 --- a/components/webui/client/src/pages/IngestPage/Details/sql.ts +++ b/components/webui/client/src/pages/IngestPage/Details/sql.ts @@ -1,4 +1,4 @@ -import {Nullable} from "src/typings/common"; +import {Nullable} from "@webui/common/utility-types" import {querySql} from "../../../api/sql"; import {SqlTableSuffix} from "../../../config/sql-table-suffix"; diff --git a/components/webui/client/src/pages/IngestPage/Jobs/sql.ts b/components/webui/client/src/pages/IngestPage/Jobs/sql.ts index d66f41238f..01fbca1baa 100644 --- a/components/webui/client/src/pages/IngestPage/Jobs/sql.ts +++ b/components/webui/client/src/pages/IngestPage/Jobs/sql.ts @@ -1,4 +1,4 @@ -import {Nullable} from "src/typings/common"; +import {Nullable} from "@webui/common/utility-types" import {settings} from "../../../settings"; import {COMPRESSION_JOBS_TABLE_COLUMN_NAMES} from "../sqlConfig"; @@ -22,7 +22,7 @@ SELECT ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.UNCOMPRESSED_SIZE}, ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.COMPRESSED_SIZE} FROM ${settings.SqlDbCompressionJobsTableName} -WHERE ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.UPDATE_TIME} >= +WHERE ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.UPDATE_TIME} >= FROM_UNIXTIME(${lastUpdateTimestampSeconds}) - 1 ORDER BY _id DESC;`; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/usePrestoSearchResults.ts b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/usePrestoSearchResults.ts index 5a2f54d232..2262830d37 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/usePrestoSearchResults.ts +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/usePrestoSearchResults.ts @@ -1,4 +1,4 @@ -import type {PrestoSearchResult} from "@webui/common"; +import type {PrestoSearchResult} from "@webui/common/presto"; import MongoSocketCollection from "../../../../../../api/socket/MongoSocketCollection"; import {useCursor} from "../../../../../../api/socket/useCursor"; diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/utils.ts b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/utils.ts index 3b37abe303..0c66dd5552 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/utils.ts +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/utils.ts @@ -1,4 +1,4 @@ -import type {PrestoSearchResult} from "@webui/common"; +import type {PrestoSearchResult} from "@webui/common/presto"; import {TableProps} from "antd"; diff --git a/components/webui/client/src/pages/SearchPage/SearchState/usePseudoProgress.ts b/components/webui/client/src/pages/SearchPage/SearchState/usePseudoProgress.ts index ac978deb33..7d97d73ba4 100644 --- a/components/webui/client/src/pages/SearchPage/SearchState/usePseudoProgress.ts +++ b/components/webui/client/src/pages/SearchPage/SearchState/usePseudoProgress.ts @@ -5,7 +5,7 @@ import { useState, } from "react"; -import {Nullable} from "../../../typings/common"; +import {Nullable} from "@webui/common/utility-types" /** diff --git a/components/webui/client/src/pages/SearchPage/SearchState/useResultsMetadata.ts b/components/webui/client/src/pages/SearchPage/SearchState/useResultsMetadata.ts index 83988797fd..876a0e905b 100644 --- a/components/webui/client/src/pages/SearchPage/SearchState/useResultsMetadata.ts +++ b/components/webui/client/src/pages/SearchPage/SearchState/useResultsMetadata.ts @@ -1,4 +1,4 @@ -import {SearchResultsMetadataDocument} from "@webui/common"; +import {SearchResultsMetadataDocument} from "@webui/common/metadata"; import MongoSocketCollection from "../../../api/socket/MongoSocketCollection"; import {useCursor} from "../../../api/socket/useCursor"; diff --git a/components/webui/client/src/pages/SearchPage/SearchState/useUpdateStateWithMetadata.ts b/components/webui/client/src/pages/SearchPage/SearchState/useUpdateStateWithMetadata.ts index c989712bbb..0bf86b47cb 100644 --- a/components/webui/client/src/pages/SearchPage/SearchState/useUpdateStateWithMetadata.ts +++ b/components/webui/client/src/pages/SearchPage/SearchState/useUpdateStateWithMetadata.ts @@ -3,7 +3,7 @@ import {useEffect} from "react"; import { PRESTO_SEARCH_SIGNAL, SEARCH_SIGNAL, -} from "@webui/common"; +} from "@webui/common/metadata"; import {notification} from "antd"; import useSearchStore from "./index"; diff --git a/components/webui/client/src/typings/query.ts b/components/webui/client/src/typings/query.ts index 0fffe772e9..f03f7bf6bd 100644 --- a/components/webui/client/src/typings/query.ts +++ b/components/webui/client/src/typings/query.ts @@ -1,5 +1,5 @@ import {Type} from "@sinclair/typebox"; - +import {QUERY_JOB_TYPE} from "@webui/common/query" /** * Enum for the different UI states of a query's loading process. @@ -22,16 +22,6 @@ const QUERY_LOADING_STATE_VALUES = Object.freeze( Object.values(QUERY_LOADING_STATE).filter((value) => "number" === typeof value) ); -/** - * Enum of job type, matching the `QueryJobType` class in - * `job_orchestration.query_scheduler.constants`. - */ -enum QUERY_JOB_TYPE { - SEARCH_OR_AGGREGATION = 0, - EXTRACT_IR, - EXTRACT_JSON, -} - /** * Mapping between extract job type enums and stream type. */ @@ -91,7 +81,6 @@ export type {ExtractStreamResp}; export { EXTRACT_JOB_TYPE, ExtractJobSearchParams, - QUERY_JOB_TYPE, QUERY_LOADING_STATE, QUERY_LOADING_STATE_DESCRIPTIONS, QUERY_LOADING_STATE_VALUES, diff --git a/components/webui/common/package.json b/components/webui/common/package.json index 78a31783f3..b5f0dd25da 100644 --- a/components/webui/common/package.json +++ b/components/webui/common/package.json @@ -1,10 +1,11 @@ { "name": "@webui/common", "version": "0.1.0", + "files": ["dist"], "exports": { - ".": { - "import": "./dist/index.js", - "types": "./dist/index.d.ts" + "./*": { + "types": "./dist/*.d.ts", + "import": "./dist/*.js" } }, "scripts": { diff --git a/components/webui/common/src/config.ts b/components/webui/common/src/config.ts new file mode 100644 index 0000000000..5bbc619752 --- /dev/null +++ b/components/webui/common/src/config.ts @@ -0,0 +1,12 @@ +/** + * CLP query engines. + */ +enum CLP_QUERY_ENGINES { + CLP = "clp", + CLP_S = "clp-s", + PRESTO = "presto", +} + +export { + CLP_QUERY_ENGINES +}; \ No newline at end of file diff --git a/components/webui/common/src/index.ts b/components/webui/common/src/index.ts index f14b49e863..c21c38e1a0 100644 --- a/components/webui/common/src/index.ts +++ b/components/webui/common/src/index.ts @@ -1,177 +1,8 @@ -import {Type} from "@sinclair/typebox"; - - -/** - * Unique ID for each active unique query. Multiple clients can subscribe to the same ID if the - * queries are identical. The ID is also used to represent the socket room, and MongoDB - * change stream. - */ -type QueryId = number; - -/** - * Error response to event. - */ -interface Err { - error: string; - queryId?: QueryId; -} - -/** - * Success response to event. - */ -interface Success { - data: T; -} - -/** - * Event response. - */ -type Response = Err | Success; - - -/** - * Events that the client can emit to the server. - */ -type ClientToServerEvents = { - "disconnect": () => void; - "collection::find::subscribe": ( - requestArgs: { - collectionName: string; - query: object; - options: object; - }, - callback: (res: Response<{queryId: QueryId; initialDocuments: object[]}>) => void) => void; - "collection::find::unsubscribe": ( - requestArgs: { - queryId: QueryId; - } - ) => Promise; -}; - -/** - * Events that the server can emit to the client. - */ -interface ServerToClientEvents { - // eslint-disable-next-line no-warning-comments - // TODO: Consider replacing this with `collection::find::update${number}`, which will - // limit callbacks being triggered in the client to their respective query IDs. - "collection::find::update": (respArgs: { - queryId: QueryId; - data: object[]; - }) => void; -} - -/** - * Empty but required by Socket IO. - */ -// eslint-disable-next-line @typescript-eslint/no-empty-object-type -interface InterServerEvents { -} - -/** - * Collection associated with each socket connection. - */ -interface SocketData { - collectionName?: string; -} - -/** - * Enum of search-related signals. - * - * This includes request and response signals for various search operations and their respective - * states. - */ -enum SEARCH_SIGNAL { - NONE = "none", - - REQ_CANCELLING = "req-cancelling", - REQ_CLEARING = "req-clearing", - REQ_QUERYING = "req-querying", - - RESP_DONE = "resp-done", - RESP_QUERYING = "resp-querying", -} - -/** - * Presto search-related signals. - */ -enum PRESTO_SEARCH_SIGNAL { - WAITING_FOR_PREREQUISITES = "WAITING_FOR_PREREQUISITES", - QUEUED = "QUEUED", - WAITING_FOR_RESOURCES = "WAITING_FOR_RESOURCES", - DISPATCHING = "DISPATCHING", - PLANNING = "PLANNING", - STARTING = "STARTING", - RUNNING = "RUNNING", - FINISHING = "FINISHING", - FINISHED = "FINISHED", - CANCELED = "CANCELED", - FAILED = "FAILED", -} - -/** - * CLP query engines. - */ -enum CLP_QUERY_ENGINES { - CLP = "clp", - CLP_S = "clp-s", - PRESTO = "presto", -} - -/** - * MongoDB document for search results metadata. `numTotalResults` is optional - * since it is only set when the search job is completed. - */ -interface SearchResultsMetadataDocument { - _id: string; - - // eslint-disable-next-line no-warning-comments - // TODO: Replace with Nullable when the `@common` directory refactoring is completed. - errorMsg: string | null; - errorName: string | null; - lastSignal: SEARCH_SIGNAL | PRESTO_SEARCH_SIGNAL; - numTotalResults?: number; - queryEngine: CLP_QUERY_ENGINES; -} - -/** - * Presto row wrapped in a `row` property to prevent conflicts with MongoDB's `_id` field. - */ -interface PrestoRowObject { - row: Record; -} - -/** - * Presto search result in MongoDB. - */ -interface PrestoSearchResult extends PrestoRowObject { - _id: string; -} - -/** - * Test TypeBox schema for testing dependency. - */ -// eslint-disable-next-line no-warning-comments -// TODO: Will be removed once shared server/client route types are migrated into common. -const TestTypeBoxSchema = Type.Object({ - type: Type.String(), -}); - -export { - CLP_QUERY_ENGINES, - PRESTO_SEARCH_SIGNAL, - SEARCH_SIGNAL, - TestTypeBoxSchema, -}; -export type { - ClientToServerEvents, - Err, - InterServerEvents, - PrestoRowObject, - PrestoSearchResult, - QueryId, - Response, - SearchResultsMetadataDocument, - ServerToClientEvents, - SocketData, -}; +export * from "./config"; +export * from "./metadata"; +export * from "./presto"; +export * from "./query"; +export * from "./socket"; +export * from "./schemas"; + +export {QUERY_JOB_TYPE} from "./query"; \ No newline at end of file diff --git a/components/webui/common/src/metadata.ts b/components/webui/common/src/metadata.ts new file mode 100644 index 0000000000..936e4a1606 --- /dev/null +++ b/components/webui/common/src/metadata.ts @@ -0,0 +1,61 @@ +import { CLP_QUERY_ENGINES } from "./config"; + +/** + * Enum of search-related signals. + * + * This includes request and response signals for various search operations and their respective + * states. + */ +enum SEARCH_SIGNAL { + NONE = "none", + + REQ_CANCELLING = "req-cancelling", + REQ_CLEARING = "req-clearing", + REQ_QUERYING = "req-querying", + + RESP_DONE = "resp-done", + RESP_QUERYING = "resp-querying", +} + +/** + * Presto search-related signals. + */ +enum PRESTO_SEARCH_SIGNAL { + WAITING_FOR_PREREQUISITES = "WAITING_FOR_PREREQUISITES", + QUEUED = "QUEUED", + WAITING_FOR_RESOURCES = "WAITING_FOR_RESOURCES", + DISPATCHING = "DISPATCHING", + PLANNING = "PLANNING", + STARTING = "STARTING", + RUNNING = "RUNNING", + FINISHING = "FINISHING", + FINISHED = "FINISHED", + CANCELED = "CANCELED", + FAILED = "FAILED", +} + + + +/** + * MongoDB document for search results metadata. `numTotalResults` is optional + * since it is only set when the search job is completed. + */ +interface SearchResultsMetadataDocument { + _id: string; + + // eslint-disable-next-line no-warning-comments + // TODO: Replace with Nullable when the `@common` directory refactoring is completed. + errorMsg: string | null; + errorName: string | null; + lastSignal: SEARCH_SIGNAL | PRESTO_SEARCH_SIGNAL; + numTotalResults?: number; + queryEngine: CLP_QUERY_ENGINES; +} + +export { + PRESTO_SEARCH_SIGNAL, + SEARCH_SIGNAL, +}; +export type { + SearchResultsMetadataDocument +}; \ No newline at end of file diff --git a/components/webui/common/src/presto.ts b/components/webui/common/src/presto.ts new file mode 100644 index 0000000000..65e06dfa5c --- /dev/null +++ b/components/webui/common/src/presto.ts @@ -0,0 +1,19 @@ +/** + * Presto row wrapped in a `row` property to prevent conflicts with MongoDB's `_id` field. + */ +interface PrestoRowObject { + row: Record; +} + +/** + * Presto search result in MongoDB. + */ +interface PrestoSearchResult extends PrestoRowObject { + _id: string; +} + +export type { + PrestoRowObject, + PrestoSearchResult +}; + diff --git a/components/webui/common/src/query.ts b/components/webui/common/src/query.ts new file mode 100644 index 0000000000..1453c68213 --- /dev/null +++ b/components/webui/common/src/query.ts @@ -0,0 +1,22 @@ +/** + * Matching the `QueryJobType` class in `job_orchestration.query_scheduler.constants`. + */ +enum QUERY_JOB_TYPE { + SEARCH_OR_AGGREGATION = 0, + EXTRACT_IR, + EXTRACT_JSON, +} + +/** + * List of valid extract job types. + */ +const EXTRACT_JOB_TYPES = new Set([ + QUERY_JOB_TYPE.EXTRACT_IR, + QUERY_JOB_TYPE.EXTRACT_JSON, +]); + +export { + EXTRACT_JOB_TYPES, + QUERY_JOB_TYPE, +}; + diff --git a/components/webui/server/src/schemas/archive-metadata.ts b/components/webui/common/src/schemas/archive-metadata.ts similarity index 100% rename from components/webui/server/src/schemas/archive-metadata.ts rename to components/webui/common/src/schemas/archive-metadata.ts diff --git a/components/webui/server/src/schemas/common.ts b/components/webui/common/src/schemas/common.ts similarity index 100% rename from components/webui/server/src/schemas/common.ts rename to components/webui/common/src/schemas/common.ts diff --git a/components/webui/server/src/schemas/error.ts b/components/webui/common/src/schemas/error.ts similarity index 100% rename from components/webui/server/src/schemas/error.ts rename to components/webui/common/src/schemas/error.ts diff --git a/components/webui/common/src/schemas/index.ts b/components/webui/common/src/schemas/index.ts new file mode 100644 index 0000000000..adca1c9913 --- /dev/null +++ b/components/webui/common/src/schemas/index.ts @@ -0,0 +1,6 @@ +export * from "./archive-metadata"; +export * from "./common"; +export * from "./error"; +export * from "./presto-search"; +export * from "./search"; +export * from "./stream-files"; \ No newline at end of file diff --git a/components/webui/server/src/schemas/presto-search.ts b/components/webui/common/src/schemas/presto-search.ts similarity index 100% rename from components/webui/server/src/schemas/presto-search.ts rename to components/webui/common/src/schemas/presto-search.ts diff --git a/components/webui/server/src/schemas/search.ts b/components/webui/common/src/schemas/search.ts similarity index 100% rename from components/webui/server/src/schemas/search.ts rename to components/webui/common/src/schemas/search.ts diff --git a/components/webui/server/src/schemas/stream-files.ts b/components/webui/common/src/schemas/stream-files.ts similarity index 91% rename from components/webui/server/src/schemas/stream-files.ts rename to components/webui/common/src/schemas/stream-files.ts index bf0c3d3edb..1e8e1b3872 100644 --- a/components/webui/server/src/schemas/stream-files.ts +++ b/components/webui/common/src/schemas/stream-files.ts @@ -1,6 +1,6 @@ import {Type} from "@sinclair/typebox"; -import {QUERY_JOB_TYPE} from "../typings/query.js"; +import {QUERY_JOB_TYPE} from "../query.js" import {StringSchema} from "./common.js"; diff --git a/components/webui/common/src/socket.ts b/components/webui/common/src/socket.ts new file mode 100644 index 0000000000..30218ec7cc --- /dev/null +++ b/components/webui/common/src/socket.ts @@ -0,0 +1,83 @@ +/** + * Unique ID for each active unique query. Multiple clients can subscribe to the same ID if the + * queries are identical. The ID is also used to represent the socket room, and MongoDB + * change stream. + */ +type QueryId = number; + +/** + * Error response to event. + */ +interface Err { + error: string; + queryId?: QueryId; +} + +/** + * Success response to event. + */ +interface Success { + data: T; +} + +/** + * Event response. + */ +type Response = Err | Success; + + +/** + * Events that the client can emit to the server. + */ +type ClientToServerEvents = { + "disconnect": () => void; + "collection::find::subscribe": ( + requestArgs: { + collectionName: string; + query: object; + options: object; + }, + callback: (res: Response<{queryId: QueryId; initialDocuments: object[]}>) => void) => void; + "collection::find::unsubscribe": ( + requestArgs: { + queryId: QueryId; + } + ) => Promise; +}; + +/** + * Events that the server can emit to the client. + */ +interface ServerToClientEvents { + // eslint-disable-next-line no-warning-comments + // TODO: Consider replacing this with `collection::find::update${number}`, which will + // limit callbacks being triggered in the client to their respective query IDs. + "collection::find::update": (respArgs: { + queryId: QueryId; + data: object[]; + }) => void; +} + +/** + * Empty but required by Socket IO. + */ +// eslint-disable-next-line @typescript-eslint/no-empty-object-type +interface InterServerEvents { +} + +/** + * Collection associated with each socket connection. + */ +interface SocketData { + collectionName?: string; +} + +export type { + ClientToServerEvents, + Err, + InterServerEvents, + QueryId, + Response, + ServerToClientEvents, + SocketData, +}; \ No newline at end of file diff --git a/components/webui/client/src/typings/common.ts b/components/webui/common/src/utility-types.ts similarity index 55% rename from components/webui/client/src/typings/common.ts rename to components/webui/common/src/utility-types.ts index 599404576f..afa0416c3d 100644 --- a/components/webui/client/src/typings/common.ts +++ b/components/webui/common/src/utility-types.ts @@ -1,3 +1,3 @@ type Nullable = T | null; -export type {Nullable}; +export type {Nullable}; \ No newline at end of file diff --git a/components/webui/server/src/plugins/app/Presto.ts b/components/webui/server/src/plugins/app/Presto.ts index 5b67e2b0ea..5ef654d5f8 100644 --- a/components/webui/server/src/plugins/app/Presto.ts +++ b/components/webui/server/src/plugins/app/Presto.ts @@ -1,4 +1,4 @@ -import {CLP_QUERY_ENGINES} from "@webui/common"; +import {CLP_QUERY_ENGINES} from "@webui/common/config"; import fp from "fastify-plugin"; import { Client, diff --git a/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts b/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts index 6e84ad5e45..63a47800d1 100644 --- a/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts +++ b/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts @@ -10,10 +10,12 @@ import settings from "../../../../settings.json" with {type: "json"}; import { QUERY_JOB_STATUS, QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, QUERY_JOBS_TABLE_COLUMN_NAMES, QueryJob, } from "../../../typings/query.js"; +import { + QUERY_JOB_TYPE, +} from "@webui/common/query"; import {JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS} from "./typings.js"; diff --git a/components/webui/server/src/plugins/app/S3Manager/index.ts b/components/webui/server/src/plugins/app/S3Manager/index.ts index 7c1b02a667..93381b9363 100644 --- a/components/webui/server/src/plugins/app/S3Manager/index.ts +++ b/components/webui/server/src/plugins/app/S3Manager/index.ts @@ -6,7 +6,7 @@ import {getSignedUrl} from "@aws-sdk/s3-request-presigner"; import fp from "fastify-plugin"; import settings from "../../../../settings.json" with {type: "json"}; -import {Nullable} from "../../../typings/common.js"; +import {Nullable} from "@webui/common/utility-types" import {PRE_SIGNED_URL_EXPIRY_TIME_SECONDS} from "./typings.js"; diff --git a/components/webui/server/src/plugins/app/StreamFileManager.ts b/components/webui/server/src/plugins/app/StreamFileManager.ts index 203f5bc873..47daed66e5 100644 --- a/components/webui/server/src/plugins/app/StreamFileManager.ts +++ b/components/webui/server/src/plugins/app/StreamFileManager.ts @@ -5,8 +5,8 @@ import { import fp from "fastify-plugin"; import settings from "../../../settings.json" with {type: "json"}; -import {Nullable} from "../../typings/common.js"; -import {QUERY_JOB_TYPE} from "../../typings/query.js"; +import {Nullable} from "@webui/common/utility-types" +import {QUERY_JOB_TYPE} from "@webui/common/query"; import { StreamFileMetadata, StreamFilesCollection, diff --git a/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/MongoWatcherCollection.ts b/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/MongoWatcherCollection.ts index 79c6d00ba6..af560d94fe 100644 --- a/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/MongoWatcherCollection.ts +++ b/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/MongoWatcherCollection.ts @@ -1,4 +1,4 @@ -import {QueryId} from "@webui/common"; +import {QueryId} from "@webui/common/socket"; import {FastifyBaseLogger} from "fastify"; import type { Collection, diff --git a/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/index.ts b/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/index.ts index c24d537413..4cd8e4ae16 100644 --- a/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/index.ts +++ b/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/index.ts @@ -13,7 +13,7 @@ import type { Response, ServerToClientEvents, SocketData, -} from "@webui/common"; +} from "@webui/common/socket"; import { FastifyBaseLogger, FastifyInstance, diff --git a/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/typings.ts b/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/typings.ts index 3a525783d9..d08ec34770 100644 --- a/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/typings.ts +++ b/components/webui/server/src/plugins/app/socket/MongoSocketIoServer/typings.ts @@ -3,7 +3,7 @@ import { InterServerEvents, ServerToClientEvents, SocketData, -} from "@webui/common"; +} from "@webui/common/socket"; import { ChangeStream, Document, diff --git a/components/webui/server/src/routes/api/archive-metadata/index.ts b/components/webui/server/src/routes/api/archive-metadata/index.ts index f52a9dc685..5b6557275d 100644 --- a/components/webui/server/src/routes/api/archive-metadata/index.ts +++ b/components/webui/server/src/routes/api/archive-metadata/index.ts @@ -4,7 +4,7 @@ import { } from "@fastify/type-provider-typebox"; import {StatusCodes} from "http-status-codes"; -import {SqlSchema} from "../../../schemas/archive-metadata.js"; +import {SqlSchema} from "@webui/common/schemas/archive-metadata" /** diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 66fb603b6a..08fd2351d1 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -3,18 +3,19 @@ import { Type, } from "@fastify/type-provider-typebox"; import { - CLP_QUERY_ENGINES, PRESTO_SEARCH_SIGNAL, type SearchResultsMetadataDocument, -} from "@webui/common"; +} from "@webui/common/metadata"; +import { + CLP_QUERY_ENGINES, +} from "@webui/common/config"; import {StatusCodes} from "http-status-codes"; - import settings from "../../../../settings.json" with {type: "json"}; -import {ErrorSchema} from "../../../schemas/error.js"; +import {ErrorSchema} from "@webui/common/schemas/error"; import { PrestoQueryJobCreationSchema, PrestoQueryJobSchema, -} from "../../../schemas/presto-search.js"; +} from "@webui/common/schemas/presto-search"; import {MAX_PRESTO_SEARCH_RESULTS} from "./typings.js"; import {insertPrestoRowsToMongo} from "./utils.js"; diff --git a/components/webui/server/src/routes/api/presto-search/utils.ts b/components/webui/server/src/routes/api/presto-search/utils.ts index 4f4f2bcfb5..316537501a 100644 --- a/components/webui/server/src/routes/api/presto-search/utils.ts +++ b/components/webui/server/src/routes/api/presto-search/utils.ts @@ -1,4 +1,4 @@ -import type {PrestoRowObject} from "@webui/common"; +import type {PrestoRowObject} from "@webui/common/presto"; import type { Db, InsertManyResult, diff --git a/components/webui/server/src/routes/api/search/index.ts b/components/webui/server/src/routes/api/search/index.ts index 1f64d5d31f..f68a011d59 100644 --- a/components/webui/server/src/routes/api/search/index.ts +++ b/components/webui/server/src/routes/api/search/index.ts @@ -3,19 +3,21 @@ import { Type, } from "@fastify/type-provider-typebox"; import { - CLP_QUERY_ENGINES, SEARCH_SIGNAL, type SearchResultsMetadataDocument, -} from "@webui/common"; +} from "@webui/common/metadata"; +import { + CLP_QUERY_ENGINES, +} from "@webui/common/config"; import {StatusCodes} from "http-status-codes"; import settings from "../../../../settings.json" with {type: "json"}; -import {ErrorSchema} from "../../../schemas/error.js"; +import {ErrorSchema} from "@webui/common/schemas/error"; import { QueryJobCreationSchema, QueryJobSchema, -} from "../../../schemas/search.js"; -import {QUERY_JOB_TYPE} from "../../../typings/query.js"; +} from "@webui/common/schemas/search"; +import {QUERY_JOB_TYPE} from "@webui/common"; import {SEARCH_MAX_NUM_RESULTS} from "./typings.js"; import { createMongoIndexes, diff --git a/components/webui/server/src/routes/api/search/typings.ts b/components/webui/server/src/routes/api/search/typings.ts index 8ab7241e5f..86ddbd4ac6 100644 --- a/components/webui/server/src/routes/api/search/typings.ts +++ b/components/webui/server/src/routes/api/search/typings.ts @@ -1,4 +1,4 @@ -import {type SearchResultsMetadataDocument} from "@webui/common"; +import {type SearchResultsMetadataDocument} from "@webui/common/metadata"; import { FastifyBaseLogger, FastifyInstance, diff --git a/components/webui/server/src/routes/api/search/utils.ts b/components/webui/server/src/routes/api/search/utils.ts index 373569860b..a37bf0e19c 100644 --- a/components/webui/server/src/routes/api/search/utils.ts +++ b/components/webui/server/src/routes/api/search/utils.ts @@ -1,4 +1,4 @@ -import {SEARCH_SIGNAL} from "@webui/common"; +import {SEARCH_SIGNAL} from "@webui/common/metadata"; import type {Db} from "mongodb"; import { diff --git a/components/webui/server/src/routes/api/stream-files/index.ts b/components/webui/server/src/routes/api/stream-files/index.ts index 599b5609f1..ced646aac2 100644 --- a/components/webui/server/src/routes/api/stream-files/index.ts +++ b/components/webui/server/src/routes/api/stream-files/index.ts @@ -2,9 +2,9 @@ import {FastifyPluginAsyncTypebox} from "@fastify/type-provider-typebox"; import {StatusCodes} from "http-status-codes"; import settings from "../../../../settings.json" with {type: "json"}; -import {ErrorSchema} from "../../../schemas/error.js"; -import {StreamFileExtractionSchema} from "../../../schemas/stream-files.js"; -import {EXTRACT_JOB_TYPES} from "../../../typings/query.js"; +import {ErrorSchema} from "@webui/common/schemas/error"; +import {StreamFileExtractionSchema} from "@webui/common/schemas/stream-files"; +import {EXTRACT_JOB_TYPES} from "@webui/common/query"; import {StreamFileMetadataSchema} from "../../../typings/stream-files.js"; diff --git a/components/webui/server/src/typings/common.ts b/components/webui/server/src/typings/common.ts deleted file mode 100644 index 10043cddc1..0000000000 --- a/components/webui/server/src/typings/common.ts +++ /dev/null @@ -1,4 +0,0 @@ -type Nullable = T | null; - - -export type {Nullable}; diff --git a/components/webui/server/src/typings/query.ts b/components/webui/server/src/typings/query.ts index 1cefbe614f..6bd4e5a3b4 100644 --- a/components/webui/server/src/typings/query.ts +++ b/components/webui/server/src/typings/query.ts @@ -1,22 +1,5 @@ import {RowDataPacket} from "mysql2/promise"; - - -/** - * Matching the `QueryJobType` class in `job_orchestration.query_scheduler.constants`. - */ -enum QUERY_JOB_TYPE { - SEARCH_OR_AGGREGATION = 0, - EXTRACT_IR, - EXTRACT_JSON, -} - -/** - * List of valid extract job types. - */ -const EXTRACT_JOB_TYPES = new Set([ - QUERY_JOB_TYPE.EXTRACT_IR, - QUERY_JOB_TYPE.EXTRACT_JSON, -]); +import {QUERY_JOB_TYPE} from "@webui/common/query" /** * Matching the `QueryJobStatus` class in @@ -64,9 +47,7 @@ interface QueryJob extends RowDataPacket { export type {QueryJob}; export { - EXTRACT_JOB_TYPES, QUERY_JOB_STATUS, QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, QUERY_JOBS_TABLE_COLUMN_NAMES, }; From 71032334e65356204048d46869097ab344ccaa5a Mon Sep 17 00:00:00 2001 From: Marco Date: Sun, 7 Sep 2025 18:08:59 +0000 Subject: [PATCH 2/6] latest --- .../client/src/api/presto-search/index.ts | 11 ++++------ .../webui/client/src/api/search/index.ts | 15 +++++-------- .../src/api/socket/MongoSocketCursor.ts | 5 ++--- .../client/src/api/socket/SocketSingleton.ts | 3 +-- .../webui/client/src/api/socket/useCursor.tsx | 3 ++- components/webui/client/src/api/sql/index.ts | 7 +------ .../client/src/api/stream-files/index.ts | 8 +++---- .../client/src/components/QueryBox/index.tsx | 2 +- .../src/pages/IngestPage/Details/Files.tsx | 2 +- .../src/pages/IngestPage/Details/Messages.tsx | 2 +- .../src/pages/IngestPage/Details/sql.ts | 2 +- .../client/src/pages/IngestPage/Jobs/sql.ts | 2 +- .../Presto/SqlQueryInput/index.tsx | 2 +- .../Presto/presto-search-requests.ts | 11 ++++++---- .../pages/SearchPage/SearchControls/index.tsx | 2 +- .../SearchControls/search-requests.ts | 10 +++++---- .../PrestoResultsVirtualTable/index.tsx | 2 +- .../pages/SearchPage/SearchState/index.tsx | 2 +- .../SearchState/usePseudoProgress.ts | 2 +- components/webui/client/src/typings/query.ts | 3 ++- components/webui/client/src/ui/Loading.tsx | 2 +- .../webui/client/src/ui/QueryStatus.tsx | 2 +- components/webui/common/eslint.config.mjs | 21 +++++++++++++++++++ components/webui/common/src/config.ts | 4 +--- components/webui/common/src/index.ts | 8 ------- components/webui/common/src/metadata.ts | 8 +++---- components/webui/common/src/presto.ts | 3 +-- components/webui/common/src/query.ts | 1 - .../common/src/schemas/archive-metadata.ts | 8 ++++++- components/webui/common/src/schemas/index.ts | 2 +- .../webui/common/src/schemas/presto-search.ts | 14 ++++++++++++- components/webui/common/src/schemas/search.ts | 13 +++++++++++- .../webui/common/src/schemas/stream-files.ts | 2 +- components/webui/common/src/socket.ts | 2 +- components/webui/common/src/utility-types.ts | 2 +- .../plugins/app/QueryJobDbManager/index.ts | 4 +--- .../server/src/plugins/app/S3Manager/index.ts | 2 +- .../src/plugins/app/StreamFileManager.ts | 4 ++-- .../src/routes/api/archive-metadata/index.ts | 3 +-- .../src/routes/api/presto-search/index.ts | 9 ++++---- .../server/src/routes/api/search/index.ts | 12 +++++------ .../src/routes/api/stream-files/index.ts | 6 +++--- components/webui/server/src/typings/query.ts | 3 ++- 43 files changed, 126 insertions(+), 105 deletions(-) delete mode 100644 components/webui/common/src/index.ts diff --git a/components/webui/client/src/api/presto-search/index.ts b/components/webui/client/src/api/presto-search/index.ts index b85cfdfa86..5b31d7bbb9 100644 --- a/components/webui/client/src/api/presto-search/index.ts +++ b/components/webui/client/src/api/presto-search/index.ts @@ -1,12 +1,9 @@ -import axios, {AxiosResponse} from "axios"; -import { Static } from '@sinclair/typebox' import { - PrestoQueryJobCreationSchema, - PrestoQueryJobSchema, -} from "@webui/common/schemas/presto-search" + type PrestoQueryJob, + type PrestoQueryJobCreation, +} from "@webui/common/schemas/presto-search"; +import axios, {AxiosResponse} from "axios"; -type PrestoQueryJobCreation = Static; -type PrestoQueryJob = Static; /** * Sends post request to server to submit presto query. diff --git a/components/webui/client/src/api/search/index.ts b/components/webui/client/src/api/search/index.ts index f4907633bc..c4e08a00e3 100644 --- a/components/webui/client/src/api/search/index.ts +++ b/components/webui/client/src/api/search/index.ts @@ -1,13 +1,8 @@ -import axios, {AxiosResponse} from "axios"; - -import { Static } from '@sinclair/typebox' import { - QueryJobCreationSchema, - QueryJobSchema, -} from "@webui/common/schemas/search" - -type QueryJobCreation = Static; -type QueryJob = Static; + type QueryJob, + type QueryJobCreation, +} from "@webui/common/schemas/search"; +import axios, {AxiosResponse} from "axios"; /** @@ -16,7 +11,7 @@ type QueryJob = Static; * @param payload * @return */ -const submitQuery = (payload: QueryJobCreation): Promise> => { +const submitQuery = (payload: QueryJobCreation): Promise> => { console.log("Submitting query:", JSON.stringify(payload)); return axios.post("/api/search/query", payload); diff --git a/components/webui/client/src/api/socket/MongoSocketCursor.ts b/components/webui/client/src/api/socket/MongoSocketCursor.ts index c66ab04b4c..52f3eae794 100644 --- a/components/webui/client/src/api/socket/MongoSocketCursor.ts +++ b/components/webui/client/src/api/socket/MongoSocketCursor.ts @@ -3,10 +3,9 @@ import { QueryId, Response, ServerToClientEvents, -} from "@webui/common/socket" +} from "@webui/common/socket"; +import {Nullable} from "@webui/common/utility-types"; import {Socket} from "socket.io-client"; -import {Nullable} from "@webui/common/utility-types" - /** diff --git a/components/webui/client/src/api/socket/SocketSingleton.ts b/components/webui/client/src/api/socket/SocketSingleton.ts index a87a7f9147..de64c88562 100644 --- a/components/webui/client/src/api/socket/SocketSingleton.ts +++ b/components/webui/client/src/api/socket/SocketSingleton.ts @@ -2,13 +2,12 @@ import { ClientToServerEvents, ServerToClientEvents, } from "@webui/common/socket"; +import {Nullable} from "@webui/common/utility-types"; import { io, Socket, } from "socket.io-client"; -import {Nullable} from "@webui/common/utility-types" - let sharedSocket: Nullable> = null; diff --git a/components/webui/client/src/api/socket/useCursor.tsx b/components/webui/client/src/api/socket/useCursor.tsx index 0022576731..c3a55547f4 100644 --- a/components/webui/client/src/api/socket/useCursor.tsx +++ b/components/webui/client/src/api/socket/useCursor.tsx @@ -4,7 +4,8 @@ import { useState, } from "react"; -import {Nullable} from "../../typings/common"; +import {Nullable} from "@webui/common/utility-types"; + import {MongoSocketCursor} from "./MongoSocketCursor.js"; diff --git a/components/webui/client/src/api/sql/index.ts b/components/webui/client/src/api/sql/index.ts index 2a65705360..7c06fa6649 100644 --- a/components/webui/client/src/api/sql/index.ts +++ b/components/webui/client/src/api/sql/index.ts @@ -1,10 +1,5 @@ +import {Sql} from "@webui/common/schemas/archive-metadata"; import axios from "axios"; -import { Static } from '@sinclair/typebox' -import { - SqlSchema, -} from "@webui/common/schemas/archive-metadata" - -type Sql = Static; /** diff --git a/components/webui/client/src/api/stream-files/index.ts b/components/webui/client/src/api/stream-files/index.ts index 933c3c20a4..b8a52be2c6 100644 --- a/components/webui/client/src/api/stream-files/index.ts +++ b/components/webui/client/src/api/stream-files/index.ts @@ -1,14 +1,12 @@ +import {QUERY_JOB_TYPE} from "@webui/common/query"; +import {Nullable} from "@webui/common/utility-types"; import axios, { AxiosProgressEvent, AxiosResponse, } from "axios"; -import {Nullable} from "@webui/common/utility-types" -import { - ExtractStreamResp, -} from "../../typings/query"; +import {ExtractStreamResp} from "../../typings/query"; -import {QUERY_JOB_TYPE} from "@webui/common/query" interface SubmitExtractStreamJobProps { dataset: Nullable; diff --git a/components/webui/client/src/components/QueryBox/index.tsx b/components/webui/client/src/components/QueryBox/index.tsx index f0b710072a..84e4e241b0 100644 --- a/components/webui/client/src/components/QueryBox/index.tsx +++ b/components/webui/client/src/components/QueryBox/index.tsx @@ -1,8 +1,8 @@ +import {Nullable} from "@webui/common/utility-types"; import { Progress, theme, } from "antd"; -import {Nullable} from "src/typings/common"; import styles from "./index.module.css"; import InputWithCaseSensitive, {InputWithCaseSensitiveProps} from "./InputWithCaseSensitive"; diff --git a/components/webui/client/src/pages/IngestPage/Details/Files.tsx b/components/webui/client/src/pages/IngestPage/Details/Files.tsx index 5e5b913719..88b47c9273 100644 --- a/components/webui/client/src/pages/IngestPage/Details/Files.tsx +++ b/components/webui/client/src/pages/IngestPage/Details/Files.tsx @@ -1,4 +1,4 @@ -import {Nullable} from "src/typings/common"; +import {Nullable} from "@webui/common/utility-types"; import {DashboardCard} from "../../../components/DashboardCard"; import Stat from "../../../components/Stat"; diff --git a/components/webui/client/src/pages/IngestPage/Details/Messages.tsx b/components/webui/client/src/pages/IngestPage/Details/Messages.tsx index a43c3c2f5d..78b5bbf332 100644 --- a/components/webui/client/src/pages/IngestPage/Details/Messages.tsx +++ b/components/webui/client/src/pages/IngestPage/Details/Messages.tsx @@ -1,4 +1,4 @@ -import {Nullable} from "src/typings/common"; +import {Nullable} from "@webui/common/utility-types"; import {DashboardCard} from "../../../components/DashboardCard"; import Stat from "../../../components/Stat"; diff --git a/components/webui/client/src/pages/IngestPage/Details/sql.ts b/components/webui/client/src/pages/IngestPage/Details/sql.ts index 99d15348d6..d20b289bd6 100644 --- a/components/webui/client/src/pages/IngestPage/Details/sql.ts +++ b/components/webui/client/src/pages/IngestPage/Details/sql.ts @@ -1,4 +1,4 @@ -import {Nullable} from "@webui/common/utility-types" +import {Nullable} from "@webui/common/utility-types"; import {querySql} from "../../../api/sql"; import {SqlTableSuffix} from "../../../config/sql-table-suffix"; diff --git a/components/webui/client/src/pages/IngestPage/Jobs/sql.ts b/components/webui/client/src/pages/IngestPage/Jobs/sql.ts index 01fbca1baa..0339113c14 100644 --- a/components/webui/client/src/pages/IngestPage/Jobs/sql.ts +++ b/components/webui/client/src/pages/IngestPage/Jobs/sql.ts @@ -1,4 +1,4 @@ -import {Nullable} from "@webui/common/utility-types" +import {Nullable} from "@webui/common/utility-types"; import {settings} from "../../../settings"; import {COMPRESSION_JOBS_TABLE_COLUMN_NAMES} from "../sqlConfig"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx index 8fe9412578..fdefc8094f 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx @@ -5,7 +5,7 @@ import { useState, } from "react"; -import {Nullable} from "src/typings/common"; +import {Nullable} from "@webui/common/utility-types"; import SqlEditor, {SqlEditorRef} from "../../../../../components/SqlEditor"; import useSearchStore from "../../../SearchState/index"; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts index 21fb7f2bdb..4661f3c7f1 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts @@ -1,8 +1,11 @@ +import { + type PrestoQueryJob, + type PrestoQueryJobCreation, +} from "@webui/common/schemas/presto-search"; + import { cancelQuery, clearQueryResults, - type PrestoQueryJobCreationSchema, - type PrestoQueryJobSchema, submitQuery, } from "../../../../api/presto-search"; import useSearchStore from "../../SearchState"; @@ -39,7 +42,7 @@ const handlePrestoClearResults = () => { * * @param payload */ -const handlePrestoQuerySubmit = (payload: PrestoQueryJobCreationSchema) => { +const handlePrestoQuerySubmit = (payload: PrestoQueryJobCreation) => { const {updateSearchJobId, updateSearchUiState, searchUiState} = useSearchStore.getState(); // User should NOT be able to submit a new query while an existing query is in progress. @@ -78,7 +81,7 @@ const handlePrestoQuerySubmit = (payload: PrestoQueryJobCreationSchema) => { * * @param payload */ -const handlePrestoQueryCancel = (payload: PrestoQueryJobSchema) => { +const handlePrestoQueryCancel = (payload: PrestoQueryJob) => { const {searchUiState, updateSearchUiState} = useSearchStore.getState(); if (searchUiState !== SEARCH_UI_STATE.QUERYING) { console.error("Cannot cancel query if there is no ongoing query."); diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx index 79413c3f77..ffd2d646ed 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -32,7 +32,7 @@ const handleSubmit = (ev: React.FormEvent) => { const SearchControls = () => { /* eslint-disable-next-line no-warning-comments */ // TODO: Remove flag and related logic when the new guide UI is fully implemented. - const isGuidedEnabled = "true" === import.meta.env[`VITE_GUIDED_DEV`]; + const isGuidedEnabled = "true" === import.meta.env["VITE_GUIDED_DEV"]; return (
diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts b/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts index 7d99fa7f9d..6329cc93be 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts +++ b/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts @@ -1,10 +1,12 @@ +import { + QueryJob, + QueryJobCreation, +} from "@webui/common/schemas/search"; import {message} from "antd"; import { cancelQuery, clearQueryResults, - QueryJobCreationSchema, - QueryJobSchema, submitQuery, } from "../../../api/search"; import { @@ -51,7 +53,7 @@ const handleClearResults = () => { * * @param payload */ -const handleQuerySubmit = (payload: QueryJobCreationSchema) => { +const handleQuerySubmit = (payload: QueryJobCreation) => { const store = useSearchStore.getState(); // User should NOT be able to submit a new query while an existing query is in progress. @@ -116,7 +118,7 @@ const handleQuerySubmit = (payload: QueryJobCreationSchema) => { * * @param payload */ -const handleQueryCancel = (payload: QueryJobSchema) => { +const handleQueryCancel = (payload: QueryJob) => { const store = useSearchStore.getState(); if (store.searchUiState !== SEARCH_UI_STATE.QUERYING) { diff --git a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/index.tsx b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/index.tsx index ca8a9657e8..ba9d5ce923 100644 --- a/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Presto/PrestoResultsVirtualTable/index.tsx @@ -3,7 +3,7 @@ import { useMemo, } from "react"; -import type {PrestoSearchResult} from "@webui/common"; +import type {PrestoSearchResult} from "@webui/common/presto"; import VirtualTable from "../../../../../../components/VirtualTable"; import useSearchStore from "../../../../SearchState/index"; diff --git a/components/webui/client/src/pages/SearchPage/SearchState/index.tsx b/components/webui/client/src/pages/SearchPage/SearchState/index.tsx index 8fc97841bf..a16ca04969 100644 --- a/components/webui/client/src/pages/SearchPage/SearchState/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchState/index.tsx @@ -1,5 +1,5 @@ +import {Nullable} from "@webui/common/utility-types"; import dayjs from "dayjs"; -import {Nullable} from "src/typings/common"; import {create} from "zustand"; import {TimelineConfig} from "../../../components/ResultsTimeline/typings"; diff --git a/components/webui/client/src/pages/SearchPage/SearchState/usePseudoProgress.ts b/components/webui/client/src/pages/SearchPage/SearchState/usePseudoProgress.ts index 7d97d73ba4..2f98ee5528 100644 --- a/components/webui/client/src/pages/SearchPage/SearchState/usePseudoProgress.ts +++ b/components/webui/client/src/pages/SearchPage/SearchState/usePseudoProgress.ts @@ -5,7 +5,7 @@ import { useState, } from "react"; -import {Nullable} from "@webui/common/utility-types" +import {Nullable} from "@webui/common/utility-types"; /** diff --git a/components/webui/client/src/typings/query.ts b/components/webui/client/src/typings/query.ts index f03f7bf6bd..0b3f386547 100644 --- a/components/webui/client/src/typings/query.ts +++ b/components/webui/client/src/typings/query.ts @@ -1,5 +1,6 @@ import {Type} from "@sinclair/typebox"; -import {QUERY_JOB_TYPE} from "@webui/common/query" +import {QUERY_JOB_TYPE} from "@webui/common/query"; + /** * Enum for the different UI states of a query's loading process. diff --git a/components/webui/client/src/ui/Loading.tsx b/components/webui/client/src/ui/Loading.tsx index e8fd77369f..eb134d7400 100644 --- a/components/webui/client/src/ui/Loading.tsx +++ b/components/webui/client/src/ui/Loading.tsx @@ -10,8 +10,8 @@ import { Typography, } from "@mui/joy"; import {DefaultColorPalette} from "@mui/joy/styles/types"; +import {Nullable} from "@webui/common/utility-types"; -import {Nullable} from "../typings/common"; import { QUERY_LOADING_STATE, QUERY_LOADING_STATE_DESCRIPTIONS, diff --git a/components/webui/client/src/ui/QueryStatus.tsx b/components/webui/client/src/ui/QueryStatus.tsx index 094da2ca83..c2a72c9342 100644 --- a/components/webui/client/src/ui/QueryStatus.tsx +++ b/components/webui/client/src/ui/QueryStatus.tsx @@ -8,10 +8,10 @@ import { AssertError, Value, } from "@sinclair/typebox/value"; +import {Nullable} from "@webui/common/utility-types"; import {isAxiosError} from "axios"; import {submitExtractStreamJob} from "../api/stream-files"; -import {Nullable} from "../typings/common"; import { EXTRACT_JOB_TYPE, ExtractJobSearchParams, diff --git a/components/webui/common/eslint.config.mjs b/components/webui/common/eslint.config.mjs index 858c71ebed..0a8a6d3f57 100644 --- a/components/webui/common/eslint.config.mjs +++ b/components/webui/common/eslint.config.mjs @@ -13,6 +13,27 @@ const EslintConfig = [ CommonConfig, ...TsConfigArray, ...StylisticConfigArray, + { + rules: { + "new-cap": [ + "error", + { + // TypeBox imports + capIsNewExceptions: [ + "Type.Any", + "Type.Enum", + "Type.Integer", + "Type.Literal", + "Type.Null", + "Type.Required", + "Type.Union", + "Value.Errors", + "Value.Parse", + ], + }, + ], + }, + }, ]; diff --git a/components/webui/common/src/config.ts b/components/webui/common/src/config.ts index 5bbc619752..1b56113857 100644 --- a/components/webui/common/src/config.ts +++ b/components/webui/common/src/config.ts @@ -7,6 +7,4 @@ enum CLP_QUERY_ENGINES { PRESTO = "presto", } -export { - CLP_QUERY_ENGINES -}; \ No newline at end of file +export {CLP_QUERY_ENGINES}; diff --git a/components/webui/common/src/index.ts b/components/webui/common/src/index.ts deleted file mode 100644 index c21c38e1a0..0000000000 --- a/components/webui/common/src/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from "./config"; -export * from "./metadata"; -export * from "./presto"; -export * from "./query"; -export * from "./socket"; -export * from "./schemas"; - -export {QUERY_JOB_TYPE} from "./query"; \ No newline at end of file diff --git a/components/webui/common/src/metadata.ts b/components/webui/common/src/metadata.ts index 936e4a1606..bc1c4d5c79 100644 --- a/components/webui/common/src/metadata.ts +++ b/components/webui/common/src/metadata.ts @@ -1,4 +1,5 @@ -import { CLP_QUERY_ENGINES } from "./config"; +import {CLP_QUERY_ENGINES} from "./config"; + /** * Enum of search-related signals. @@ -35,7 +36,6 @@ enum PRESTO_SEARCH_SIGNAL { } - /** * MongoDB document for search results metadata. `numTotalResults` is optional * since it is only set when the search job is completed. @@ -56,6 +56,4 @@ export { PRESTO_SEARCH_SIGNAL, SEARCH_SIGNAL, }; -export type { - SearchResultsMetadataDocument -}; \ No newline at end of file +export type {SearchResultsMetadataDocument}; diff --git a/components/webui/common/src/presto.ts b/components/webui/common/src/presto.ts index 65e06dfa5c..f01c5126da 100644 --- a/components/webui/common/src/presto.ts +++ b/components/webui/common/src/presto.ts @@ -14,6 +14,5 @@ interface PrestoSearchResult extends PrestoRowObject { export type { PrestoRowObject, - PrestoSearchResult + PrestoSearchResult, }; - diff --git a/components/webui/common/src/query.ts b/components/webui/common/src/query.ts index 1453c68213..8ba5eaa74c 100644 --- a/components/webui/common/src/query.ts +++ b/components/webui/common/src/query.ts @@ -19,4 +19,3 @@ export { EXTRACT_JOB_TYPES, QUERY_JOB_TYPE, }; - diff --git a/components/webui/common/src/schemas/archive-metadata.ts b/components/webui/common/src/schemas/archive-metadata.ts index 338d798197..6f65fcb4fc 100644 --- a/components/webui/common/src/schemas/archive-metadata.ts +++ b/components/webui/common/src/schemas/archive-metadata.ts @@ -1,4 +1,7 @@ -import {Type} from "@sinclair/typebox"; +import { + Static, + Type, +} from "@sinclair/typebox"; import {StringSchema} from "./common.js"; @@ -10,4 +13,7 @@ const SqlSchema = Type.Object({ queryString: StringSchema, }); +type Sql = Static; + export {SqlSchema}; +export type {Sql}; diff --git a/components/webui/common/src/schemas/index.ts b/components/webui/common/src/schemas/index.ts index adca1c9913..0437e3e63f 100644 --- a/components/webui/common/src/schemas/index.ts +++ b/components/webui/common/src/schemas/index.ts @@ -3,4 +3,4 @@ export * from "./common"; export * from "./error"; export * from "./presto-search"; export * from "./search"; -export * from "./stream-files"; \ No newline at end of file +export * from "./stream-files"; diff --git a/components/webui/common/src/schemas/presto-search.ts b/components/webui/common/src/schemas/presto-search.ts index ec4a92002d..097806b7f7 100644 --- a/components/webui/common/src/schemas/presto-search.ts +++ b/components/webui/common/src/schemas/presto-search.ts @@ -1,4 +1,7 @@ -import {Type} from "@sinclair/typebox"; +import { + Static, + Type, +} from "@sinclair/typebox"; import {StringSchema} from "./common.js"; @@ -17,7 +20,16 @@ const PrestoQueryJobSchema = Type.Object({ searchJobId: StringSchema, }); +type PrestoQueryJobCreation = Static; + +type PrestoQueryJob = Static; + + export { PrestoQueryJobCreationSchema, PrestoQueryJobSchema, }; +export type { + PrestoQueryJob, + PrestoQueryJobCreation, +}; diff --git a/components/webui/common/src/schemas/search.ts b/components/webui/common/src/schemas/search.ts index 9be62a4b66..07b6d33d84 100644 --- a/components/webui/common/src/schemas/search.ts +++ b/components/webui/common/src/schemas/search.ts @@ -1,4 +1,7 @@ -import {Type} from "@sinclair/typebox"; +import { + Static, + Type, +} from "@sinclair/typebox"; import { IdSchema, @@ -29,7 +32,15 @@ const QueryJobSchema = Type.Object({ aggregationJobId: IdSchema, }); +type QueryJobCreation = Static; + +type QueryJob = Static; + export { QueryJobCreationSchema, QueryJobSchema, }; +export type { + QueryJob, + QueryJobCreation, +}; diff --git a/components/webui/common/src/schemas/stream-files.ts b/components/webui/common/src/schemas/stream-files.ts index 1e8e1b3872..def378bfd4 100644 --- a/components/webui/common/src/schemas/stream-files.ts +++ b/components/webui/common/src/schemas/stream-files.ts @@ -1,6 +1,6 @@ import {Type} from "@sinclair/typebox"; -import {QUERY_JOB_TYPE} from "../query.js" +import {QUERY_JOB_TYPE} from "../query.js"; import {StringSchema} from "./common.js"; diff --git a/components/webui/common/src/socket.ts b/components/webui/common/src/socket.ts index 30218ec7cc..d3e102667c 100644 --- a/components/webui/common/src/socket.ts +++ b/components/webui/common/src/socket.ts @@ -80,4 +80,4 @@ export type { Response, ServerToClientEvents, SocketData, -}; \ No newline at end of file +}; diff --git a/components/webui/common/src/utility-types.ts b/components/webui/common/src/utility-types.ts index afa0416c3d..599404576f 100644 --- a/components/webui/common/src/utility-types.ts +++ b/components/webui/common/src/utility-types.ts @@ -1,3 +1,3 @@ type Nullable = T | null; -export type {Nullable}; \ No newline at end of file +export type {Nullable}; diff --git a/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts b/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts index 63a47800d1..f14f728ec1 100644 --- a/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts +++ b/components/webui/server/src/plugins/app/QueryJobDbManager/index.ts @@ -2,6 +2,7 @@ import {setTimeout} from "node:timers/promises"; import type {MySQLPromisePool} from "@fastify/mysql"; import {encode} from "@msgpack/msgpack"; +import {QUERY_JOB_TYPE} from "@webui/common/query"; import {FastifyInstance} from "fastify"; import fp from "fastify-plugin"; import {ResultSetHeader} from "mysql2"; @@ -13,9 +14,6 @@ import { QUERY_JOBS_TABLE_COLUMN_NAMES, QueryJob, } from "../../../typings/query.js"; -import { - QUERY_JOB_TYPE, -} from "@webui/common/query"; import {JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS} from "./typings.js"; diff --git a/components/webui/server/src/plugins/app/S3Manager/index.ts b/components/webui/server/src/plugins/app/S3Manager/index.ts index 93381b9363..bea5c995bf 100644 --- a/components/webui/server/src/plugins/app/S3Manager/index.ts +++ b/components/webui/server/src/plugins/app/S3Manager/index.ts @@ -3,10 +3,10 @@ import { S3Client, } from "@aws-sdk/client-s3"; import {getSignedUrl} from "@aws-sdk/s3-request-presigner"; +import {Nullable} from "@webui/common/utility-types"; import fp from "fastify-plugin"; import settings from "../../../../settings.json" with {type: "json"}; -import {Nullable} from "@webui/common/utility-types" import {PRE_SIGNED_URL_EXPIRY_TIME_SECONDS} from "./typings.js"; diff --git a/components/webui/server/src/plugins/app/StreamFileManager.ts b/components/webui/server/src/plugins/app/StreamFileManager.ts index 47daed66e5..aff49ef87f 100644 --- a/components/webui/server/src/plugins/app/StreamFileManager.ts +++ b/components/webui/server/src/plugins/app/StreamFileManager.ts @@ -1,3 +1,5 @@ +import {QUERY_JOB_TYPE} from "@webui/common/query"; +import {Nullable} from "@webui/common/utility-types"; import { FastifyBaseLogger, FastifyInstance, @@ -5,8 +7,6 @@ import { import fp from "fastify-plugin"; import settings from "../../../settings.json" with {type: "json"}; -import {Nullable} from "@webui/common/utility-types" -import {QUERY_JOB_TYPE} from "@webui/common/query"; import { StreamFileMetadata, StreamFilesCollection, diff --git a/components/webui/server/src/routes/api/archive-metadata/index.ts b/components/webui/server/src/routes/api/archive-metadata/index.ts index 5b6557275d..8d1076ab26 100644 --- a/components/webui/server/src/routes/api/archive-metadata/index.ts +++ b/components/webui/server/src/routes/api/archive-metadata/index.ts @@ -2,10 +2,9 @@ import { FastifyPluginAsyncTypebox, Type, } from "@fastify/type-provider-typebox"; +import {SqlSchema} from "@webui/common/schemas/archive-metadata"; import {StatusCodes} from "http-status-codes"; -import {SqlSchema} from "@webui/common/schemas/archive-metadata" - /** * Archive metadata API routes. diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 08fd2351d1..4dd4331a4c 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -2,20 +2,19 @@ import { FastifyPluginAsyncTypebox, Type, } from "@fastify/type-provider-typebox"; +import {CLP_QUERY_ENGINES} from "@webui/common/config"; import { PRESTO_SEARCH_SIGNAL, type SearchResultsMetadataDocument, } from "@webui/common/metadata"; -import { - CLP_QUERY_ENGINES, -} from "@webui/common/config"; -import {StatusCodes} from "http-status-codes"; -import settings from "../../../../settings.json" with {type: "json"}; import {ErrorSchema} from "@webui/common/schemas/error"; import { PrestoQueryJobCreationSchema, PrestoQueryJobSchema, } from "@webui/common/schemas/presto-search"; +import {StatusCodes} from "http-status-codes"; + +import settings from "../../../../settings.json" with {type: "json"}; import {MAX_PRESTO_SEARCH_RESULTS} from "./typings.js"; import {insertPrestoRowsToMongo} from "./utils.js"; diff --git a/components/webui/server/src/routes/api/search/index.ts b/components/webui/server/src/routes/api/search/index.ts index f68a011d59..1090415966 100644 --- a/components/webui/server/src/routes/api/search/index.ts +++ b/components/webui/server/src/routes/api/search/index.ts @@ -2,22 +2,20 @@ import { FastifyPluginAsyncTypebox, Type, } from "@fastify/type-provider-typebox"; +import {CLP_QUERY_ENGINES} from "@webui/common/config"; import { SEARCH_SIGNAL, type SearchResultsMetadataDocument, } from "@webui/common/metadata"; -import { - CLP_QUERY_ENGINES, -} from "@webui/common/config"; -import {StatusCodes} from "http-status-codes"; - -import settings from "../../../../settings.json" with {type: "json"}; +import {QUERY_JOB_TYPE} from "@webui/common/query"; import {ErrorSchema} from "@webui/common/schemas/error"; import { QueryJobCreationSchema, QueryJobSchema, } from "@webui/common/schemas/search"; -import {QUERY_JOB_TYPE} from "@webui/common"; +import {StatusCodes} from "http-status-codes"; + +import settings from "../../../../settings.json" with {type: "json"}; import {SEARCH_MAX_NUM_RESULTS} from "./typings.js"; import { createMongoIndexes, diff --git a/components/webui/server/src/routes/api/stream-files/index.ts b/components/webui/server/src/routes/api/stream-files/index.ts index ced646aac2..ac6002cfca 100644 --- a/components/webui/server/src/routes/api/stream-files/index.ts +++ b/components/webui/server/src/routes/api/stream-files/index.ts @@ -1,10 +1,10 @@ import {FastifyPluginAsyncTypebox} from "@fastify/type-provider-typebox"; +import {EXTRACT_JOB_TYPES} from "@webui/common/query"; +import {ErrorSchema} from "@webui/common/schemas/error"; +import {StreamFileExtractionSchema} from "@webui/common/schemas/stream-files"; import {StatusCodes} from "http-status-codes"; import settings from "../../../../settings.json" with {type: "json"}; -import {ErrorSchema} from "@webui/common/schemas/error"; -import {StreamFileExtractionSchema} from "@webui/common/schemas/stream-files"; -import {EXTRACT_JOB_TYPES} from "@webui/common/query"; import {StreamFileMetadataSchema} from "../../../typings/stream-files.js"; diff --git a/components/webui/server/src/typings/query.ts b/components/webui/server/src/typings/query.ts index 6bd4e5a3b4..5a08635ced 100644 --- a/components/webui/server/src/typings/query.ts +++ b/components/webui/server/src/typings/query.ts @@ -1,5 +1,6 @@ +import {QUERY_JOB_TYPE} from "@webui/common/query"; import {RowDataPacket} from "mysql2/promise"; -import {QUERY_JOB_TYPE} from "@webui/common/query" + /** * Matching the `QueryJobStatus` class in From 90e72fe75ed204d6454b4b0e62c41601cd36d31a Mon Sep 17 00:00:00 2001 From: Marco Date: Sun, 7 Sep 2025 18:14:09 +0000 Subject: [PATCH 3/6] latest --- components/webui/client/src/pages/IngestPage/Jobs/sql.ts | 2 +- .../client/src/pages/SearchPage/SearchControls/index.tsx | 2 +- .../src/pages/SearchPage/SearchControls/search-requests.ts | 4 ++-- components/webui/common/package.json | 1 - components/webui/common/src/schemas/index.ts | 6 ------ 5 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 components/webui/common/src/schemas/index.ts diff --git a/components/webui/client/src/pages/IngestPage/Jobs/sql.ts b/components/webui/client/src/pages/IngestPage/Jobs/sql.ts index 0339113c14..cd1dc20e68 100644 --- a/components/webui/client/src/pages/IngestPage/Jobs/sql.ts +++ b/components/webui/client/src/pages/IngestPage/Jobs/sql.ts @@ -22,7 +22,7 @@ SELECT ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.UNCOMPRESSED_SIZE}, ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.COMPRESSED_SIZE} FROM ${settings.SqlDbCompressionJobsTableName} -WHERE ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.UPDATE_TIME} >= +WHERE ${COMPRESSION_JOBS_TABLE_COLUMN_NAMES.UPDATE_TIME} >= FROM_UNIXTIME(${lastUpdateTimestampSeconds}) - 1 ORDER BY _id DESC;`; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx index ffd2d646ed..79413c3f77 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -32,7 +32,7 @@ const handleSubmit = (ev: React.FormEvent) => { const SearchControls = () => { /* eslint-disable-next-line no-warning-comments */ // TODO: Remove flag and related logic when the new guide UI is fully implemented. - const isGuidedEnabled = "true" === import.meta.env["VITE_GUIDED_DEV"]; + const isGuidedEnabled = "true" === import.meta.env[`VITE_GUIDED_DEV`]; return ( diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts b/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts index 6329cc93be..6938a49aef 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts +++ b/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts @@ -1,6 +1,6 @@ import { - QueryJob, - QueryJobCreation, + type QueryJob, + type QueryJobCreation, } from "@webui/common/schemas/search"; import {message} from "antd"; diff --git a/components/webui/common/package.json b/components/webui/common/package.json index b5f0dd25da..9e9e612d52 100644 --- a/components/webui/common/package.json +++ b/components/webui/common/package.json @@ -1,7 +1,6 @@ { "name": "@webui/common", "version": "0.1.0", - "files": ["dist"], "exports": { "./*": { "types": "./dist/*.d.ts", diff --git a/components/webui/common/src/schemas/index.ts b/components/webui/common/src/schemas/index.ts deleted file mode 100644 index 0437e3e63f..0000000000 --- a/components/webui/common/src/schemas/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from "./archive-metadata"; -export * from "./common"; -export * from "./error"; -export * from "./presto-search"; -export * from "./search"; -export * from "./stream-files"; From 85d27dfdc04551eae19c40dd3286f24ce4e55e0e Mon Sep 17 00:00:00 2001 From: Marco Date: Mon, 8 Sep 2025 14:38:38 +0000 Subject: [PATCH 4/6] latest --- components/webui/client/src/api/sql/index.ts | 7 +++---- .../SearchControls/SearchButton/CancelButton.tsx | 5 ++++- .../client/src/pages/SearchPage/SearchControls/index.tsx | 2 +- .../pages/SearchPage/SearchControls/search-requests.ts | 9 ++++++--- components/webui/common/src/metadata.ts | 8 +++----- components/webui/common/src/schemas/archive-metadata.ts | 7 +------ 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/components/webui/client/src/api/sql/index.ts b/components/webui/client/src/api/sql/index.ts index 7c06fa6649..3fd2b39928 100644 --- a/components/webui/client/src/api/sql/index.ts +++ b/components/webui/client/src/api/sql/index.ts @@ -1,15 +1,14 @@ -import {Sql} from "@webui/common/schemas/archive-metadata"; import axios from "axios"; /** * Query the SQL server with the queryString. * - * @param payload + * @param queryString * @return */ -const querySql = async (payload: Sql) => { - return axios.post("/api/archive-metadata/sql", payload); +const querySql = async (queryString: string) => { + return axios.post("/api/archive-metadata/sql", {queryString}); }; export {querySql}; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx index 36984d801d..6416e2be04 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx @@ -32,7 +32,10 @@ const CancelButton = () => { return; } handleQueryCancel( - {searchJobId, aggregationJobId} + { + searchJobId: Number(searchJobId), + aggregationJobId: Number(aggregationJobId), + } ); }, [searchJobId, aggregationJobId]); diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx index 79413c3f77..ffd2d646ed 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -32,7 +32,7 @@ const handleSubmit = (ev: React.FormEvent) => { const SearchControls = () => { /* eslint-disable-next-line no-warning-comments */ // TODO: Remove flag and related logic when the new guide UI is fully implemented. - const isGuidedEnabled = "true" === import.meta.env[`VITE_GUIDED_DEV`]; + const isGuidedEnabled = "true" === import.meta.env["VITE_GUIDED_DEV"]; return ( diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts b/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts index 6938a49aef..998e49ec3a 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts +++ b/components/webui/client/src/pages/SearchPage/SearchControls/search-requests.ts @@ -42,7 +42,10 @@ const handleClearResults = () => { } clearQueryResults( - {searchJobId, aggregationJobId} + { + searchJobId: Number(searchJobId), + aggregationJobId: Number(aggregationJobId), + } ).catch((err: unknown) => { console.error("Failed to clear query results:", err); }); @@ -97,8 +100,8 @@ const handleQuerySubmit = (payload: QueryJobCreation) => { submitQuery(payload) .then((result) => { const {searchJobId, aggregationJobId} = result.data; - store.updateSearchJobId(searchJobId); - store.updateAggregationJobId(aggregationJobId); + store.updateSearchJobId(searchJobId.toString()); + store.updateAggregationJobId(aggregationJobId.toString()); store.updateSearchUiState(SEARCH_UI_STATE.QUERYING); console.debug( "Search job created - ", diff --git a/components/webui/common/src/metadata.ts b/components/webui/common/src/metadata.ts index bc1c4d5c79..38c3dcbcf2 100644 --- a/components/webui/common/src/metadata.ts +++ b/components/webui/common/src/metadata.ts @@ -1,4 +1,5 @@ import {CLP_QUERY_ENGINES} from "./config"; +import {Nullable} from "./utility-types"; /** @@ -42,11 +43,8 @@ enum PRESTO_SEARCH_SIGNAL { */ interface SearchResultsMetadataDocument { _id: string; - - // eslint-disable-next-line no-warning-comments - // TODO: Replace with Nullable when the `@common` directory refactoring is completed. - errorMsg: string | null; - errorName: string | null; + errorMsg: Nullable; + errorName: Nullable; lastSignal: SEARCH_SIGNAL | PRESTO_SEARCH_SIGNAL; numTotalResults?: number; queryEngine: CLP_QUERY_ENGINES; diff --git a/components/webui/common/src/schemas/archive-metadata.ts b/components/webui/common/src/schemas/archive-metadata.ts index 6f65fcb4fc..a9fc2ccab5 100644 --- a/components/webui/common/src/schemas/archive-metadata.ts +++ b/components/webui/common/src/schemas/archive-metadata.ts @@ -1,7 +1,4 @@ -import { - Static, - Type, -} from "@sinclair/typebox"; +import {Type} from "@sinclair/typebox"; import {StringSchema} from "./common.js"; @@ -13,7 +10,5 @@ const SqlSchema = Type.Object({ queryString: StringSchema, }); -type Sql = Static; export {SqlSchema}; -export type {Sql}; From 9a6001514add9449bb90e4697a53e96186834d01 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 11 Sep 2025 14:40:50 +0000 Subject: [PATCH 5/6] latest --- components/webui/client/eslint.config.mjs | 8 ++------ components/webui/common/eslint.config.mjs | 12 +----------- components/webui/server/eslint.config.mjs | 12 +----------- 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/components/webui/client/eslint.config.mjs b/components/webui/client/eslint.config.mjs index ea24b4121c..9dc0ed82c5 100644 --- a/components/webui/client/eslint.config.mjs +++ b/components/webui/client/eslint.config.mjs @@ -31,16 +31,12 @@ const EslintConfig = [ "new-cap": [ "error", { + // TypeBox imports capIsNewExceptions: [ - // TypeBox imports "Decode", "Encode", - "Type.Literal", - "Type.Optional", - "Type.Transform", - "Type.Union", - "Value.Parse", ], + capIsNewExceptionPattern: "^(Type|Value)\\.", }, ], }, diff --git a/components/webui/common/eslint.config.mjs b/components/webui/common/eslint.config.mjs index 0a8a6d3f57..8388c255b3 100644 --- a/components/webui/common/eslint.config.mjs +++ b/components/webui/common/eslint.config.mjs @@ -19,17 +19,7 @@ const EslintConfig = [ "error", { // TypeBox imports - capIsNewExceptions: [ - "Type.Any", - "Type.Enum", - "Type.Integer", - "Type.Literal", - "Type.Null", - "Type.Required", - "Type.Union", - "Value.Errors", - "Value.Parse", - ], + capIsNewExceptionPattern: "^(Type|Value)\\.", }, ], }, diff --git a/components/webui/server/eslint.config.mjs b/components/webui/server/eslint.config.mjs index 214f3aae69..5782f411c4 100644 --- a/components/webui/server/eslint.config.mjs +++ b/components/webui/server/eslint.config.mjs @@ -19,17 +19,7 @@ const EslintConfig = [ "error", { // TypeBox imports - capIsNewExceptions: [ - "Type.Any", - "Type.Enum", - "Type.Integer", - "Type.Literal", - "Type.Null", - "Type.Required", - "Type.Union", - "Value.Errors", - "Value.Parse", - ], + capIsNewExceptionPattern: "^(Type|Value)\\.", }, ], }, From c09333da1e052505688a70f7341fcb7330069430 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 11 Sep 2025 20:20:17 +0000 Subject: [PATCH 6/6] latest --- .../SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx index b335575f51..283d07ca8a 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/SqlQueryInput/index.tsx @@ -4,8 +4,10 @@ import { useRef, useState, } from "react"; + import {Nullable} from "@webui/common/utility-types"; -import SqlEditor, {SqlEditorRef} from "../../../../../components/SqlEditor"; + +import SqlEditor, {SqlEditorType} from "../../../../../components/SqlEditor"; import useSearchStore from "../../../SearchState/index"; import {SEARCH_UI_STATE} from "../../../SearchState/typings"; import styles from "./index.module.css";