From 206f75557d6d4cddf079aff1142b810ebb8d8919 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 8 May 2025 01:25:04 +0000 Subject: [PATCH 01/55] saving --- .../client/tsconfig/tsconfig.app.json | 2 +- .../log-viewer-webui/client/vite.config.ts | 4 +- components/log-viewer-webui/server/.env | 13 +- .../log-viewer-webui/server/package-lock.json | 119 +++++++++++++++ .../log-viewer-webui/server/package.json | 9 +- .../log-viewer-webui/server/settings.json | 18 ++- .../log-viewer-webui/server/src/app.test.ts | 8 +- components/log-viewer-webui/server/src/app.ts | 46 ++---- .../log-viewer-webui/server/src/main.ts | 17 ++- .../server/src/newWebui/app.ts | 101 +++++++++++++ .../server/src/newWebui/old.ts | 111 ++++++++++++++ .../plugins/app/QueryJobsDbManager.ts | 143 ++++++++++++++++++ .../app/SearchJobCollectionsManager.ts | 71 +++++++++ .../src/newWebui/plugins/app/constants.ts | 131 ++++++++++++++++ .../src/newWebui/plugins/external/env.ts | 100 ++++++++++++ .../src/newWebui/plugins/external/mongoDb.ts | 21 +++ .../src/newWebui/plugins/external/mySql.ts | 31 ++++ .../newWebui/plugins/external/rate-limit.ts | 17 +++ .../server/src/newWebui/schemas/QueryJobs.ts | 11 ++ .../server/src/newWebui/schemas/common.ts | 5 + .../server/src/newWebui/schemas/test. | 11 ++ .../server/src/plugins/DbManager.ts | 7 +- .../log-viewer-webui/server/src/server.ts | 67 ++++++++ .../log-viewer-webui/server/src/utils/misc.ts | 37 +++++ .../log-viewer-webui/server/tsconfig.json | 2 +- 25 files changed, 1051 insertions(+), 51 deletions(-) create mode 100644 components/log-viewer-webui/server/src/newWebui/app.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/old.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/app/SearchJobCollectionsManager.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/app/constants.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/external/env.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/external/mongoDb.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/external/mySql.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/external/rate-limit.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/schemas/QueryJobs.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/schemas/common.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/schemas/test. create mode 100644 components/log-viewer-webui/server/src/server.ts create mode 100644 components/log-viewer-webui/server/src/utils/misc.ts diff --git a/components/log-viewer-webui/client/tsconfig/tsconfig.app.json b/components/log-viewer-webui/client/tsconfig/tsconfig.app.json index 059a6f365a..10d7cc2e4b 100644 --- a/components/log-viewer-webui/client/tsconfig/tsconfig.app.json +++ b/components/log-viewer-webui/client/tsconfig/tsconfig.app.json @@ -2,7 +2,7 @@ "extends": "./tsconfig.base.json", "include": [ "../src" - ], +, "../../IngestPage3" ], "compilerOptions": { "lib": [ diff --git a/components/log-viewer-webui/client/vite.config.ts b/components/log-viewer-webui/client/vite.config.ts index 9b2428dc87..05473e40ba 100644 --- a/components/log-viewer-webui/client/vite.config.ts +++ b/components/log-viewer-webui/client/vite.config.ts @@ -8,12 +8,12 @@ export default defineConfig({ plugins: [react()], publicDir: "public", server: { - port: 8080, + port: 3001, proxy: { "/query/": { // Below target should match the server's configuration in // `components/log-viewer-webui/server/.env` (or `.env.local` if overridden) - target: "http://localhost:3000/", + target: "http://localhost:3001/", changeOrigin: true, }, }, diff --git a/components/log-viewer-webui/server/.env b/components/log-viewer-webui/server/.env index 95e3054a83..fee1e78126 100644 --- a/components/log-viewer-webui/server/.env +++ b/components/log-viewer-webui/server/.env @@ -1,8 +1,15 @@ +# Must always set to production +# @see {@link https://www.youtube.com/watch?v=HMM7GJC5E2o} +NODE_ENV=production + CLIENT_DIR=../client/dist -STREAMS_DATA_DIR=../../../build/clp-package/var/data/streams +STREAMS_DATA_DIR=../../../../build/clp-package/var/data/streams LOG_VIEWER_DIR=../yscope-log-viewer/dist HOST=localhost -PORT=3000 +PORT=3001 CLP_DB_USER=clp-user -CLP_DB_PASS= +CLP_DB_PASS=mGltCMuLSaM + + +/home/davidmawk/clp/build \ No newline at end of file diff --git a/components/log-viewer-webui/server/package-lock.json b/components/log-viewer-webui/server/package-lock.json index 2c940e588c..c886322bbb 100644 --- a/components/log-viewer-webui/server/package-lock.json +++ b/components/log-viewer-webui/server/package-lock.json @@ -11,13 +11,17 @@ "dependencies": { "@aws-sdk/client-s3": "^3.758.0", "@aws-sdk/s3-request-presigner": "^3.758.0", + "@fastify/autoload": "^6.3.0", + "@fastify/env": "^5.0.2", "@fastify/mongodb": "^9.0.2", "@fastify/mysql": "^5.0.2", + "@fastify/rate-limit": "^10.2.2", "@fastify/static": "^8.1.1", "@fastify/type-provider-typebox": "^5.1.0", "@msgpack/msgpack": "^3.1.0", "@sinclair/typebox": "^0.34.28", "@types/node": "^22.13.9", + "close-with-grace": "^2.2.0", "dotenv": "^16.4.7", "fastify": "^5.2.1", "fastify-plugin": "^5.0.1", @@ -1264,6 +1268,42 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "license": "MIT" }, + "node_modules/@fastify/autoload": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/@fastify/autoload/-/autoload-6.3.0.tgz", + "integrity": "sha512-VBYApgvqX5e+M2BAWKCxrYk9yTo8z127xsSzsSePQZlxt97tbQay74n//NDFK4ovFMgyhKSjoWtHTbdGw/RDvQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/@fastify/env": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@fastify/env/-/env-5.0.2.tgz", + "integrity": "sha512-4m/jHS3s/G/DBJVODob9sxGUei/Ij8JFbA2PYqBfoihTm+Qqae2xD9xhez68UFZu1d4SNJPIb6uAOwbNvRYw+A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "env-schema": "^6.0.0", + "fastify-plugin": "^5.0.0" + } + }, "node_modules/@fastify/error": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/@fastify/error/-/error-4.0.0.tgz", @@ -1364,6 +1404,27 @@ "ipaddr.js": "^2.1.0" } }, + "node_modules/@fastify/rate-limit": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/@fastify/rate-limit/-/rate-limit-10.2.2.tgz", + "integrity": "sha512-45vXZImiYthKlMohF4XoHXYiBXCyRYY+zmtjLZuQrGraW0Zj9hYPYNOIa47012+5A65M0KJQxIVbzYCNP90hcg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@lukeed/ms": "^2.0.2", + "fastify-plugin": "^5.0.0", + "toad-cache": "^3.7.0" + } + }, "node_modules/@fastify/send": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/@fastify/send/-/send-3.3.1.tgz", @@ -5047,6 +5108,12 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/close-with-grace": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/close-with-grace/-/close-with-grace-2.2.0.tgz", + "integrity": "sha512-OdcFxnxTm/AMLPHA4Aq3J1BLpkojXP7I4G5QBQLN5TT55ED/rk04rAoDbtfNnfZ988kGXPxh1bdRLeIU9bz/lA==", + "license": "MIT" + }, "node_modules/code-excerpt": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-4.0.0.tgz", @@ -5373,6 +5440,15 @@ "url": "https://dotenvx.com" } }, + "node_modules/dotenv-expand": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", + "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -5446,6 +5522,49 @@ "node": ">=6" } }, + "node_modules/env-schema": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/env-schema/-/env-schema-6.0.1.tgz", + "integrity": "sha512-WRD40Q25pP4NUbI3g3CNU5PPzcaiX7YYcPwiCZlfR4qGsKmTlckRixgHww0/fOXiXSNKA87pwshzq0ULTK/48A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "ajv": "^8.12.0", + "dotenv": "^16.4.5", + "dotenv-expand": "10.0.0" + } + }, + "node_modules/env-schema/node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/env-schema/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, "node_modules/environment": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", diff --git a/components/log-viewer-webui/server/package.json b/components/log-viewer-webui/server/package.json index d34c7c9572..a562ae9bf3 100644 --- a/components/log-viewer-webui/server/package.json +++ b/components/log-viewer-webui/server/package.json @@ -8,9 +8,10 @@ "build:watch": "npm run build -- --watch", "lint:check": "eslint . --max-warnings 0", "lint:fix": "npm run lint:check -- --fix", - "prod": "NODE_ENV=production node dist/src/main.js", + "prod": "NODE_ENV=production node dist/src/server.js", "start": "NODE_ENV=development concurrently \"npm run build:watch\" \"nodemon dist/src/main.js\"", - "test": "NODE_ENV=test tap" + "test": "NODE_ENV=test tap", + "standalone": "NODE_ENV=production npm run build && node --env-file=.env dist/src/server.js" }, "author": "YScope Inc. ", "license": "Apache-2.0", @@ -18,13 +19,17 @@ "dependencies": { "@aws-sdk/client-s3": "^3.758.0", "@aws-sdk/s3-request-presigner": "^3.758.0", + "@fastify/autoload": "^6.3.0", + "@fastify/env": "^5.0.2", "@fastify/mongodb": "^9.0.2", "@fastify/mysql": "^5.0.2", + "@fastify/rate-limit": "^10.2.2", "@fastify/static": "^8.1.1", "@fastify/type-provider-typebox": "^5.1.0", "@msgpack/msgpack": "^3.1.0", "@sinclair/typebox": "^0.34.28", "@types/node": "^22.13.9", + "close-with-grace": "^2.2.0", "dotenv": "^16.4.7", "fastify": "^5.2.1", "fastify-plugin": "^5.0.1", diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json index 0372e662c6..70e8a40bb3 100644 --- a/components/log-viewer-webui/server/settings.json +++ b/components/log-viewer-webui/server/settings.json @@ -3,6 +3,10 @@ "SqlDbPort": 3306, "SqlDbName": "clp-db", "SqlDbQueryJobsTableName": "query_jobs", + "SqlDbClpArchivesTableName": "clp_archives", + "SqlDbClpFilesTableName": "clp_files", + "SqlDbCompressionJobsTableName": "compression_jobs", + "MongoDbHost": "localhost", "MongoDbPort": 27017, "MongoDbName": "clp-query-results", @@ -14,5 +18,17 @@ "StreamTargetUncompressedSize": 134217728, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, - "StreamFilesS3Profile": null + "StreamFilesS3Profile": null, + + + "public": { + "AggregationResultsCollectionName": "aggregation-results", + "ClpStorageEngine": "clp", + "CompressionJobsCollectionName": "compression-jobs", + "LogViewerWebuiUrl": "http://localhost:8080", + "SearchResultsCollectionName": "search-results", + "SearchResultsMetadataCollectionName": "results-metadata", + "StatsCollectionName": "stats", + "SupportUrl": "https://github.com/y-scope/clp/issues/new/choose" + } } diff --git a/components/log-viewer-webui/server/src/app.test.ts b/components/log-viewer-webui/server/src/app.test.ts index 76ad84673e..f57954c0b6 100644 --- a/components/log-viewer-webui/server/src/app.test.ts +++ b/components/log-viewer-webui/server/src/app.test.ts @@ -2,10 +2,16 @@ import {StatusCodes} from "http-status-codes"; import tap from "tap"; import app from "./app.js"; +import fastify from 'fastify'; tap.test("Tests the example routes", async (t) => { - const server = await app({fastifyOptions: {}, sqlDbPass: "", sqlDbUser: ""}); + const server = fastify(); + + await server.register(app, { + sqlDbUser: process.env.CLP_DB_USER!, + sqlDbPass: process.env.CLP_DB_PASS!, + }); t.teardown(() => server.close()); let resp = await server.inject({ diff --git a/components/log-viewer-webui/server/src/app.ts b/components/log-viewer-webui/server/src/app.ts index b5b928c2d8..bac5d2c343 100644 --- a/components/log-viewer-webui/server/src/app.ts +++ b/components/log-viewer-webui/server/src/app.ts @@ -1,9 +1,7 @@ import process from "node:process"; import { - fastify, - FastifyInstance, - FastifyServerOptions, + FastifyPluginAsync } from "fastify"; import settings from "../settings.json" with {type: "json"}; @@ -13,30 +11,17 @@ import exampleRoutes from "./routes/example.js"; import queryRoutes from "./routes/query.js"; import staticRoutes from "./routes/static.js"; - -interface AppProps { - fastifyOptions: FastifyServerOptions; +interface AppPluginOptions { sqlDbUser: string; sqlDbPass: string; } -/** - * Creates the Fastify app with the given options. - * - * @param props - * @param props.fastifyOptions - * @param props.sqlDbUser - * @param props.sqlDbPass - * @return The created Fastify instance. - */ -const app = async ({ - fastifyOptions, - sqlDbUser, - sqlDbPass, -}: AppProps): Promise => { - const server = fastify(fastifyOptions); +const AppPlugin: FastifyPluginAsync = async (server, opts) => { + const { sqlDbUser, sqlDbPass } = opts; - if ("test" !== process.env.NODE_ENV) { + // Ensure environment is not "test" + if (process.env.NODE_ENV !== "test") { + // Register DbManager and S3Manager plugins await server.register(DbManager, { mysqlConfig: { database: settings.SqlDbName, @@ -53,20 +38,17 @@ const app = async ({ port: settings.MongoDbPort, }, }); - await server.register( - S3Manager, - { - region: settings.StreamFilesS3Region, - profile: settings.StreamFilesS3Profile, - } - ); + + await server.register(S3Manager, { + region: settings.StreamFilesS3Region, + profile: settings.StreamFilesS3Profile, + }); } + // Register the routes await server.register(staticRoutes); await server.register(exampleRoutes); await server.register(queryRoutes); - - return server; }; -export default app; +export default AppPlugin; \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/main.ts b/components/log-viewer-webui/server/src/main.ts index fc9e6fd33c..29ec326f9a 100644 --- a/components/log-viewer-webui/server/src/main.ts +++ b/components/log-viewer-webui/server/src/main.ts @@ -6,6 +6,9 @@ import { parseEnvVars, } from "./utils/env.js"; +import fastify from 'fastify'; + + /** * Sets up and runs the server. @@ -13,14 +16,16 @@ import { const main = async () => { const envVars = parseEnvVars(); const loggerConfig = ENV_TO_LOGGER_MAP[envVars.NODE_ENV]; - const server = await app({ - fastifyOptions: { - ...(loggerConfig && {logger: loggerConfig}), - }, - sqlDbPass: envVars.CLP_DB_PASS, - sqlDbUser: envVars.CLP_DB_USER, + + const server = fastify({ + ...(loggerConfig && { logger: loggerConfig }), }); + await server.register(app, { + sqlDbUser: process.env.CLP_DB_USER!, + sqlDbPass: process.env.CLP_DB_PASS!, + }); + try { await server.listen({host: envVars.HOST, port: Number(envVars.PORT)}); } catch (e) { diff --git a/components/log-viewer-webui/server/src/newWebui/app.ts b/components/log-viewer-webui/server/src/newWebui/app.ts new file mode 100644 index 0000000000..83892266be --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/app.ts @@ -0,0 +1,101 @@ +//https://github.com/fastify/demo/blob/main/src/app.ts + +import path from 'node:path' +import fastifyAutoload from '@fastify/autoload' +import { FastifyInstance, FastifyPluginOptions } from 'fastify' + +import OldApp from "../app.js"; + +export const options = { + ajv: { + customOptions: { + coerceTypes: 'array', + removeAdditional: 'all' + } + } +} + +export default async function serviceApp ( + fastify: FastifyInstance, + opts: FastifyPluginOptions +) { + delete opts.skipOverride // This option only serves testing purpose + // This loads all external plugins defined in plugins/external + // those should be registered first as your application plugins might depend on them + await fastify.register(fastifyAutoload, { + dir: path.join(import.meta.dirname, 'plugins/external'), + options: { ...opts } + }) + + await fastify.register(OldApp, { + sqlDbUser: fastify.config.CLP_DB_USER, + sqlDbPass: fastify.config.CLP_DB_PASS, + }) + + // This loads all your application plugins defined in plugins/app + // those should be support plugins that are reused + // through your application + //fastify.register(fastifyAutoload, { + // dir: path.join(import.meta.dirname, 'plugins/app'), + // options: { ...opts } + //}) + + // This loads all plugins defined in routes + // define your routes in one of these + //fastify.register(fastifyAutoload, { + // dir: path.join(import.meta.dirname, 'routes'), + // autoHooks: true, + // cascadeHooks: true, + // options: { ...opts } + //}) + + fastify.setErrorHandler((err, request, reply) => { + fastify.log.error( + { + err, + request: { + method: request.method, + url: request.url, + query: request.query, + params: request.params + } + }, + 'Unhandled error occurred' + ) + + reply.code(err.statusCode ?? 500) + + let message = 'Internal Server Error' + if (err.statusCode && err.statusCode < 500) { + message = err.message + } + + return { message } + }) + + // An attacker could search for valid URLs if your 404 error handling is not rate limited. + fastify.setNotFoundHandler( + { + preHandler: fastify.rateLimit({ + max: 3, + timeWindow: 500 + }) + }, + (request, reply) => { + request.log.warn( + { + request: { + method: request.method, + url: request.url, + query: request.query, + params: request.params + } + }, + 'Resource not found' + ) + + reply.code(404) + + return { message: 'Not Found' } + }) +} \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/old.ts b/components/log-viewer-webui/server/src/newWebui/old.ts new file mode 100644 index 0000000000..d381a75997 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/old.ts @@ -0,0 +1,111 @@ +/*import process from "node:process"; + +import { + fastify, + FastifyInstance, + FastifyServerOptions, +} from "fastify"; + +import settings from "../settings.json" with {type: "json"}; +import DbManager from "./plugins/DbManager.js"; +import S3Manager from "./plugins/S3Manager.js"; +import exampleRoutes from "./routes/example.js"; +import queryRoutes from "./routes/query.js"; +import staticRoutes from "./routes/static.js"; + +import fastifyMongo from '@fastify/mongodb'; + +import { + fastifyMysql, + isMySQLPromisePool, + MySQLPromisePool, +} from "@fastify/mysql"; + + +interface AppProps { + fastifyOptions: FastifyServerOptions; + sqlDbUser: string; + sqlDbPass: string; +} + +/** + * Creates the Fastify app with the given options. + * + * @param props + * @param props.fastifyOptions + * @param props.sqlDbUser + * @param props.sqlDbPass + * @return The created Fastify instance. + */ +/* +const app = async ({ + fastifyOptions, + sqlDbUser, + sqlDbPass, +}: AppProps): Promise => { + const server = fastify(fastifyOptions); + + if ("test" !== process.env.NODE_ENV) { + await server.register(DbManager, { + mysqlConfig: { + database: settings.SqlDbName, + host: settings.SqlDbHost, + password: sqlDbPass, + port: settings.SqlDbPort, + queryJobsTableName: settings.SqlDbQueryJobsTableName, + user: sqlDbUser, + }, + mongoConfig: { + database: settings.MongoDbName, + host: settings.MongoDbHost, + streamFilesCollectionName: settings.MongoDbStreamFilesCollectionName, + port: settings.MongoDbPort, + }, + }); + await server.register( + S3Manager, + { + region: settings.StreamFilesS3Region, + profile: settings.StreamFilesS3Profile, + } + ); + await server.register(fastifyMongo, { + forceClose: true, + url: `mongodb://${settings.MongoDbHost}:${settings.MongoDbPort}/${settings.MongoDbName}`, + name: 'MONGO2' + }) + + server + .register(fastifyMysql, { + promise: true, + connectionString: `mysql://${sqlDbUser}:${sqlDbPass}@${settings.SqlDbHost}:` + + `${settings.SqlDbPort}/${settings.SqlDbName}`, + name: 'SQL2' + }) + .after(async function () { + if (isMySQLPromisePool(server.mysql)) { + const mysql = server.mysql + const con = await mysql.getConnection(); + con.release(); + mysql.pool.end(); + } + }); + + } + + await server.register(staticRoutes); + await server.register(exampleRoutes); + await server.register(queryRoutes); + + return server; +}; + +// if you passed promise = true +declare module 'fastify' { + interface FastifyInstance { + mysql: MySQLPromisePool + } + } + +export default app; +*/ \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts b/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts new file mode 100644 index 0000000000..e5cb6eafe9 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts @@ -0,0 +1,143 @@ +import {encode} from "@msgpack/msgpack"; + +import { ResultSetHeader, RowDataPacket } from 'mysql2'; + +import { + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, +} from "./constants.js"; + +import { FastifyInstance } from 'fastify' +import fp from 'fastify-plugin' + +import settings from "../../../../settings.json" with {type: "json"}; + +import { sleep } from "../../../utils/misc.js"; + +/** + * Interval in milliseconds for polling the completion status of a job. + */ +const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 0.5; + +/** + * Enum of the `query_jobs` table's column names. + * + * @enum {string} + */ +const QUERY_JOBS_TABLE_COLUMN_NAMES = Object.freeze({ + ID: "id", + STATUS: "status", + TYPE: "type", + JOB_CONFIG: "job_config", +}); + + +declare module 'fastify' { + export interface FastifyInstance { + QueryJobsDbManager: ReturnType; + } +} + +/** + * Class for submitting and monitoring query jobs in the database. + */ +function createQueryJobsDbManager (fastify: FastifyInstance) { + const sqlDbConnPool = fastify.MYSQL2 + + return { + /** + * Submits a search job to the database. + * + * @param {object} searchConfig The arguments for the query. + * @return {Promise} The job's ID. + * @throws {Error} on error. + */ + async submitSearchJob (searchConfig: object) { + const [queryInsertResults] = await sqlDbConnPool.query( + `INSERT INTO ${settings.SqlDbQueryJobsTableName} + (${QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG}, + ${QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE}) + VALUES (?, ?)`, + [Buffer.from(encode(searchConfig)), + QUERY_JOB_TYPE.SEARCH_OR_AGGREGATION], + ); + + return queryInsertResults.insertId; + }, + + + /** + * Submits an aggregation job to the database. + * + * @param {object} searchConfig The arguments for the query. + * @param {number} timeRangeBucketSizeMillis + * @return {Promise} The aggregation job's ID. + * @throws {Error} on error. + */ + async submitAggregationJob (searchConfig: object, timeRangeBucketSizeMillis: number) { + const searchAggregationConfig = { + ...searchConfig, + aggregation_config: { + count_by_time_bucket_size: timeRangeBucketSizeMillis, + }, + }; + + return await this.submitSearchJob(searchAggregationConfig); + }, + + /** + * Waits for the job to complete. + * + * @param {number} jobId + * @return {Promise} + * @throws {Error} on MySQL error, if the job wasn't found in the database, if the job was + * cancelled, or if the job completed in an unexpected state. + */ + async awaitJobCompletion (jobId: number) { + while (true) { + let rows; + try { + const [queryRows] = await sqlDbConnPool.query( + ` + SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} + FROM ${settings.SqlDbQueryJobsTableName} + WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? + `, + jobId, + ); + + rows = queryRows; + } catch (e) { + throw new Error(`Failed to query status for job ${jobId} - ${e}`); + } + if (0 === rows.length) { + throw new Error(`Job ${jobId} not found in database.`); + } + const status = rows[0]![QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; + + if (false === QUERY_JOB_STATUS_WAITING_STATES.includes(status)) { + if (QUERY_JOB_STATUS.CANCELLED === status) { + throw new Error(`Job ${jobId} was cancelled.`); + } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { + throw new Error(`Job ${jobId} exited with unexpected status=${status}: ` + + `${Object.keys(QUERY_JOB_STATUS)[status]}.`); + } + break; + } + + await sleep(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); + } + } + } +} + +export default fp( + function (fastify) { + fastify.decorate('QueryJobsDbManager', createQueryJobsDbManager(fastify)) + }, + { + name: 'QueryJobsDbManager', + dependencies: ['SearchJobCollectionsManager'] + } +) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/SearchJobCollectionsManager.ts b/components/log-viewer-webui/server/src/newWebui/plugins/app/SearchJobCollectionsManager.ts new file mode 100644 index 0000000000..88e6fb885a --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/app/SearchJobCollectionsManager.ts @@ -0,0 +1,71 @@ + +import { FastifyInstance } from 'fastify' +import fp from 'fastify-plugin' +import {Nullable} from "../../../typings/common.js"; +import {Collection, Document} from "mongodb"; + +declare module 'fastify' { + export interface FastifyInstance { + SearchJobCollectionsManager: ReturnType; + } +} + +/** + * Class to keep track of MongoDB collections created for search jobs, ensuring all collections + * have unique names. + */ +function createSearchJobCollectionsManager (fastify: FastifyInstance) { + const mongo = fastify.mongo.MONGO2 + if ("undefined" === typeof mongo) { + throw new Error("MongoDB fastify plugin not found") + } + + const db = mongo.db + if ("undefined" === typeof db) { + throw new Error("MongoDB database not found") + } + + const collections: Map>> = new Map(); + + return { + /** + * Gets, or if it doesn't exist, creates a MongoDB collection named with the given job ID. + * + * @param jobId + * @return MongoDB collection + * @throws {Error} if the collection was already dropped. + */ + async getOrCreateCollection (jobId: number) { + const name = jobId.toString(); + if ("undefined" === typeof collections.get(name)) { + collections.set(name, db.collection(name)); + } else if (null === collections.get(name)) { + throw new Error( + `Collection ${name} has been dropped.` + ); + } + return collections.get(name); + }, + + + async dropCollection (jobId: number) { + const name = jobId.toString(); + const collection = collections.get(name); + if ("undefined" === typeof collection || null === collection) { + throw new Error(`Collection ${name} not found`); + } + + await collection.drop(); + collections.set(name, null); + }, + } +} + +export default fp( + function (fastify) { + fastify.decorate('SearchJobCollectionsManager', createSearchJobCollectionsManager(fastify)) + }, + { + name: 'SearchJobCollectionsManager', + } +) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/constants.ts b/components/log-viewer-webui/server/src/newWebui/plugins/app/constants.ts new file mode 100644 index 0000000000..557509e818 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/app/constants.ts @@ -0,0 +1,131 @@ +/** + * Enum of search-related signals. + * + * This includes request and response signals for various search operations and their respective + * states. + * + * @enum {string} + */ +enum SEARCH_SIGNAL { + NONE = "none", + + REQ_CANCELLING = "req-cancelling", + REQ_CLEARING = "req-clearing", + REQ_QUERYING = "req-querying", + + RESP_DONE = "resp-done", + RESP_QUERYING = "resp-querying", +} + +/** + * Checks if the given search signal is a search signal request. + * + * @param {SEARCH_SIGNAL} s + * @return {boolean} + */ +const isSearchSignalReq = (s: SEARCH_SIGNAL): boolean => s.startsWith("req-"); + +/** + * Checks if the given search signal is a search signal response. + * + * @param {SEARCH_SIGNAL} s + * @return {boolean} + */ +const isSearchSignalResp = (s: SEARCH_SIGNAL): boolean => s.startsWith("resp-"); + +/** + * Checks if the given search signal is a querying request / response. + * + * @param {SEARCH_SIGNAL} s + * @return {boolean} + */ +const isSearchSignalQuerying = (s: SEARCH_SIGNAL): boolean => s.endsWith("-querying"); + +/** + * Checks if the given search signal is an operation in progress, which can be used as a + * condition to disable UI elements. + * + * @param {SEARCH_SIGNAL} s + * @return {boolean} + */ +const isOperationInProgress = (s: SEARCH_SIGNAL): boolean => ( + (true === isSearchSignalReq(s)) || + (true === isSearchSignalQuerying(s)) +); + +/* eslint-disable sort-keys */ +let enumQueryJobStatus; +/** + * Enum of job statuses, matching the `QueryJobStatus` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +const QUERY_JOB_STATUS = Object.freeze({ + PENDING: (enumQueryJobStatus = 0), + RUNNING: ++enumQueryJobStatus, + SUCCEEDED: ++enumQueryJobStatus, + FAILED: ++enumQueryJobStatus, + CANCELLING: ++enumQueryJobStatus, + CANCELLED: ++enumQueryJobStatus, +}); +/* eslint-enable sort-keys */ + +const QUERY_JOB_STATUS_WAITING_STATES = [ + QUERY_JOB_STATUS.PENDING, + QUERY_JOB_STATUS.RUNNING, + QUERY_JOB_STATUS.CANCELLING, +]; + +/* eslint-disable sort-keys */ +let enumQueryType; +/** + * Enum of job type, matching the `QueryJobType` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +const QUERY_JOB_TYPE = Object.freeze({ + SEARCH_OR_AGGREGATION: (enumQueryType = 0), + EXTRACT_IR: ++enumQueryType, +}); +/* eslint-enable sort-keys */ + +/** + * Enum of Mongo Collection sort orders. + * + * @enum {string} + */ +const MONGO_SORT_ORDER = Object.freeze({ + ASCENDING: "asc", + DESCENDING: "desc", +}); + +/** + * Enum of search results cache fields. + * + * @enum {string} + */ +const SEARCH_RESULTS_FIELDS = Object.freeze({ + ID: "_id", + TIMESTAMP: "timestamp", +}); + +/** + * The maximum number of results to retrieve for a search. + */ +const SEARCH_MAX_NUM_RESULTS = 1000; + +export { + isOperationInProgress, + isSearchSignalQuerying, + isSearchSignalReq, + isSearchSignalResp, + MONGO_SORT_ORDER, + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, + SEARCH_MAX_NUM_RESULTS, + SEARCH_RESULTS_FIELDS, + SEARCH_SIGNAL, +}; diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/env.ts b/components/log-viewer-webui/server/src/newWebui/plugins/external/env.ts new file mode 100644 index 0000000000..9f624d9692 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/external/env.ts @@ -0,0 +1,100 @@ +import env from '@fastify/env' + +declare module 'fastify' { + export interface FastifyInstance { + config: { + PORT: number; + HOST: string + CLP_DB_USER: string + CLP_DB_PASS: string + + CLIENT_DIR: string + STREAMS_DATA_DIR: string + LOG_VIEWER_DIR: string + + RATE_LIMIT_MAX: number; + }; + } +} + +const schema = { + type: 'object', + required: [ + 'PORT', + 'HOST', + 'CLP_DB_USER', + 'CLP_DB_PASS', + 'CLIENT_DIR', + 'STREAMS_DATA_DIR', + 'LOG_VIEWER_DIR', + ], + properties: { + PORT: { + type: 'number', + default: '3000' + }, + HOST: { + type: 'string', + minLength: 1, + default: 'localhost' + }, + // Databases + CLP_DB_USER: { + type: 'string', + minLength: 1, + default: 'clp-user' + }, + CLP_DB_PASS: { + type: 'string', + }, + // Directories + CLIENT_DIR: { + type: 'string', + minLength: 1, + default: '../client/dist' + }, + STREAMS_DATA_DIR: { + type: 'string', + minLength: 1, + default: '../../../../build/clp-package/var/data/streams' + }, + LOG_VIEWER_DIR: { + type: 'string', + minLength: 1, + default: '../yscope-log-viewer/dist' + }, + // Security + RATE_LIMIT_MAX: { + type: 'number', + default: 100 // Put it to 4 in your .env file for tests + }, + } +} + +export const autoConfig = { + // Decorate Fastify instance with `config` key + // Optional, default: 'config' + confKey: 'config', + + // Schema to validate + schema, + + // Needed to read .env in root folder + dotenv: true, + // or, pass config options available on dotenv module + // dotenv: { + // path: `${import.meta.dirname}/.env`, + // debug: true + // } + + // Source for the configuration data + // Optional, default: process.env + data: process.env +} + +/** + * This plugins helps to check environment variables. + * + * @see {@link https://github.com/fastify/fastify-env} + */ +export default env \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/mongoDb.ts b/components/log-viewer-webui/server/src/newWebui/plugins/external/mongoDb.ts new file mode 100644 index 0000000000..4a26f9896a --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/external/mongoDb.ts @@ -0,0 +1,21 @@ +import fastifyMongoDb from "@fastify/mongodb"; + +import settings from "../../../../settings.json" with {type: "json"}; + +// use MySQLPromisePool if passed promise = true +//declare module 'fastify' { +// interface FastifyInstance { +// mongo: MySQLPromisePool +// } +//} + +export const autoConfig = () => { + return { + forceClose: true, + url: `mongodb://${settings.MongoDbHost}:${settings.MongoDbPort}/${settings.MongoDbName}`, + // Temporary until old webui refactored + name: "MONGO2" + } +} + +export default fastifyMongoDb diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/mySql.ts b/components/log-viewer-webui/server/src/newWebui/plugins/external/mySql.ts new file mode 100644 index 0000000000..636a7c4c80 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/external/mySql.ts @@ -0,0 +1,31 @@ +import { + fastifyMysql, + MySQLPromisePool, +} from "@fastify/mysql"; +import { FastifyInstance } from 'fastify' + +import settings from "../../../../settings.json" with {type: "json"}; +// use MySQLPromisePool if passed promise = true +declare module 'fastify' { + interface FastifyInstance { + mysql: MySQLPromisePool; + MYSQL2: MySQLPromisePool; + } +} + + +export const autoConfig = (fastify: FastifyInstance) => { + return { + promise: true, + host: settings.SqlDbHost, + user: fastify.config.CLP_DB_USER, + password: fastify.config.CLP_DB_PASS, + database: settings.SqlDbName, + port: Number(settings.SqlDbPort), + // Temporary until old webui refactored + name: "MYSQL2" + } +} + +export default fastifyMysql + diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/rate-limit.ts b/components/log-viewer-webui/server/src/newWebui/plugins/external/rate-limit.ts new file mode 100644 index 0000000000..2a275a5240 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/external/rate-limit.ts @@ -0,0 +1,17 @@ +// Reference: https://github.com/fastify/demo/blob/main/src/plugins/external/rate-limit.ts +import fastifyRateLimit from '@fastify/rate-limit' +import { FastifyInstance } from 'fastify' + +export const autoConfig = (fastify: FastifyInstance) => { + return { + max: fastify.config.RATE_LIMIT_MAX, + timeWindow: '1 minute' + } +} + +/** + * This plugins is low overhead rate limiter for your routes. + * + * @see {@link https://github.com/fastify/fastify-rate-limit} + */ +export default fastifyRateLimit \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/QueryJobs.ts b/components/log-viewer-webui/server/src/newWebui/schemas/QueryJobs.ts new file mode 100644 index 0000000000..35453686d0 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/schemas/QueryJobs.ts @@ -0,0 +1,11 @@ +import { Static, Type } from '@sinclair/typebox' +import { StringSchema } from './common.js' + +export const QueryJobsSchema = Type.Object({ + id: StringSchema, + status: StringSchema, + type: StringSchema, + job_config: StringSchema, +}) + +export interface QueryJobs extends Static {} diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/common.ts b/components/log-viewer-webui/server/src/newWebui/schemas/common.ts new file mode 100644 index 0000000000..746af1805b --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/schemas/common.ts @@ -0,0 +1,5 @@ +import { Type } from '@sinclair/typebox' + +export const StringSchema = Type.String({ + minLength: 1, +}) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/test. b/components/log-viewer-webui/server/src/newWebui/schemas/test. new file mode 100644 index 0000000000..8268124e3b --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/schemas/test. @@ -0,0 +1,11 @@ +/** + * Enum of the `query_jobs` table's column names. + * + * @enum {string} + */ +const QUERY_JOBS_TABLE_COLUMN_NAMES = Object.freeze({ + ID: "id", + STATUS: "status", + TYPE: "type", + JOB_CONFIG: "job_config", +}); \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/plugins/DbManager.ts b/components/log-viewer-webui/server/src/plugins/DbManager.ts index 32968fabbd..c2a8d76364 100644 --- a/components/log-viewer-webui/server/src/plugins/DbManager.ts +++ b/components/log-viewer-webui/server/src/plugins/DbManager.ts @@ -89,6 +89,7 @@ class DbManager { try { await app.register(fastifyMysql, { promise: true, + name: '2', connectionString: `mysql://${config.user}:${config.password}@${config.host}:` + `${config.port}/${config.database}`, }); @@ -119,6 +120,7 @@ class DbManager { await app.register(fastifyMongoDb, { forceClose: true, url: `mongodb://${config.host}:${config.port}/${config.database}`, + name: "MONGO1", }); } catch (e: unknown) { throw new Error( @@ -128,11 +130,12 @@ class DbManager { {cause: e} ); } - if ("undefined" === typeof app.mongo.db) { + // Check if the MongoDB client was properly initialized + if ("undefined" === typeof app.mongo.MONGO1 || "undefined" === typeof app.mongo.MONGO1.db) { throw new Error("Failed to initialize MongoDB plugin."); } - return {streamFilesCollection: app.mongo.db.collection(config.streamFilesCollectionName)}; + return {streamFilesCollection: app.mongo.MONGO1.db.collection(config.streamFilesCollectionName)}; } diff --git a/components/log-viewer-webui/server/src/server.ts b/components/log-viewer-webui/server/src/server.ts new file mode 100644 index 0000000000..99868a8568 --- /dev/null +++ b/components/log-viewer-webui/server/src/server.ts @@ -0,0 +1,67 @@ +// https://github.com/fastify/demo/blob/main/src/server.ts + +import Fastify from 'fastify' +import fp from 'fastify-plugin' + +import closeWithGrace from 'close-with-grace' + +import serviceApp from './newWebui/app.js' + +/** + * Do not use NODE_ENV to determine what logger (or any env related feature) to use + * @see {@link https://www.youtube.com/watch?v=HMM7GJC5E2o} + */ +function getLoggerOptions () { + // Only if the program is running in an interactive terminal + if (process.stdout.isTTY) { + return { + level: 'info', + transport: { + target: 'pino-pretty', + options: { + translateTime: 'HH:MM:ss Z', + ignore: 'pid,hostname' + } + } + } + } + + return { level: process.env.LOG_LEVEL ?? 'silent' } +} + +const app = Fastify({ + logger: getLoggerOptions(), + ajv: { + customOptions: { + coerceTypes: 'array', // change type of data to match type keyword + removeAdditional: 'all' // Remove additional body properties + } + } +}) + +async function init () { + // fp must be used to override default error handler + app.register(fp(serviceApp)) + + closeWithGrace( + { delay: Number(process.env.FASTIFY_CLOSE_GRACE_DELAY || 500) }, + async ({ err }) => { + if (err != null) { + app.log.error(err) + } + + await app.close() + } + ) + + await app.ready() + + try { + await app.listen({ port: Number(process.env.PORT ?? 3000) }) + } catch (err) { + app.log.error(err) + process.exit(1) + } +} + +init() \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/utils/misc.ts b/components/log-viewer-webui/server/src/utils/misc.ts new file mode 100644 index 0000000000..e36d040484 --- /dev/null +++ b/components/log-viewer-webui/server/src/utils/misc.ts @@ -0,0 +1,37 @@ +const MILLIS_PER_SECOND = 1000; +const BYTES_PER_KIBIBYTE = 1024; + +/** + * Creates a promise that resolves after a specified number of seconds. + * + * @param {number} seconds to wait before resolving the promise + * @return {Promise} that resolves after the specified delay + */ +const sleep = (seconds: number) => new Promise((resolve) => { + setTimeout(resolve, seconds * MILLIS_PER_SECOND); +}); + +/** + * Computes a human-readable representation of a size in bytes. + * + * @param {number} num + * @return {string} + */ +const computeHumanSize = (num: number) => { + // eslint-disable-next-line @stylistic/js/array-element-newline + const siPrefixes = ["", "K", "M", "G", "T", "P", "E", "Z"]; + for (let i = 0; i < siPrefixes.length; ++i) { + if (BYTES_PER_KIBIBYTE > Math.abs(num)) { + return `${Math.round(num)} ${siPrefixes[i]}B`; + } + num /= BYTES_PER_KIBIBYTE; + } + + return `${Math.round(num)} B`; +}; + + +export { + computeHumanSize, + sleep, +}; diff --git a/components/log-viewer-webui/server/tsconfig.json b/components/log-viewer-webui/server/tsconfig.json index 39f4e911e9..e3b6d81a53 100644 --- a/components/log-viewer-webui/server/tsconfig.json +++ b/components/log-viewer-webui/server/tsconfig.json @@ -35,5 +35,5 @@ }, "include": [ "src/**/*" - ] +, "src/server.ts" ] } From 8a00cc78567e13fddda217b417cfc8ac45cf5fc7 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 8 May 2025 18:38:01 +0000 Subject: [PATCH 02/55] latest --- .../src/newWebui/routes/api/search/index.ts | 401 ++++++++++++++++++ .../server/src/newWebui/schemas/common.ts | 4 +- .../server/src/newWebui/schemas/search.ts | 15 + 3 files changed, 419 insertions(+), 1 deletion(-) create mode 100644 components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/schemas/search.ts diff --git a/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts b/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts new file mode 100644 index 0000000000..0dcd319c4d --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts @@ -0,0 +1,401 @@ +import {Meteor} from "meteor/meteor"; + +import {logger} from "/imports/utils/logger"; + +import {SearchResultsMetadataCollection} from "../collections"; +import { + SEARCH_MAX_NUM_RESULTS, + SEARCH_SIGNAL, +} from "../constants"; +import {ERROR_NAME_COLLECTION_DROPPED} from "../SearchJobCollectionsManager"; +import {searchJobCollectionsManager} from "./collections"; +import QueryJobsDbManager from "./QueryJobsDbManager"; + + +/** + * @type {QueryJobsDbManager|null} + */ +let queryJobsDbManager = null; + +/** + * Initializes the QueryJobsDbManager. + * + * @param {import("mysql2/promise").Pool} sqlDbConnPool + * @param {object} tableNames + * @param {string} tableNames.queryJobsTableName + * @throws {Error} on error. + */ +const initQueryJobsDbManager = (sqlDbConnPool, {queryJobsTableName}) => { + queryJobsDbManager = new QueryJobsDbManager(sqlDbConnPool, {queryJobsTableName}); +}; + +/** + * Modifies the search results metadata for a given job ID. + * + * @param {object} filter + * @param {number} filter.jobId + * @param {string} filter.lastSignal + * @param {SearchResultsMetadata} fields The fields to be updated in the search results metadata. + */ +const updateSearchResultsMeta = ({ + jobId, + lastSignal, +}, fields) => { + const filter = { + _id: jobId.toString(), + lastSignal: lastSignal, + }; + + const modifier = { + $set: fields, + }; + + logger.debug("SearchResultsMetadataCollection modifier = ", modifier); + SearchResultsMetadataCollection.update(filter, modifier); +}; + +/** + * Updates the search signal when the specified job finishes. + * + * @param {object} props + * @param {number} props.searchJobId of the job to monitor + * @param {number} props.aggregationJobId of the job to monitor + */ +const updateSearchSignalWhenJobsFinish = async ({ + searchJobId, + aggregationJobId, +}) => { + let errorMsg; + try { + await queryJobsDbManager.awaitJobCompletion(searchJobId); + await queryJobsDbManager.awaitJobCompletion(aggregationJobId); + } catch (e) { + errorMsg = e.message; + } + + let numResultsInCollection = -1; + try { + numResultsInCollection = await searchJobCollectionsManager + .getOrCreateCollection(searchJobId) + .countDocuments(); + } catch (e) { + if (ERROR_NAME_COLLECTION_DROPPED === e.error) { + logger.warn(`Collection ${searchJobId} has been dropped.`); + + return; + } + throw e; + } + + updateSearchResultsMeta({ + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + }, { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: errorMsg, + numTotalResults: Math.min( + numResultsInCollection, + SEARCH_MAX_NUM_RESULTS + ), + }); +}; + +/** + * Creates MongoDB indexes for a specific job's collection. + * + * @param {number} searchJobId used to identify the Mongo Collection to add indexes + */ +const createMongoIndexes = async (searchJobId) => { + const timestampAscendingIndex = { + key: { + timestamp: 1, + _id: 1, + }, + name: "timestamp-ascending", + }; + const timestampDescendingIndex = { + key: { + timestamp: -1, + _id: -1, + }, + name: "timestamp-descending", + }; + + const queryJobCollection = searchJobCollectionsManager.getOrCreateCollection(searchJobId); + const queryJobRawCollection = queryJobCollection.rawCollection(); + await queryJobRawCollection.createIndexes([ + timestampAscendingIndex, + timestampDescendingIndex, + ]); +}; + +Meteor.methods({ + /** + * @typedef {object} SubmitQueryResp + * @property {number} searchJobId + * @property {number} aggregationJobId + */ + /** + * Submits a search query and initiates the search process. + * + * @param {object} props + * @param {string} props.queryString + * @param {number} props.timestampBegin + * @param {number} props.timestampEnd + * @param {boolean} props.ignoreCase + * @param {number} props.timeRangeBucketSizeMillis + * @return {SubmitQueryResp} + */ + async "search.submitQuery" ({ + queryString, + timestampBegin, + timestampEnd, + ignoreCase, + timeRangeBucketSizeMillis, + }) { + this.unblock(); + + const args = { + query_string: queryString, + + begin_timestamp: timestampBegin, + end_timestamp: timestampEnd, + ignore_case: ignoreCase, + max_num_results: SEARCH_MAX_NUM_RESULTS, + }; + + logger.info("search.submitQuery args =", args); + + let searchJobId; + let aggregationJobId; + try { + searchJobId = await queryJobsDbManager.submitSearchJob(args); + aggregationJobId = + await queryJobsDbManager.submitAggregationJob(args, timeRangeBucketSizeMillis); + } catch (e) { + const errorMsg = "Unable to submit search/aggregation job to the SQL database."; + logger.error(errorMsg, e.toString()); + throw new Meteor.Error("query-submit-error", errorMsg); + } + + SearchResultsMetadataCollection.insert({ + _id: searchJobId.toString(), + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + errorMsg: null, + }); + + Meteor.defer(async () => { + await updateSearchSignalWhenJobsFinish({ + searchJobId, + aggregationJobId, + }); + }); + + await createMongoIndexes(searchJobId); + + return {searchJobId, aggregationJobId}; + }, + + /** + * Clears the results of a search operation identified by jobId. + * + * @param {object} props + * @param {number} props.searchJobId of the search results to clear + * @param {number} props.aggregationJobId of the search results to clear + */ + async "search.clearResults" ({ + searchJobId, + aggregationJobId, + }) { + this.unblock(); + + logger.info(`search.clearResults searchJobId=${searchJobId}, ` + + `aggregationJobId=${aggregationJobId}`); + + try { + await searchJobCollectionsManager.dropCollection(searchJobId); + await searchJobCollectionsManager.dropCollection(aggregationJobId); + } catch (e) { + const errorMsg = `Failed to clear search results for searchJobId=${searchJobId}, ` + + `aggregationJobId=${aggregationJobId}`; + + logger.error(errorMsg, e.toString()); + throw new Meteor.Error("clear-results-error", errorMsg); + } + }, + + /** + * Cancels an ongoing search operation identified by jobId. + * + * @param {object} props + * @param {number} props.searchJobId + * @param {number} props.aggregationJobId + */ + async "search.cancelOperation" ({ + searchJobId, + aggregationJobId, + }) { + this.unblock(); + + logger.info(`search.cancelOperation searchJobId=${searchJobId}, ` + + `aggregationJobId=${aggregationJobId}`); + + try { + await queryJobsDbManager.submitQueryCancellation(searchJobId); + await queryJobsDbManager.submitQueryCancellation(aggregationJobId); + updateSearchResultsMeta({ + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + }, { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: "Query cancelled before it could be completed.", + }); + } catch (e) { + const errorMsg = `Failed to submit cancel request for searchJobId=${searchJobId},` + + `aggregationJobId=${aggregationJobId}.`; + + logger.error(errorMsg, e.toString()); + throw new Meteor.Error("query-cancel-error", errorMsg); + } + }, +}); + +export {initQueryJobsDbManager}; + + + + +import { + FastifyPluginAsyncTypebox, + Type + } from '@fastify/type-provider-typebox' + //import { CredentialsSchema } from '../../../schemas/auth.js' + import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/search.js' + + const plugin: FastifyPluginAsyncTypebox = async (fastify) => { + const { QueryJobsDbManager, SearchJobCollectionsManager } = fastify + + fastify.post( + '/search', + { + schema: { + body: CreateSearchJobSchema, + response: { + 200: SearchJobResultSchema, + 404: Type.Object({ message: Type.String() }), + 400: Type.Object({ message: Type.String() }) + }, + tags: ['Search'] + }, + }, + async function (request, reply) { + const { + queryString, + timestampBegin, + timestampEnd, + ignoreCase, + timeRangeBucketSizeMillis + } = request.body; + + const args = { + query_string: queryString, + begin_timestamp: timestampBegin, + end_timestamp: timestampEnd, + ignore_case: ignoreCase, + max_num_results: SEARCH_MAX_NUM_RESULTS + }; + + request.log.info({ args }, 'Submitting search query'); + + let searchJobId: number; + let aggregationJobId: number; + + try { + searchJobId = await QueryJobsDbManager.submitSearchJob(args); + aggregationJobId = await QueryJobsDbManager.submitAggregationJob( + args, + timeRangeBucketSizeMillis + ); + } catch (err: any) { + const message = 'Unable to submit search/aggregation job to the SQL database'; + request.log.error(err, message); + return reply.internalServerError(message); + } + + SearchResultsMetadataCollection.insert({ + _id: searchJobId.toString(), + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + errorMsg: null + }); + + // Defer signal update until after response is sent + setImmediate(() => { + updateSearchSignalWhenJobsFinish({ searchJobId, aggregationJobId }).catch(err => + request.log.error(err, 'Deferred update failed') + ); + }); + + await createMongoIndexes(searchJobId); + + return { searchJobId, aggregationJobId }; + } + + + + + + + + fastify.post( + '/login', + { + schema: { + body: CredentialsSchema, + response: { + 200: Type.Object({ + success: Type.Boolean(), + message: Type.Optional(Type.String()) + }), + 401: Type.Object({ + message: Type.String() + }) + }, + tags: ['Authentication'] + } + }, + async function (request, reply) { + const { email, password } = request.body + + return fastify.knex.transaction(async (trx) => { + const user = await usersRepository.findByEmail(email, trx) + + if (user) { + const isPasswordValid = await passwordManager.compare( + password, + user.password + ) + if (isPasswordValid) { + const roles = await usersRepository.findUserRolesByEmail(email, trx) + + request.session.user = { + id: user.id, + email: user.email, + username: user.username, + roles: roles.map((role) => role.name) + } + + await request.session.save() + + return { success: true } + } + } + + reply.status(401) + + return { message: 'Invalid email or password.' } + }) + } + ) + } + + export default plugin \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/common.ts b/components/log-viewer-webui/server/src/newWebui/schemas/common.ts index 746af1805b..206b01b8e6 100644 --- a/components/log-viewer-webui/server/src/newWebui/schemas/common.ts +++ b/components/log-viewer-webui/server/src/newWebui/schemas/common.ts @@ -2,4 +2,6 @@ import { Type } from '@sinclair/typebox' export const StringSchema = Type.String({ minLength: 1, -}) \ No newline at end of file +}) + +export const IdSchema = Type.Integer({ minimum: 1 }) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/search.ts b/components/log-viewer-webui/server/src/newWebui/schemas/search.ts new file mode 100644 index 0000000000..3ac45d0056 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/schemas/search.ts @@ -0,0 +1,15 @@ +import { Type } from '@sinclair/typebox' +import { StringSchema, IdSchema } from './common.js' + +export const CreateSearchJobSchema = Type.Object({ + queryString: StringSchema, + timestampBegin: Type.Integer(), + timestampEnd: Type.Integer(), + ignoreCase: Type.Boolean(), + timeRangeBucketSizeMillis: Type.Integer(), +}) + +export const SearchJobResultSchema = Type.Object({ + searchJobId: IdSchema, + aggregationJobId: IdSchema +}) \ No newline at end of file From 592f5e42962df2d4a4fa7733d62d40c08ec48fd7 Mon Sep 17 00:00:00 2001 From: Marco Date: Mon, 12 May 2025 16:44:35 +0000 Subject: [PATCH 03/55] saving --- .../log-viewer-webui/server/package-lock.json | 87 +++++ .../log-viewer-webui/server/package.json | 1 + .../plugins/app/QueryJobsDbManager.ts | 25 +- .../app/SearchResultsMetadataCollection.ts | 81 +++++ .../src/newWebui/plugins/external/sensible.ts | 12 + .../src/newWebui/routes/api/search/index.ts | 322 ++++++------------ 6 files changed, 312 insertions(+), 216 deletions(-) create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/app/SearchResultsMetadataCollection.ts create mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/external/sensible.ts diff --git a/components/log-viewer-webui/server/package-lock.json b/components/log-viewer-webui/server/package-lock.json index c886322bbb..3e9c85cded 100644 --- a/components/log-viewer-webui/server/package-lock.json +++ b/components/log-viewer-webui/server/package-lock.json @@ -16,6 +16,7 @@ "@fastify/mongodb": "^9.0.2", "@fastify/mysql": "^5.0.2", "@fastify/rate-limit": "^10.2.2", + "@fastify/sensible": "^6.0.3", "@fastify/static": "^8.1.1", "@fastify/type-provider-typebox": "^5.1.0", "@msgpack/msgpack": "^3.1.0", @@ -1448,6 +1449,31 @@ "mime": "^3" } }, + "node_modules/@fastify/sensible": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@fastify/sensible/-/sensible-6.0.3.tgz", + "integrity": "sha512-Iyn8698hp/e5+v8SNBBruTa7UfrMEP52R16dc9jMpqSyEcPsvWFQo+R6WwHCUnJiLIsuci2ZoEZ7ilrSSCPIVg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@lukeed/ms": "^2.0.2", + "dequal": "^2.0.3", + "fastify-plugin": "^5.0.0", + "forwarded": "^0.2.0", + "http-errors": "^2.0.0", + "type-is": "^1.6.18", + "vary": "^1.1.2" + } + }, "node_modules/@fastify/static": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/@fastify/static/-/static-8.1.1.tgz", @@ -6755,6 +6781,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fromentries": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", @@ -8427,6 +8462,15 @@ "node": ">= 0.4" } }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/memory-pager": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", @@ -8485,6 +8529,27 @@ "node": ">=10.0.0" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -11562,6 +11627,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/typed-array-buffer": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", @@ -11949,6 +12027,15 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/walk-up-path": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/walk-up-path/-/walk-up-path-4.0.0.tgz", diff --git a/components/log-viewer-webui/server/package.json b/components/log-viewer-webui/server/package.json index a562ae9bf3..11d7d4ddfc 100644 --- a/components/log-viewer-webui/server/package.json +++ b/components/log-viewer-webui/server/package.json @@ -24,6 +24,7 @@ "@fastify/mongodb": "^9.0.2", "@fastify/mysql": "^5.0.2", "@fastify/rate-limit": "^10.2.2", + "@fastify/sensible": "^6.0.3", "@fastify/static": "^8.1.1", "@fastify/type-provider-typebox": "^5.1.0", "@msgpack/msgpack": "^3.1.0", diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts b/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts index e5cb6eafe9..7fead8a9b7 100644 --- a/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts +++ b/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts @@ -15,6 +15,8 @@ import settings from "../../../../settings.json" with {type: "json"}; import { sleep } from "../../../utils/misc.js"; +import { Collection, Document } from "mongodb"; + /** * Interval in milliseconds for polling the completion status of a job. */ @@ -39,6 +41,8 @@ declare module 'fastify' { } } + + /** * Class for submitting and monitoring query jobs in the database. */ @@ -67,6 +71,7 @@ function createQueryJobsDbManager (fastify: FastifyInstance) { }, + /** * Submits an aggregation job to the database. * @@ -86,6 +91,24 @@ function createQueryJobsDbManager (fastify: FastifyInstance) { return await this.submitSearchJob(searchAggregationConfig); }, + /** + * Submits a query cancellation request to the database. + * + * @param {number} jobId ID of the job to cancel. + * @return {Promise} + * @throws {Error} on error. + */ + async submitQueryCancellation (jobId: number) { + await sqlDbConnPool.query( + `UPDATE ${settings.SqlDbQueryJobsTableName } + SET ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${QUERY_JOB_STATUS.CANCELLING} + WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? + AND ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} + IN (${QUERY_JOB_STATUS.PENDING}, ${QUERY_JOB_STATUS.RUNNING})`, + jobId, + ); + }, + /** * Waits for the job to complete. * @@ -138,6 +161,6 @@ export default fp( }, { name: 'QueryJobsDbManager', - dependencies: ['SearchJobCollectionsManager'] + dependencies: ['SearchJobCollectionsManager', 'SearchResultsMetadataCollection'], } ) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/SearchResultsMetadataCollection.ts b/components/log-viewer-webui/server/src/newWebui/plugins/app/SearchResultsMetadataCollection.ts new file mode 100644 index 0000000000..10be615f07 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/app/SearchResultsMetadataCollection.ts @@ -0,0 +1,81 @@ +import {encode} from "@msgpack/msgpack"; + +import { ResultSetHeader, RowDataPacket } from 'mysql2'; + +import { + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, +} from "./constants.js"; + +import { FastifyInstance } from 'fastify' +import fp from 'fastify-plugin' + +import settings from "../../../../settings.json" with {type: "json"}; + +import { sleep } from "../../../utils/misc.js"; + +import { Collection, Document } from "mongodb"; + +/** + * Interval in milliseconds for polling the completion status of a job. + */ +const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 0.5; + +/** + * Enum of the `query_jobs` table's column names. + * + * @enum {string} + */ +const QUERY_JOBS_TABLE_COLUMN_NAMES = Object.freeze({ + ID: "id", + STATUS: "status", + TYPE: "type", + JOB_CONFIG: "job_config", +}); + + +declare module 'fastify' { + export interface FastifyInstance { + SearchResultsMetadataCollection: ReturnType; + } +} +interface SearchResultsMetadata { + _id: string; + lastSignal: string; + errorMsg: string | null; + } + + +/** + * Class for submitting and monitoring query jobs in the database. + */ +function createSearchResultsMetadataCollection (fastify: FastifyInstance) { + const mongo = fastify.mongo.MONGO2 + + if ("undefined" === typeof mongo) { + throw new Error("MongoDB fastify plugin not found") + } + + const db = mongo.db + if ("undefined" === typeof db) { + throw new Error("MongoDB database not found") + } + + /** + * A MongoDB collection for storing metadata about search results. + */ + const SearchResultsMetadataCollection: Collection = db.collection("test"); + + return SearchResultsMetadataCollection; +} + +export default fp( + function (fastify) { + fastify.decorate('SearchResultsMetadataCollection', createSearchResultsMetadataCollection(fastify)) + }, + { + name: 'QueryJobsDbManager', + dependencies: ['SearchJobCollectionsManager'] + } +) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/sensible.ts b/components/log-viewer-webui/server/src/newWebui/plugins/external/sensible.ts new file mode 100644 index 0000000000..b68a1155f1 --- /dev/null +++ b/components/log-viewer-webui/server/src/newWebui/plugins/external/sensible.ts @@ -0,0 +1,12 @@ +import sensible from '@fastify/sensible' + +export const autoConfig = { + // Set plugin options here +} + +/** + * This plugin adds some utilities to handle http errors + * + * @see {@link https://github.com/fastify/fastify-sensible} + */ +export default sensible \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts b/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts index 0dcd319c4d..668144bab0 100644 --- a/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts +++ b/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts @@ -1,46 +1,36 @@ -import {Meteor} from "meteor/meteor"; - -import {logger} from "/imports/utils/logger"; - -import {SearchResultsMetadataCollection} from "../collections"; import { SEARCH_MAX_NUM_RESULTS, SEARCH_SIGNAL, } from "../constants"; -import {ERROR_NAME_COLLECTION_DROPPED} from "../SearchJobCollectionsManager"; -import {searchJobCollectionsManager} from "./collections"; -import QueryJobsDbManager from "./QueryJobsDbManager"; +import { Collection } from 'mongodb'; // Assuming MongoDB is used for SearchResultsMetadataCollection -/** - * @type {QueryJobsDbManager|null} - */ -let queryJobsDbManager = null; +type UpdateSearchResultsMetaParams = { + jobId: number; + lastSignal: string; + SearchResultsMetadataCollection: Collection; +}; -/** - * Initializes the QueryJobsDbManager. - * - * @param {import("mysql2/promise").Pool} sqlDbConnPool - * @param {object} tableNames - * @param {string} tableNames.queryJobsTableName - * @throws {Error} on error. - */ -const initQueryJobsDbManager = (sqlDbConnPool, {queryJobsTableName}) => { - queryJobsDbManager = new QueryJobsDbManager(sqlDbConnPool, {queryJobsTableName}); +type SearchResultsMetadataFields = { + lastSignal?: string; + errorMsg?: string | null; + numTotalResults?: number; }; /** * Modifies the search results metadata for a given job ID. * - * @param {object} filter - * @param {number} filter.jobId - * @param {string} filter.lastSignal - * @param {SearchResultsMetadata} fields The fields to be updated in the search results metadata. + * @param {UpdateSearchResultsMetaParams} params + * @param {SearchResultsMetadataFields} fields The fields to be updated in the search results metadata. */ -const updateSearchResultsMeta = ({ - jobId, - lastSignal, -}, fields) => { +const updateSearchResultsMeta = ( + { + jobId, + lastSignal, + SearchResultsMetadataCollection, + }: UpdateSearchResultsMetaParams, + fields: SearchResultsMetadataFields +) => { const filter = { _id: jobId.toString(), lastSignal: lastSignal, @@ -57,13 +47,19 @@ const updateSearchResultsMeta = ({ /** * Updates the search signal when the specified job finishes. * - * @param {object} props - * @param {number} props.searchJobId of the job to monitor - * @param {number} props.aggregationJobId of the job to monitor + * @param {object} params + * @param {number} params.searchJobId of the job to monitor + * @param {number} params.aggregationJobId of the job to monitor + * @param {QueryJobsDbManager} queryJobsDbManager + * @param {SearchJobCollectionsManager} searchJobCollectionsManager + * @param {SearchResultsMetadataCollection} SearchResultsMetadataCollection */ const updateSearchSignalWhenJobsFinish = async ({ searchJobId, aggregationJobId, + queryJobsDbManager, + searchJobCollectionsManager, + SearchResultsMetadataCollection, }) => { let errorMsg; try { @@ -90,6 +86,7 @@ const updateSearchSignalWhenJobsFinish = async ({ updateSearchResultsMeta({ jobId: searchJobId, lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + SearchResultsMetadataCollection, }, { lastSignal: SEARCH_SIGNAL.RESP_DONE, errorMsg: errorMsg, @@ -104,8 +101,9 @@ const updateSearchSignalWhenJobsFinish = async ({ * Creates MongoDB indexes for a specific job's collection. * * @param {number} searchJobId used to identify the Mongo Collection to add indexes + * @param {SearchJobCollectionsManager} searchJobCollectionsManager */ -const createMongoIndexes = async (searchJobId) => { +const createMongoIndexes = async (searchJobId, searchJobCollectionsManager) => { const timestampAscendingIndex = { key: { timestamp: 1, @@ -129,142 +127,6 @@ const createMongoIndexes = async (searchJobId) => { ]); }; -Meteor.methods({ - /** - * @typedef {object} SubmitQueryResp - * @property {number} searchJobId - * @property {number} aggregationJobId - */ - /** - * Submits a search query and initiates the search process. - * - * @param {object} props - * @param {string} props.queryString - * @param {number} props.timestampBegin - * @param {number} props.timestampEnd - * @param {boolean} props.ignoreCase - * @param {number} props.timeRangeBucketSizeMillis - * @return {SubmitQueryResp} - */ - async "search.submitQuery" ({ - queryString, - timestampBegin, - timestampEnd, - ignoreCase, - timeRangeBucketSizeMillis, - }) { - this.unblock(); - - const args = { - query_string: queryString, - - begin_timestamp: timestampBegin, - end_timestamp: timestampEnd, - ignore_case: ignoreCase, - max_num_results: SEARCH_MAX_NUM_RESULTS, - }; - - logger.info("search.submitQuery args =", args); - - let searchJobId; - let aggregationJobId; - try { - searchJobId = await queryJobsDbManager.submitSearchJob(args); - aggregationJobId = - await queryJobsDbManager.submitAggregationJob(args, timeRangeBucketSizeMillis); - } catch (e) { - const errorMsg = "Unable to submit search/aggregation job to the SQL database."; - logger.error(errorMsg, e.toString()); - throw new Meteor.Error("query-submit-error", errorMsg); - } - - SearchResultsMetadataCollection.insert({ - _id: searchJobId.toString(), - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - errorMsg: null, - }); - - Meteor.defer(async () => { - await updateSearchSignalWhenJobsFinish({ - searchJobId, - aggregationJobId, - }); - }); - - await createMongoIndexes(searchJobId); - - return {searchJobId, aggregationJobId}; - }, - - /** - * Clears the results of a search operation identified by jobId. - * - * @param {object} props - * @param {number} props.searchJobId of the search results to clear - * @param {number} props.aggregationJobId of the search results to clear - */ - async "search.clearResults" ({ - searchJobId, - aggregationJobId, - }) { - this.unblock(); - - logger.info(`search.clearResults searchJobId=${searchJobId}, ` + - `aggregationJobId=${aggregationJobId}`); - - try { - await searchJobCollectionsManager.dropCollection(searchJobId); - await searchJobCollectionsManager.dropCollection(aggregationJobId); - } catch (e) { - const errorMsg = `Failed to clear search results for searchJobId=${searchJobId}, ` + - `aggregationJobId=${aggregationJobId}`; - - logger.error(errorMsg, e.toString()); - throw new Meteor.Error("clear-results-error", errorMsg); - } - }, - - /** - * Cancels an ongoing search operation identified by jobId. - * - * @param {object} props - * @param {number} props.searchJobId - * @param {number} props.aggregationJobId - */ - async "search.cancelOperation" ({ - searchJobId, - aggregationJobId, - }) { - this.unblock(); - - logger.info(`search.cancelOperation searchJobId=${searchJobId}, ` + - `aggregationJobId=${aggregationJobId}`); - - try { - await queryJobsDbManager.submitQueryCancellation(searchJobId); - await queryJobsDbManager.submitQueryCancellation(aggregationJobId); - updateSearchResultsMeta({ - jobId: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - }, { - lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: "Query cancelled before it could be completed.", - }); - } catch (e) { - const errorMsg = `Failed to submit cancel request for searchJobId=${searchJobId},` + - `aggregationJobId=${aggregationJobId}.`; - - logger.error(errorMsg, e.toString()); - throw new Meteor.Error("query-cancel-error", errorMsg); - } - }, -}); - -export {initQueryJobsDbManager}; - - - - import { FastifyPluginAsyncTypebox, Type @@ -273,7 +135,10 @@ import { import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/search.js' const plugin: FastifyPluginAsyncTypebox = async (fastify) => { - const { QueryJobsDbManager, SearchJobCollectionsManager } = fastify + const { QueryJobsDbManager, + SearchJobCollectionsManager, + SearchResultsMetadataCollection + } = fastify fastify.post( '/search', @@ -282,8 +147,7 @@ import { body: CreateSearchJobSchema, response: { 200: SearchJobResultSchema, - 404: Type.Object({ message: Type.String() }), - 400: Type.Object({ message: Type.String() }) + 500: Type.Object({ message: Type.String() }), }, tags: ['Search'] }, @@ -322,7 +186,8 @@ import { return reply.internalServerError(message); } - SearchResultsMetadataCollection.insert({ + // I need to update the schema. + SearchResultsMetadataCollection.insertOne({ _id: searchJobId.toString(), lastSignal: SEARCH_SIGNAL.RESP_QUERYING, errorMsg: null @@ -330,72 +195,99 @@ import { // Defer signal update until after response is sent setImmediate(() => { - updateSearchSignalWhenJobsFinish({ searchJobId, aggregationJobId }).catch(err => + updateSearchSignalWhenJobsFinish({ + searchJobId, + aggregationJobId, + queryJobsDbManager: QueryJobsDbManager, + searchJobCollectionsManager: SearchJobCollectionsManager, + SearchResultsMetadataCollection, + }).catch(err => request.log.error(err, 'Deferred update failed') ); }); - await createMongoIndexes(searchJobId); + await createMongoIndexes(searchJobId, SearchJobCollectionsManager); return { searchJobId, aggregationJobId }; } + ); + fastify.delete( + '/search/results', + { + schema: { + body: SearchJobResultSchema, + response: { + 204: Type.Null, + 500: Type.Object({ message: Type.String() }), + }, + tags: ['Search'], + }, + }, + async function (request, reply) { + const { searchJobId, aggregationJobId } = request.body; + request.log.info( + `search.clearResults searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` + ); - - + try { + await SearchJobCollectionsManager.dropCollection(searchJobId); + await SearchJobCollectionsManager.dropCollection(aggregationJobId); + + reply.code(204); + } catch (err: any) { + const message = `Failed to clear search results for searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}`; + request.log.error(err, message); + return reply.internalServerError(message); + } + } + ); fastify.post( - '/login', + '/search/cancel', { schema: { - body: CredentialsSchema, + body: SearchJobResultSchema, response: { - 200: Type.Object({ - success: Type.Boolean(), - message: Type.Optional(Type.String()) - }), - 401: Type.Object({ - message: Type.String() - }) + 204: Type.Null, + 500: Type.Object({ message: Type.String()}), }, - tags: ['Authentication'] - } + tags: ['Search'], + }, }, async function (request, reply) { - const { email, password } = request.body - - return fastify.knex.transaction(async (trx) => { - const user = await usersRepository.findByEmail(email, trx) + const { searchJobId, aggregationJobId } = request.body; - if (user) { - const isPasswordValid = await passwordManager.compare( - password, - user.password - ) - if (isPasswordValid) { - const roles = await usersRepository.findUserRolesByEmail(email, trx) + request.log.info( + `search.cancelOperation searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` + ); - request.session.user = { - id: user.id, - email: user.email, - username: user.username, - roles: roles.map((role) => role.name) - } - - await request.session.save() + try { + await QueryJobsDbManager.submitQueryCancellation(searchJobId); + await QueryJobsDbManager.submitQueryCancellation(aggregationJobId); - return { success: true } + updateSearchResultsMeta( + { + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + SearchResultsMetadataCollection, + }, + { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: 'Query cancelled before it could be completed.', } - } - - reply.status(401) + ); - return { message: 'Invalid email or password.' } - }) + reply.code(204); + } catch (err: any) { + const message = `Failed to submit cancel request for searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}`; + request.log.error(err, message); + return reply.internalServerError(message); + } } - ) + ); } export default plugin \ No newline at end of file From c93574e5a98e3bf2bdde675ac223799ce58d1d77 Mon Sep 17 00:00:00 2001 From: Marco Date: Mon, 12 May 2025 18:29:46 +0000 Subject: [PATCH 04/55] latest --- .../src/{newWebui => fastify-v2}/app.ts | 47 ++--- .../plugins/app/QueryJobsDbManager.ts} | 0 .../app/SearchJobCollectionsManager.ts | 71 ++++++++ .../app/SearchResultsMetadataCollection.ts | 0 .../plugins/app/constants.ts | 0 .../src/fastify-v2/plugins/app/sample.ts | 37 ++++ .../plugins/external/env.ts | 0 .../plugins/external/mongoDb.ts | 9 - .../plugins/external/mySql.ts | 3 +- .../plugins/external/rate-limit.ts | 0 .../plugins/external/sensible.ts | 0 .../routes/api/search/index.ts | 2 +- .../server/src/fastify-v2/routes/sample.ts | 31 ++++ .../schemas/QueryJobs.ts | 0 .../schemas/common.ts | 0 .../schemas/search.ts | 0 .../{newWebui => fastify-v2}/schemas/test. | 0 .../log-viewer-webui/server/src/main.ts | 2 +- .../server/src/newWebui/old.ts | 111 ------------ .../plugins/app/QueryJobsDbManager.ts | 166 ------------------ .../server/src/plugins/DbManager.ts | 68 +------ .../log-viewer-webui/server/src/server.ts | 2 +- 22 files changed, 175 insertions(+), 374 deletions(-) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/app.ts (62%) rename components/log-viewer-webui/server/src/{newWebui/plugins/app/SearchJobCollectionsManager.ts => fastify-v2/plugins/app/QueryJobsDbManager.ts} (100%) create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchJobCollectionsManager.ts rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/plugins/app/SearchResultsMetadataCollection.ts (100%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/plugins/app/constants.ts (100%) create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/sample.ts rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/plugins/external/env.ts (100%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/plugins/external/mongoDb.ts (58%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/plugins/external/mySql.ts (90%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/plugins/external/rate-limit.ts (100%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/plugins/external/sensible.ts (100%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/routes/api/search/index.ts (99%) create mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/sample.ts rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/schemas/QueryJobs.ts (100%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/schemas/common.ts (100%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/schemas/search.ts (100%) rename components/log-viewer-webui/server/src/{newWebui => fastify-v2}/schemas/test. (100%) delete mode 100644 components/log-viewer-webui/server/src/newWebui/old.ts delete mode 100644 components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts diff --git a/components/log-viewer-webui/server/src/newWebui/app.ts b/components/log-viewer-webui/server/src/fastify-v2/app.ts similarity index 62% rename from components/log-viewer-webui/server/src/newWebui/app.ts rename to components/log-viewer-webui/server/src/fastify-v2/app.ts index 83892266be..0a24171abf 100644 --- a/components/log-viewer-webui/server/src/newWebui/app.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/app.ts @@ -1,10 +1,10 @@ -//https://github.com/fastify/demo/blob/main/src/app.ts +// Reference: https://github.com/fastify/demo/blob/main/src/app.ts import path from 'node:path' import fastifyAutoload from '@fastify/autoload' import { FastifyInstance, FastifyPluginOptions } from 'fastify' -import OldApp from "../app.js"; +import appV1 from "../app.js"; export const options = { ajv: { @@ -21,33 +21,40 @@ export default async function serviceApp ( ) { delete opts.skipOverride // This option only serves testing purpose // This loads all external plugins defined in plugins/external - // those should be registered first as your application plugins might depend on them + // those should be registered first as application plugins might depend on them await fastify.register(fastifyAutoload, { dir: path.join(import.meta.dirname, 'plugins/external'), options: { ...opts } }) - await fastify.register(OldApp, { - sqlDbUser: fastify.config.CLP_DB_USER, - sqlDbPass: fastify.config.CLP_DB_PASS, + /* eslint-disable no-warning-comments */ + // TODO: Refactor to old webui code to use more modular fastify style. For now the + // old webui code is temporary loaded as a separate plugin. + await fastify.register(function (fastify) { + fastify.register(appV1, { + sqlDbUser: fastify.config.CLP_DB_USER, + sqlDbPass: fastify.config.CLP_DB_PASS, + }) }) - // This loads all your application plugins defined in plugins/app - // those should be support plugins that are reused - // through your application - //fastify.register(fastifyAutoload, { - // dir: path.join(import.meta.dirname, 'plugins/app'), - // options: { ...opts } + //await fastify.register(appV1, { + // sqlDbUser: fastify.config.CLP_DB_USER, + // sqlDbPass: fastify.config.CLP_DB_PASS, //}) - // This loads all plugins defined in routes - // define your routes in one of these - //fastify.register(fastifyAutoload, { - // dir: path.join(import.meta.dirname, 'routes'), - // autoHooks: true, - // cascadeHooks: true, - // options: { ...opts } - //}) + // Loads all application plugins defined in plugins/app + fastify.register(fastifyAutoload, { + dir: path.join(import.meta.dirname, 'plugins/app'), + options: { ...opts } + }) + + // loads all plugins routes + fastify.register(fastifyAutoload, { + dir: path.join(import.meta.dirname, 'routes'), + autoHooks: true, + cascadeHooks: true, + options: { ...opts } + }) fastify.setErrorHandler((err, request, reply) => { fastify.log.error( diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/SearchJobCollectionsManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/QueryJobsDbManager.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/plugins/app/SearchJobCollectionsManager.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/app/QueryJobsDbManager.ts diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchJobCollectionsManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchJobCollectionsManager.ts new file mode 100644 index 0000000000..88e6fb885a --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchJobCollectionsManager.ts @@ -0,0 +1,71 @@ + +import { FastifyInstance } from 'fastify' +import fp from 'fastify-plugin' +import {Nullable} from "../../../typings/common.js"; +import {Collection, Document} from "mongodb"; + +declare module 'fastify' { + export interface FastifyInstance { + SearchJobCollectionsManager: ReturnType; + } +} + +/** + * Class to keep track of MongoDB collections created for search jobs, ensuring all collections + * have unique names. + */ +function createSearchJobCollectionsManager (fastify: FastifyInstance) { + const mongo = fastify.mongo.MONGO2 + if ("undefined" === typeof mongo) { + throw new Error("MongoDB fastify plugin not found") + } + + const db = mongo.db + if ("undefined" === typeof db) { + throw new Error("MongoDB database not found") + } + + const collections: Map>> = new Map(); + + return { + /** + * Gets, or if it doesn't exist, creates a MongoDB collection named with the given job ID. + * + * @param jobId + * @return MongoDB collection + * @throws {Error} if the collection was already dropped. + */ + async getOrCreateCollection (jobId: number) { + const name = jobId.toString(); + if ("undefined" === typeof collections.get(name)) { + collections.set(name, db.collection(name)); + } else if (null === collections.get(name)) { + throw new Error( + `Collection ${name} has been dropped.` + ); + } + return collections.get(name); + }, + + + async dropCollection (jobId: number) { + const name = jobId.toString(); + const collection = collections.get(name); + if ("undefined" === typeof collection || null === collection) { + throw new Error(`Collection ${name} not found`); + } + + await collection.drop(); + collections.set(name, null); + }, + } +} + +export default fp( + function (fastify) { + fastify.decorate('SearchJobCollectionsManager', createSearchJobCollectionsManager(fastify)) + }, + { + name: 'SearchJobCollectionsManager', + } +) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/SearchResultsMetadataCollection.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchResultsMetadataCollection.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/plugins/app/SearchResultsMetadataCollection.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchResultsMetadataCollection.ts diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/constants.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/constants.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/plugins/app/constants.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/app/constants.ts diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/sample.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/sample.ts new file mode 100644 index 0000000000..64f6154fce --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/sample.ts @@ -0,0 +1,37 @@ + +import { FastifyInstance } from 'fastify' +import fp from 'fastify-plugin' + +declare module 'fastify' { + export interface FastifyInstance { + sampleApp: ReturnType; + } +} + +/** + * Sample code to demonstrate how to create a Fastify plugin. + * + * TODO: Remove sample code when new webui app code is ready. + */ +function createSamplePlugin (fastify: FastifyInstance) { + return { + /** + * Returns a sample string. + * + * @returns A sample string. + */ + async getSample (): Promise { + return `sample`; + }, + } + } + + +export default fp( + function (fastify) { + fastify.decorate('samplePlugin', createSamplePlugin(fastify)) + }, + { + name: 'samplePlugin', + } +) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/env.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/env.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/plugins/external/env.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/external/env.ts diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/mongoDb.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mongoDb.ts similarity index 58% rename from components/log-viewer-webui/server/src/newWebui/plugins/external/mongoDb.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/external/mongoDb.ts index 4a26f9896a..699b06a1e4 100644 --- a/components/log-viewer-webui/server/src/newWebui/plugins/external/mongoDb.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mongoDb.ts @@ -2,19 +2,10 @@ import fastifyMongoDb from "@fastify/mongodb"; import settings from "../../../../settings.json" with {type: "json"}; -// use MySQLPromisePool if passed promise = true -//declare module 'fastify' { -// interface FastifyInstance { -// mongo: MySQLPromisePool -// } -//} - export const autoConfig = () => { return { forceClose: true, url: `mongodb://${settings.MongoDbHost}:${settings.MongoDbPort}/${settings.MongoDbName}`, - // Temporary until old webui refactored - name: "MONGO2" } } diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/mySql.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mySql.ts similarity index 90% rename from components/log-viewer-webui/server/src/newWebui/plugins/external/mySql.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/external/mySql.ts index 636a7c4c80..38bc17cbe3 100644 --- a/components/log-viewer-webui/server/src/newWebui/plugins/external/mySql.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mySql.ts @@ -5,6 +5,7 @@ import { import { FastifyInstance } from 'fastify' import settings from "../../../../settings.json" with {type: "json"}; + // use MySQLPromisePool if passed promise = true declare module 'fastify' { interface FastifyInstance { @@ -22,8 +23,6 @@ export const autoConfig = (fastify: FastifyInstance) => { password: fastify.config.CLP_DB_PASS, database: settings.SqlDbName, port: Number(settings.SqlDbPort), - // Temporary until old webui refactored - name: "MYSQL2" } } diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/rate-limit.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/rate-limit.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/plugins/external/rate-limit.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/external/rate-limit.ts diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/external/sensible.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/sensible.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/plugins/external/sensible.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/external/sensible.ts diff --git a/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts similarity index 99% rename from components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts rename to components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts index 668144bab0..a890441c61 100644 --- a/components/log-viewer-webui/server/src/newWebui/routes/api/search/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts @@ -1,7 +1,7 @@ import { SEARCH_MAX_NUM_RESULTS, SEARCH_SIGNAL, -} from "../constants"; +} from "../constants.js"; import { Collection } from 'mongodb'; // Assuming MongoDB is used for SearchResultsMetadataCollection diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/sample.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/sample.ts new file mode 100644 index 0000000000..4288c99ff7 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/sample.ts @@ -0,0 +1,31 @@ +import { + FastifyPluginAsyncTypebox, + Type + } from '@fastify/type-provider-typebox' + +/** + * Sample code to demonstrate how to create a Fastify route. + * + * TODO: Remove sample code when new webui app code is ready. + */ + const plugin: FastifyPluginAsyncTypebox = async (fastify) => { + const { sampleApp } = fastify + fastify.get( + '/sample', + { + schema: { + response: { + 200: Type.Object({ + message: Type.String() + }) + } + } + }, + async function () { + const sampleMessage = await sampleApp.getSample(); + return { message: sampleMessage }; + } + ) + } + + export default plugin \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/QueryJobs.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/QueryJobs.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/schemas/QueryJobs.ts rename to components/log-viewer-webui/server/src/fastify-v2/schemas/QueryJobs.ts diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/common.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/schemas/common.ts rename to components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/search.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/schemas/search.ts rename to components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts diff --git a/components/log-viewer-webui/server/src/newWebui/schemas/test. b/components/log-viewer-webui/server/src/fastify-v2/schemas/test. similarity index 100% rename from components/log-viewer-webui/server/src/newWebui/schemas/test. rename to components/log-viewer-webui/server/src/fastify-v2/schemas/test. diff --git a/components/log-viewer-webui/server/src/main.ts b/components/log-viewer-webui/server/src/main.ts index 29ec326f9a..bd2ce5b833 100644 --- a/components/log-viewer-webui/server/src/main.ts +++ b/components/log-viewer-webui/server/src/main.ts @@ -24,7 +24,7 @@ const main = async () => { await server.register(app, { sqlDbUser: process.env.CLP_DB_USER!, sqlDbPass: process.env.CLP_DB_PASS!, - }); + }); try { await server.listen({host: envVars.HOST, port: Number(envVars.PORT)}); diff --git a/components/log-viewer-webui/server/src/newWebui/old.ts b/components/log-viewer-webui/server/src/newWebui/old.ts deleted file mode 100644 index d381a75997..0000000000 --- a/components/log-viewer-webui/server/src/newWebui/old.ts +++ /dev/null @@ -1,111 +0,0 @@ -/*import process from "node:process"; - -import { - fastify, - FastifyInstance, - FastifyServerOptions, -} from "fastify"; - -import settings from "../settings.json" with {type: "json"}; -import DbManager from "./plugins/DbManager.js"; -import S3Manager from "./plugins/S3Manager.js"; -import exampleRoutes from "./routes/example.js"; -import queryRoutes from "./routes/query.js"; -import staticRoutes from "./routes/static.js"; - -import fastifyMongo from '@fastify/mongodb'; - -import { - fastifyMysql, - isMySQLPromisePool, - MySQLPromisePool, -} from "@fastify/mysql"; - - -interface AppProps { - fastifyOptions: FastifyServerOptions; - sqlDbUser: string; - sqlDbPass: string; -} - -/** - * Creates the Fastify app with the given options. - * - * @param props - * @param props.fastifyOptions - * @param props.sqlDbUser - * @param props.sqlDbPass - * @return The created Fastify instance. - */ -/* -const app = async ({ - fastifyOptions, - sqlDbUser, - sqlDbPass, -}: AppProps): Promise => { - const server = fastify(fastifyOptions); - - if ("test" !== process.env.NODE_ENV) { - await server.register(DbManager, { - mysqlConfig: { - database: settings.SqlDbName, - host: settings.SqlDbHost, - password: sqlDbPass, - port: settings.SqlDbPort, - queryJobsTableName: settings.SqlDbQueryJobsTableName, - user: sqlDbUser, - }, - mongoConfig: { - database: settings.MongoDbName, - host: settings.MongoDbHost, - streamFilesCollectionName: settings.MongoDbStreamFilesCollectionName, - port: settings.MongoDbPort, - }, - }); - await server.register( - S3Manager, - { - region: settings.StreamFilesS3Region, - profile: settings.StreamFilesS3Profile, - } - ); - await server.register(fastifyMongo, { - forceClose: true, - url: `mongodb://${settings.MongoDbHost}:${settings.MongoDbPort}/${settings.MongoDbName}`, - name: 'MONGO2' - }) - - server - .register(fastifyMysql, { - promise: true, - connectionString: `mysql://${sqlDbUser}:${sqlDbPass}@${settings.SqlDbHost}:` + - `${settings.SqlDbPort}/${settings.SqlDbName}`, - name: 'SQL2' - }) - .after(async function () { - if (isMySQLPromisePool(server.mysql)) { - const mysql = server.mysql - const con = await mysql.getConnection(); - con.release(); - mysql.pool.end(); - } - }); - - } - - await server.register(staticRoutes); - await server.register(exampleRoutes); - await server.register(queryRoutes); - - return server; -}; - -// if you passed promise = true -declare module 'fastify' { - interface FastifyInstance { - mysql: MySQLPromisePool - } - } - -export default app; -*/ \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts b/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts deleted file mode 100644 index 7fead8a9b7..0000000000 --- a/components/log-viewer-webui/server/src/newWebui/plugins/app/QueryJobsDbManager.ts +++ /dev/null @@ -1,166 +0,0 @@ -import {encode} from "@msgpack/msgpack"; - -import { ResultSetHeader, RowDataPacket } from 'mysql2'; - -import { - QUERY_JOB_STATUS, - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, -} from "./constants.js"; - -import { FastifyInstance } from 'fastify' -import fp from 'fastify-plugin' - -import settings from "../../../../settings.json" with {type: "json"}; - -import { sleep } from "../../../utils/misc.js"; - -import { Collection, Document } from "mongodb"; - -/** - * Interval in milliseconds for polling the completion status of a job. - */ -const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 0.5; - -/** - * Enum of the `query_jobs` table's column names. - * - * @enum {string} - */ -const QUERY_JOBS_TABLE_COLUMN_NAMES = Object.freeze({ - ID: "id", - STATUS: "status", - TYPE: "type", - JOB_CONFIG: "job_config", -}); - - -declare module 'fastify' { - export interface FastifyInstance { - QueryJobsDbManager: ReturnType; - } -} - - - -/** - * Class for submitting and monitoring query jobs in the database. - */ -function createQueryJobsDbManager (fastify: FastifyInstance) { - const sqlDbConnPool = fastify.MYSQL2 - - return { - /** - * Submits a search job to the database. - * - * @param {object} searchConfig The arguments for the query. - * @return {Promise} The job's ID. - * @throws {Error} on error. - */ - async submitSearchJob (searchConfig: object) { - const [queryInsertResults] = await sqlDbConnPool.query( - `INSERT INTO ${settings.SqlDbQueryJobsTableName} - (${QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG}, - ${QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE}) - VALUES (?, ?)`, - [Buffer.from(encode(searchConfig)), - QUERY_JOB_TYPE.SEARCH_OR_AGGREGATION], - ); - - return queryInsertResults.insertId; - }, - - - - /** - * Submits an aggregation job to the database. - * - * @param {object} searchConfig The arguments for the query. - * @param {number} timeRangeBucketSizeMillis - * @return {Promise} The aggregation job's ID. - * @throws {Error} on error. - */ - async submitAggregationJob (searchConfig: object, timeRangeBucketSizeMillis: number) { - const searchAggregationConfig = { - ...searchConfig, - aggregation_config: { - count_by_time_bucket_size: timeRangeBucketSizeMillis, - }, - }; - - return await this.submitSearchJob(searchAggregationConfig); - }, - - /** - * Submits a query cancellation request to the database. - * - * @param {number} jobId ID of the job to cancel. - * @return {Promise} - * @throws {Error} on error. - */ - async submitQueryCancellation (jobId: number) { - await sqlDbConnPool.query( - `UPDATE ${settings.SqlDbQueryJobsTableName } - SET ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${QUERY_JOB_STATUS.CANCELLING} - WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? - AND ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} - IN (${QUERY_JOB_STATUS.PENDING}, ${QUERY_JOB_STATUS.RUNNING})`, - jobId, - ); - }, - - /** - * Waits for the job to complete. - * - * @param {number} jobId - * @return {Promise} - * @throws {Error} on MySQL error, if the job wasn't found in the database, if the job was - * cancelled, or if the job completed in an unexpected state. - */ - async awaitJobCompletion (jobId: number) { - while (true) { - let rows; - try { - const [queryRows] = await sqlDbConnPool.query( - ` - SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} - FROM ${settings.SqlDbQueryJobsTableName} - WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? - `, - jobId, - ); - - rows = queryRows; - } catch (e) { - throw new Error(`Failed to query status for job ${jobId} - ${e}`); - } - if (0 === rows.length) { - throw new Error(`Job ${jobId} not found in database.`); - } - const status = rows[0]![QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; - - if (false === QUERY_JOB_STATUS_WAITING_STATES.includes(status)) { - if (QUERY_JOB_STATUS.CANCELLED === status) { - throw new Error(`Job ${jobId} was cancelled.`); - } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { - throw new Error(`Job ${jobId} exited with unexpected status=${status}: ` + - `${Object.keys(QUERY_JOB_STATUS)[status]}.`); - } - break; - } - - await sleep(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); - } - } - } -} - -export default fp( - function (fastify) { - fastify.decorate('QueryJobsDbManager', createQueryJobsDbManager(fastify)) - }, - { - name: 'QueryJobsDbManager', - dependencies: ['SearchJobCollectionsManager', 'SearchResultsMetadataCollection'], - } -) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/plugins/DbManager.ts b/components/log-viewer-webui/server/src/plugins/DbManager.ts index c2a8d76364..950aebe7c4 100644 --- a/components/log-viewer-webui/server/src/plugins/DbManager.ts +++ b/components/log-viewer-webui/server/src/plugins/DbManager.ts @@ -68,8 +68,11 @@ class DbManager { } static async create (app: FastifyInstance, dbConfig: DbManagerOptions) { - const mysqlConnectionPool = await DbManager.#initMySql(app, dbConfig.mysqlConfig); - const {streamFilesCollection} = await DbManager.#initMongo(app, dbConfig.mongoConfig); + const mysqlConnectionPool = app.mysql.pool; + if ("undefined" === typeof app.mongo.db) { + throw new Error("MongoDB database not found"); + } + const streamFilesCollection = app.mongo.db.collection(dbConfig.mongoConfig.streamFilesCollectionName); return new DbManager({ app: app, mysqlConnectionPool: mysqlConnectionPool, @@ -78,67 +81,6 @@ class DbManager { }); } - /** - * Initializes the MySQL plugin. - * - * @param app - * @param config - * @return The MySQL connection pool created during initialization. - */ - static async #initMySql (app: FastifyInstance, config: DbManagerOptions["mysqlConfig"]) { - try { - await app.register(fastifyMysql, { - promise: true, - name: '2', - connectionString: `mysql://${config.user}:${config.password}@${config.host}:` + - `${config.port}/${config.database}`, - }); - } catch (e: unknown) { - throw new Error( - `Failed to init MySql: ${e instanceof Error ? - e.message : - JSON.stringify(e)}`, - {cause: e} - ); - } - - return app.mysql.pool; - } - - /** - * Initializes the MongoDB plugin. - * - * @param app - * @param config - * @return The MongoDB Collection objects created during initialization. - */ - static async #initMongo (app: FastifyInstance, config: DbManagerOptions["mongoConfig"]) - : Promise<{ - streamFilesCollection: StreamFilesCollection; - }> { - try { - await app.register(fastifyMongoDb, { - forceClose: true, - url: `mongodb://${config.host}:${config.port}/${config.database}`, - name: "MONGO1", - }); - } catch (e: unknown) { - throw new Error( - `Failed to init MongoDB: ${e instanceof Error ? - e.message : - JSON.stringify(e)}`, - {cause: e} - ); - } - // Check if the MongoDB client was properly initialized - if ("undefined" === typeof app.mongo.MONGO1 || "undefined" === typeof app.mongo.MONGO1.db) { - throw new Error("Failed to initialize MongoDB plugin."); - } - - return {streamFilesCollection: app.mongo.MONGO1.db.collection(config.streamFilesCollectionName)}; - } - - /** * Submits a stream extraction job to the scheduler and waits for it to finish. * diff --git a/components/log-viewer-webui/server/src/server.ts b/components/log-viewer-webui/server/src/server.ts index 99868a8568..8bb1855856 100644 --- a/components/log-viewer-webui/server/src/server.ts +++ b/components/log-viewer-webui/server/src/server.ts @@ -5,7 +5,7 @@ import fp from 'fastify-plugin' import closeWithGrace from 'close-with-grace' -import serviceApp from './newWebui/app.js' +import serviceApp from './fastify-v2/app.js' /** * Do not use NODE_ENV to determine what logger (or any env related feature) to use From f22304ea0eae1a100d7a10bd9e6c41bd633ef5ce Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 14 May 2025 19:19:59 +0000 Subject: [PATCH 05/55] latest --- .../log-viewer-webui/server/settings.json | 1 + .../plugins/app/QueryJobsDbManager.ts | 71 ------- .../app/SearchJobCollectionsManager.ts | 71 ------- .../app/SearchResultsMetadataCollection.ts | 81 -------- .../src/fastify-v2/plugins/app/constants.ts | 131 ------------- .../src/fastify-v2/routes/api/search/index.ts | 178 +++--------------- .../src/fastify-v2/routes/api/search/utils.ts | 129 +++++++++++++ 7 files changed, 158 insertions(+), 504 deletions(-) delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/QueryJobsDbManager.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchJobCollectionsManager.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchResultsMetadataCollection.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/constants.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json index 70e8a40bb3..8010a8f38b 100644 --- a/components/log-viewer-webui/server/settings.json +++ b/components/log-viewer-webui/server/settings.json @@ -11,6 +11,7 @@ "MongoDbPort": 27017, "MongoDbName": "clp-query-results", "MongoDbStreamFilesCollectionName": "stream-files", + "MongoDbSearchResultsMetadataCollectionName": "results-metadata" "ClientDir": "../client/dist", "LogViewerDir": "../yscope-log-viewer/dist", diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/QueryJobsDbManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/QueryJobsDbManager.ts deleted file mode 100644 index 88e6fb885a..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/QueryJobsDbManager.ts +++ /dev/null @@ -1,71 +0,0 @@ - -import { FastifyInstance } from 'fastify' -import fp from 'fastify-plugin' -import {Nullable} from "../../../typings/common.js"; -import {Collection, Document} from "mongodb"; - -declare module 'fastify' { - export interface FastifyInstance { - SearchJobCollectionsManager: ReturnType; - } -} - -/** - * Class to keep track of MongoDB collections created for search jobs, ensuring all collections - * have unique names. - */ -function createSearchJobCollectionsManager (fastify: FastifyInstance) { - const mongo = fastify.mongo.MONGO2 - if ("undefined" === typeof mongo) { - throw new Error("MongoDB fastify plugin not found") - } - - const db = mongo.db - if ("undefined" === typeof db) { - throw new Error("MongoDB database not found") - } - - const collections: Map>> = new Map(); - - return { - /** - * Gets, or if it doesn't exist, creates a MongoDB collection named with the given job ID. - * - * @param jobId - * @return MongoDB collection - * @throws {Error} if the collection was already dropped. - */ - async getOrCreateCollection (jobId: number) { - const name = jobId.toString(); - if ("undefined" === typeof collections.get(name)) { - collections.set(name, db.collection(name)); - } else if (null === collections.get(name)) { - throw new Error( - `Collection ${name} has been dropped.` - ); - } - return collections.get(name); - }, - - - async dropCollection (jobId: number) { - const name = jobId.toString(); - const collection = collections.get(name); - if ("undefined" === typeof collection || null === collection) { - throw new Error(`Collection ${name} not found`); - } - - await collection.drop(); - collections.set(name, null); - }, - } -} - -export default fp( - function (fastify) { - fastify.decorate('SearchJobCollectionsManager', createSearchJobCollectionsManager(fastify)) - }, - { - name: 'SearchJobCollectionsManager', - } -) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchJobCollectionsManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchJobCollectionsManager.ts deleted file mode 100644 index 88e6fb885a..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchJobCollectionsManager.ts +++ /dev/null @@ -1,71 +0,0 @@ - -import { FastifyInstance } from 'fastify' -import fp from 'fastify-plugin' -import {Nullable} from "../../../typings/common.js"; -import {Collection, Document} from "mongodb"; - -declare module 'fastify' { - export interface FastifyInstance { - SearchJobCollectionsManager: ReturnType; - } -} - -/** - * Class to keep track of MongoDB collections created for search jobs, ensuring all collections - * have unique names. - */ -function createSearchJobCollectionsManager (fastify: FastifyInstance) { - const mongo = fastify.mongo.MONGO2 - if ("undefined" === typeof mongo) { - throw new Error("MongoDB fastify plugin not found") - } - - const db = mongo.db - if ("undefined" === typeof db) { - throw new Error("MongoDB database not found") - } - - const collections: Map>> = new Map(); - - return { - /** - * Gets, or if it doesn't exist, creates a MongoDB collection named with the given job ID. - * - * @param jobId - * @return MongoDB collection - * @throws {Error} if the collection was already dropped. - */ - async getOrCreateCollection (jobId: number) { - const name = jobId.toString(); - if ("undefined" === typeof collections.get(name)) { - collections.set(name, db.collection(name)); - } else if (null === collections.get(name)) { - throw new Error( - `Collection ${name} has been dropped.` - ); - } - return collections.get(name); - }, - - - async dropCollection (jobId: number) { - const name = jobId.toString(); - const collection = collections.get(name); - if ("undefined" === typeof collection || null === collection) { - throw new Error(`Collection ${name} not found`); - } - - await collection.drop(); - collections.set(name, null); - }, - } -} - -export default fp( - function (fastify) { - fastify.decorate('SearchJobCollectionsManager', createSearchJobCollectionsManager(fastify)) - }, - { - name: 'SearchJobCollectionsManager', - } -) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchResultsMetadataCollection.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchResultsMetadataCollection.ts deleted file mode 100644 index 10be615f07..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/SearchResultsMetadataCollection.ts +++ /dev/null @@ -1,81 +0,0 @@ -import {encode} from "@msgpack/msgpack"; - -import { ResultSetHeader, RowDataPacket } from 'mysql2'; - -import { - QUERY_JOB_STATUS, - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, -} from "./constants.js"; - -import { FastifyInstance } from 'fastify' -import fp from 'fastify-plugin' - -import settings from "../../../../settings.json" with {type: "json"}; - -import { sleep } from "../../../utils/misc.js"; - -import { Collection, Document } from "mongodb"; - -/** - * Interval in milliseconds for polling the completion status of a job. - */ -const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 0.5; - -/** - * Enum of the `query_jobs` table's column names. - * - * @enum {string} - */ -const QUERY_JOBS_TABLE_COLUMN_NAMES = Object.freeze({ - ID: "id", - STATUS: "status", - TYPE: "type", - JOB_CONFIG: "job_config", -}); - - -declare module 'fastify' { - export interface FastifyInstance { - SearchResultsMetadataCollection: ReturnType; - } -} -interface SearchResultsMetadata { - _id: string; - lastSignal: string; - errorMsg: string | null; - } - - -/** - * Class for submitting and monitoring query jobs in the database. - */ -function createSearchResultsMetadataCollection (fastify: FastifyInstance) { - const mongo = fastify.mongo.MONGO2 - - if ("undefined" === typeof mongo) { - throw new Error("MongoDB fastify plugin not found") - } - - const db = mongo.db - if ("undefined" === typeof db) { - throw new Error("MongoDB database not found") - } - - /** - * A MongoDB collection for storing metadata about search results. - */ - const SearchResultsMetadataCollection: Collection = db.collection("test"); - - return SearchResultsMetadataCollection; -} - -export default fp( - function (fastify) { - fastify.decorate('SearchResultsMetadataCollection', createSearchResultsMetadataCollection(fastify)) - }, - { - name: 'QueryJobsDbManager', - dependencies: ['SearchJobCollectionsManager'] - } -) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/constants.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/constants.ts deleted file mode 100644 index 557509e818..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/constants.ts +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Enum of search-related signals. - * - * This includes request and response signals for various search operations and their respective - * states. - * - * @enum {string} - */ -enum SEARCH_SIGNAL { - NONE = "none", - - REQ_CANCELLING = "req-cancelling", - REQ_CLEARING = "req-clearing", - REQ_QUERYING = "req-querying", - - RESP_DONE = "resp-done", - RESP_QUERYING = "resp-querying", -} - -/** - * Checks if the given search signal is a search signal request. - * - * @param {SEARCH_SIGNAL} s - * @return {boolean} - */ -const isSearchSignalReq = (s: SEARCH_SIGNAL): boolean => s.startsWith("req-"); - -/** - * Checks if the given search signal is a search signal response. - * - * @param {SEARCH_SIGNAL} s - * @return {boolean} - */ -const isSearchSignalResp = (s: SEARCH_SIGNAL): boolean => s.startsWith("resp-"); - -/** - * Checks if the given search signal is a querying request / response. - * - * @param {SEARCH_SIGNAL} s - * @return {boolean} - */ -const isSearchSignalQuerying = (s: SEARCH_SIGNAL): boolean => s.endsWith("-querying"); - -/** - * Checks if the given search signal is an operation in progress, which can be used as a - * condition to disable UI elements. - * - * @param {SEARCH_SIGNAL} s - * @return {boolean} - */ -const isOperationInProgress = (s: SEARCH_SIGNAL): boolean => ( - (true === isSearchSignalReq(s)) || - (true === isSearchSignalQuerying(s)) -); - -/* eslint-disable sort-keys */ -let enumQueryJobStatus; -/** - * Enum of job statuses, matching the `QueryJobStatus` class in - * `job_orchestration.query_scheduler.constants`. - * - * @enum {number} - */ -const QUERY_JOB_STATUS = Object.freeze({ - PENDING: (enumQueryJobStatus = 0), - RUNNING: ++enumQueryJobStatus, - SUCCEEDED: ++enumQueryJobStatus, - FAILED: ++enumQueryJobStatus, - CANCELLING: ++enumQueryJobStatus, - CANCELLED: ++enumQueryJobStatus, -}); -/* eslint-enable sort-keys */ - -const QUERY_JOB_STATUS_WAITING_STATES = [ - QUERY_JOB_STATUS.PENDING, - QUERY_JOB_STATUS.RUNNING, - QUERY_JOB_STATUS.CANCELLING, -]; - -/* eslint-disable sort-keys */ -let enumQueryType; -/** - * Enum of job type, matching the `QueryJobType` class in - * `job_orchestration.query_scheduler.constants`. - * - * @enum {number} - */ -const QUERY_JOB_TYPE = Object.freeze({ - SEARCH_OR_AGGREGATION: (enumQueryType = 0), - EXTRACT_IR: ++enumQueryType, -}); -/* eslint-enable sort-keys */ - -/** - * Enum of Mongo Collection sort orders. - * - * @enum {string} - */ -const MONGO_SORT_ORDER = Object.freeze({ - ASCENDING: "asc", - DESCENDING: "desc", -}); - -/** - * Enum of search results cache fields. - * - * @enum {string} - */ -const SEARCH_RESULTS_FIELDS = Object.freeze({ - ID: "_id", - TIMESTAMP: "timestamp", -}); - -/** - * The maximum number of results to retrieve for a search. - */ -const SEARCH_MAX_NUM_RESULTS = 1000; - -export { - isOperationInProgress, - isSearchSignalQuerying, - isSearchSignalReq, - isSearchSignalResp, - MONGO_SORT_ORDER, - QUERY_JOB_STATUS, - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, - SEARCH_MAX_NUM_RESULTS, - SEARCH_RESULTS_FIELDS, - SEARCH_SIGNAL, -}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts index a890441c61..de2dd03a48 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts @@ -1,144 +1,18 @@ -import { - SEARCH_MAX_NUM_RESULTS, - SEARCH_SIGNAL, -} from "../constants.js"; - -import { Collection } from 'mongodb'; // Assuming MongoDB is used for SearchResultsMetadataCollection - -type UpdateSearchResultsMetaParams = { - jobId: number; - lastSignal: string; - SearchResultsMetadataCollection: Collection; -}; - -type SearchResultsMetadataFields = { - lastSignal?: string; - errorMsg?: string | null; - numTotalResults?: number; -}; - -/** - * Modifies the search results metadata for a given job ID. - * - * @param {UpdateSearchResultsMetaParams} params - * @param {SearchResultsMetadataFields} fields The fields to be updated in the search results metadata. - */ -const updateSearchResultsMeta = ( - { - jobId, - lastSignal, - SearchResultsMetadataCollection, - }: UpdateSearchResultsMetaParams, - fields: SearchResultsMetadataFields -) => { - const filter = { - _id: jobId.toString(), - lastSignal: lastSignal, - }; - - const modifier = { - $set: fields, - }; - - logger.debug("SearchResultsMetadataCollection modifier = ", modifier); - SearchResultsMetadataCollection.update(filter, modifier); -}; - -/** - * Updates the search signal when the specified job finishes. - * - * @param {object} params - * @param {number} params.searchJobId of the job to monitor - * @param {number} params.aggregationJobId of the job to monitor - * @param {QueryJobsDbManager} queryJobsDbManager - * @param {SearchJobCollectionsManager} searchJobCollectionsManager - * @param {SearchResultsMetadataCollection} SearchResultsMetadataCollection - */ -const updateSearchSignalWhenJobsFinish = async ({ - searchJobId, - aggregationJobId, - queryJobsDbManager, - searchJobCollectionsManager, - SearchResultsMetadataCollection, -}) => { - let errorMsg; - try { - await queryJobsDbManager.awaitJobCompletion(searchJobId); - await queryJobsDbManager.awaitJobCompletion(aggregationJobId); - } catch (e) { - errorMsg = e.message; - } - - let numResultsInCollection = -1; - try { - numResultsInCollection = await searchJobCollectionsManager - .getOrCreateCollection(searchJobId) - .countDocuments(); - } catch (e) { - if (ERROR_NAME_COLLECTION_DROPPED === e.error) { - logger.warn(`Collection ${searchJobId} has been dropped.`); - - return; - } - throw e; - } - - updateSearchResultsMeta({ - jobId: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - SearchResultsMetadataCollection, - }, { - lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: errorMsg, - numTotalResults: Math.min( - numResultsInCollection, - SEARCH_MAX_NUM_RESULTS - ), - }); -}; - -/** - * Creates MongoDB indexes for a specific job's collection. - * - * @param {number} searchJobId used to identify the Mongo Collection to add indexes - * @param {SearchJobCollectionsManager} searchJobCollectionsManager - */ -const createMongoIndexes = async (searchJobId, searchJobCollectionsManager) => { - const timestampAscendingIndex = { - key: { - timestamp: 1, - _id: 1, - }, - name: "timestamp-ascending", - }; - const timestampDescendingIndex = { - key: { - timestamp: -1, - _id: -1, - }, - name: "timestamp-descending", - }; - - const queryJobCollection = searchJobCollectionsManager.getOrCreateCollection(searchJobId); - const queryJobRawCollection = queryJobCollection.rawCollection(); - await queryJobRawCollection.createIndexes([ - timestampAscendingIndex, - timestampDescendingIndex, - ]); -}; +import { SEARCH_SIGNAL, SEARCH_MAX_NUM_RESULTS } from '../../../plugins/app/search/constants.js'; +import { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes } from './utils.js'; import { FastifyPluginAsyncTypebox, Type } from '@fastify/type-provider-typebox' - //import { CredentialsSchema } from '../../../schemas/auth.js' - import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/search.js' +import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/search.js' const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const { QueryJobsDbManager, SearchJobCollectionsManager, - SearchResultsMetadataCollection - } = fastify + SearchResultsMetadataCollection, + log: logger + } = fastify; fastify.post( '/search', @@ -153,6 +27,7 @@ import { }, }, async function (request, reply) { + const { queryString, timestampBegin, @@ -180,13 +55,12 @@ import { args, timeRangeBucketSizeMillis ); - } catch (err: any) { - const message = 'Unable to submit search/aggregation job to the SQL database'; - request.log.error(err, message); - return reply.internalServerError(message); + } catch (err: unknown) { + const errorMsg = 'Unable to submit search/aggregation job to the SQL database'; + request.log.error(errorMsg, err); + return reply.internalServerError(errorMsg); } - // I need to update the schema. SearchResultsMetadataCollection.insertOne({ _id: searchJobId.toString(), lastSignal: SEARCH_SIGNAL.RESP_QUERYING, @@ -195,18 +69,23 @@ import { // Defer signal update until after response is sent setImmediate(() => { - updateSearchSignalWhenJobsFinish({ + updateSearchSignalWhenJobsFinish({ searchJobId, aggregationJobId, queryJobsDbManager: QueryJobsDbManager, searchJobCollectionsManager: SearchJobCollectionsManager, SearchResultsMetadataCollection, - }).catch(err => + logger: request.log, + }).catch(err => request.log.error(err, 'Deferred update failed') - ); + ); }); - await createMongoIndexes(searchJobId, SearchJobCollectionsManager); + await createMongoIndexes({ + searchJobId, + searchJobCollectionsManager: SearchJobCollectionsManager, + logger, + }); return { searchJobId, aggregationJobId }; } @@ -268,17 +147,16 @@ import { await QueryJobsDbManager.submitQueryCancellation(searchJobId); await QueryJobsDbManager.submitQueryCancellation(aggregationJobId); - updateSearchResultsMeta( - { - jobId: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - SearchResultsMetadataCollection, - }, - { + updateSearchResultsMeta({ + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + SearchResultsMetadataCollection, + logger, + fields: { lastSignal: SEARCH_SIGNAL.RESP_DONE, errorMsg: 'Query cancelled before it could be completed.', - } - ); + }, + }); reply.code(204); } catch (err: any) { diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts new file mode 100644 index 0000000000..fecfa3815e --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts @@ -0,0 +1,129 @@ +import { FastifyBaseLogger } from "fastify"; +import { Collection } from "mongodb"; + +import { SearchResultsMetadataDocument } from '../../../plugins/app/search/SearchResultsMetadataCollection/typings.js'; +import { SEARCH_SIGNAL } from '../../../plugins/app/search/constants.js'; + +type UpdateSearchResultsMetaProps = { + jobId: number; + lastSignal: SEARCH_SIGNAL; + SearchResultsMetadataCollection: Collection; + logger: FastifyBaseLogger; + fields: Partial; +}; + +type UpdateSearchSignalWhenJobsFinishProps = { + searchJobId: number; + aggregationJobId: number; + queryJobsDbManager: any; // Replace with actual type + searchJobCollectionsManager: any; // Replace with actual type + SearchResultsMetadataCollection: Collection; // Replace with actual type + logger: FastifyBaseLogger; +}; + +type CreateMongoIndexesProps = { + searchJobId: number; + searchJobCollectionsManager: any; // Replace with actual type + logger: FastifyBaseLogger; +}; + +/** + * Modifies the search results metadata for a given job ID. + * + * @param {UpdateSearchResultsMetaProps} params + */ +const updateSearchResultsMeta = ({ + jobId, + lastSignal, + SearchResultsMetadataCollection, + logger, + fields, +}: UpdateSearchResultsMetaProps) => { + const filter = { + _id: jobId.toString(), + lastSignal: lastSignal, + }; + + const modifier = { + $set: fields, + }; + + logger.debug("SearchResultsMetadataCollection modifier = ", modifier); + SearchResultsMetadataCollection.updateOne(filter, modifier); +}; + +const updateSearchSignalWhenJobsFinish = async ({ + searchJobId, + aggregationJobId, + queryJobsDbManager, + searchJobCollectionsManager, + SearchResultsMetadataCollection, + logger, +}: UpdateSearchSignalWhenJobsFinishProps) => { + let errorMsg; + try { + await queryJobsDbManager.awaitJobCompletion(searchJobId); + await queryJobsDbManager.awaitJobCompletion(aggregationJobId); + } catch (e) { + errorMsg = e.message; + } + + let numResultsInCollection = -1; + try { + numResultsInCollection = await searchJobCollectionsManager + .getOrCreateCollection(searchJobId) + .countDocuments(); + } catch (e) { + if (ERROR_NAME_COLLECTION_DROPPED === e.error) { + logger.warn(`Collection ${searchJobId} has been dropped.`); + return; + } + throw e; + } + + updateSearchResultsMeta({ + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + SearchResultsMetadataCollection, + logger, + fields: { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: errorMsg, + numTotalResults: Math.min( + numResultsInCollection, + SEARCH_MAX_NUM_RESULTS + ), + }, + }); +}; + +const createMongoIndexes = async ({ + searchJobId, + searchJobCollectionsManager, + logger, +}: CreateMongoIndexesProps) => { + const timestampAscendingIndex = { + key: { + timestamp: 1, + _id: 1, + }, + name: "timestamp-ascending", + }; + const timestampDescendingIndex = { + key: { + timestamp: -1, + _id: -1, + }, + name: "timestamp-descending", + }; + + const queryJobCollection = searchJobCollectionsManager.getOrCreateCollection(searchJobId); + const queryJobRawCollection = queryJobCollection.rawCollection(); + logger.info(`Creating indexes for collection ${searchJobId}`); + await queryJobRawCollection.createIndexes([ + timestampAscendingIndex, + timestampDescendingIndex, + ]); +}; + +export { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes }; \ No newline at end of file From 14c2f00ec61bdee978efc6f94ace5ee83c517ec7 Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 14 May 2025 19:21:32 +0000 Subject: [PATCH 06/55] latest --- .../plugins/app/search/QueryJobsDbManager.ts | 177 ++++++++++++++++++ .../app/search/SearchJobCollectionsManager.ts | 89 +++++++++ .../SearchResultsMetadataCollection/index.ts | 36 ++++ .../typings.ts | 11 ++ .../plugins/app/search/constants.ts | 133 +++++++++++++ 5 files changed, 446 insertions(+) create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts new file mode 100644 index 0000000000..4a1232b489 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts @@ -0,0 +1,177 @@ +import { encode } from "@msgpack/msgpack"; +import { ResultSetHeader, RowDataPacket } from "mysql2"; +// Fix types to modern typescripts +// Use shared types with DB manager, and move to new file. +import { + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, +} from "./constants.js"; +import { FastifyInstance } from "fastify"; +import fp from "fastify-plugin"; +import settings from "../../../../../settings.json" with { type: "json" }; +import { setTimeout } from 'timers/promises'; + +import type { + MySQLPromisePool, +} from "@fastify/mysql"; + +/** + * Interval in milliseconds for polling the completion status of a job. + */ +const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 500; + +/** + * Enum of the `query_jobs` table's column names. + */ +enum QUERY_JOBS_TABLE_COLUMN_NAMES { + ID = "id", + STATUS = "status", + TYPE = "type", + JOB_CONFIG = "job_config", +} + +/** + * Class for submitting and monitoring query jobs in the database. + */ +class QueryJobsDbManager { + #sqlDbConnPool: MySQLPromisePool; + + private constructor(sqlDbConnPool: MySQLPromisePool) { + this.#sqlDbConnPool = sqlDbConnPool; + } + + /** + * Creates a new QueryJobsDbManager. + * + * @param fastify + * @returns + */ + static create(fastify: FastifyInstance): QueryJobsDbManager { + const sqlDbConnPool = fastify.mysql; + return new QueryJobsDbManager(sqlDbConnPool); + } + + /** + * Submits a search job to the database. + * + * @param searchConfig The arguments for the query. + * @return The job's ID. + * @throws {Error} on error. + */ + async submitSearchJob(searchConfig: object): Promise { + const [queryInsertResults] = await this.#sqlDbConnPool.query( + `INSERT INTO ${settings.SqlDbQueryJobsTableName} + (${QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG}, + ${QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE}) + VALUES (?, ?)`, + [ + Buffer.from(encode(searchConfig)), + QUERY_JOB_TYPE.SEARCH_OR_AGGREGATION, + ] + ); + + return queryInsertResults.insertId; + } + + /** + * Submits an aggregation job to the database. + * + * @param searchConfig + * @param timeRangeBucketSizeMillis + * @return The aggregation job's ID. + * @throws {Error} on error. + */ + async submitAggregationJob( + searchConfig: object, + timeRangeBucketSizeMillis: number + ): Promise { + const searchAggregationConfig = { + ...searchConfig, + aggregation_config: { + count_by_time_bucket_size: timeRangeBucketSizeMillis, + }, + }; + + return await this.submitSearchJob(searchAggregationConfig); + } + + /** + * Submits a query cancellation request to the database. + * + * @param jobId ID of the job to cancel. + * @return + * @throws {Error} on error. + */ + async submitQueryCancellation (jobId: number): Promise { + await this.#sqlDbConnPool.query( + `UPDATE ${settings.SqlDbQueryJobsTableName} + SET ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${QUERY_JOB_STATUS.CANCELLING} + WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? + AND ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} + IN (${QUERY_JOB_STATUS.PENDING}, ${QUERY_JOB_STATUS.RUNNING})`, + jobId, + ); + } + + /** + * Waits for the job to complete. + * + * @param jobId + * @return + * @throws {Error} on MySQL error, if the job wasn't found in the database, if the job was + * cancelled, or if the job completed in an unexpected state. + */ + async awaitJobCompletion(jobId: number): Promise { + while (true) { + let rows; + try { + const [queryRows] = await this.#sqlDbConnPool.query( + ` + SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} + FROM ${settings.SqlDbQueryJobsTableName} + WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? + `, + jobId + ); + + rows = queryRows; + } catch (e) { + throw new Error(`Failed to query status for job ${jobId} - ${e}`); + } + if (rows.length === 0) { + throw new Error(`Job ${jobId} not found in database.`); + } + const status = rows[0]![QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; + + if (false === QUERY_JOB_STATUS_WAITING_STATES.includes(status)) { + if (QUERY_JOB_STATUS.CANCELLED === status) { + throw new Error(`Job ${jobId} was cancelled.`); + } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { + throw new Error( + `Job ${jobId} exited with unexpected status=${status}: ` + + `${Object.keys(QUERY_JOB_STATUS)[status]}.` + ); + } + break; + } + + await setTimeout(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); + } + } +} + +declare module "fastify" { + export interface FastifyInstance { + QueryJobsDbManager: QueryJobsDbManager; + } +} + +export default fp( + function (fastify) { + fastify.decorate("QueryJobsDbManager", QueryJobsDbManager.create(fastify)); + }, + { + name: "QueryJobsDbManager", + } +); \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager.ts new file mode 100644 index 0000000000..926101ddf2 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager.ts @@ -0,0 +1,89 @@ +import { FastifyInstance } from 'fastify'; +import fp from 'fastify-plugin'; +import { Nullable } from "../../../../typings/common.js"; +import type { Db, Collection } from "mongodb"; + +interface SearchResultsMetadataDocument { + _id: string; + orig_file_id: string; + orig_file_path: string; + log_event_ix: number; + timestamp: number; + message: string; +} + +/** + * Class to keep track of MongoDB collections created for search jobs, ensuring all collections + * have unique names. + */ +class SearchJobCollectionsManager { + #collections: Map>> = new Map(); + #db: Db; + + private constructor (db: Db) { + this.#db = db; + } + + /** + * Creates a new SearchJobCollectionsManager. + * + * @param fastify + * @returns + */ + static create(fastify: FastifyInstance): SearchJobCollectionsManager { + if ("undefined" === typeof fastify.mongo.db) { + throw new Error("MongoDB database not found"); + } + + return new SearchJobCollectionsManager(fastify.mongo.db); + } + + /** + * Gets, or if it doesn't exist, creates a MongoDB collection named with the given job ID. + * + * @param jobId + * @return MongoDB collection + * @throws {Error} if the collection was already dropped. + */ + async getOrCreateCollection(jobId: number) { + const name = jobId.toString(); + if (false === this.#collections.has(name)) { + this.#collections.set(name, this.#db.collection(name)); + } else if (this.#collections.get(name) === null) { + throw new Error(`Collection ${name} has been dropped.`); + } + return this.#collections.get(name); + } + + /** + * Drops the MongoDB collection associated with the given job ID. + * + * @param jobId + * @throws {Error} if the collection does not exist or has already been dropped. + */ + async dropCollection(jobId: number) { + const name = jobId.toString(); + const collection = this.#collections.get(name); + if ("undefined" === typeof collection || null === collection) { + throw new Error(`Collection ${name} not found`); + } + + await collection.drop(); + this.#collections.set(name, null); + } +} + +declare module 'fastify' { + export interface FastifyInstance { + SearchJobCollectionsManager: SearchJobCollectionsManager; + } +} + +export default fp( + function (fastify) { + fastify.decorate('SearchJobCollectionsManager', SearchJobCollectionsManager.create(fastify)); + }, + { + name: 'SearchJobCollectionsManager', + } +); \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts new file mode 100644 index 0000000000..d1774c173b --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts @@ -0,0 +1,36 @@ +import { FastifyInstance } from 'fastify' +import fp from 'fastify-plugin' + +import settings from "../../../../../../settings.json" with {type: "json"}; +import { SearchResultsMetadataDocument } from './typings.js'; + +/** + * Creates a MongoDB collection for search results metadata. + */ +const createSearchResultsMetadataCollection = (fastify: FastifyInstance) => { + if ("undefined" === typeof fastify.mongo.db) { + throw new Error("MongoDB database not found"); + } + + return fastify.mongo.db.collection( + settings.MongoDbSearchResultsMetadataCollectionName + ); +}; + +declare module 'fastify' { + export interface FastifyInstance { + SearchResultsMetadataCollection: ReturnType; + } +} + +export default fp( + function (fastify) { + fastify.decorate( + 'SearchResultsMetadataCollection', + createSearchResultsMetadataCollection(fastify) + ) + }, + { + name: 'SearchResultsMetadataCollection', + } +) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts new file mode 100644 index 0000000000..a10a424974 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts @@ -0,0 +1,11 @@ +import { Nullable } from "../../../../../typings/common.js"; +import { SEARCH_SIGNAL } from "../constants.js"; + +interface SearchResultsMetadataDocument { + _id: string; + errorMsg: Nullable; + lastSignal: SEARCH_SIGNAL; + numTotalResults?: number; +} + +export { SearchResultsMetadataDocument }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts new file mode 100644 index 0000000000..ed0090e2b6 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts @@ -0,0 +1,133 @@ +// Maybe move shared stuff to a more common location. + +/** + * Enum of search-related signals. + * + * This includes request and response signals for various search operations and their respective + * states. + * + * @enum {string} + */ +enum SEARCH_SIGNAL { + NONE = "none", + + REQ_CANCELLING = "req-cancelling", + REQ_CLEARING = "req-clearing", + REQ_QUERYING = "req-querying", + + RESP_DONE = "resp-done", + RESP_QUERYING = "resp-querying", +} + +/** + * Checks if the given search signal is a search signal request. + * + * @param {SEARCH_SIGNAL} s + * @return {boolean} + */ +const isSearchSignalReq = (s: SEARCH_SIGNAL): boolean => s.startsWith("req-"); + +/** + * Checks if the given search signal is a search signal response. + * + * @param {SEARCH_SIGNAL} s + * @return {boolean} + */ +const isSearchSignalResp = (s: SEARCH_SIGNAL): boolean => s.startsWith("resp-"); + +/** + * Checks if the given search signal is a querying request / response. + * + * @param {SEARCH_SIGNAL} s + * @return {boolean} + */ +const isSearchSignalQuerying = (s: SEARCH_SIGNAL): boolean => s.endsWith("-querying"); + +/** + * Checks if the given search signal is an operation in progress, which can be used as a + * condition to disable UI elements. + * + * @param {SEARCH_SIGNAL} s + * @return {boolean} + */ +const isOperationInProgress = (s: SEARCH_SIGNAL): boolean => ( + (true === isSearchSignalReq(s)) || + (true === isSearchSignalQuerying(s)) +); + +/* eslint-disable sort-keys */ +let enumQueryJobStatus; +/** + * Enum of job statuses, matching the `QueryJobStatus` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +const QUERY_JOB_STATUS = Object.freeze({ + PENDING: (enumQueryJobStatus = 0), + RUNNING: ++enumQueryJobStatus, + SUCCEEDED: ++enumQueryJobStatus, + FAILED: ++enumQueryJobStatus, + CANCELLING: ++enumQueryJobStatus, + CANCELLED: ++enumQueryJobStatus, +}); +/* eslint-enable sort-keys */ + +const QUERY_JOB_STATUS_WAITING_STATES = [ + QUERY_JOB_STATUS.PENDING, + QUERY_JOB_STATUS.RUNNING, + QUERY_JOB_STATUS.CANCELLING, +]; + +/* eslint-disable sort-keys */ +let enumQueryType; +/** + * Enum of job type, matching the `QueryJobType` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +const QUERY_JOB_TYPE = Object.freeze({ + SEARCH_OR_AGGREGATION: (enumQueryType = 0), + EXTRACT_IR: ++enumQueryType, +}); +/* eslint-enable sort-keys */ + +/** + * Enum of Mongo Collection sort orders. + * + * @enum {string} + */ +const MONGO_SORT_ORDER = Object.freeze({ + ASCENDING: "asc", + DESCENDING: "desc", +}); + +/** + * Enum of search results cache fields. + * + * @enum {string} + */ +const SEARCH_RESULTS_FIELDS = Object.freeze({ + ID: "_id", + TIMESTAMP: "timestamp", +}); + +/** + * The maximum number of results to retrieve for a search. + */ +const SEARCH_MAX_NUM_RESULTS = 1000; + +export { + isOperationInProgress, + isSearchSignalQuerying, + isSearchSignalReq, + isSearchSignalResp, + MONGO_SORT_ORDER, + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, + SEARCH_MAX_NUM_RESULTS, + SEARCH_RESULTS_FIELDS, + SEARCH_SIGNAL, +}; From 4a740639453a4f335d8d0e6239cb684646167237 Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 14 May 2025 21:21:56 +0000 Subject: [PATCH 07/55] latest --- .../src/fastify-v2/routes/api/search/utils.ts | 28 ++++--------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts index fecfa3815e..a6cdee2f55 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts @@ -3,29 +3,11 @@ import { Collection } from "mongodb"; import { SearchResultsMetadataDocument } from '../../../plugins/app/search/SearchResultsMetadataCollection/typings.js'; import { SEARCH_SIGNAL } from '../../../plugins/app/search/constants.js'; - -type UpdateSearchResultsMetaProps = { - jobId: number; - lastSignal: SEARCH_SIGNAL; - SearchResultsMetadataCollection: Collection; - logger: FastifyBaseLogger; - fields: Partial; -}; - -type UpdateSearchSignalWhenJobsFinishProps = { - searchJobId: number; - aggregationJobId: number; - queryJobsDbManager: any; // Replace with actual type - searchJobCollectionsManager: any; // Replace with actual type - SearchResultsMetadataCollection: Collection; // Replace with actual type - logger: FastifyBaseLogger; -}; - -type CreateMongoIndexesProps = { - searchJobId: number; - searchJobCollectionsManager: any; // Replace with actual type - logger: FastifyBaseLogger; -}; +import { + UpdateSearchResultsMetaProps, + UpdateSearchSignalWhenJobsFinishProps, + CreateMongoIndexesProps, +} from './typings.js'; /** * Modifies the search results metadata for a given job ID. From 3be40cadeb81bf8a24f614c910d900145e99115b Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 15 May 2025 14:48:23 +0000 Subject: [PATCH 08/55] latest --- .../fastify-v2/routes/api/search/typings.ts | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts new file mode 100644 index 0000000000..fc082045fd --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts @@ -0,0 +1,37 @@ +import { + FastifyBaseLogger, + FastifyInstance, +} from "fastify"; + +import {SEARCH_SIGNAL} from "../../../plugins/app/search/constants.js"; +import {SearchResultsMetadataDocument} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; + + +type UpdateSearchResultsMetaProps = { + jobId: number; + lastSignal: SEARCH_SIGNAL; + SearchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; + logger: FastifyBaseLogger; + fields: Partial; +}; + +type UpdateSearchSignalWhenJobsFinishProps = { + searchJobId: number; + aggregationJobId: number; + queryJobsDbManager: FastifyInstance["QueryJobsDbManager"]; + searchJobCollectionsManager: FastifyInstance["SearchJobCollectionsManager"]; + SearchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; + logger: FastifyBaseLogger; +}; + +type CreateMongoIndexesProps = { + searchJobId: number; + searchJobCollectionsManager: FastifyInstance["SearchJobCollectionsManager"]; + logger: FastifyBaseLogger; +}; + +export { + CreateMongoIndexesProps, + UpdateSearchResultsMetaProps, + UpdateSearchSignalWhenJobsFinishProps, +}; From 4ba04b3bf73852c3cfb16ac3e59b29e47e66a76f Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 14:45:16 +0000 Subject: [PATCH 09/55] latest --- .../log-viewer-webui/client/src/App.tsx | 126 ++++++++++++++++++ .../log-viewer-webui/server/settings.json | 2 +- .../server/src/fastify-v2/app.ts | 6 + .../src/fastify-v2/plugins/app/sample.ts | 5 +- .../plugins/app/search/QueryJobsDbManager.ts | 19 +-- .../index.ts} | 25 ++-- .../SearchJobCollectionsManager/typings.ts | 17 +++ .../typings.ts | 24 +++- .../plugins/app/search/constants.ts | 41 ------ .../fastify-v2/routes/api/search/constants.ts | 8 ++ .../fastify-v2/routes/api/search/errors.ts | 24 ++++ .../src/fastify-v2/routes/api/search/index.ts | 103 ++++++++------ .../fastify-v2/routes/api/search/typings.ts | 3 +- .../src/fastify-v2/routes/api/search/utils.ts | 36 ++--- .../server/src/fastify-v2/schemas/error.ts | 22 +++ .../server/src/fastify-v2/schemas/search.ts | 2 +- .../server/src/plugins/DbManager.ts | 13 +- .../server/src/typings/DbManager.ts | 72 +--------- .../server/src/typings/query.ts | 73 ++++++++++ 19 files changed, 409 insertions(+), 212 deletions(-) rename components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/{SearchJobCollectionsManager.ts => SearchJobCollectionsManager/index.ts} (79%) create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/constants.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/errors.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts create mode 100644 components/log-viewer-webui/server/src/typings/query.ts diff --git a/components/log-viewer-webui/client/src/App.tsx b/components/log-viewer-webui/client/src/App.tsx index b54609e539..9e7c0e5db3 100644 --- a/components/log-viewer-webui/client/src/App.tsx +++ b/components/log-viewer-webui/client/src/App.tsx @@ -1,4 +1,6 @@ import {CssVarsProvider} from "@mui/joy"; +import {useState} from "react"; +import axios from "axios"; import {LOCAL_STORAGE_KEY} from "./typings/config"; import QueryStatus from "./ui/QueryStatus"; @@ -10,9 +12,133 @@ import QueryStatus from "./ui/QueryStatus"; * @return */ const App = () => { + const [output, setOutput] = useState(""); + const [queryString, setQueryString] = useState("test query"); + const [timestampBegin, setTimestampBegin] = useState(0); + const [timestampEnd, setTimestampEnd] = useState(Date.now()); + const [ignoreCase, setIgnoreCase] = useState(true); + const [timeRangeBucketSizeMillis, setTimeRangeBucketSizeMillis] = useState(60000); + const [searchJobId, setSearchJobId] = useState(1); + const [aggregationJobId, setAggregationJobId] = useState(2); + + const sendRequest = async (url: string, method: string, body: object) => { + try { + const response = await axios({ + url, + method, + data: body, + headers: {"Content-Type": "application/json"}, + }); + setOutput(JSON.stringify(response.data, null, 2)); + } catch (error: unknown) { + if (axios.isAxiosError(error)) { + setOutput(`Error: ${error.response?.data?.message || error.message}`); + } else { + setOutput(`Error: ${String(error)}`); + } + } + }; + return ( +
+

Search API Test

+
+ + + + + + + +
+ + + + 
+                    {output}
+
+
); }; diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json index 8010a8f38b..630a49d8c2 100644 --- a/components/log-viewer-webui/server/settings.json +++ b/components/log-viewer-webui/server/settings.json @@ -11,7 +11,7 @@ "MongoDbPort": 27017, "MongoDbName": "clp-query-results", "MongoDbStreamFilesCollectionName": "stream-files", - "MongoDbSearchResultsMetadataCollectionName": "results-metadata" + "MongoDbSearchResultsMetadataCollectionName": "results-metadata", "ClientDir": "../client/dist", "LogViewerDir": "../yscope-log-viewer/dist", diff --git a/components/log-viewer-webui/server/src/fastify-v2/app.ts b/components/log-viewer-webui/server/src/fastify-v2/app.ts index 0a24171abf..2fa6ca57bd 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/app.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/app.ts @@ -45,17 +45,23 @@ export default async function serviceApp ( // Loads all application plugins defined in plugins/app fastify.register(fastifyAutoload, { dir: path.join(import.meta.dirname, 'plugins/app'), + ignorePattern: /^.*(?:constants|typings|utils).js$/, options: { ...opts } }) // loads all plugins routes fastify.register(fastifyAutoload, { dir: path.join(import.meta.dirname, 'routes'), + ignorePattern: /^.*(?:constants|typings|utils).js$/, autoHooks: true, cascadeHooks: true, options: { ...opts } }) + fastify.addHook('onRoute', (routeOptions) => { + console.log(`Route registered: ${routeOptions.method} ${routeOptions.url}`); + }); + fastify.setErrorHandler((err, request, reply) => { fastify.log.error( { diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/sample.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/sample.ts index 64f6154fce..62c27c3a05 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/sample.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/sample.ts @@ -1,5 +1,4 @@ -import { FastifyInstance } from 'fastify' import fp from 'fastify-plugin' declare module 'fastify' { @@ -13,7 +12,7 @@ declare module 'fastify' { * * TODO: Remove sample code when new webui app code is ready. */ -function createSamplePlugin (fastify: FastifyInstance) { +function createSamplePlugin () { return { /** * Returns a sample string. @@ -29,7 +28,7 @@ function createSamplePlugin (fastify: FastifyInstance) { export default fp( function (fastify) { - fastify.decorate('samplePlugin', createSamplePlugin(fastify)) + fastify.decorate('samplePlugin', createSamplePlugin()) }, { name: 'samplePlugin', diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts index 4a1232b489..607b186e56 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts @@ -1,12 +1,11 @@ import { encode } from "@msgpack/msgpack"; import { ResultSetHeader, RowDataPacket } from "mysql2"; -// Fix types to modern typescripts -// Use shared types with DB manager, and move to new file. import { - QUERY_JOB_STATUS, QUERY_JOB_STATUS_WAITING_STATES, QUERY_JOB_TYPE, -} from "./constants.js"; + QUERY_JOB_STATUS, + QUERY_JOBS_TABLE_COLUMN_NAMES, +} from "../../../../typings/query.js"; import { FastifyInstance } from "fastify"; import fp from "fastify-plugin"; import settings from "../../../../../settings.json" with { type: "json" }; @@ -21,16 +20,6 @@ import type { */ const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 500; -/** - * Enum of the `query_jobs` table's column names. - */ -enum QUERY_JOBS_TABLE_COLUMN_NAMES { - ID = "id", - STATUS = "status", - TYPE = "type", - JOB_CONFIG = "job_config", -} - /** * Class for submitting and monitoring query jobs in the database. */ @@ -144,7 +133,7 @@ class QueryJobsDbManager { } const status = rows[0]![QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; - if (false === QUERY_JOB_STATUS_WAITING_STATES.includes(status)) { + if (false === QUERY_JOB_STATUS_WAITING_STATES.has(status)) { if (QUERY_JOB_STATUS.CANCELLED === status) { throw new Error(`Job ${jobId} was cancelled.`); } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts similarity index 79% rename from components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager.ts rename to components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts index 926101ddf2..c55c6e6d80 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts @@ -1,16 +1,9 @@ import { FastifyInstance } from 'fastify'; import fp from 'fastify-plugin'; -import { Nullable } from "../../../../typings/common.js"; +import { Nullable } from "../../../../../typings/common.js"; import type { Db, Collection } from "mongodb"; - -interface SearchResultsMetadataDocument { - _id: string; - orig_file_id: string; - orig_file_path: string; - log_event_ix: number; - timestamp: number; - message: string; -} +import type { SearchResultsMetadataDocument } from "./typings.js"; +import { CollectionDroppedError } from "./typings.js"; /** * Class to keep track of MongoDB collections created for search jobs, ensuring all collections @@ -43,16 +36,16 @@ class SearchJobCollectionsManager { * * @param jobId * @return MongoDB collection - * @throws {Error} if the collection was already dropped. + * @throws {CollectionDroppedError} if the collection was already dropped. */ - async getOrCreateCollection(jobId: number) { + async getOrCreateCollection(jobId: number): Promise> { const name = jobId.toString(); if (false === this.#collections.has(name)) { this.#collections.set(name, this.#db.collection(name)); } else if (this.#collections.get(name) === null) { - throw new Error(`Collection ${name} has been dropped.`); + throw new CollectionDroppedError(name); } - return this.#collections.get(name); + return this.#collections.get(name) as Collection; } /** @@ -86,4 +79,6 @@ export default fp( { name: 'SearchJobCollectionsManager', } -); \ No newline at end of file +); + +export {CollectionDroppedError} \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts new file mode 100644 index 0000000000..e032e8bc83 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts @@ -0,0 +1,17 @@ +interface SearchResultsMetadataDocument { + _id: string; + orig_file_id: string; + orig_file_path: string; + log_event_ix: number; + timestamp: number; + message: string; +} + +class CollectionDroppedError extends Error { + constructor(collectionName: string) { + super(`Collection ${collectionName} has been dropped.`); + this.name = "CollectionDroppedError"; + } +} + +export { SearchResultsMetadataDocument, CollectionDroppedError }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts index a10a424974..6223dbd17e 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts @@ -1,5 +1,4 @@ import { Nullable } from "../../../../../typings/common.js"; -import { SEARCH_SIGNAL } from "../constants.js"; interface SearchResultsMetadataDocument { _id: string; @@ -8,4 +7,25 @@ interface SearchResultsMetadataDocument { numTotalResults?: number; } -export { SearchResultsMetadataDocument }; +/** + * Enum of search-related signals. + * + * This includes request and response signals for various search operations and their respective + * states. + * + * @enum {string} + */ +enum SEARCH_SIGNAL { + NONE = "none", + + REQ_CANCELLING = "req-cancelling", + REQ_CLEARING = "req-clearing", + REQ_QUERYING = "req-querying", + + RESP_DONE = "resp-done", + RESP_QUERYING = "resp-querying", +} + + + +export { SearchResultsMetadataDocument, SEARCH_SIGNAL }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts index ed0090e2b6..978ac225a5 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts @@ -55,44 +55,6 @@ const isOperationInProgress = (s: SEARCH_SIGNAL): boolean => ( (true === isSearchSignalQuerying(s)) ); -/* eslint-disable sort-keys */ -let enumQueryJobStatus; -/** - * Enum of job statuses, matching the `QueryJobStatus` class in - * `job_orchestration.query_scheduler.constants`. - * - * @enum {number} - */ -const QUERY_JOB_STATUS = Object.freeze({ - PENDING: (enumQueryJobStatus = 0), - RUNNING: ++enumQueryJobStatus, - SUCCEEDED: ++enumQueryJobStatus, - FAILED: ++enumQueryJobStatus, - CANCELLING: ++enumQueryJobStatus, - CANCELLED: ++enumQueryJobStatus, -}); -/* eslint-enable sort-keys */ - -const QUERY_JOB_STATUS_WAITING_STATES = [ - QUERY_JOB_STATUS.PENDING, - QUERY_JOB_STATUS.RUNNING, - QUERY_JOB_STATUS.CANCELLING, -]; - -/* eslint-disable sort-keys */ -let enumQueryType; -/** - * Enum of job type, matching the `QueryJobType` class in - * `job_orchestration.query_scheduler.constants`. - * - * @enum {number} - */ -const QUERY_JOB_TYPE = Object.freeze({ - SEARCH_OR_AGGREGATION: (enumQueryType = 0), - EXTRACT_IR: ++enumQueryType, -}); -/* eslint-enable sort-keys */ - /** * Enum of Mongo Collection sort orders. * @@ -124,9 +86,6 @@ export { isSearchSignalReq, isSearchSignalResp, MONGO_SORT_ORDER, - QUERY_JOB_STATUS, - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, SEARCH_MAX_NUM_RESULTS, SEARCH_RESULTS_FIELDS, SEARCH_SIGNAL, diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/constants.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/constants.ts new file mode 100644 index 0000000000..42bf06f673 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/constants.ts @@ -0,0 +1,8 @@ +/** + * The maximum number of results to retrieve for a search. + */ +const SEARCH_MAX_NUM_RESULTS = 1000; + +export { + SEARCH_MAX_NUM_RESULTS, +}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/errors.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/errors.ts new file mode 100644 index 0000000000..ae2fe76006 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/errors.ts @@ -0,0 +1,24 @@ +import createError from '@fastify/error'; + +enum ErrorCodes { + QuerySubmitError = 'FST_QUERY_SUBMIT_ERROR', + ClearResultsError = 'FST_CLEAR_RESULTS_ERROR', + QueryCancelError = 'FST_QUERY_CANCEL_ERROR', +} + +const QuerySubmitError = createError( + ErrorCodes.QuerySubmitError, + 'Unable to submit search/aggregation job to the SQL database', +); + +const ClearResultsError = createError( + ErrorCodes.ClearResultsError, + 'Failed to clear search results for searchJobId=%s, aggregationJobId=%s', +); + +const QueryCancelError = createError( + ErrorCodes.QueryCancelError, + 'Failed to submit cancel request for searchJobId=%s, aggregationJobId=%s', +); + +export {QuerySubmitError, ClearResultsError, QueryCancelError }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts index de2dd03a48..59c7266213 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts @@ -1,27 +1,32 @@ -import { SEARCH_SIGNAL, SEARCH_MAX_NUM_RESULTS } from '../../../plugins/app/search/constants.js'; - +import { SEARCH_MAX_NUM_RESULTS } from './constants.js'; +import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; import { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes } from './utils.js'; import { FastifyPluginAsyncTypebox, Type } from '@fastify/type-provider-typebox' -import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/search.js' +import { CreateSearchJobSchema, SearchJobSchema } from '../../../schemas/search.js' +import { FastifyErrorSchema } from '../../../schemas/error.js' +import { QuerySubmitError, ClearResultsError, QueryCancelError } from './errors.js'; const plugin: FastifyPluginAsyncTypebox = async (fastify) => { - const { QueryJobsDbManager, + const { + QueryJobsDbManager, SearchJobCollectionsManager, SearchResultsMetadataCollection, - log: logger } = fastify; + /** + * Submits a search query and initiates the search process. + */ fastify.post( '/search', { schema: { body: CreateSearchJobSchema, response: { - 200: SearchJobResultSchema, - 500: Type.Object({ message: Type.String() }), + 201: SearchJobSchema, + 500: FastifyErrorSchema, }, tags: ['Search'] }, @@ -44,7 +49,7 @@ import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/s max_num_results: SEARCH_MAX_NUM_RESULTS }; - request.log.info({ args }, 'Submitting search query'); + request.log.info({ args }, "/search args"); let searchJobId: number; let aggregationJobId: number; @@ -56,9 +61,9 @@ import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/s timeRangeBucketSizeMillis ); } catch (err: unknown) { - const errorMsg = 'Unable to submit search/aggregation job to the SQL database'; - request.log.error(errorMsg, err); - return reply.internalServerError(errorMsg); + request.log.error( err , 'Unable to submit search/aggregation job to the SQL database'); + reply.code(500) + return QuerySubmitError(); } SearchResultsMetadataCollection.insertOne({ @@ -68,37 +73,40 @@ import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/s }); // Defer signal update until after response is sent - setImmediate(() => { - updateSearchSignalWhenJobsFinish({ + setImmediate( async () => { + await updateSearchSignalWhenJobsFinish({ searchJobId, aggregationJobId, queryJobsDbManager: QueryJobsDbManager, searchJobCollectionsManager: SearchJobCollectionsManager, SearchResultsMetadataCollection, logger: request.log, - }).catch(err => - request.log.error(err, 'Deferred update failed') - ); + }); }); await createMongoIndexes({ searchJobId, searchJobCollectionsManager: SearchJobCollectionsManager, - logger, + logger: request.log, }); + reply.code(201) + return { searchJobId, aggregationJobId }; } ); + /** + * Clears the results of a search operation identified by jobId. + */ fastify.delete( '/search/results', { schema: { - body: SearchJobResultSchema, + body: SearchJobSchema, response: { - 204: Type.Null, - 500: Type.Object({ message: Type.String() }), + 204: Type.Null(), + 500: FastifyErrorSchema, }, tags: ['Search'], }, @@ -106,20 +114,30 @@ import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/s async function (request, reply) { const { searchJobId, aggregationJobId } = request.body; - request.log.info( - `search.clearResults searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` - ); + request.log.info({ + searchJobId, + aggregationJobId, + }, "/search/results args"); try { await SearchJobCollectionsManager.dropCollection(searchJobId); await SearchJobCollectionsManager.dropCollection(aggregationJobId); + } catch (err: unknown) { + request.log.error( + { + err, + searchJobId, + aggregationJobId, + }, + 'Failed to clear search results' + ); - reply.code(204); - } catch (err: any) { - const message = `Failed to clear search results for searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}`; - request.log.error(err, message); - return reply.internalServerError(message); + reply.code(500); + return ClearResultsError(searchJobId, aggregationJobId); } + + reply.code(204); + return; } ); @@ -128,10 +146,10 @@ import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/s '/search/cancel', { schema: { - body: SearchJobResultSchema, + body: SearchJobSchema, response: { - 204: Type.Null, - 500: Type.Object({ message: Type.String()}), + 204: Type.Null(), + 500: FastifyErrorSchema, }, tags: ['Search'], }, @@ -140,7 +158,7 @@ import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/s const { searchJobId, aggregationJobId } = request.body; request.log.info( - `search.cancelOperation searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` + `/search/cancel searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` ); try { @@ -151,19 +169,28 @@ import { CreateSearchJobSchema, SearchJobResultSchema } from '../../../schemas/s jobId: searchJobId, lastSignal: SEARCH_SIGNAL.RESP_QUERYING, SearchResultsMetadataCollection, - logger, + logger: request.log, fields: { lastSignal: SEARCH_SIGNAL.RESP_DONE, errorMsg: 'Query cancelled before it could be completed.', }, }); - reply.code(204); - } catch (err: any) { - const message = `Failed to submit cancel request for searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}`; - request.log.error(err, message); - return reply.internalServerError(message); + } catch (err: unknown) { + request.log.error( + { + err, + searchJobId, + aggregationJobId, + }, + 'Failed to submit cancel request' + ); + reply.code(500); + return QueryCancelError(searchJobId, aggregationJobId); } + + reply.code(204); + return; } ); } diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts index fc082045fd..3f8c5f4ae3 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts @@ -3,10 +3,9 @@ import { FastifyInstance, } from "fastify"; -import {SEARCH_SIGNAL} from "../../../plugins/app/search/constants.js"; +import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; import {SearchResultsMetadataDocument} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; - type UpdateSearchResultsMetaProps = { jobId: number; lastSignal: SEARCH_SIGNAL; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts index a6cdee2f55..725e84e950 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts @@ -1,14 +1,11 @@ -import { FastifyBaseLogger } from "fastify"; -import { Collection } from "mongodb"; - -import { SearchResultsMetadataDocument } from '../../../plugins/app/search/SearchResultsMetadataCollection/typings.js'; -import { SEARCH_SIGNAL } from '../../../plugins/app/search/constants.js'; +import { SEARCH_MAX_NUM_RESULTS } from '../../../plugins/app/search/constants.js'; +import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; import { UpdateSearchResultsMetaProps, UpdateSearchSignalWhenJobsFinishProps, CreateMongoIndexesProps, } from './typings.js'; - +import { CollectionDroppedError } from '../../../plugins/app/search/SearchJobCollectionsManager/index.js'; /** * Modifies the search results metadata for a given job ID. * @@ -46,17 +43,23 @@ const updateSearchSignalWhenJobsFinish = async ({ try { await queryJobsDbManager.awaitJobCompletion(searchJobId); await queryJobsDbManager.awaitJobCompletion(aggregationJobId); - } catch (e) { - errorMsg = e.message; + } catch (e: unknown) { + if (e instanceof Error) { + errorMsg = e.message; + } + errorMsg = "Error while waiting for job completion"; } let numResultsInCollection = -1; try { - numResultsInCollection = await searchJobCollectionsManager - .getOrCreateCollection(searchJobId) - .countDocuments(); - } catch (e) { - if (ERROR_NAME_COLLECTION_DROPPED === e.error) { + const collection = await searchJobCollectionsManager + .getOrCreateCollection(searchJobId); + numResultsInCollection = await collection.countDocuments(); + //Need this here in new method. I believe publication used to do this? + await searchJobCollectionsManager.getOrCreateCollection(aggregationJobId); + + } catch (e: unknown) { + if (e instanceof CollectionDroppedError) { logger.warn(`Collection ${searchJobId} has been dropped.`); return; } @@ -70,7 +73,7 @@ const updateSearchSignalWhenJobsFinish = async ({ logger, fields: { lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: errorMsg, + errorMsg: errorMsg ?? null, numTotalResults: Math.min( numResultsInCollection, SEARCH_MAX_NUM_RESULTS @@ -99,10 +102,9 @@ const createMongoIndexes = async ({ name: "timestamp-descending", }; - const queryJobCollection = searchJobCollectionsManager.getOrCreateCollection(searchJobId); - const queryJobRawCollection = queryJobCollection.rawCollection(); + const queryJobCollection = await searchJobCollectionsManager.getOrCreateCollection(searchJobId); logger.info(`Creating indexes for collection ${searchJobId}`); - await queryJobRawCollection.createIndexes([ + await queryJobCollection.createIndexes([ timestampAscendingIndex, timestampDescendingIndex, ]); diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts new file mode 100644 index 0000000000..42ab049455 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts @@ -0,0 +1,22 @@ +import { Type } from '@sinclair/typebox' + + + + +// Base error schema — minimal + const BaseErrorSchema = Type.Object({ + message: Type.String(), + }); + + // Extended error — adds code and statusCode + const FastifyErrorSchema = Type.Intersect([ + BaseErrorSchema, + Type.Object({ + code: Type.String(), + name: Type.String(), + statusCode: Type.Optional(Type.Integer({ minimum: 100, maximum: 599 })), + }), + ]); + + + export {BaseErrorSchema, FastifyErrorSchema}; \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts index 3ac45d0056..9af46285cd 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts @@ -9,7 +9,7 @@ export const CreateSearchJobSchema = Type.Object({ timeRangeBucketSizeMillis: Type.Integer(), }) -export const SearchJobResultSchema = Type.Object({ +export const SearchJobSchema = Type.Object({ searchJobId: IdSchema, aggregationJobId: IdSchema }) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/plugins/DbManager.ts b/components/log-viewer-webui/server/src/plugins/DbManager.ts index 950aebe7c4..1f0b4a2be2 100644 --- a/components/log-viewer-webui/server/src/plugins/DbManager.ts +++ b/components/log-viewer-webui/server/src/plugins/DbManager.ts @@ -1,6 +1,5 @@ -import fastifyMongoDb from "@fastify/mongodb"; + import { - fastifyMysql, MySQLPromisePool, } from "@fastify/mysql"; import {encode as msgpackEncode} from "@msgpack/msgpack"; @@ -17,14 +16,16 @@ import { import {Nullable} from "../typings/common.js"; import { DbManagerOptions, - QUERY_JOB_STATUS, + StreamFileMongoDocument, + StreamFilesCollection, +} from "../typings/DbManager.js"; +import { QUERY_JOB_STATUS_WAITING_STATES, QUERY_JOB_TYPE, QUERY_JOBS_TABLE_COLUMN_NAMES, QueryJob, - StreamFileMongoDocument, - StreamFilesCollection, -} from "../typings/DbManager.js"; + QUERY_JOB_STATUS, +} from "../typings/query.js"; import {sleep} from "../utils/time.js"; diff --git a/components/log-viewer-webui/server/src/typings/DbManager.ts b/components/log-viewer-webui/server/src/typings/DbManager.ts index 6464ccd4d3..ae6567b748 100644 --- a/components/log-viewer-webui/server/src/typings/DbManager.ts +++ b/components/log-viewer-webui/server/src/typings/DbManager.ts @@ -1,66 +1,4 @@ import {Collection} from "mongodb"; -import {RowDataPacket} from "mysql2/promise"; - - -/** - * Matching the `QueryJobType` class in `job_orchestration.query_scheduler.constants`. - */ -enum QUERY_JOB_TYPE { - SEARCH_OR_AGGREGATION = 0, - EXTRACT_IR, - EXTRACT_JSON, -} - -/** - * List of valid extract job types. - */ -const EXTRACT_JOB_TYPES = new Set([ - QUERY_JOB_TYPE.EXTRACT_IR, - QUERY_JOB_TYPE.EXTRACT_JSON, -]); - -/** - * Matching the `QueryJobStatus` class in - * `job_orchestration.query_scheduler.constants`. - * - * @enum {number} - */ -enum QUERY_JOB_STATUS { - PENDING = 0, - RUNNING, - SUCCEEDED, - FAILED, - CANCELLING, - CANCELLED, -} - -/** - * List of states that indicate the job is either pending or in progress. - */ -const QUERY_JOB_STATUS_WAITING_STATES = new Set([ - QUERY_JOB_STATUS.PENDING, - QUERY_JOB_STATUS.RUNNING, - QUERY_JOB_STATUS.CANCELLING, -]); - -/** - * The `query_jobs` table's column names. - * - * @enum {string} - */ -enum QUERY_JOBS_TABLE_COLUMN_NAMES { - ID = "id", - TYPE = "type", - STATUS = "status", - JOB_CONFIG = "job_config", -} - -interface QueryJob extends RowDataPacket { - [QUERY_JOBS_TABLE_COLUMN_NAMES.ID]: number; - [QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE]: QUERY_JOB_TYPE; - [QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]: QUERY_JOB_STATUS; - [QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG]: string; -} interface DbManagerOptions { mysqlConfig: { @@ -92,14 +30,6 @@ type StreamFilesCollection = Collection; export type { DbManagerOptions, - QueryJob, StreamFileMongoDocument, StreamFilesCollection, -}; -export { - EXTRACT_JOB_TYPES, - QUERY_JOB_STATUS, - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, - QUERY_JOBS_TABLE_COLUMN_NAMES, -}; +}; \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/typings/query.ts b/components/log-viewer-webui/server/src/typings/query.ts new file mode 100644 index 0000000000..93833956b9 --- /dev/null +++ b/components/log-viewer-webui/server/src/typings/query.ts @@ -0,0 +1,73 @@ +import {RowDataPacket} from "mysql2/promise"; + +/** + * Matching the `QueryJobType` class in `job_orchestration.query_scheduler.constants`. + */ +enum QUERY_JOB_TYPE { + SEARCH_OR_AGGREGATION = 0, + EXTRACT_IR, + EXTRACT_JSON, +} + +/** + * List of valid extract job types. + */ +const EXTRACT_JOB_TYPES = new Set([ + QUERY_JOB_TYPE.EXTRACT_IR, + QUERY_JOB_TYPE.EXTRACT_JSON, +]); + +/** + * Matching the `QueryJobStatus` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +enum QUERY_JOB_STATUS { + PENDING = 0, + RUNNING, + SUCCEEDED, + FAILED, + CANCELLING, + CANCELLED, +} + +/** + * List of states that indicate the job is either pending or in progress. + */ +const QUERY_JOB_STATUS_WAITING_STATES = new Set([ + QUERY_JOB_STATUS.PENDING, + QUERY_JOB_STATUS.RUNNING, + QUERY_JOB_STATUS.CANCELLING, +]); + +/** + * The `query_jobs` table's column names. + * + * @enum {string} + */ +enum QUERY_JOBS_TABLE_COLUMN_NAMES { + ID = "id", + TYPE = "type", + STATUS = "status", + JOB_CONFIG = "job_config", +} + +interface QueryJob extends RowDataPacket { + [QUERY_JOBS_TABLE_COLUMN_NAMES.ID]: number; + [QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE]: QUERY_JOB_TYPE; + [QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]: QUERY_JOB_STATUS; + [QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG]: string; +} + +export type { + QueryJob, +}; +export { + EXTRACT_JOB_TYPES, + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, + QUERY_JOBS_TABLE_COLUMN_NAMES, +}; + From 50e2ce7af44da0eddffe5004b73772b462e566c5 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 15:56:49 +0000 Subject: [PATCH 10/55] latest --- .../log-viewer-webui/client/vite.config.ts | 4 +- components/log-viewer-webui/server/.env | 7 +- .../log-viewer-webui/server/settings.json | 14 +- components/log-viewer-webui/server/src/app.ts | 1 + .../plugins/app/search/QueryJobsDbManager.ts | 166 ------------------ .../SearchJobCollectionsManager/index.ts | 24 ++- .../SearchJobCollectionsManager/typings.ts | 12 +- .../SearchResultsMetadataCollection/index.ts | 6 +- .../typings.ts | 9 +- .../fastify-v2/plugins/external/mongoDb.ts | 12 -- .../src/fastify-v2/plugins/external/mySql.ts | 30 ---- .../fastify-v2/plugins/external/rate-limit.ts | 17 -- .../fastify-v2/routes/api/search/constants.ts | 8 - .../fastify-v2/routes/api/search/typings.ts | 6 + .../log-viewer-webui/server/tsconfig.json | 3 +- 15 files changed, 48 insertions(+), 271 deletions(-) delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/external/mongoDb.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/external/mySql.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/external/rate-limit.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/constants.ts diff --git a/components/log-viewer-webui/client/vite.config.ts b/components/log-viewer-webui/client/vite.config.ts index 9f9007621f..30afef3431 100644 --- a/components/log-viewer-webui/client/vite.config.ts +++ b/components/log-viewer-webui/client/vite.config.ts @@ -12,12 +12,12 @@ export default defineConfig({ ], publicDir: "public", server: { - port: 3001, + port: 3000, proxy: { "/query/": { // Below target should match the server's configuration in // `components/log-viewer-webui/server/.env` (or `.env.local` if overridden) - target: "http://localhost:3001/", + target: "http://localhost:3000/", changeOrigin: true, }, "/socket.io/": { diff --git a/components/log-viewer-webui/server/.env b/components/log-viewer-webui/server/.env index 43caf205d3..0f5aa2268d 100644 --- a/components/log-viewer-webui/server/.env +++ b/components/log-viewer-webui/server/.env @@ -4,9 +4,6 @@ NODE_ENV=production HOST=localhost -PORT=3001 +PORT=3000 CLP_DB_USER=clp-user -CLP_DB_PASS=mGltCMuLSaM - - -/home/davidmawk/clp/build \ No newline at end of file +CLP_DB_PASS= \ No newline at end of file diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json index a2cd939b5f..293f2c8df7 100644 --- a/components/log-viewer-webui/server/settings.json +++ b/components/log-viewer-webui/server/settings.json @@ -19,17 +19,5 @@ "StreamTargetUncompressedSize": 134217728, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, - "StreamFilesS3Profile": null, - - - "public": { - "AggregationResultsCollectionName": "aggregation-results", - "ClpStorageEngine": "clp", - "CompressionJobsCollectionName": "compression-jobs", - "LogViewerWebuiUrl": "http://localhost:8080", - "SearchResultsCollectionName": "search-results", - "SearchResultsMetadataCollectionName": "results-metadata", - "StatsCollectionName": "stats", - "SupportUrl": "https://github.com/y-scope/clp/issues/new/choose" - } + "StreamFilesS3Profile": null } diff --git a/components/log-viewer-webui/server/src/app.ts b/components/log-viewer-webui/server/src/app.ts index 1e267a2b35..1f4a35cafd 100644 --- a/components/log-viewer-webui/server/src/app.ts +++ b/components/log-viewer-webui/server/src/app.ts @@ -11,6 +11,7 @@ import exampleRoutes from "./routes/example.js"; import queryRoutes from "./routes/query.js"; import staticRoutes from "./routes/static.js"; + interface AppPluginOptions { sqlDbUser: string; sqlDbPass: string; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts deleted file mode 100644 index 607b186e56..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { encode } from "@msgpack/msgpack"; -import { ResultSetHeader, RowDataPacket } from "mysql2"; -import { - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, - QUERY_JOB_STATUS, - QUERY_JOBS_TABLE_COLUMN_NAMES, -} from "../../../../typings/query.js"; -import { FastifyInstance } from "fastify"; -import fp from "fastify-plugin"; -import settings from "../../../../../settings.json" with { type: "json" }; -import { setTimeout } from 'timers/promises'; - -import type { - MySQLPromisePool, -} from "@fastify/mysql"; - -/** - * Interval in milliseconds for polling the completion status of a job. - */ -const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 500; - -/** - * Class for submitting and monitoring query jobs in the database. - */ -class QueryJobsDbManager { - #sqlDbConnPool: MySQLPromisePool; - - private constructor(sqlDbConnPool: MySQLPromisePool) { - this.#sqlDbConnPool = sqlDbConnPool; - } - - /** - * Creates a new QueryJobsDbManager. - * - * @param fastify - * @returns - */ - static create(fastify: FastifyInstance): QueryJobsDbManager { - const sqlDbConnPool = fastify.mysql; - return new QueryJobsDbManager(sqlDbConnPool); - } - - /** - * Submits a search job to the database. - * - * @param searchConfig The arguments for the query. - * @return The job's ID. - * @throws {Error} on error. - */ - async submitSearchJob(searchConfig: object): Promise { - const [queryInsertResults] = await this.#sqlDbConnPool.query( - `INSERT INTO ${settings.SqlDbQueryJobsTableName} - (${QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG}, - ${QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE}) - VALUES (?, ?)`, - [ - Buffer.from(encode(searchConfig)), - QUERY_JOB_TYPE.SEARCH_OR_AGGREGATION, - ] - ); - - return queryInsertResults.insertId; - } - - /** - * Submits an aggregation job to the database. - * - * @param searchConfig - * @param timeRangeBucketSizeMillis - * @return The aggregation job's ID. - * @throws {Error} on error. - */ - async submitAggregationJob( - searchConfig: object, - timeRangeBucketSizeMillis: number - ): Promise { - const searchAggregationConfig = { - ...searchConfig, - aggregation_config: { - count_by_time_bucket_size: timeRangeBucketSizeMillis, - }, - }; - - return await this.submitSearchJob(searchAggregationConfig); - } - - /** - * Submits a query cancellation request to the database. - * - * @param jobId ID of the job to cancel. - * @return - * @throws {Error} on error. - */ - async submitQueryCancellation (jobId: number): Promise { - await this.#sqlDbConnPool.query( - `UPDATE ${settings.SqlDbQueryJobsTableName} - SET ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${QUERY_JOB_STATUS.CANCELLING} - WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? - AND ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} - IN (${QUERY_JOB_STATUS.PENDING}, ${QUERY_JOB_STATUS.RUNNING})`, - jobId, - ); - } - - /** - * Waits for the job to complete. - * - * @param jobId - * @return - * @throws {Error} on MySQL error, if the job wasn't found in the database, if the job was - * cancelled, or if the job completed in an unexpected state. - */ - async awaitJobCompletion(jobId: number): Promise { - while (true) { - let rows; - try { - const [queryRows] = await this.#sqlDbConnPool.query( - ` - SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} - FROM ${settings.SqlDbQueryJobsTableName} - WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? - `, - jobId - ); - - rows = queryRows; - } catch (e) { - throw new Error(`Failed to query status for job ${jobId} - ${e}`); - } - if (rows.length === 0) { - throw new Error(`Job ${jobId} not found in database.`); - } - const status = rows[0]![QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; - - if (false === QUERY_JOB_STATUS_WAITING_STATES.has(status)) { - if (QUERY_JOB_STATUS.CANCELLED === status) { - throw new Error(`Job ${jobId} was cancelled.`); - } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { - throw new Error( - `Job ${jobId} exited with unexpected status=${status}: ` + - `${Object.keys(QUERY_JOB_STATUS)[status]}.` - ); - } - break; - } - - await setTimeout(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); - } - } -} - -declare module "fastify" { - export interface FastifyInstance { - QueryJobsDbManager: QueryJobsDbManager; - } -} - -export default fp( - function (fastify) { - fastify.decorate("QueryJobsDbManager", QueryJobsDbManager.create(fastify)); - }, - { - name: "QueryJobsDbManager", - } -); \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts index c55c6e6d80..59830afc75 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts @@ -2,7 +2,7 @@ import { FastifyInstance } from 'fastify'; import fp from 'fastify-plugin'; import { Nullable } from "../../../../../typings/common.js"; import type { Db, Collection } from "mongodb"; -import type { SearchResultsMetadataDocument } from "./typings.js"; +import type { SearchResultsDocument } from "./typings.js"; import { CollectionDroppedError } from "./typings.js"; /** @@ -10,7 +10,9 @@ import { CollectionDroppedError } from "./typings.js"; * have unique names. */ class SearchJobCollectionsManager { - #collections: Map>> = new Map(); + #jobIdToCollectionsMap: Map>> + = new Map(); + #db: Db; private constructor (db: Db) { @@ -38,14 +40,18 @@ class SearchJobCollectionsManager { * @return MongoDB collection * @throws {CollectionDroppedError} if the collection was already dropped. */ - async getOrCreateCollection(jobId: number): Promise> { + async getOrCreateCollection(jobId: number): Promise> { const name = jobId.toString(); - if (false === this.#collections.has(name)) { - this.#collections.set(name, this.#db.collection(name)); - } else if (this.#collections.get(name) === null) { + if (false === this.#jobIdToCollectionsMap.has(name)) { + this.#jobIdToCollectionsMap.set(name, this.#db.collection(name)); + } else if (this.#jobIdToCollectionsMap.get(name) === null) { throw new CollectionDroppedError(name); } - return this.#collections.get(name) as Collection; + // `collections.get(name)` will always return a valid collection since: + // - Function creates a new collection if it doesn't exist in map. + // - Throws an error if the collection was already dropped. + // Therefore, we can safely cast to `Collection`. + return this.#jobIdToCollectionsMap.get(name) as Collection; } /** @@ -56,13 +62,13 @@ class SearchJobCollectionsManager { */ async dropCollection(jobId: number) { const name = jobId.toString(); - const collection = this.#collections.get(name); + const collection = this.#jobIdToCollectionsMap.get(name); if ("undefined" === typeof collection || null === collection) { throw new Error(`Collection ${name} not found`); } await collection.drop(); - this.#collections.set(name, null); + this.#jobIdToCollectionsMap.set(name, null); } } diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts index e032e8bc83..08a11dc984 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts @@ -1,4 +1,7 @@ -interface SearchResultsMetadataDocument { +/** + * MongoDB document for search results. + */ +interface SearchResultsDocument { _id: string; orig_file_id: string; orig_file_path: string; @@ -7,6 +10,9 @@ interface SearchResultsMetadataDocument { message: string; } +/** + * Error thrown when a MongoDB collection has been dropped unexpectedly. + */ class CollectionDroppedError extends Error { constructor(collectionName: string) { super(`Collection ${collectionName} has been dropped.`); @@ -14,4 +20,6 @@ class CollectionDroppedError extends Error { } } -export { SearchResultsMetadataDocument, CollectionDroppedError }; +export type { SearchResultsDocument }; + +export { CollectionDroppedError }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts index d1774c173b..0f42cd702e 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts @@ -1,8 +1,8 @@ import { FastifyInstance } from 'fastify' import fp from 'fastify-plugin' -import settings from "../../../../../../settings.json" with {type: "json"}; -import { SearchResultsMetadataDocument } from './typings.js'; +import settings from "@settings" with {type: "json"}; +import type { SearchResultsMetadataDocument } from './typings.js'; /** * Creates a MongoDB collection for search results metadata. @@ -33,4 +33,4 @@ export default fp( { name: 'SearchResultsMetadataCollection', } -) \ No newline at end of file +) diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts index 6223dbd17e..35dc37fcdd 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts @@ -1,5 +1,9 @@ import { Nullable } from "../../../../../typings/common.js"; +/** + * MongoDB document for search results metadata. `numTotalResults` is optional + * since it is only set when the search job is completed. + */ interface SearchResultsMetadataDocument { _id: string; errorMsg: Nullable; @@ -12,8 +16,6 @@ interface SearchResultsMetadataDocument { * * This includes request and response signals for various search operations and their respective * states. - * - * @enum {string} */ enum SEARCH_SIGNAL { NONE = "none", @@ -28,4 +30,5 @@ enum SEARCH_SIGNAL { -export { SearchResultsMetadataDocument, SEARCH_SIGNAL }; +export type { SearchResultsMetadataDocument }; +export { SEARCH_SIGNAL }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mongoDb.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mongoDb.ts deleted file mode 100644 index 699b06a1e4..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mongoDb.ts +++ /dev/null @@ -1,12 +0,0 @@ -import fastifyMongoDb from "@fastify/mongodb"; - -import settings from "../../../../settings.json" with {type: "json"}; - -export const autoConfig = () => { - return { - forceClose: true, - url: `mongodb://${settings.MongoDbHost}:${settings.MongoDbPort}/${settings.MongoDbName}`, - } -} - -export default fastifyMongoDb diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mySql.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mySql.ts deleted file mode 100644 index 38bc17cbe3..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/external/mySql.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { - fastifyMysql, - MySQLPromisePool, -} from "@fastify/mysql"; -import { FastifyInstance } from 'fastify' - -import settings from "../../../../settings.json" with {type: "json"}; - -// use MySQLPromisePool if passed promise = true -declare module 'fastify' { - interface FastifyInstance { - mysql: MySQLPromisePool; - MYSQL2: MySQLPromisePool; - } -} - - -export const autoConfig = (fastify: FastifyInstance) => { - return { - promise: true, - host: settings.SqlDbHost, - user: fastify.config.CLP_DB_USER, - password: fastify.config.CLP_DB_PASS, - database: settings.SqlDbName, - port: Number(settings.SqlDbPort), - } -} - -export default fastifyMysql - diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/external/rate-limit.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/rate-limit.ts deleted file mode 100644 index 2a275a5240..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/external/rate-limit.ts +++ /dev/null @@ -1,17 +0,0 @@ -// Reference: https://github.com/fastify/demo/blob/main/src/plugins/external/rate-limit.ts -import fastifyRateLimit from '@fastify/rate-limit' -import { FastifyInstance } from 'fastify' - -export const autoConfig = (fastify: FastifyInstance) => { - return { - max: fastify.config.RATE_LIMIT_MAX, - timeWindow: '1 minute' - } -} - -/** - * This plugins is low overhead rate limiter for your routes. - * - * @see {@link https://github.com/fastify/fastify-rate-limit} - */ -export default fastifyRateLimit \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/constants.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/constants.ts deleted file mode 100644 index 42bf06f673..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * The maximum number of results to retrieve for a search. - */ -const SEARCH_MAX_NUM_RESULTS = 1000; - -export { - SEARCH_MAX_NUM_RESULTS, -}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts index 3f8c5f4ae3..e69ad951aa 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts @@ -29,8 +29,14 @@ type CreateMongoIndexesProps = { logger: FastifyBaseLogger; }; +/** + * The maximum number of results to retrieve for a search. + */ +const SEARCH_MAX_NUM_RESULTS = 1000; + export { CreateMongoIndexesProps, UpdateSearchResultsMetaProps, UpdateSearchSignalWhenJobsFinishProps, + SEARCH_MAX_NUM_RESULTS, }; diff --git a/components/log-viewer-webui/server/tsconfig.json b/components/log-viewer-webui/server/tsconfig.json index f21550a911..7eb501605f 100644 --- a/components/log-viewer-webui/server/tsconfig.json +++ b/components/log-viewer-webui/server/tsconfig.json @@ -18,7 +18,8 @@ "baseUrl": ".", "paths": { // Map imports from "@common/*" to the common server-client folder - "@common/*": ["../common/*"] + "@common/*": ["../common/*"], + "@settings": ["settings.json"] }, "alwaysStrict": true, From 45e623ca3b470997f1deb098111480af2fb8077f Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 16:02:11 +0000 Subject: [PATCH 11/55] latest --- components/log-viewer-webui/server/src/fastify-v2/app.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/log-viewer-webui/server/src/fastify-v2/app.ts b/components/log-viewer-webui/server/src/fastify-v2/app.ts index c5b6420bb4..a5f823ed65 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/app.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/app.ts @@ -14,6 +14,7 @@ import FastifyV1App from "../app.js"; const INTERNAL_SERVER_ERROR_CODE = 500; const RATE_LIMIT_MAX_REQUESTS = 3; const RATE_LIMIT_TIME_WINDOW_MS = 500; +const IGNORED_FILES_REGEX = /^.*(?:utils|typings)\.js$/; /** * Registers all plugins and routes. @@ -47,12 +48,14 @@ export default async function serviceApp ( // Loads all application plugins. fastify.register(fastifyAutoload, { dir: path.join(import.meta.dirname, "plugins/app"), + ignorePattern: IGNORED_FILES_REGEX, options: {...opts}, }); // Loads all routes. fastify.register(fastifyAutoload, { dir: path.join(import.meta.dirname, "routes"), + ignorePattern: IGNORED_FILES_REGEX, autoHooks: true, cascadeHooks: true, options: {...opts}, From c9dea3b49762a14c4bbd9c175577c6161fe1374d Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 16:07:16 +0000 Subject: [PATCH 12/55] latest --- .../log-viewer-webui/client/vite.config.ts | 2 +- components/log-viewer-webui/server/.env | 2 +- .../SearchJobCollectionsManager/index.ts | 2 - .../fastify-v2/routes/api/search/errors.ts | 24 --- .../src/fastify-v2/routes/api/search/index.ts | 198 ------------------ .../fastify-v2/routes/api/search/typings.ts | 42 ---- .../src/fastify-v2/routes/api/search/utils.ts | 113 ---------- .../src/fastify-v2/schemas/QueryJobs.ts | 11 - .../server/src/fastify-v2/schemas/common.ts | 7 - .../server/src/fastify-v2/schemas/error.ts | 22 -- .../server/src/fastify-v2/schemas/search.ts | 15 -- .../server/src/fastify-v2/schemas/test. | 11 - 12 files changed, 2 insertions(+), 447 deletions(-) delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/errors.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/QueryJobs.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/test. diff --git a/components/log-viewer-webui/client/vite.config.ts b/components/log-viewer-webui/client/vite.config.ts index 30afef3431..f3e6c5d260 100644 --- a/components/log-viewer-webui/client/vite.config.ts +++ b/components/log-viewer-webui/client/vite.config.ts @@ -12,7 +12,7 @@ export default defineConfig({ ], publicDir: "public", server: { - port: 3000, + port: 8080, proxy: { "/query/": { // Below target should match the server's configuration in diff --git a/components/log-viewer-webui/server/.env b/components/log-viewer-webui/server/.env index 0f5aa2268d..13ed2e61b9 100644 --- a/components/log-viewer-webui/server/.env +++ b/components/log-viewer-webui/server/.env @@ -6,4 +6,4 @@ NODE_ENV=production HOST=localhost PORT=3000 CLP_DB_USER=clp-user -CLP_DB_PASS= \ No newline at end of file +CLP_DB_PASS= diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts index 59830afc75..5a01b45ce1 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts @@ -86,5 +86,3 @@ export default fp( name: 'SearchJobCollectionsManager', } ); - -export {CollectionDroppedError} \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/errors.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/errors.ts deleted file mode 100644 index ae2fe76006..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/errors.ts +++ /dev/null @@ -1,24 +0,0 @@ -import createError from '@fastify/error'; - -enum ErrorCodes { - QuerySubmitError = 'FST_QUERY_SUBMIT_ERROR', - ClearResultsError = 'FST_CLEAR_RESULTS_ERROR', - QueryCancelError = 'FST_QUERY_CANCEL_ERROR', -} - -const QuerySubmitError = createError( - ErrorCodes.QuerySubmitError, - 'Unable to submit search/aggregation job to the SQL database', -); - -const ClearResultsError = createError( - ErrorCodes.ClearResultsError, - 'Failed to clear search results for searchJobId=%s, aggregationJobId=%s', -); - -const QueryCancelError = createError( - ErrorCodes.QueryCancelError, - 'Failed to submit cancel request for searchJobId=%s, aggregationJobId=%s', -); - -export {QuerySubmitError, ClearResultsError, QueryCancelError }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts deleted file mode 100644 index 59c7266213..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { SEARCH_MAX_NUM_RESULTS } from './constants.js'; -import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; -import { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes } from './utils.js'; -import { - FastifyPluginAsyncTypebox, - Type - } from '@fastify/type-provider-typebox' -import { CreateSearchJobSchema, SearchJobSchema } from '../../../schemas/search.js' -import { FastifyErrorSchema } from '../../../schemas/error.js' -import { QuerySubmitError, ClearResultsError, QueryCancelError } from './errors.js'; - - const plugin: FastifyPluginAsyncTypebox = async (fastify) => { - const { - QueryJobsDbManager, - SearchJobCollectionsManager, - SearchResultsMetadataCollection, - } = fastify; - - /** - * Submits a search query and initiates the search process. - */ - fastify.post( - '/search', - { - schema: { - body: CreateSearchJobSchema, - response: { - 201: SearchJobSchema, - 500: FastifyErrorSchema, - }, - tags: ['Search'] - }, - }, - async function (request, reply) { - - const { - queryString, - timestampBegin, - timestampEnd, - ignoreCase, - timeRangeBucketSizeMillis - } = request.body; - - const args = { - query_string: queryString, - begin_timestamp: timestampBegin, - end_timestamp: timestampEnd, - ignore_case: ignoreCase, - max_num_results: SEARCH_MAX_NUM_RESULTS - }; - - request.log.info({ args }, "/search args"); - - let searchJobId: number; - let aggregationJobId: number; - - try { - searchJobId = await QueryJobsDbManager.submitSearchJob(args); - aggregationJobId = await QueryJobsDbManager.submitAggregationJob( - args, - timeRangeBucketSizeMillis - ); - } catch (err: unknown) { - request.log.error( err , 'Unable to submit search/aggregation job to the SQL database'); - reply.code(500) - return QuerySubmitError(); - } - - SearchResultsMetadataCollection.insertOne({ - _id: searchJobId.toString(), - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - errorMsg: null - }); - - // Defer signal update until after response is sent - setImmediate( async () => { - await updateSearchSignalWhenJobsFinish({ - searchJobId, - aggregationJobId, - queryJobsDbManager: QueryJobsDbManager, - searchJobCollectionsManager: SearchJobCollectionsManager, - SearchResultsMetadataCollection, - logger: request.log, - }); - }); - - await createMongoIndexes({ - searchJobId, - searchJobCollectionsManager: SearchJobCollectionsManager, - logger: request.log, - }); - - reply.code(201) - - return { searchJobId, aggregationJobId }; - } - ); - - /** - * Clears the results of a search operation identified by jobId. - */ - fastify.delete( - '/search/results', - { - schema: { - body: SearchJobSchema, - response: { - 204: Type.Null(), - 500: FastifyErrorSchema, - }, - tags: ['Search'], - }, - }, - async function (request, reply) { - const { searchJobId, aggregationJobId } = request.body; - - request.log.info({ - searchJobId, - aggregationJobId, - }, "/search/results args"); - - try { - await SearchJobCollectionsManager.dropCollection(searchJobId); - await SearchJobCollectionsManager.dropCollection(aggregationJobId); - } catch (err: unknown) { - request.log.error( - { - err, - searchJobId, - aggregationJobId, - }, - 'Failed to clear search results' - ); - - reply.code(500); - return ClearResultsError(searchJobId, aggregationJobId); - } - - reply.code(204); - return; - } - ); - - - fastify.post( - '/search/cancel', - { - schema: { - body: SearchJobSchema, - response: { - 204: Type.Null(), - 500: FastifyErrorSchema, - }, - tags: ['Search'], - }, - }, - async function (request, reply) { - const { searchJobId, aggregationJobId } = request.body; - - request.log.info( - `/search/cancel searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` - ); - - try { - await QueryJobsDbManager.submitQueryCancellation(searchJobId); - await QueryJobsDbManager.submitQueryCancellation(aggregationJobId); - - updateSearchResultsMeta({ - jobId: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - SearchResultsMetadataCollection, - logger: request.log, - fields: { - lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: 'Query cancelled before it could be completed.', - }, - }); - - } catch (err: unknown) { - request.log.error( - { - err, - searchJobId, - aggregationJobId, - }, - 'Failed to submit cancel request' - ); - reply.code(500); - return QueryCancelError(searchJobId, aggregationJobId); - } - - reply.code(204); - return; - } - ); - } - - export default plugin \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts deleted file mode 100644 index e69ad951aa..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { - FastifyBaseLogger, - FastifyInstance, -} from "fastify"; - -import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; -import {SearchResultsMetadataDocument} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; - -type UpdateSearchResultsMetaProps = { - jobId: number; - lastSignal: SEARCH_SIGNAL; - SearchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; - logger: FastifyBaseLogger; - fields: Partial; -}; - -type UpdateSearchSignalWhenJobsFinishProps = { - searchJobId: number; - aggregationJobId: number; - queryJobsDbManager: FastifyInstance["QueryJobsDbManager"]; - searchJobCollectionsManager: FastifyInstance["SearchJobCollectionsManager"]; - SearchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; - logger: FastifyBaseLogger; -}; - -type CreateMongoIndexesProps = { - searchJobId: number; - searchJobCollectionsManager: FastifyInstance["SearchJobCollectionsManager"]; - logger: FastifyBaseLogger; -}; - -/** - * The maximum number of results to retrieve for a search. - */ -const SEARCH_MAX_NUM_RESULTS = 1000; - -export { - CreateMongoIndexesProps, - UpdateSearchResultsMetaProps, - UpdateSearchSignalWhenJobsFinishProps, - SEARCH_MAX_NUM_RESULTS, -}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts deleted file mode 100644 index 725e84e950..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { SEARCH_MAX_NUM_RESULTS } from '../../../plugins/app/search/constants.js'; -import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; -import { - UpdateSearchResultsMetaProps, - UpdateSearchSignalWhenJobsFinishProps, - CreateMongoIndexesProps, -} from './typings.js'; -import { CollectionDroppedError } from '../../../plugins/app/search/SearchJobCollectionsManager/index.js'; -/** - * Modifies the search results metadata for a given job ID. - * - * @param {UpdateSearchResultsMetaProps} params - */ -const updateSearchResultsMeta = ({ - jobId, - lastSignal, - SearchResultsMetadataCollection, - logger, - fields, -}: UpdateSearchResultsMetaProps) => { - const filter = { - _id: jobId.toString(), - lastSignal: lastSignal, - }; - - const modifier = { - $set: fields, - }; - - logger.debug("SearchResultsMetadataCollection modifier = ", modifier); - SearchResultsMetadataCollection.updateOne(filter, modifier); -}; - -const updateSearchSignalWhenJobsFinish = async ({ - searchJobId, - aggregationJobId, - queryJobsDbManager, - searchJobCollectionsManager, - SearchResultsMetadataCollection, - logger, -}: UpdateSearchSignalWhenJobsFinishProps) => { - let errorMsg; - try { - await queryJobsDbManager.awaitJobCompletion(searchJobId); - await queryJobsDbManager.awaitJobCompletion(aggregationJobId); - } catch (e: unknown) { - if (e instanceof Error) { - errorMsg = e.message; - } - errorMsg = "Error while waiting for job completion"; - } - - let numResultsInCollection = -1; - try { - const collection = await searchJobCollectionsManager - .getOrCreateCollection(searchJobId); - numResultsInCollection = await collection.countDocuments(); - //Need this here in new method. I believe publication used to do this? - await searchJobCollectionsManager.getOrCreateCollection(aggregationJobId); - - } catch (e: unknown) { - if (e instanceof CollectionDroppedError) { - logger.warn(`Collection ${searchJobId} has been dropped.`); - return; - } - throw e; - } - - updateSearchResultsMeta({ - jobId: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - SearchResultsMetadataCollection, - logger, - fields: { - lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: errorMsg ?? null, - numTotalResults: Math.min( - numResultsInCollection, - SEARCH_MAX_NUM_RESULTS - ), - }, - }); -}; - -const createMongoIndexes = async ({ - searchJobId, - searchJobCollectionsManager, - logger, -}: CreateMongoIndexesProps) => { - const timestampAscendingIndex = { - key: { - timestamp: 1, - _id: 1, - }, - name: "timestamp-ascending", - }; - const timestampDescendingIndex = { - key: { - timestamp: -1, - _id: -1, - }, - name: "timestamp-descending", - }; - - const queryJobCollection = await searchJobCollectionsManager.getOrCreateCollection(searchJobId); - logger.info(`Creating indexes for collection ${searchJobId}`); - await queryJobCollection.createIndexes([ - timestampAscendingIndex, - timestampDescendingIndex, - ]); -}; - -export { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes }; \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/QueryJobs.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/QueryJobs.ts deleted file mode 100644 index 35453686d0..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/QueryJobs.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Static, Type } from '@sinclair/typebox' -import { StringSchema } from './common.js' - -export const QueryJobsSchema = Type.Object({ - id: StringSchema, - status: StringSchema, - type: StringSchema, - job_config: StringSchema, -}) - -export interface QueryJobs extends Static {} diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts deleted file mode 100644 index 206b01b8e6..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Type } from '@sinclair/typebox' - -export const StringSchema = Type.String({ - minLength: 1, -}) - -export const IdSchema = Type.Integer({ minimum: 1 }) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts deleted file mode 100644 index 42ab049455..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Type } from '@sinclair/typebox' - - - - -// Base error schema — minimal - const BaseErrorSchema = Type.Object({ - message: Type.String(), - }); - - // Extended error — adds code and statusCode - const FastifyErrorSchema = Type.Intersect([ - BaseErrorSchema, - Type.Object({ - code: Type.String(), - name: Type.String(), - statusCode: Type.Optional(Type.Integer({ minimum: 100, maximum: 599 })), - }), - ]); - - - export {BaseErrorSchema, FastifyErrorSchema}; \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts deleted file mode 100644 index 9af46285cd..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Type } from '@sinclair/typebox' -import { StringSchema, IdSchema } from './common.js' - -export const CreateSearchJobSchema = Type.Object({ - queryString: StringSchema, - timestampBegin: Type.Integer(), - timestampEnd: Type.Integer(), - ignoreCase: Type.Boolean(), - timeRangeBucketSizeMillis: Type.Integer(), -}) - -export const SearchJobSchema = Type.Object({ - searchJobId: IdSchema, - aggregationJobId: IdSchema -}) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/test. b/components/log-viewer-webui/server/src/fastify-v2/schemas/test. deleted file mode 100644 index 8268124e3b..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/test. +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Enum of the `query_jobs` table's column names. - * - * @enum {string} - */ -const QUERY_JOBS_TABLE_COLUMN_NAMES = Object.freeze({ - ID: "id", - STATUS: "status", - TYPE: "type", - JOB_CONFIG: "job_config", -}); \ No newline at end of file From 8dc38837eb62db882881dfd02d99a6acd064abe9 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 16:14:16 +0000 Subject: [PATCH 13/55] latest --- .../log-viewer-webui/client/src/App.tsx | 126 -------------- .../log-viewer-webui/server/settings.json | 3 - .../app/search/QueryJobsDbManager/index.ts | 162 ++++++++++++++++++ .../app/search/QueryJobsDbManager/typings.ts | 4 + .../plugins/app/search/constants.ts | 92 ---------- .../server/src/plugins/DbManager.ts | 10 +- .../log-viewer-webui/server/src/server.ts | 67 -------- .../server/src/typings/DbManager.ts | 72 +++++++- .../server/src/typings/query.ts | 73 -------- .../log-viewer-webui/server/src/utils/misc.ts | 37 ---- tools/yscope-dev-utils | 2 +- 11 files changed, 242 insertions(+), 406 deletions(-) create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/typings.ts delete mode 100644 components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts delete mode 100644 components/log-viewer-webui/server/src/server.ts delete mode 100644 components/log-viewer-webui/server/src/typings/query.ts delete mode 100644 components/log-viewer-webui/server/src/utils/misc.ts diff --git a/components/log-viewer-webui/client/src/App.tsx b/components/log-viewer-webui/client/src/App.tsx index 9e7c0e5db3..b54609e539 100644 --- a/components/log-viewer-webui/client/src/App.tsx +++ b/components/log-viewer-webui/client/src/App.tsx @@ -1,6 +1,4 @@ import {CssVarsProvider} from "@mui/joy"; -import {useState} from "react"; -import axios from "axios"; import {LOCAL_STORAGE_KEY} from "./typings/config"; import QueryStatus from "./ui/QueryStatus"; @@ -12,133 +10,9 @@ import QueryStatus from "./ui/QueryStatus"; * @return */ const App = () => { - const [output, setOutput] = useState(""); - const [queryString, setQueryString] = useState("test query"); - const [timestampBegin, setTimestampBegin] = useState(0); - const [timestampEnd, setTimestampEnd] = useState(Date.now()); - const [ignoreCase, setIgnoreCase] = useState(true); - const [timeRangeBucketSizeMillis, setTimeRangeBucketSizeMillis] = useState(60000); - const [searchJobId, setSearchJobId] = useState(1); - const [aggregationJobId, setAggregationJobId] = useState(2); - - const sendRequest = async (url: string, method: string, body: object) => { - try { - const response = await axios({ - url, - method, - data: body, - headers: {"Content-Type": "application/json"}, - }); - setOutput(JSON.stringify(response.data, null, 2)); - } catch (error: unknown) { - if (axios.isAxiosError(error)) { - setOutput(`Error: ${error.response?.data?.message || error.message}`); - } else { - setOutput(`Error: ${String(error)}`); - } - } - }; - return ( -
-

Search API Test

-
- - - - - - - -
- - - - 
-                    {output}
-
-
); }; diff --git a/components/log-viewer-webui/server/settings.json b/components/log-viewer-webui/server/settings.json index 293f2c8df7..ac3e244fa3 100644 --- a/components/log-viewer-webui/server/settings.json +++ b/components/log-viewer-webui/server/settings.json @@ -3,9 +3,6 @@ "SqlDbPort": 3306, "SqlDbName": "clp-db", "SqlDbQueryJobsTableName": "query_jobs", - "SqlDbClpArchivesTableName": "clp_archives", - "SqlDbClpFilesTableName": "clp_files", - "SqlDbCompressionJobsTableName": "compression_jobs", "MongoDbHost": "localhost", "MongoDbPort": 27017, diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts new file mode 100644 index 0000000000..40217bec60 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts @@ -0,0 +1,162 @@ +import { encode } from "@msgpack/msgpack"; +import { ResultSetHeader, RowDataPacket } from "mysql2"; +import { + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, + QUERY_JOB_STATUS, + QUERY_JOBS_TABLE_COLUMN_NAMES, +} from "../../../../../typings/query.js"; +import { JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS } from "./typings.js"; +import { FastifyInstance } from "fastify"; +import fp from "fastify-plugin"; +import settings from "@settings" with { type: "json" }; +import { setTimeout } from 'timers/promises'; + +import type { + MySQLPromisePool, +} from "@fastify/mysql"; + +/** + * Class for submitting and monitoring query jobs in the database. + */ +class QueryJobsDbManager { + #sqlDbConnPool: MySQLPromisePool; + + private constructor(sqlDbConnPool: MySQLPromisePool) { + this.#sqlDbConnPool = sqlDbConnPool; + } + + /** + * Creates a new QueryJobsDbManager. + * + * @param fastify + * @returns + */ + static create(fastify: FastifyInstance): QueryJobsDbManager { + const sqlDbConnPool = fastify.mysql; + return new QueryJobsDbManager(sqlDbConnPool); + } + + /** + * Submits a search job to the database. + * + * @param searchConfig The arguments for the query. + * @return The job's ID. + * @throws {Error} on error. + */ + async submitSearchJob(searchConfig: object): Promise { + const [queryInsertResults] = await this.#sqlDbConnPool.query( + `INSERT INTO ${settings.SqlDbQueryJobsTableName} + (${QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG}, + ${QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE}) + VALUES (?, ?)`, + [ + Buffer.from(encode(searchConfig)), + QUERY_JOB_TYPE.SEARCH_OR_AGGREGATION, + ] + ); + + return queryInsertResults.insertId; + } + + /** + * Submits an aggregation job to the database. + * + * @param searchConfig The arguments for the query. + * @param timeRangeBucketSizeMillis + * @return The aggregation job's ID. + * @throws {Error} on error. + */ + async submitAggregationJob( + searchConfig: object, + timeRangeBucketSizeMillis: number + ): Promise { + const searchAggregationConfig = { + ...searchConfig, + aggregation_config: { + count_by_time_bucket_size: timeRangeBucketSizeMillis, + }, + }; + + return await this.submitSearchJob(searchAggregationConfig); + } + + /** + * Submits a query cancellation request to the database. + * + * @param jobId ID of the job to cancel. + * @return + * @throws {Error} on error. + */ + async submitQueryCancellation (jobId: number): Promise { + await this.#sqlDbConnPool.query( + `UPDATE ${settings.SqlDbQueryJobsTableName} + SET ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} = ${QUERY_JOB_STATUS.CANCELLING} + WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? + AND ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} + IN (${QUERY_JOB_STATUS.PENDING}, ${QUERY_JOB_STATUS.RUNNING})`, + jobId, + ); + } + + /** + * Waits for the job to complete. + * + * @param jobId + * @return + * @throws {Error} on MySQL error, if the job wasn't found in the database, if the job was + * cancelled, or if the job completed in an unexpected state. + */ + async awaitJobCompletion(jobId: number): Promise { + while (true) { + let rows; + try { + const [queryRows] = await this.#sqlDbConnPool.query( + ` + SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} + FROM ${settings.SqlDbQueryJobsTableName} + WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? + `, + jobId + ); + + rows = queryRows; + } catch (e) { + throw new Error(`Failed to query status for job ${jobId} - ${e}`); + } + if (0 === rows.length) { + throw new Error(`Job ${jobId} not found in database.`); + } + const status = rows[0]![QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; + + if (false === QUERY_JOB_STATUS_WAITING_STATES.has(status)) { + if (QUERY_JOB_STATUS.CANCELLED === status) { + throw new Error(`Job ${jobId} was cancelled.`); + } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { + throw new Error( + `Job ${jobId} exited with unexpected status=${status}: ` + + `${Object.keys(QUERY_JOB_STATUS)[status]}.` + ); + } + break; + } + + await setTimeout(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); + } + } +} + +declare module "fastify" { + export interface FastifyInstance { + QueryJobsDbManager: QueryJobsDbManager; + } +} + +export default fp( + function (fastify) { + fastify.decorate("QueryJobsDbManager", QueryJobsDbManager.create(fastify)); + }, + { + name: "QueryJobsDbManager", + } +); diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/typings.ts new file mode 100644 index 0000000000..9ce73c0f4d --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/typings.ts @@ -0,0 +1,4 @@ +/** + * Interval in milliseconds for polling the completion status of a job. + */ +export const JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS = 500; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts deleted file mode 100644 index 978ac225a5..0000000000 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/constants.ts +++ /dev/null @@ -1,92 +0,0 @@ -// Maybe move shared stuff to a more common location. - -/** - * Enum of search-related signals. - * - * This includes request and response signals for various search operations and their respective - * states. - * - * @enum {string} - */ -enum SEARCH_SIGNAL { - NONE = "none", - - REQ_CANCELLING = "req-cancelling", - REQ_CLEARING = "req-clearing", - REQ_QUERYING = "req-querying", - - RESP_DONE = "resp-done", - RESP_QUERYING = "resp-querying", -} - -/** - * Checks if the given search signal is a search signal request. - * - * @param {SEARCH_SIGNAL} s - * @return {boolean} - */ -const isSearchSignalReq = (s: SEARCH_SIGNAL): boolean => s.startsWith("req-"); - -/** - * Checks if the given search signal is a search signal response. - * - * @param {SEARCH_SIGNAL} s - * @return {boolean} - */ -const isSearchSignalResp = (s: SEARCH_SIGNAL): boolean => s.startsWith("resp-"); - -/** - * Checks if the given search signal is a querying request / response. - * - * @param {SEARCH_SIGNAL} s - * @return {boolean} - */ -const isSearchSignalQuerying = (s: SEARCH_SIGNAL): boolean => s.endsWith("-querying"); - -/** - * Checks if the given search signal is an operation in progress, which can be used as a - * condition to disable UI elements. - * - * @param {SEARCH_SIGNAL} s - * @return {boolean} - */ -const isOperationInProgress = (s: SEARCH_SIGNAL): boolean => ( - (true === isSearchSignalReq(s)) || - (true === isSearchSignalQuerying(s)) -); - -/** - * Enum of Mongo Collection sort orders. - * - * @enum {string} - */ -const MONGO_SORT_ORDER = Object.freeze({ - ASCENDING: "asc", - DESCENDING: "desc", -}); - -/** - * Enum of search results cache fields. - * - * @enum {string} - */ -const SEARCH_RESULTS_FIELDS = Object.freeze({ - ID: "_id", - TIMESTAMP: "timestamp", -}); - -/** - * The maximum number of results to retrieve for a search. - */ -const SEARCH_MAX_NUM_RESULTS = 1000; - -export { - isOperationInProgress, - isSearchSignalQuerying, - isSearchSignalReq, - isSearchSignalResp, - MONGO_SORT_ORDER, - SEARCH_MAX_NUM_RESULTS, - SEARCH_RESULTS_FIELDS, - SEARCH_SIGNAL, -}; diff --git a/components/log-viewer-webui/server/src/plugins/DbManager.ts b/components/log-viewer-webui/server/src/plugins/DbManager.ts index 1562bfc590..28a39f6651 100644 --- a/components/log-viewer-webui/server/src/plugins/DbManager.ts +++ b/components/log-viewer-webui/server/src/plugins/DbManager.ts @@ -12,16 +12,14 @@ import { import {Nullable} from "../typings/common.js"; import { DbManagerOptions, - StreamFileMongoDocument, - StreamFilesCollection, -} from "../typings/DbManager.js"; -import { + QUERY_JOB_STATUS, QUERY_JOB_STATUS_WAITING_STATES, QUERY_JOB_TYPE, QUERY_JOBS_TABLE_COLUMN_NAMES, QueryJob, - QUERY_JOB_STATUS, -} from "../typings/query.js"; + StreamFileMongoDocument, + StreamFilesCollection, +} from "../typings/DbManager.js"; import {sleep} from "../utils/time.js"; diff --git a/components/log-viewer-webui/server/src/server.ts b/components/log-viewer-webui/server/src/server.ts deleted file mode 100644 index 8bb1855856..0000000000 --- a/components/log-viewer-webui/server/src/server.ts +++ /dev/null @@ -1,67 +0,0 @@ -// https://github.com/fastify/demo/blob/main/src/server.ts - -import Fastify from 'fastify' -import fp from 'fastify-plugin' - -import closeWithGrace from 'close-with-grace' - -import serviceApp from './fastify-v2/app.js' - -/** - * Do not use NODE_ENV to determine what logger (or any env related feature) to use - * @see {@link https://www.youtube.com/watch?v=HMM7GJC5E2o} - */ -function getLoggerOptions () { - // Only if the program is running in an interactive terminal - if (process.stdout.isTTY) { - return { - level: 'info', - transport: { - target: 'pino-pretty', - options: { - translateTime: 'HH:MM:ss Z', - ignore: 'pid,hostname' - } - } - } - } - - return { level: process.env.LOG_LEVEL ?? 'silent' } -} - -const app = Fastify({ - logger: getLoggerOptions(), - ajv: { - customOptions: { - coerceTypes: 'array', // change type of data to match type keyword - removeAdditional: 'all' // Remove additional body properties - } - } -}) - -async function init () { - // fp must be used to override default error handler - app.register(fp(serviceApp)) - - closeWithGrace( - { delay: Number(process.env.FASTIFY_CLOSE_GRACE_DELAY || 500) }, - async ({ err }) => { - if (err != null) { - app.log.error(err) - } - - await app.close() - } - ) - - await app.ready() - - try { - await app.listen({ port: Number(process.env.PORT ?? 3000) }) - } catch (err) { - app.log.error(err) - process.exit(1) - } -} - -init() \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/typings/DbManager.ts b/components/log-viewer-webui/server/src/typings/DbManager.ts index ae6567b748..6464ccd4d3 100644 --- a/components/log-viewer-webui/server/src/typings/DbManager.ts +++ b/components/log-viewer-webui/server/src/typings/DbManager.ts @@ -1,4 +1,66 @@ import {Collection} from "mongodb"; +import {RowDataPacket} from "mysql2/promise"; + + +/** + * Matching the `QueryJobType` class in `job_orchestration.query_scheduler.constants`. + */ +enum QUERY_JOB_TYPE { + SEARCH_OR_AGGREGATION = 0, + EXTRACT_IR, + EXTRACT_JSON, +} + +/** + * List of valid extract job types. + */ +const EXTRACT_JOB_TYPES = new Set([ + QUERY_JOB_TYPE.EXTRACT_IR, + QUERY_JOB_TYPE.EXTRACT_JSON, +]); + +/** + * Matching the `QueryJobStatus` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +enum QUERY_JOB_STATUS { + PENDING = 0, + RUNNING, + SUCCEEDED, + FAILED, + CANCELLING, + CANCELLED, +} + +/** + * List of states that indicate the job is either pending or in progress. + */ +const QUERY_JOB_STATUS_WAITING_STATES = new Set([ + QUERY_JOB_STATUS.PENDING, + QUERY_JOB_STATUS.RUNNING, + QUERY_JOB_STATUS.CANCELLING, +]); + +/** + * The `query_jobs` table's column names. + * + * @enum {string} + */ +enum QUERY_JOBS_TABLE_COLUMN_NAMES { + ID = "id", + TYPE = "type", + STATUS = "status", + JOB_CONFIG = "job_config", +} + +interface QueryJob extends RowDataPacket { + [QUERY_JOBS_TABLE_COLUMN_NAMES.ID]: number; + [QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE]: QUERY_JOB_TYPE; + [QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]: QUERY_JOB_STATUS; + [QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG]: string; +} interface DbManagerOptions { mysqlConfig: { @@ -30,6 +92,14 @@ type StreamFilesCollection = Collection; export type { DbManagerOptions, + QueryJob, StreamFileMongoDocument, StreamFilesCollection, -}; \ No newline at end of file +}; +export { + EXTRACT_JOB_TYPES, + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, + QUERY_JOBS_TABLE_COLUMN_NAMES, +}; diff --git a/components/log-viewer-webui/server/src/typings/query.ts b/components/log-viewer-webui/server/src/typings/query.ts deleted file mode 100644 index 93833956b9..0000000000 --- a/components/log-viewer-webui/server/src/typings/query.ts +++ /dev/null @@ -1,73 +0,0 @@ -import {RowDataPacket} from "mysql2/promise"; - -/** - * Matching the `QueryJobType` class in `job_orchestration.query_scheduler.constants`. - */ -enum QUERY_JOB_TYPE { - SEARCH_OR_AGGREGATION = 0, - EXTRACT_IR, - EXTRACT_JSON, -} - -/** - * List of valid extract job types. - */ -const EXTRACT_JOB_TYPES = new Set([ - QUERY_JOB_TYPE.EXTRACT_IR, - QUERY_JOB_TYPE.EXTRACT_JSON, -]); - -/** - * Matching the `QueryJobStatus` class in - * `job_orchestration.query_scheduler.constants`. - * - * @enum {number} - */ -enum QUERY_JOB_STATUS { - PENDING = 0, - RUNNING, - SUCCEEDED, - FAILED, - CANCELLING, - CANCELLED, -} - -/** - * List of states that indicate the job is either pending or in progress. - */ -const QUERY_JOB_STATUS_WAITING_STATES = new Set([ - QUERY_JOB_STATUS.PENDING, - QUERY_JOB_STATUS.RUNNING, - QUERY_JOB_STATUS.CANCELLING, -]); - -/** - * The `query_jobs` table's column names. - * - * @enum {string} - */ -enum QUERY_JOBS_TABLE_COLUMN_NAMES { - ID = "id", - TYPE = "type", - STATUS = "status", - JOB_CONFIG = "job_config", -} - -interface QueryJob extends RowDataPacket { - [QUERY_JOBS_TABLE_COLUMN_NAMES.ID]: number; - [QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE]: QUERY_JOB_TYPE; - [QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]: QUERY_JOB_STATUS; - [QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG]: string; -} - -export type { - QueryJob, -}; -export { - EXTRACT_JOB_TYPES, - QUERY_JOB_STATUS, - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, - QUERY_JOBS_TABLE_COLUMN_NAMES, -}; - diff --git a/components/log-viewer-webui/server/src/utils/misc.ts b/components/log-viewer-webui/server/src/utils/misc.ts deleted file mode 100644 index e36d040484..0000000000 --- a/components/log-viewer-webui/server/src/utils/misc.ts +++ /dev/null @@ -1,37 +0,0 @@ -const MILLIS_PER_SECOND = 1000; -const BYTES_PER_KIBIBYTE = 1024; - -/** - * Creates a promise that resolves after a specified number of seconds. - * - * @param {number} seconds to wait before resolving the promise - * @return {Promise} that resolves after the specified delay - */ -const sleep = (seconds: number) => new Promise((resolve) => { - setTimeout(resolve, seconds * MILLIS_PER_SECOND); -}); - -/** - * Computes a human-readable representation of a size in bytes. - * - * @param {number} num - * @return {string} - */ -const computeHumanSize = (num: number) => { - // eslint-disable-next-line @stylistic/js/array-element-newline - const siPrefixes = ["", "K", "M", "G", "T", "P", "E", "Z"]; - for (let i = 0; i < siPrefixes.length; ++i) { - if (BYTES_PER_KIBIBYTE > Math.abs(num)) { - return `${Math.round(num)} ${siPrefixes[i]}B`; - } - num /= BYTES_PER_KIBIBYTE; - } - - return `${Math.round(num)} B`; -}; - - -export { - computeHumanSize, - sleep, -}; diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 56740d3316..c19656be5c 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 56740d33164f80c00daef37542a27dffcef3a7e1 +Subproject commit c19656be5c2c495c5c06c81687225f2540f1fffe From 440b674c39a96cf6098234b8b85401f4cacf8ae3 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 16:42:06 +0000 Subject: [PATCH 14/55] latest --- .../server/src/fastify-v2/app.ts | 4 +- .../app/search/QueryJobsDbManager/index.ts | 246 +++++++++--------- .../SearchJobCollectionsManager/index.ts | 147 ++++++----- .../SearchJobCollectionsManager/typings.ts | 26 +- .../SearchResultsMetadataCollection/index.ts | 53 ++-- .../typings.ts | 27 +- .../server/src/plugins/DbManager.ts | 8 +- .../server/src/routes/query.ts | 6 +- .../server/src/typings/DbManager.ts | 69 ----- .../server/src/typings/query.ts | 71 +++++ 10 files changed, 345 insertions(+), 312 deletions(-) create mode 100644 components/log-viewer-webui/server/src/typings/query.ts diff --git a/components/log-viewer-webui/server/src/fastify-v2/app.ts b/components/log-viewer-webui/server/src/fastify-v2/app.ts index a5f823ed65..ae9f0a2ba2 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/app.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/app.ts @@ -54,10 +54,10 @@ export default async function serviceApp ( // Loads all routes. fastify.register(fastifyAutoload, { - dir: path.join(import.meta.dirname, "routes"), - ignorePattern: IGNORED_FILES_REGEX, autoHooks: true, cascadeHooks: true, + dir: path.join(import.meta.dirname, "routes"), + ignorePattern: IGNORED_FILES_REGEX, options: {...opts}, }); diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts index 40217bec60..be824004f8 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts @@ -1,85 +1,85 @@ -import { encode } from "@msgpack/msgpack"; -import { ResultSetHeader, RowDataPacket } from "mysql2"; +import type {MySQLPromisePool} from "@fastify/mysql"; +import {encode} from "@msgpack/msgpack"; +import settings from "@settings" with { type: "json" }; +import {FastifyInstance} from "fastify"; +import fp from "fastify-plugin"; +import {ResultSetHeader} from "mysql2"; +import {setTimeout} from "timers/promises"; + import { - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, - QUERY_JOB_STATUS, - QUERY_JOBS_TABLE_COLUMN_NAMES, + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, + QUERY_JOBS_TABLE_COLUMN_NAMES, + QueryJob, } from "../../../../../typings/query.js"; -import { JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS } from "./typings.js"; -import { FastifyInstance } from "fastify"; -import fp from "fastify-plugin"; -import settings from "@settings" with { type: "json" }; -import { setTimeout } from 'timers/promises'; +import {JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS} from "./typings.js"; -import type { - MySQLPromisePool, -} from "@fastify/mysql"; /** * Class for submitting and monitoring query jobs in the database. */ class QueryJobsDbManager { - #sqlDbConnPool: MySQLPromisePool; - - private constructor(sqlDbConnPool: MySQLPromisePool) { - this.#sqlDbConnPool = sqlDbConnPool; - } - - /** - * Creates a new QueryJobsDbManager. - * - * @param fastify - * @returns - */ - static create(fastify: FastifyInstance): QueryJobsDbManager { - const sqlDbConnPool = fastify.mysql; - return new QueryJobsDbManager(sqlDbConnPool); - } - - /** - * Submits a search job to the database. - * - * @param searchConfig The arguments for the query. - * @return The job's ID. - * @throws {Error} on error. - */ - async submitSearchJob(searchConfig: object): Promise { - const [queryInsertResults] = await this.#sqlDbConnPool.query( - `INSERT INTO ${settings.SqlDbQueryJobsTableName} + #sqlDbConnPool: MySQLPromisePool; + + private constructor (sqlDbConnPool: MySQLPromisePool) { + this.#sqlDbConnPool = sqlDbConnPool; + } + + /** + * Creates a new QueryJobsDbManager. + * + * @param fastify + * @return + */ + static create (fastify: FastifyInstance): QueryJobsDbManager { + const sqlDbConnPool = fastify.mysql; + return new QueryJobsDbManager(sqlDbConnPool); + } + + /** + * Submits a search job to the database. + * + * @param searchConfig The arguments for the query. + * @return The job's ID. + * @throws {Error} on error. + */ + async submitSearchJob (searchConfig: object): Promise { + const [queryInsertResults] = await this.#sqlDbConnPool.query( + `INSERT INTO ${settings.SqlDbQueryJobsTableName} (${QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG}, ${QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE}) VALUES (?, ?)`, - [ - Buffer.from(encode(searchConfig)), - QUERY_JOB_TYPE.SEARCH_OR_AGGREGATION, - ] - ); - - return queryInsertResults.insertId; - } - - /** - * Submits an aggregation job to the database. - * - * @param searchConfig The arguments for the query. - * @param timeRangeBucketSizeMillis - * @return The aggregation job's ID. - * @throws {Error} on error. - */ - async submitAggregationJob( - searchConfig: object, - timeRangeBucketSizeMillis: number - ): Promise { - const searchAggregationConfig = { - ...searchConfig, - aggregation_config: { - count_by_time_bucket_size: timeRangeBucketSizeMillis, - }, - }; - - return await this.submitSearchJob(searchAggregationConfig); - } + [ + Buffer.from(encode(searchConfig)), + QUERY_JOB_TYPE.SEARCH_OR_AGGREGATION, + ] + ); + + return queryInsertResults.insertId; + } + + /** + * Submits an aggregation job to the database. + * + * @param searchConfig The arguments for the query. + * @param timeRangeBucketSizeMillis + * @return The aggregation job's ID. + * @throws {Error} on error. + */ + async submitAggregationJob ( + searchConfig: object, + timeRangeBucketSizeMillis: number + ): Promise { + const searchAggregationConfig = { + ...searchConfig, + aggregation_config: { + count_by_time_bucket_size: timeRangeBucketSizeMillis, + }, + }; + + return await this.submitSearchJob(searchAggregationConfig); + } /** * Submits a query cancellation request to the database. @@ -99,64 +99,74 @@ class QueryJobsDbManager { ); } - /** - * Waits for the job to complete. - * - * @param jobId - * @return - * @throws {Error} on MySQL error, if the job wasn't found in the database, if the job was - * cancelled, or if the job completed in an unexpected state. - */ - async awaitJobCompletion(jobId: number): Promise { - while (true) { - let rows; - try { - const [queryRows] = await this.#sqlDbConnPool.query( - ` + /** + * Waits for the job to complete. + * + * @param jobId + * @return + * @throws {Error} on MySQL error, if the job wasn't found in the database, if the job was + * cancelled, or if the job completed in an unexpected state. + */ + async awaitJobCompletion (jobId: number): Promise { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + while (true) { + let rows: QueryJob[]; + try { + const [queryRows] = await this.#sqlDbConnPool.query( + ` SELECT ${QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS} FROM ${settings.SqlDbQueryJobsTableName} WHERE ${QUERY_JOBS_TABLE_COLUMN_NAMES.ID} = ? `, - jobId - ); + jobId + ); - rows = queryRows; - } catch (e) { - throw new Error(`Failed to query status for job ${jobId} - ${e}`); - } - if (0 === rows.length) { - throw new Error(`Job ${jobId} not found in database.`); - } - const status = rows[0]![QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; - - if (false === QUERY_JOB_STATUS_WAITING_STATES.has(status)) { - if (QUERY_JOB_STATUS.CANCELLED === status) { - throw new Error(`Job ${jobId} was cancelled.`); - } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { - throw new Error( - `Job ${jobId} exited with unexpected status=${status}: ` + - `${Object.keys(QUERY_JOB_STATUS)[status]}.` - ); - } - break; - } + rows = queryRows; + } catch (e: unknown) { + let errorMessage: string; + + if (e instanceof Error) { + errorMessage = e.message; + } else { + errorMessage = String(e); + } + + throw new Error(`Failed to query status for job ${jobId} - ${errorMessage}`); + } + if (0 === rows.length) { + throw new Error(`Job ${jobId} not found in database.`); + } - await setTimeout(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); + const status = (rows[0] as QueryJob)[QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]; + + if (false === QUERY_JOB_STATUS_WAITING_STATES.has(status)) { + if (QUERY_JOB_STATUS.CANCELLED === status) { + throw new Error(`Job ${jobId} was cancelled.`); + } else if (QUERY_JOB_STATUS.SUCCEEDED !== status) { + throw new Error( + `Job ${jobId} exited with unexpected status=${status}: ` + + `${Object.keys(QUERY_JOB_STATUS)[status]}.` + ); + } + break; + } + + await setTimeout(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); + } } - } } declare module "fastify" { - export interface FastifyInstance { - QueryJobsDbManager: QueryJobsDbManager; - } + export interface FastifyInstance { + QueryJobsDbManager: QueryJobsDbManager; + } } export default fp( - function (fastify) { - fastify.decorate("QueryJobsDbManager", QueryJobsDbManager.create(fastify)); - }, - { - name: "QueryJobsDbManager", - } + (fastify) => { + fastify.decorate("QueryJobsDbManager", QueryJobsDbManager.create(fastify)); + }, + { + name: "QueryJobsDbManager", + } ); diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts index 5a01b45ce1..cb203abbd1 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/index.ts @@ -1,88 +1,101 @@ -import { FastifyInstance } from 'fastify'; -import fp from 'fastify-plugin'; -import { Nullable } from "../../../../../typings/common.js"; -import type { Db, Collection } from "mongodb"; -import type { SearchResultsDocument } from "./typings.js"; -import { CollectionDroppedError } from "./typings.js"; +import {FastifyInstance} from "fastify"; +import fp from "fastify-plugin"; +import type { + Collection, + Db, +} from "mongodb"; + +import {Nullable} from "../../../../../typings/common.js"; +import { + CollectionDroppedError, + SearchResultsDocument, +} from "./typings.js"; + /** * Class to keep track of MongoDB collections created for search jobs, ensuring all collections * have unique names. */ class SearchJobCollectionsManager { - #jobIdToCollectionsMap: Map>> - = new Map(); - - #db: Db; + #jobIdToCollectionsMap: Map>> = + new Map(); - private constructor (db: Db) { - this.#db = db; - } + #db: Db; - /** - * Creates a new SearchJobCollectionsManager. - * - * @param fastify - * @returns - */ - static create(fastify: FastifyInstance): SearchJobCollectionsManager { - if ("undefined" === typeof fastify.mongo.db) { - throw new Error("MongoDB database not found"); + private constructor (db: Db) { + this.#db = db; } - return new SearchJobCollectionsManager(fastify.mongo.db); - } + /** + * Creates a new SearchJobCollectionsManager. + * + * @param fastify + * @return + * @throws {Error} if the MongoDB database is not found. + */ + static create (fastify: FastifyInstance): SearchJobCollectionsManager { + if ("undefined" === typeof fastify.mongo.db) { + throw new Error("MongoDB database not found"); + } - /** - * Gets, or if it doesn't exist, creates a MongoDB collection named with the given job ID. - * - * @param jobId - * @return MongoDB collection - * @throws {CollectionDroppedError} if the collection was already dropped. - */ - async getOrCreateCollection(jobId: number): Promise> { - const name = jobId.toString(); - if (false === this.#jobIdToCollectionsMap.has(name)) { - this.#jobIdToCollectionsMap.set(name, this.#db.collection(name)); - } else if (this.#jobIdToCollectionsMap.get(name) === null) { - throw new CollectionDroppedError(name); + return new SearchJobCollectionsManager(fastify.mongo.db); } - // `collections.get(name)` will always return a valid collection since: - // - Function creates a new collection if it doesn't exist in map. - // - Throws an error if the collection was already dropped. - // Therefore, we can safely cast to `Collection`. - return this.#jobIdToCollectionsMap.get(name) as Collection; - } - /** - * Drops the MongoDB collection associated with the given job ID. - * - * @param jobId - * @throws {Error} if the collection does not exist or has already been dropped. - */ - async dropCollection(jobId: number) { - const name = jobId.toString(); - const collection = this.#jobIdToCollectionsMap.get(name); - if ("undefined" === typeof collection || null === collection) { - throw new Error(`Collection ${name} not found`); + /** + * Gets, or if it doesn't exist, creates a MongoDB collection named with the given job ID. + * + * @param jobId + * @return MongoDB collection + * @throws {CollectionDroppedError} if the collection was already dropped. + */ + async getOrCreateCollection (jobId: number): Promise> { + const name = jobId.toString(); + if (false === this.#jobIdToCollectionsMap.has(name)) { + this.#jobIdToCollectionsMap.set(name, this.#db.collection(name)); + } else if (null === this.#jobIdToCollectionsMap.get(name)) { + throw new CollectionDroppedError(name); + } + + + // `collections.get(name)` will always return a valid collection since: + // - Function creates a new collection if it doesn't exist in map. + // - Throws an error if the collection was already dropped. + // Therefore, we can safely cast to `Collection`. + return this.#jobIdToCollectionsMap.get(name) as Collection; } - await collection.drop(); - this.#jobIdToCollectionsMap.set(name, null); - } + /** + * Drops the MongoDB collection associated with the given job ID. + * + * @param jobId + * @throws {Error} if the collection does not exist or has already been dropped. + */ + async dropCollection (jobId: number) { + const name = jobId.toString(); + const collection = this.#jobIdToCollectionsMap.get(name); + if ("undefined" === typeof collection || null === collection) { + throw new Error(`Collection ${name} not found`); + } + + await collection.drop(); + this.#jobIdToCollectionsMap.set(name, null); + } } -declare module 'fastify' { - export interface FastifyInstance { - SearchJobCollectionsManager: SearchJobCollectionsManager; - } +declare module "fastify" { + export interface FastifyInstance { + SearchJobCollectionsManager: SearchJobCollectionsManager; + } } export default fp( - function (fastify) { - fastify.decorate('SearchJobCollectionsManager', SearchJobCollectionsManager.create(fastify)); - }, - { - name: 'SearchJobCollectionsManager', - } + (fastify) => { + fastify.decorate( + "SearchJobCollectionsManager", + SearchJobCollectionsManager.create(fastify) + ); + }, + { + name: "SearchJobCollectionsManager", + } ); diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts index 08a11dc984..01bb54b354 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchJobCollectionsManager/typings.ts @@ -2,24 +2,24 @@ * MongoDB document for search results. */ interface SearchResultsDocument { - _id: string; - orig_file_id: string; - orig_file_path: string; - log_event_ix: number; - timestamp: number; - message: string; + _id: string; + orig_file_id: string; + orig_file_path: string; + log_event_ix: number; + timestamp: number; + message: string; } /** * Error thrown when a MongoDB collection has been dropped unexpectedly. */ class CollectionDroppedError extends Error { - constructor(collectionName: string) { - super(`Collection ${collectionName} has been dropped.`); - this.name = "CollectionDroppedError"; - } + constructor (collectionName: string) { + super(`Collection ${collectionName} has been dropped.`); + this.name = "CollectionDroppedError"; + } } -export type { SearchResultsDocument }; - -export { CollectionDroppedError }; +export { + CollectionDroppedError, SearchResultsDocument, +}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts index 0f42cd702e..1f68c41328 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts @@ -1,36 +1,41 @@ -import { FastifyInstance } from 'fastify' -import fp from 'fastify-plugin' - import settings from "@settings" with {type: "json"}; -import type { SearchResultsMetadataDocument } from './typings.js'; +import {FastifyInstance} from "fastify"; +import fp from "fastify-plugin"; + +import type {SearchResultsMetadataDocument} from "./typings.js"; + /** * Creates a MongoDB collection for search results metadata. + * + * @param fastify + * @return MongoDB collection + * @throws {Error} if the MongoDB database is not found. */ const createSearchResultsMetadataCollection = (fastify: FastifyInstance) => { - if ("undefined" === typeof fastify.mongo.db) { - throw new Error("MongoDB database not found"); - } + if ("undefined" === typeof fastify.mongo.db) { + throw new Error("MongoDB database not found"); + } - return fastify.mongo.db.collection( - settings.MongoDbSearchResultsMetadataCollectionName - ); + return fastify.mongo.db.collection( + settings.MongoDbSearchResultsMetadataCollectionName + ); }; -declare module 'fastify' { - export interface FastifyInstance { - SearchResultsMetadataCollection: ReturnType; - } +declare module "fastify" { + export interface FastifyInstance { + SearchResultsMetadataCollection: ReturnType; + } } export default fp( - function (fastify) { - fastify.decorate( - 'SearchResultsMetadataCollection', - createSearchResultsMetadataCollection(fastify) - ) - }, - { - name: 'SearchResultsMetadataCollection', - } -) + (fastify) => { + fastify.decorate( + "SearchResultsMetadataCollection", + createSearchResultsMetadataCollection(fastify) + ); + }, + { + name: "SearchResultsMetadataCollection", + } +); diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts index 35dc37fcdd..b4998d4045 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/typings.ts @@ -1,15 +1,5 @@ -import { Nullable } from "../../../../../typings/common.js"; +import {Nullable} from "../../../../../typings/common.js"; -/** - * MongoDB document for search results metadata. `numTotalResults` is optional - * since it is only set when the search job is completed. - */ -interface SearchResultsMetadataDocument { - _id: string; - errorMsg: Nullable; - lastSignal: SEARCH_SIGNAL; - numTotalResults?: number; -} /** * Enum of search-related signals. @@ -28,7 +18,16 @@ enum SEARCH_SIGNAL { RESP_QUERYING = "resp-querying", } +/** + * MongoDB document for search results metadata. `numTotalResults` is optional + * since it is only set when the search job is completed. + */ +interface SearchResultsMetadataDocument { + _id: string; + errorMsg: Nullable; + lastSignal: SEARCH_SIGNAL; + numTotalResults?: number; +} - -export type { SearchResultsMetadataDocument }; -export { SEARCH_SIGNAL }; +export type {SearchResultsMetadataDocument}; +export {SEARCH_SIGNAL}; diff --git a/components/log-viewer-webui/server/src/plugins/DbManager.ts b/components/log-viewer-webui/server/src/plugins/DbManager.ts index 28a39f6651..a1f863c21e 100644 --- a/components/log-viewer-webui/server/src/plugins/DbManager.ts +++ b/components/log-viewer-webui/server/src/plugins/DbManager.ts @@ -12,14 +12,16 @@ import { import {Nullable} from "../typings/common.js"; import { DbManagerOptions, + StreamFileMongoDocument, + StreamFilesCollection, +} from "../typings/DbManager.js"; +import { QUERY_JOB_STATUS, QUERY_JOB_STATUS_WAITING_STATES, QUERY_JOB_TYPE, QUERY_JOBS_TABLE_COLUMN_NAMES, QueryJob, - StreamFileMongoDocument, - StreamFilesCollection, -} from "../typings/DbManager.js"; +} from "../typings/query.js"; import {sleep} from "../utils/time.js"; diff --git a/components/log-viewer-webui/server/src/routes/query.ts b/components/log-viewer-webui/server/src/routes/query.ts index f43a724b07..a8500e53c5 100644 --- a/components/log-viewer-webui/server/src/routes/query.ts +++ b/components/log-viewer-webui/server/src/routes/query.ts @@ -4,8 +4,10 @@ import {FastifyPluginAsync} from "fastify"; import {StatusCodes} from "http-status-codes"; import settings from "../../settings.json" with {type: "json"}; -import {QUERY_JOB_TYPE} from "../plugins/DbManager.js"; -import {EXTRACT_JOB_TYPES} from "../typings/DbManager.js"; +import { + EXTRACT_JOB_TYPES, + QUERY_JOB_TYPE, +} from "../typings/query.js"; /** diff --git a/components/log-viewer-webui/server/src/typings/DbManager.ts b/components/log-viewer-webui/server/src/typings/DbManager.ts index 6464ccd4d3..c294c782bf 100644 --- a/components/log-viewer-webui/server/src/typings/DbManager.ts +++ b/components/log-viewer-webui/server/src/typings/DbManager.ts @@ -1,67 +1,6 @@ import {Collection} from "mongodb"; -import {RowDataPacket} from "mysql2/promise"; -/** - * Matching the `QueryJobType` class in `job_orchestration.query_scheduler.constants`. - */ -enum QUERY_JOB_TYPE { - SEARCH_OR_AGGREGATION = 0, - EXTRACT_IR, - EXTRACT_JSON, -} - -/** - * List of valid extract job types. - */ -const EXTRACT_JOB_TYPES = new Set([ - QUERY_JOB_TYPE.EXTRACT_IR, - QUERY_JOB_TYPE.EXTRACT_JSON, -]); - -/** - * Matching the `QueryJobStatus` class in - * `job_orchestration.query_scheduler.constants`. - * - * @enum {number} - */ -enum QUERY_JOB_STATUS { - PENDING = 0, - RUNNING, - SUCCEEDED, - FAILED, - CANCELLING, - CANCELLED, -} - -/** - * List of states that indicate the job is either pending or in progress. - */ -const QUERY_JOB_STATUS_WAITING_STATES = new Set([ - QUERY_JOB_STATUS.PENDING, - QUERY_JOB_STATUS.RUNNING, - QUERY_JOB_STATUS.CANCELLING, -]); - -/** - * The `query_jobs` table's column names. - * - * @enum {string} - */ -enum QUERY_JOBS_TABLE_COLUMN_NAMES { - ID = "id", - TYPE = "type", - STATUS = "status", - JOB_CONFIG = "job_config", -} - -interface QueryJob extends RowDataPacket { - [QUERY_JOBS_TABLE_COLUMN_NAMES.ID]: number; - [QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE]: QUERY_JOB_TYPE; - [QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]: QUERY_JOB_STATUS; - [QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG]: string; -} - interface DbManagerOptions { mysqlConfig: { user: string; @@ -92,14 +31,6 @@ type StreamFilesCollection = Collection; export type { DbManagerOptions, - QueryJob, StreamFileMongoDocument, StreamFilesCollection, }; -export { - EXTRACT_JOB_TYPES, - QUERY_JOB_STATUS, - QUERY_JOB_STATUS_WAITING_STATES, - QUERY_JOB_TYPE, - QUERY_JOBS_TABLE_COLUMN_NAMES, -}; diff --git a/components/log-viewer-webui/server/src/typings/query.ts b/components/log-viewer-webui/server/src/typings/query.ts new file mode 100644 index 0000000000..1b000699fb --- /dev/null +++ b/components/log-viewer-webui/server/src/typings/query.ts @@ -0,0 +1,71 @@ +import {RowDataPacket} from "mysql2/promise"; + + +/** + * Matching the `QueryJobType` class in `job_orchestration.query_scheduler.constants`. + */ +enum QUERY_JOB_TYPE { + SEARCH_OR_AGGREGATION = 0, + EXTRACT_IR, + EXTRACT_JSON, +} + +/** + * List of valid extract job types. + */ +const EXTRACT_JOB_TYPES = new Set([ + QUERY_JOB_TYPE.EXTRACT_IR, + QUERY_JOB_TYPE.EXTRACT_JSON, +]); + +/** + * Matching the `QueryJobStatus` class in + * `job_orchestration.query_scheduler.constants`. + * + * @enum {number} + */ +enum QUERY_JOB_STATUS { + PENDING = 0, + RUNNING, + SUCCEEDED, + FAILED, + CANCELLING, + CANCELLED, +} + +/** + * List of states that indicate the job is either pending or in progress. + */ +const QUERY_JOB_STATUS_WAITING_STATES = new Set([ + QUERY_JOB_STATUS.PENDING, + QUERY_JOB_STATUS.RUNNING, + QUERY_JOB_STATUS.CANCELLING, +]); + +/** + * The `query_jobs` table's column names. + * + * @enum {string} + */ +enum QUERY_JOBS_TABLE_COLUMN_NAMES { + ID = "id", + TYPE = "type", + STATUS = "status", + JOB_CONFIG = "job_config", +} + +interface QueryJob extends RowDataPacket { + [QUERY_JOBS_TABLE_COLUMN_NAMES.ID]: number; + [QUERY_JOBS_TABLE_COLUMN_NAMES.TYPE]: QUERY_JOB_TYPE; + [QUERY_JOBS_TABLE_COLUMN_NAMES.STATUS]: QUERY_JOB_STATUS; + [QUERY_JOBS_TABLE_COLUMN_NAMES.JOB_CONFIG]: string; +} + +export type {QueryJob}; +export { + EXTRACT_JOB_TYPES, + QUERY_JOB_STATUS, + QUERY_JOB_STATUS_WAITING_STATES, + QUERY_JOB_TYPE, + QUERY_JOBS_TABLE_COLUMN_NAMES, +}; From 8dd1341274b80479815e479f8703a40bbcbb7338 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 16:47:08 +0000 Subject: [PATCH 15/55] latest --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index c19656be5c..2693d8b690 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit c19656be5c2c495c5c06c81687225f2540f1fffe +Subproject commit 2693d8b69050b8d48f96ae60cfce7be8626d3875 From 23d3f4d06cdfbfe2cfdf4c62f8036b5d1c823ac7 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 18:19:32 +0000 Subject: [PATCH 16/55] latest --- .../src/fastify-v2/routes/api/search/index.ts | 200 ++++++++++++++++++ .../fastify-v2/routes/api/search/typings.ts | 42 ++++ .../src/fastify-v2/routes/api/search/utils.ts | 113 ++++++++++ .../server/src/fastify-v2/schemas/common.ts | 7 + .../server/src/fastify-v2/schemas/error.ts | 7 + .../server/src/fastify-v2/schemas/search.ts | 15 ++ 6 files changed, 384 insertions(+) create mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts create mode 100644 components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts new file mode 100644 index 0000000000..6d952aba2e --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts @@ -0,0 +1,200 @@ +import { SEARCH_MAX_NUM_RESULTS } from './typings.js'; +import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; +import { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes } from './utils.js'; +import { + FastifyPluginAsyncTypebox, + Type + } from '@fastify/type-provider-typebox' +import { CreateSearchJobSchema, SearchJobSchema } from '../../../schemas/search.js' +import { ErrorSchema } from '../../../schemas/error.js' + + const plugin: FastifyPluginAsyncTypebox = async (fastify) => { + const { + QueryJobsDbManager, + SearchJobCollectionsManager, + SearchResultsMetadataCollection, + } = fastify; + + /** + * Submits a search query and initiates the search process. + */ + fastify.post( + '/query', + { + schema: { + body: CreateSearchJobSchema, + response: { + 201: SearchJobSchema, + 500: ErrorSchema, + }, + tags: ['Search'] + }, + }, + async function (request, reply) { + + const { + queryString, + timestampBegin, + timestampEnd, + ignoreCase, + timeRangeBucketSizeMillis + } = request.body; + + const args = { + query_string: queryString, + begin_timestamp: timestampBegin, + end_timestamp: timestampEnd, + ignore_case: ignoreCase, + max_num_results: SEARCH_MAX_NUM_RESULTS + }; + + request.log.info({ args }, "/search args"); + + let searchJobId: number; + let aggregationJobId: number; + + try { + searchJobId = await QueryJobsDbManager.submitSearchJob(args); + aggregationJobId = await QueryJobsDbManager.submitAggregationJob( + args, + timeRangeBucketSizeMillis + ); + } catch (err: unknown) { + const errMsg = 'Unable to submit search/aggregation job to the SQL database'; + request.log.error( err , errMsg); + reply.code(500); + return {message: errMsg}; + } + + SearchResultsMetadataCollection.insertOne({ + _id: searchJobId.toString(), + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + errorMsg: null + }); + + // Defer signal update until after response is sent + setImmediate( async () => { + await updateSearchSignalWhenJobsFinish({ + searchJobId, + aggregationJobId, + queryJobsDbManager: QueryJobsDbManager, + searchJobCollectionsManager: SearchJobCollectionsManager, + SearchResultsMetadataCollection, + logger: request.log, + }); + }); + + await createMongoIndexes({ + searchJobId, + searchJobCollectionsManager: SearchJobCollectionsManager, + logger: request.log, + }); + + reply.code(201) + + return { searchJobId, aggregationJobId }; + } + ); + + /** + * Clears the results of a search operation identified by jobId. + */ + fastify.delete( + '/search/results', + { + schema: { + body: SearchJobSchema, + response: { + 204: Type.Null(), + 500: ErrorSchema, + }, + tags: ['Search'], + }, + }, + async function (request, reply) { + const { searchJobId, aggregationJobId } = request.body; + + request.log.info({ + searchJobId, + aggregationJobId, + }, "/search/results args"); + + try { + await SearchJobCollectionsManager.dropCollection(searchJobId); + await SearchJobCollectionsManager.dropCollection(aggregationJobId); + } catch (err: unknown) { + const errMsg = 'Failed to clear search results'; + request.log.error( + { + err, + searchJobId, + aggregationJobId, + }, + errMsg + ); + reply.code(500); + return {message: `${errMsg} for searchJobId=${searchJobId}, ` + + `aggregationJobId=${aggregationJobId}`}; + } + reply.code(204); + return; + } + ); + + + fastify.post( + '/search/cancel', + { + schema: { + body: SearchJobSchema, + response: { + 204: Type.Null(), + 500: ErrorSchema, + }, + tags: ['Search'], + }, + }, + async function (request, reply) { + const { searchJobId, aggregationJobId } = request.body; + + request.log.info( + `/search/cancel searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` + ); + + try { + await QueryJobsDbManager.submitQueryCancellation(searchJobId); + await QueryJobsDbManager.submitQueryCancellation(aggregationJobId); + + updateSearchResultsMeta({ + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + SearchResultsMetadataCollection, + logger: request.log, + fields: { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: 'Query cancelled before it could be completed.', + }, + }); + + } catch (err: unknown) { + const errMsg = 'Failed to submit cancel request'; + request.log.error( + { + err, + searchJobId, + aggregationJobId, + }, + errMsg + ); + reply.code(500); + return {message: `${errMsg} for searchJobId=${searchJobId}, ` + + `aggregationJobId=${aggregationJobId}`}; + } + + reply.code(204); + return; + } + ); + } + + export default plugin \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts new file mode 100644 index 0000000000..8062fcd74b --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts @@ -0,0 +1,42 @@ +import { + FastifyBaseLogger, + FastifyInstance, +} from "fastify"; + +import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; +import {SearchResultsMetadataDocument} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; + +type UpdateSearchResultsMetaProps = { + jobId: number; + lastSignal: SEARCH_SIGNAL; + SearchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; + logger: FastifyBaseLogger; + fields: Partial; +}; + +type UpdateSearchSignalWhenJobsFinishProps = { + searchJobId: number; + aggregationJobId: number; + queryJobsDbManager: FastifyInstance["QueryJobsDbManager"]; + searchJobCollectionsManager: FastifyInstance["SearchJobCollectionsManager"]; + SearchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; + logger: FastifyBaseLogger; +}; + +type CreateMongoIndexesProps = { + searchJobId: number; + searchJobCollectionsManager: FastifyInstance["SearchJobCollectionsManager"]; + logger: FastifyBaseLogger; +}; + +/** + * The maximum number of results to retrieve for a search. + */ +const SEARCH_MAX_NUM_RESULTS = 1000; + +export { + CreateMongoIndexesProps, + UpdateSearchResultsMetaProps, + UpdateSearchSignalWhenJobsFinishProps, + SEARCH_MAX_NUM_RESULTS, +}; \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts new file mode 100644 index 0000000000..31600c7173 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts @@ -0,0 +1,113 @@ +import { SEARCH_MAX_NUM_RESULTS } from './typings.js'; +import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; +import { + UpdateSearchResultsMetaProps, + UpdateSearchSignalWhenJobsFinishProps, + CreateMongoIndexesProps, +} from './typings.js'; +import { CollectionDroppedError } from '../../../plugins/app/search/SearchJobCollectionsManager/typings.js'; +/** + * Modifies the search results metadata for a given job ID. + * + * @param {UpdateSearchResultsMetaProps} params + */ +const updateSearchResultsMeta = ({ + jobId, + lastSignal, + SearchResultsMetadataCollection, + logger, + fields, +}: UpdateSearchResultsMetaProps) => { + const filter = { + _id: jobId.toString(), + lastSignal: lastSignal, + }; + + const modifier = { + $set: fields, + }; + + logger.debug("SearchResultsMetadataCollection modifier = ", modifier); + SearchResultsMetadataCollection.updateOne(filter, modifier); +}; + +const updateSearchSignalWhenJobsFinish = async ({ + searchJobId, + aggregationJobId, + queryJobsDbManager, + searchJobCollectionsManager, + SearchResultsMetadataCollection, + logger, +}: UpdateSearchSignalWhenJobsFinishProps) => { + let errorMsg; + try { + await queryJobsDbManager.awaitJobCompletion(searchJobId); + await queryJobsDbManager.awaitJobCompletion(aggregationJobId); + } catch (e: unknown) { + if (e instanceof Error) { + errorMsg = e.message; + } + errorMsg = "Error while waiting for job completion"; + } + + let numResultsInCollection = -1; + try { + const collection = await searchJobCollectionsManager + .getOrCreateCollection(searchJobId); + numResultsInCollection = await collection.countDocuments(); + //Need this here in new method. I believe publication used to do this? + await searchJobCollectionsManager.getOrCreateCollection(aggregationJobId); + + } catch (e: unknown) { + if (e instanceof CollectionDroppedError) { + logger.warn(`Collection ${searchJobId} has been dropped.`); + return; + } + throw e; + } + + updateSearchResultsMeta({ + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + SearchResultsMetadataCollection, + logger, + fields: { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: errorMsg ?? null, + numTotalResults: Math.min( + numResultsInCollection, + SEARCH_MAX_NUM_RESULTS + ), + }, + }); +}; + +const createMongoIndexes = async ({ + searchJobId, + searchJobCollectionsManager, + logger, +}: CreateMongoIndexesProps) => { + const timestampAscendingIndex = { + key: { + timestamp: 1, + _id: 1, + }, + name: "timestamp-ascending", + }; + const timestampDescendingIndex = { + key: { + timestamp: -1, + _id: -1, + }, + name: "timestamp-descending", + }; + + const queryJobCollection = await searchJobCollectionsManager.getOrCreateCollection(searchJobId); + logger.info(`Creating indexes for collection ${searchJobId}`); + await queryJobCollection.createIndexes([ + timestampAscendingIndex, + timestampDescendingIndex, + ]); +}; + +export { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes }; \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts new file mode 100644 index 0000000000..206b01b8e6 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts @@ -0,0 +1,7 @@ +import { Type } from '@sinclair/typebox' + +export const StringSchema = Type.String({ + minLength: 1, +}) + +export const IdSchema = Type.Integer({ minimum: 1 }) \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts new file mode 100644 index 0000000000..35f0efef49 --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts @@ -0,0 +1,7 @@ +import { Type } from '@sinclair/typebox' + +const ErrorSchema = Type.Object({ + message: Type.String(), +}); + +export {ErrorSchema}; \ No newline at end of file diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts new file mode 100644 index 0000000000..9af46285cd --- /dev/null +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts @@ -0,0 +1,15 @@ +import { Type } from '@sinclair/typebox' +import { StringSchema, IdSchema } from './common.js' + +export const CreateSearchJobSchema = Type.Object({ + queryString: StringSchema, + timestampBegin: Type.Integer(), + timestampEnd: Type.Integer(), + ignoreCase: Type.Boolean(), + timeRangeBucketSizeMillis: Type.Integer(), +}) + +export const SearchJobSchema = Type.Object({ + searchJobId: IdSchema, + aggregationJobId: IdSchema +}) \ No newline at end of file From 32b8d1962885e81d4ee90e13f38bd813e7b4c003 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 18:31:55 +0000 Subject: [PATCH 17/55] latest --- .../src/fastify-v2/routes/api/search/index.ts | 300 +++++++++--------- .../fastify-v2/routes/api/search/typings.ts | 20 +- .../src/fastify-v2/routes/api/search/utils.ts | 50 ++- .../server/src/fastify-v2/schemas/common.ts | 9 +- .../server/src/fastify-v2/schemas/error.ts | 5 +- .../server/src/fastify-v2/schemas/search.ts | 27 +- 6 files changed, 232 insertions(+), 179 deletions(-) diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts index 6d952aba2e..d8fceba551 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts @@ -1,200 +1,212 @@ -import { SEARCH_MAX_NUM_RESULTS } from './typings.js'; -import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; -import { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes } from './utils.js'; import { FastifyPluginAsyncTypebox, - Type - } from '@fastify/type-provider-typebox' -import { CreateSearchJobSchema, SearchJobSchema } from '../../../schemas/search.js' -import { ErrorSchema } from '../../../schemas/error.js' + Type, +} from "@fastify/type-provider-typebox"; + +import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; +import {ErrorSchema} from "../../../schemas/error.js"; +import { + CreateSearchJobSchema, + SearchJobSchema, +} from "../../../schemas/search.js"; +import {SEARCH_MAX_NUM_RESULTS} from "./typings.js"; +import { + createMongoIndexes, + updateSearchResultsMeta, + updateSearchSignalWhenJobsFinish, +} from "./utils.js"; - const plugin: FastifyPluginAsyncTypebox = async (fastify) => { + +/** + * + * @param fastify + */ +const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const { - QueryJobsDbManager, - SearchJobCollectionsManager, - SearchResultsMetadataCollection, + QueryJobsDbManager, + SearchJobCollectionsManager, + SearchResultsMetadataCollection, } = fastify; - /** + /** * Submits a search query and initiates the search process. */ fastify.post( - '/query', + "/query", { - schema: { - body: CreateSearchJobSchema, - response: { - 201: SearchJobSchema, - 500: ErrorSchema, + schema: { + body: CreateSearchJobSchema, + response: { + 201: SearchJobSchema, + 500: ErrorSchema, + }, + tags: ["Search"], }, - tags: ['Search'] - }, }, - async function (request, reply) { - + async (request, reply) => { const { - queryString, - timestampBegin, - timestampEnd, - ignoreCase, - timeRangeBucketSizeMillis + queryString, + timestampBegin, + timestampEnd, + ignoreCase, + timeRangeBucketSizeMillis, } = request.body; const args = { - query_string: queryString, - begin_timestamp: timestampBegin, - end_timestamp: timestampEnd, - ignore_case: ignoreCase, - max_num_results: SEARCH_MAX_NUM_RESULTS + query_string: queryString, + begin_timestamp: timestampBegin, + end_timestamp: timestampEnd, + ignore_case: ignoreCase, + max_num_results: SEARCH_MAX_NUM_RESULTS, }; - request.log.info({ args }, "/search args"); + request.log.info({args}, "/search args"); let searchJobId: number; let aggregationJobId: number; try { - searchJobId = await QueryJobsDbManager.submitSearchJob(args); - aggregationJobId = await QueryJobsDbManager.submitAggregationJob( - args, - timeRangeBucketSizeMillis - ); + searchJobId = await QueryJobsDbManager.submitSearchJob(args); + aggregationJobId = await QueryJobsDbManager.submitAggregationJob( + args, + timeRangeBucketSizeMillis + ); } catch (err: unknown) { - const errMsg = 'Unable to submit search/aggregation job to the SQL database'; - request.log.error( err , errMsg); - reply.code(500); - return {message: errMsg}; + const errMsg = "Unable to submit search/aggregation job to the SQL database"; + request.log.error(err, errMsg); + reply.code(500); + + return {message: errMsg}; } SearchResultsMetadataCollection.insertOne({ - _id: searchJobId.toString(), - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - errorMsg: null + _id: searchJobId.toString(), + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + errorMsg: null, }); // Defer signal update until after response is sent - setImmediate( async () => { + setImmediate(async () => { await updateSearchSignalWhenJobsFinish({ - searchJobId, - aggregationJobId, - queryJobsDbManager: QueryJobsDbManager, - searchJobCollectionsManager: SearchJobCollectionsManager, - SearchResultsMetadataCollection, - logger: request.log, + searchJobId, + aggregationJobId, + queryJobsDbManager: QueryJobsDbManager, + searchJobCollectionsManager: SearchJobCollectionsManager, + SearchResultsMetadataCollection, + logger: request.log, }); }); await createMongoIndexes({ - searchJobId, - searchJobCollectionsManager: SearchJobCollectionsManager, - logger: request.log, + searchJobId, + searchJobCollectionsManager: SearchJobCollectionsManager, + logger: request.log, }); - reply.code(201) + reply.code(201); - return { searchJobId, aggregationJobId }; - } + return {searchJobId, aggregationJobId}; + } ); /** * Clears the results of a search operation identified by jobId. */ fastify.delete( - '/search/results', - { - schema: { - body: SearchJobSchema, - response: { - 204: Type.Null(), - 500: ErrorSchema, - }, - tags: ['Search'], - }, - }, - async function (request, reply) { - const { searchJobId, aggregationJobId } = request.body; - - request.log.info({ - searchJobId, - aggregationJobId, - }, "/search/results args"); - - try { - await SearchJobCollectionsManager.dropCollection(searchJobId); - await SearchJobCollectionsManager.dropCollection(aggregationJobId); - } catch (err: unknown) { - const errMsg = 'Failed to clear search results'; - request.log.error( - { - err, - searchJobId, - aggregationJobId, + "/search/results", + { + schema: { + body: SearchJobSchema, + response: { + 204: Type.Null(), + 500: ErrorSchema, + }, + tags: ["Search"], }, - errMsg - ); - reply.code(500); - return {message: `${errMsg} for searchJobId=${searchJobId}, ` + + }, + async (request, reply) => { + const {searchJobId, aggregationJobId} = request.body; + + request.log.info({ + searchJobId, + aggregationJobId, + }, "/search/results args"); + + try { + await SearchJobCollectionsManager.dropCollection(searchJobId); + await SearchJobCollectionsManager.dropCollection(aggregationJobId); + } catch (err: unknown) { + const errMsg = "Failed to clear search results"; + request.log.error( + { + err, + searchJobId, + aggregationJobId, + }, + errMsg + ); + reply.code(500); + + return {message: `${errMsg} for searchJobId=${searchJobId}, ` + `aggregationJobId=${aggregationJobId}`}; + } + reply.code(204); } - reply.code(204); - return; - } ); fastify.post( - '/search/cancel', - { - schema: { - body: SearchJobSchema, - response: { - 204: Type.Null(), - 500: ErrorSchema, - }, - tags: ['Search'], - }, - }, - async function (request, reply) { - const { searchJobId, aggregationJobId } = request.body; - - request.log.info( - `/search/cancel searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` - ); - - try { - await QueryJobsDbManager.submitQueryCancellation(searchJobId); - await QueryJobsDbManager.submitQueryCancellation(aggregationJobId); - - updateSearchResultsMeta({ - jobId: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - SearchResultsMetadataCollection, - logger: request.log, - fields: { - lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: 'Query cancelled before it could be completed.', - }, - }); - - } catch (err: unknown) { - const errMsg = 'Failed to submit cancel request'; - request.log.error( - { - err, - searchJobId, - aggregationJobId, + "/search/cancel", + { + schema: { + body: SearchJobSchema, + response: { + 204: Type.Null(), + 500: ErrorSchema, + }, + tags: ["Search"], }, - errMsg - ); - reply.code(500); - return {message: `${errMsg} for searchJobId=${searchJobId}, ` + + }, + async (request, reply) => { + const {searchJobId, aggregationJobId} = request.body; + + request.log.info( + `/search/cancel searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` + ); + + try { + await QueryJobsDbManager.submitQueryCancellation(searchJobId); + await QueryJobsDbManager.submitQueryCancellation(aggregationJobId); + + updateSearchResultsMeta({ + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + SearchResultsMetadataCollection, + logger: request.log, + fields: { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: "Query cancelled before it could be completed.", + }, + }); + } catch (err: unknown) { + const errMsg = "Failed to submit cancel request"; + request.log.error( + { + err, + searchJobId, + aggregationJobId, + }, + errMsg + ); + reply.code(500); + + return {message: `${errMsg} for searchJobId=${searchJobId}, ` + `aggregationJobId=${aggregationJobId}`}; - } + } - reply.code(204); - return; - } + reply.code(204); + } ); - } +}; - export default plugin \ No newline at end of file +export default plugin; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts index 8062fcd74b..dc47e5ad95 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts @@ -3,8 +3,15 @@ import { FastifyInstance, } from "fastify"; -import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; -import {SearchResultsMetadataDocument} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; +import { + SEARCH_SIGNAL, + SearchResultsMetadataDocument, +} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; + +/** + * The maximum number of results to retrieve for a search. + */ +const SEARCH_MAX_NUM_RESULTS = 1000; type UpdateSearchResultsMetaProps = { jobId: number; @@ -29,14 +36,9 @@ type CreateMongoIndexesProps = { logger: FastifyBaseLogger; }; -/** - * The maximum number of results to retrieve for a search. - */ -const SEARCH_MAX_NUM_RESULTS = 1000; - export { CreateMongoIndexesProps, + SEARCH_MAX_NUM_RESULTS, UpdateSearchResultsMetaProps, UpdateSearchSignalWhenJobsFinishProps, - SEARCH_MAX_NUM_RESULTS, -}; \ No newline at end of file +}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts index 31600c7173..0e250ee2dc 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts @@ -1,15 +1,22 @@ -import { SEARCH_MAX_NUM_RESULTS } from './typings.js'; +import {CollectionDroppedError} from "../../../plugins/app/search/SearchJobCollectionsManager/typings.js"; import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; import { + CreateMongoIndexesProps, + SEARCH_MAX_NUM_RESULTS, UpdateSearchResultsMetaProps, UpdateSearchSignalWhenJobsFinishProps, - CreateMongoIndexesProps, -} from './typings.js'; -import { CollectionDroppedError } from '../../../plugins/app/search/SearchJobCollectionsManager/typings.js'; +} from "./typings.js"; + + /** * Modifies the search results metadata for a given job ID. * - * @param {UpdateSearchResultsMetaProps} params + * @param props + * @param props.jobId + * @param props.lastSignal + * @param props.SearchResultsMetadataCollection + * @param props.logger + * @param props.fields */ const updateSearchResultsMeta = ({ jobId, @@ -31,6 +38,17 @@ const updateSearchResultsMeta = ({ SearchResultsMetadataCollection.updateOne(filter, modifier); }; +/** + * Updates the search signal when the specified job finishes. + * + * @param props + * @param props.searchJobId + * @param props.aggregationJobId + * @param props.queryJobsDbManager + * @param props.searchJobCollectionsManager + * @param props.SearchResultsMetadataCollection + * @param props.logger + */ const updateSearchSignalWhenJobsFinish = async ({ searchJobId, aggregationJobId, @@ -54,13 +72,15 @@ const updateSearchSignalWhenJobsFinish = async ({ try { const collection = await searchJobCollectionsManager .getOrCreateCollection(searchJobId); - numResultsInCollection = await collection.countDocuments(); - //Need this here in new method. I believe publication used to do this? - await searchJobCollectionsManager.getOrCreateCollection(aggregationJobId); + numResultsInCollection = await collection.countDocuments(); + + // Need this here in new method. I believe publication used to do this? + await searchJobCollectionsManager.getOrCreateCollection(aggregationJobId); } catch (e: unknown) { if (e instanceof CollectionDroppedError) { logger.warn(`Collection ${searchJobId} has been dropped.`); + return; } throw e; @@ -82,6 +102,14 @@ const updateSearchSignalWhenJobsFinish = async ({ }); }; +/** + * Creates MongoDB indexes for a specific job's collection. + * + * @param props + * @param props.searchJobId + * @param props.searchJobCollectionsManager + * @param props.logger + */ const createMongoIndexes = async ({ searchJobId, searchJobCollectionsManager, @@ -110,4 +138,8 @@ const createMongoIndexes = async ({ ]); }; -export { updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, createMongoIndexes }; \ No newline at end of file +export { + createMongoIndexes, + updateSearchResultsMeta, + updateSearchSignalWhenJobsFinish, +}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts index 206b01b8e6..1b824f76e8 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts @@ -1,7 +1,8 @@ -import { Type } from '@sinclair/typebox' +import {Type} from "@sinclair/typebox"; + export const StringSchema = Type.String({ - minLength: 1, -}) + minLength: 1, +}); -export const IdSchema = Type.Integer({ minimum: 1 }) \ No newline at end of file +export const IdSchema = Type.Integer({minimum: 1}); diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts index 35f0efef49..b7551173de 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/error.ts @@ -1,7 +1,8 @@ -import { Type } from '@sinclair/typebox' +import {Type} from "@sinclair/typebox"; + const ErrorSchema = Type.Object({ message: Type.String(), }); -export {ErrorSchema}; \ No newline at end of file +export {ErrorSchema}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts index 9af46285cd..4b608dbe38 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts @@ -1,15 +1,20 @@ -import { Type } from '@sinclair/typebox' -import { StringSchema, IdSchema } from './common.js' +import {Type} from "@sinclair/typebox"; + +import { + IdSchema, + StringSchema, +} from "./common.js"; + export const CreateSearchJobSchema = Type.Object({ - queryString: StringSchema, - timestampBegin: Type.Integer(), - timestampEnd: Type.Integer(), - ignoreCase: Type.Boolean(), - timeRangeBucketSizeMillis: Type.Integer(), -}) + queryString: StringSchema, + timestampBegin: Type.Integer(), + timestampEnd: Type.Integer(), + ignoreCase: Type.Boolean(), + timeRangeBucketSizeMillis: Type.Integer(), +}); export const SearchJobSchema = Type.Object({ - searchJobId: IdSchema, - aggregationJobId: IdSchema -}) \ No newline at end of file + searchJobId: IdSchema, + aggregationJobId: IdSchema, +}); From 87e2313b4d0f9b5157c5d43234cdc1cf5ca576cc Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 19:38:31 +0000 Subject: [PATCH 18/55] latest --- .../log-viewer-webui/server/eslint.config.mjs | 9 ++ .../server/src/fastify-v2/app.ts | 6 +- .../src/fastify-v2/routes/api/search/index.ts | 95 ++++++++++--------- .../fastify-v2/routes/api/search/typings.ts | 5 +- .../src/fastify-v2/routes/api/search/utils.ts | 43 +++++---- .../server/src/fastify-v2/schemas/search.ts | 16 +++- 6 files changed, 99 insertions(+), 75 deletions(-) diff --git a/components/log-viewer-webui/server/eslint.config.mjs b/components/log-viewer-webui/server/eslint.config.mjs index 96b1e8f77b..c56977dd14 100644 --- a/components/log-viewer-webui/server/eslint.config.mjs +++ b/components/log-viewer-webui/server/eslint.config.mjs @@ -25,6 +25,7 @@ const EslintConfig = [ "Type.Enum", "Type.Integer", "Type.Literal", + "Type.Null", "Type.Required", "Type.Union", "Value.Errors", @@ -32,6 +33,14 @@ const EslintConfig = [ ], }, ], + "@stylistic/max-len": [ + "error", + { + code: 180, + ignorePattern: "^(import\\s.+\\sfrom\\s.+|\\} from)", + ignoreUrls: true, + }, + ], }, }, { diff --git a/components/log-viewer-webui/server/src/fastify-v2/app.ts b/components/log-viewer-webui/server/src/fastify-v2/app.ts index ae9f0a2ba2..b11146fed2 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/app.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/app.ts @@ -7,11 +7,11 @@ import { FastifyInstance, FastifyPluginOptions, } from "fastify"; +import {StatusCodes} from "http-status-codes"; import FastifyV1App from "../app.js"; -const INTERNAL_SERVER_ERROR_CODE = 500; const RATE_LIMIT_MAX_REQUESTS = 3; const RATE_LIMIT_TIME_WINDOW_MS = 500; const IGNORED_FILES_REGEX = /^.*(?:utils|typings)\.js$/; @@ -75,7 +75,9 @@ export default async function serviceApp ( "Unhandled error occurred" ); - if ("undefined" !== typeof err.statusCode && INTERNAL_SERVER_ERROR_CODE > err.statusCode) { + if ("undefined" !== typeof err.statusCode && + StatusCodes.INTERNAL_SERVER_ERROR as number > err.statusCode + ) { reply.code(err.statusCode); return err.message; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts index d8fceba551..6c221d36f5 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts @@ -2,12 +2,13 @@ import { FastifyPluginAsyncTypebox, Type, } from "@fastify/type-provider-typebox"; +import {StatusCodes} from "http-status-codes"; import {SEARCH_SIGNAL} from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; import {ErrorSchema} from "../../../schemas/error.js"; import { - CreateSearchJobSchema, - SearchJobSchema, + CreateQueryJobSchema, + QueryJobSchema, } from "../../../schemas/search.js"; import {SEARCH_MAX_NUM_RESULTS} from "./typings.js"; import { @@ -18,9 +19,11 @@ import { /** + * Search API routes. * * @param fastify */ +// eslint-disable-next-line max-lines-per-function const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const { QueryJobsDbManager, @@ -35,32 +38,33 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { "/query", { schema: { - body: CreateSearchJobSchema, + body: CreateQueryJobSchema, response: { - 201: SearchJobSchema, - 500: ErrorSchema, + [StatusCodes.CREATED]: QueryJobSchema, + [StatusCodes.INTERNAL_SERVER_ERROR]: ErrorSchema, }, tags: ["Search"], }, }, async (request, reply) => { const { - queryString, + timestampBegin, timestampEnd, ignoreCase, timeRangeBucketSizeMillis, + queryString, } = request.body; const args = { - query_string: queryString, begin_timestamp: timestampBegin, end_timestamp: timestampEnd, ignore_case: ignoreCase, max_num_results: SEARCH_MAX_NUM_RESULTS, + query_string: queryString, }; - request.log.info({args}, "/search args"); + request.log.info({args}, "/api/search/query args"); let searchJobId: number; let aggregationJobId: number; @@ -74,36 +78,37 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { } catch (err: unknown) { const errMsg = "Unable to submit search/aggregation job to the SQL database"; request.log.error(err, errMsg); - reply.code(500); - return {message: errMsg}; + return reply.internalServerError(errMsg); } - SearchResultsMetadataCollection.insertOne({ + await SearchResultsMetadataCollection.insertOne({ _id: searchJobId.toString(), lastSignal: SEARCH_SIGNAL.RESP_QUERYING, errorMsg: null, }); // Defer signal update until after response is sent - setImmediate(async () => { - await updateSearchSignalWhenJobsFinish({ - searchJobId, - aggregationJobId, + setImmediate(() => { + updateSearchSignalWhenJobsFinish({ + aggregationJobId: aggregationJobId, + logger: request.log, queryJobsDbManager: QueryJobsDbManager, searchJobCollectionsManager: SearchJobCollectionsManager, - SearchResultsMetadataCollection, - logger: request.log, + searchJobId: searchJobId, + searchResultsMetadataCollection: SearchResultsMetadataCollection, + }).catch((err: unknown) => { + request.log.error(err, "Error updating search signal when jobs finish"); }); }); await createMongoIndexes({ - searchJobId, - searchJobCollectionsManager: SearchJobCollectionsManager, logger: request.log, + searchJobId: searchJobId, + searchJobCollectionsManager: SearchJobCollectionsManager, }); - reply.code(201); + reply.code(StatusCodes.CREATED); return {searchJobId, aggregationJobId}; } @@ -113,13 +118,13 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { * Clears the results of a search operation identified by jobId. */ fastify.delete( - "/search/results", + "/results", { schema: { - body: SearchJobSchema, + body: QueryJobSchema, response: { - 204: Type.Null(), - 500: ErrorSchema, + [StatusCodes.NO_CONTENT]: Type.Null(), + [StatusCodes.INTERNAL_SERVER_ERROR]: ErrorSchema, }, tags: ["Search"], }, @@ -130,7 +135,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { request.log.info({ searchJobId, aggregationJobId, - }, "/search/results args"); + }, "api/search/results args"); try { await SearchJobCollectionsManager.dropCollection(searchJobId); @@ -145,24 +150,24 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { }, errMsg ); - reply.code(500); - return {message: `${errMsg} for searchJobId=${searchJobId}, ` + - `aggregationJobId=${aggregationJobId}`}; + return reply.internalServerError(`${errMsg} for searchJobId=${searchJobId}, ` + + `aggregationJobId=${aggregationJobId}`); } - reply.code(204); + + return reply.code(StatusCodes.NO_CONTENT); } ); fastify.post( - "/search/cancel", + "/cancel", { schema: { - body: SearchJobSchema, + body: QueryJobSchema, response: { - 204: Type.Null(), - 500: ErrorSchema, + [StatusCodes.NO_CONTENT]: Type.Null(), + [StatusCodes.INTERNAL_SERVER_ERROR]: ErrorSchema, }, tags: ["Search"], }, @@ -170,23 +175,24 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { async (request, reply) => { const {searchJobId, aggregationJobId} = request.body; - request.log.info( - `/search/cancel searchJobId=${searchJobId}, aggregationJobId=${aggregationJobId}` - ); + request.log.info({ + searchJobId, + aggregationJobId, + }, "api/search/cancel args"); try { await QueryJobsDbManager.submitQueryCancellation(searchJobId); await QueryJobsDbManager.submitQueryCancellation(aggregationJobId); - updateSearchResultsMeta({ - jobId: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - SearchResultsMetadataCollection, - logger: request.log, + await updateSearchResultsMeta({ fields: { lastSignal: SEARCH_SIGNAL.RESP_DONE, errorMsg: "Query cancelled before it could be completed.", }, + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + logger: request.log, + searchResultsMetadataCollection: SearchResultsMetadataCollection, }); } catch (err: unknown) { const errMsg = "Failed to submit cancel request"; @@ -198,13 +204,12 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { }, errMsg ); - reply.code(500); - return {message: `${errMsg} for searchJobId=${searchJobId}, ` + - `aggregationJobId=${aggregationJobId}`}; + return reply.internalServerError(`${errMsg} for searchJobId=${searchJobId}, ` + + `aggregationJobId=${aggregationJobId}`); } - reply.code(204); + return reply.code(StatusCodes.NO_CONTENT); } ); }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts index dc47e5ad95..2bcc02d264 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/typings.ts @@ -8,6 +8,7 @@ import { SearchResultsMetadataDocument, } from "../../../plugins/app/search/SearchResultsMetadataCollection/typings.js"; + /** * The maximum number of results to retrieve for a search. */ @@ -16,7 +17,7 @@ const SEARCH_MAX_NUM_RESULTS = 1000; type UpdateSearchResultsMetaProps = { jobId: number; lastSignal: SEARCH_SIGNAL; - SearchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; + searchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; logger: FastifyBaseLogger; fields: Partial; }; @@ -26,7 +27,7 @@ type UpdateSearchSignalWhenJobsFinishProps = { aggregationJobId: number; queryJobsDbManager: FastifyInstance["QueryJobsDbManager"]; searchJobCollectionsManager: FastifyInstance["SearchJobCollectionsManager"]; - SearchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; + searchResultsMetadataCollection: FastifyInstance["SearchResultsMetadataCollection"]; logger: FastifyBaseLogger; }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts index 0e250ee2dc..ceca40adca 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts @@ -12,18 +12,18 @@ import { * Modifies the search results metadata for a given job ID. * * @param props + * @param props.fields * @param props.jobId * @param props.lastSignal - * @param props.SearchResultsMetadataCollection * @param props.logger - * @param props.fields + * @param props.searchResultsMetadataCollection */ -const updateSearchResultsMeta = ({ +const updateSearchResultsMeta = async ({ + fields, jobId, lastSignal, - SearchResultsMetadataCollection, logger, - fields, + searchResultsMetadataCollection, }: UpdateSearchResultsMetaProps) => { const filter = { _id: jobId.toString(), @@ -35,29 +35,30 @@ const updateSearchResultsMeta = ({ }; logger.debug("SearchResultsMetadataCollection modifier = ", modifier); - SearchResultsMetadataCollection.updateOne(filter, modifier); + await searchResultsMetadataCollection.updateOne(filter, modifier); }; /** * Updates the search signal when the specified job finishes. * * @param props - * @param props.searchJobId * @param props.aggregationJobId + * @param props.logger * @param props.queryJobsDbManager + * @param props.searchJobId * @param props.searchJobCollectionsManager - * @param props.SearchResultsMetadataCollection - * @param props.logger + * @param props.searchResultsMetadataCollection */ const updateSearchSignalWhenJobsFinish = async ({ - searchJobId, aggregationJobId, queryJobsDbManager, - searchJobCollectionsManager, - SearchResultsMetadataCollection, logger, + searchJobId, + searchJobCollectionsManager, + searchResultsMetadataCollection, + }: UpdateSearchSignalWhenJobsFinishProps) => { - let errorMsg; + let errorMsg = null; try { await queryJobsDbManager.awaitJobCompletion(searchJobId); await queryJobsDbManager.awaitJobCompletion(aggregationJobId); @@ -86,19 +87,19 @@ const updateSearchSignalWhenJobsFinish = async ({ throw e; } - updateSearchResultsMeta({ - jobId: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - SearchResultsMetadataCollection, - logger, + await updateSearchResultsMeta({ fields: { lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: errorMsg ?? null, + errorMsg: errorMsg, numTotalResults: Math.min( numResultsInCollection, SEARCH_MAX_NUM_RESULTS ), }, + jobId: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + logger: logger, + searchResultsMetadataCollection: searchResultsMetadataCollection, }); }; @@ -106,14 +107,14 @@ const updateSearchSignalWhenJobsFinish = async ({ * Creates MongoDB indexes for a specific job's collection. * * @param props + * @param props.logger * @param props.searchJobId * @param props.searchJobCollectionsManager - * @param props.logger */ const createMongoIndexes = async ({ + logger, searchJobId, searchJobCollectionsManager, - logger, }: CreateMongoIndexesProps) => { const timestampAscendingIndex = { key: { diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts index 4b608dbe38..9c7267649d 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts @@ -5,16 +5,22 @@ import { StringSchema, } from "./common.js"; - -export const CreateSearchJobSchema = Type.Object({ +/* Schema for request to create a new query job */ +const CreateQueryJobSchema = Type.Object({ + ignoreCase: Type.Boolean(), queryString: StringSchema, timestampBegin: Type.Integer(), - timestampEnd: Type.Integer(), - ignoreCase: Type.Boolean(), timeRangeBucketSizeMillis: Type.Integer(), + timestampEnd: Type.Integer(), }); -export const SearchJobSchema = Type.Object({ +/* Schema to identify query job */ +const QueryJobSchema = Type.Object({ searchJobId: IdSchema, aggregationJobId: IdSchema, }); + +export { + CreateQueryJobSchema, + QueryJobSchema, +}; \ No newline at end of file From 7ccda94940e1c7d83605debaf62dcde1051d9e56 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 20:06:50 +0000 Subject: [PATCH 19/55] latest --- .../log-viewer-webui/server/eslint.config.mjs | 13 +++++++++-- .../src/fastify-v2/routes/api/search/index.ts | 3 +++ .../src/fastify-v2/routes/api/search/utils.ts | 22 ++++++++----------- .../server/src/fastify-v2/schemas/search.ts | 5 +++-- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/components/log-viewer-webui/server/eslint.config.mjs b/components/log-viewer-webui/server/eslint.config.mjs index c56977dd14..b575d6bc3a 100644 --- a/components/log-viewer-webui/server/eslint.config.mjs +++ b/components/log-viewer-webui/server/eslint.config.mjs @@ -33,12 +33,21 @@ const EslintConfig = [ ], }, ], + // eslint-disable-next-line no-warning-comments + // TODO: update eslint-config-yscope "@stylistic/max-len": [ - "error", + "warn", { - code: 180, + code: 100, + comments: 100, + ignoreComments: false, ignorePattern: "^(import\\s.+\\sfrom\\s.+|\\} from)", + ignoreRegExpLiterals: true, + ignoreStrings: false, + ignoreTemplateLiterals: false, + ignoreTrailingComments: false, ignoreUrls: true, + tabWidth: 4, }, ], }, diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts index 6c221d36f5..bb30cfe02a 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts @@ -82,6 +82,9 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { return reply.internalServerError(errMsg); } + await SearchJobCollectionsManager.getOrCreateCollection(searchJobId); + await SearchJobCollectionsManager.getOrCreateCollection(aggregationJobId); + await SearchResultsMetadataCollection.insertOne({ _id: searchJobId.toString(), lastSignal: SEARCH_SIGNAL.RESP_QUERYING, diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts index ceca40adca..40d1fe8745 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts @@ -58,29 +58,25 @@ const updateSearchSignalWhenJobsFinish = async ({ searchResultsMetadataCollection, }: UpdateSearchSignalWhenJobsFinishProps) => { - let errorMsg = null; + let errorMsg: string | null = null; + try { await queryJobsDbManager.awaitJobCompletion(searchJobId); await queryJobsDbManager.awaitJobCompletion(aggregationJobId); } catch (e: unknown) { - if (e instanceof Error) { - errorMsg = e.message; - } - errorMsg = "Error while waiting for job completion"; + errorMsg = e instanceof Error ? + e.message : + "Error while waiting for job completion"; } - let numResultsInCollection = -1; - try { - const collection = await searchJobCollectionsManager - .getOrCreateCollection(searchJobId); + let numResultsInCollection: number; + try { + const collection = await searchJobCollectionsManager.getOrCreateCollection(searchJobId); numResultsInCollection = await collection.countDocuments(); - - // Need this here in new method. I believe publication used to do this? - await searchJobCollectionsManager.getOrCreateCollection(aggregationJobId); } catch (e: unknown) { if (e instanceof CollectionDroppedError) { - logger.warn(`Collection ${searchJobId} has been dropped.`); + logger.warn({e, errorMsg}); return; } diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts index 9c7267649d..354c292250 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/search.ts @@ -5,12 +5,13 @@ import { StringSchema, } from "./common.js"; + /* Schema for request to create a new query job */ const CreateQueryJobSchema = Type.Object({ ignoreCase: Type.Boolean(), queryString: StringSchema, - timestampBegin: Type.Integer(), timeRangeBucketSizeMillis: Type.Integer(), + timestampBegin: Type.Integer(), timestampEnd: Type.Integer(), }); @@ -23,4 +24,4 @@ const QueryJobSchema = Type.Object({ export { CreateQueryJobSchema, QueryJobSchema, -}; \ No newline at end of file +}; From 99d96123b60071f39d06418ce6405068ce57f951 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 16 May 2025 20:10:41 +0000 Subject: [PATCH 20/55] latest --- .../server/src/fastify-v2/routes/api/search/index.ts | 2 +- .../server/src/fastify-v2/schemas/common.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts index bb30cfe02a..de6a09cc86 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/index.ts @@ -64,7 +64,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { query_string: queryString, }; - request.log.info({args}, "/api/search/query args"); + request.log.info(args, "/api/search/query args"); let searchJobId: number; let aggregationJobId: number; diff --git a/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts b/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts index 1b824f76e8..9af42bc685 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/schemas/common.ts @@ -1,8 +1,10 @@ import {Type} from "@sinclair/typebox"; -export const StringSchema = Type.String({ +const StringSchema = Type.String({ minLength: 1, }); -export const IdSchema = Type.Integer({minimum: 1}); +const IdSchema = Type.Integer({minimum: 1}); + +export { StringSchema, IdSchema }; From 028835b6b13e0d4afe68f3850d2ac8ca5ac3ca9f Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 22 May 2025 14:37:36 +0000 Subject: [PATCH 21/55] saving --- components/QueryBox/index.d.ts | 12 +++ .../log-viewer-webui/client/package-lock.json | 41 +++++++--- .../log-viewer-webui/client/package.json | 5 +- .../client/src/api/search/queryCancel.ts | 31 +++++++ .../client/src/api/search/queryClear.ts | 31 +++++++ .../client/src/api/search/querySubmit.ts | 49 ++++++++++++ .../client/src/api/socket/useCursor.tsx | 6 +- .../src/components/QueryBox/index.module.css | 11 +-- .../client/src/components/QueryBox/index.tsx | 12 ++- .../SearchPage/SearchControls/QueryInput.tsx | 75 +++++++++++++++++ .../SearchControls/SearchButton.tsx | 16 +++- .../pages/SearchPage/SearchControls/index.tsx | 15 +--- .../SearchResultsTable/Message/index.tsx | 69 +++++++++++----- .../SearchResultsTable/index.tsx | 39 ++------- .../SearchResultsTable/typings.tsx | 16 +++- .../src/pages/SearchPage/SearchState.tsx | 38 --------- .../pages/SearchPage/SearchState/index.tsx | 61 ++++++++++++++ .../src/pages/SearchPage/SearchState/query.ts | 80 +++++++++++++++++++ .../client/src/pages/SearchPage/metadata.ts | 30 +++++++ .../src/pages/SearchPage/searchResults.ts | 27 +++++++ .../client/tsconfig/tsconfig.app.json | 2 +- .../log-viewer-webui/client/vite.config.ts | 18 ++++- .../common/searchResultsMetadata.ts | 70 ++++++++++++++++ .../app/search/QueryJobsDbManager/index.ts | 2 +- .../SearchResultsMetadataCollection/index.ts | 2 +- .../src/plugins/MongoSocketIoServer/index.ts | 22 +++-- .../src/plugins/MongoSocketIoServer/utils.ts | 2 +- .../log-viewer-webui/server/tsconfig.json | 1 - 28 files changed, 630 insertions(+), 153 deletions(-) create mode 100644 components/QueryBox/index.d.ts create mode 100644 components/log-viewer-webui/client/src/api/search/queryCancel.ts create mode 100644 components/log-viewer-webui/client/src/api/search/queryClear.ts create mode 100644 components/log-viewer-webui/client/src/api/search/querySubmit.ts create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx delete mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState.tsx create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState/index.tsx create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/searchResults.ts create mode 100644 components/log-viewer-webui/common/searchResultsMetadata.ts diff --git a/components/QueryBox/index.d.ts b/components/QueryBox/index.d.ts new file mode 100644 index 0000000000..228b45cea7 --- /dev/null +++ b/components/QueryBox/index.d.ts @@ -0,0 +1,12 @@ +import { FC } from "react"; + +export interface QueryBoxProps { + placeholder?: string; + progress?: number; + size?: "large" | "small" | "default"; + value: string; + onChange: (e: React.ChangeEvent) => void; +} + +declare const QueryBox: FC; +export default QueryBox; diff --git a/components/log-viewer-webui/client/package-lock.json b/components/log-viewer-webui/client/package-lock.json index b1408ded38..0ae29b1221 100644 --- a/components/log-viewer-webui/client/package-lock.json +++ b/components/log-viewer-webui/client/package-lock.json @@ -21,17 +21,18 @@ "react-router": "^7.4.1", "react-syntax-highlighter": "^15.6.1", "socket.io-client": "^4.8.1", - "vite-tsconfig-paths": "^5.1.4", "zustand": "^5.0.3" }, "devDependencies": { + "@types/node": "^22.15.20", "@types/react": "^19.0.10", "@types/react-dom": "^19.0.4", "@types/react-syntax-highlighter": "^15.5.13", "@vitejs/plugin-react": "^4.3.4", "eslint-config-yscope": "latest", "typescript": "~5.6.2", - "vite": "^6.2.6" + "vite": "^6.2.6", + "vite-tsconfig-paths": "^5.1.4" } }, "node_modules/@ampproject/remapping": { @@ -2249,7 +2250,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", - "devOptional": true, + "dev": true, "license": "MIT" }, "node_modules/@types/hast": { @@ -2277,6 +2278,16 @@ "license": "MIT", "peer": true }, + "node_modules/@types/node": { + "version": "22.15.20", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.20.tgz", + "integrity": "sha512-A6BohGFRGHAscJsTslDCA9JG7qSJr/DWUvrvY8yi9IgnGtMxCyat7vvQ//MFa0DnLsyuS3wYTpLdw4Hf+Q5JXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -4006,7 +4017,7 @@ "version": "0.25.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.1.tgz", "integrity": "sha512-BGO5LtrGC7vxnqucAe/rmvKdJllfGaYWdyABvyMoXQlfYMb2bbRuReWR5tEGE//4LcNJj9XrkovTqNYRFZHAMQ==", - "devOptional": true, + "dev": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -5053,6 +5064,7 @@ "version": "0.1.2", "resolved": "https://registry.npmjs.org/globrex/-/globrex-0.1.2.tgz", "integrity": "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg==", + "dev": true, "license": "MIT" }, "node_modules/gopd": { @@ -6107,7 +6119,7 @@ "version": "3.3.11", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", - "devOptional": true, + "dev": true, "funding": [ { "type": "github", @@ -6472,7 +6484,7 @@ "version": "8.5.3", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", "integrity": "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==", - "devOptional": true, + "dev": true, "funding": [ { "type": "opencollective", @@ -7423,7 +7435,7 @@ "version": "4.34.8", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.34.8.tgz", "integrity": "sha512-489gTVMzAYdiZHFVA/ig/iYFllCcWFHMvUHI1rpFmkoUtRlQxqh6/yiNqnYibjMZ2b/+FUQwldG+aLsEt6bglQ==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "@types/estree": "1.0.6" @@ -7816,7 +7828,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "devOptional": true, + "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" @@ -8128,6 +8140,7 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/tsconfck/-/tsconfck-3.1.5.tgz", "integrity": "sha512-CLDfGgUp7XPswWnezWwsCRxNmgQjhYq3VXHM0/XIRxhVrKw0M1if9agzryh1QS3nxjCROvV+xWxoJO1YctzzWg==", + "dev": true, "license": "MIT", "bin": { "tsconfck": "bin/tsconfck.js" @@ -8286,7 +8299,7 @@ "version": "5.6.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -8460,6 +8473,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, "node_modules/update-browserslist-db": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.2.tgz", @@ -8506,7 +8526,7 @@ "version": "6.2.6", "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.6.tgz", "integrity": "sha512-9xpjNl3kR4rVDZgPNdTL0/c6ao4km69a/2ihNQbcANz8RuCOK3hQBmLSJf3bRKVQjVMda+YvizNE8AwvogcPbw==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "esbuild": "^0.25.0", @@ -8578,6 +8598,7 @@ "version": "5.1.4", "resolved": "https://registry.npmjs.org/vite-tsconfig-paths/-/vite-tsconfig-paths-5.1.4.tgz", "integrity": "sha512-cYj0LRuLV2c2sMqhqhGpaO3LretdtMn/BVX4cPLanIZuwwrkVl+lK84E/miEXkCHWXuq65rhNN4rXsBcOB3S4w==", + "dev": true, "license": "MIT", "dependencies": { "debug": "^4.1.1", diff --git a/components/log-viewer-webui/client/package.json b/components/log-viewer-webui/client/package.json index 2f2488cb69..602305c329 100644 --- a/components/log-viewer-webui/client/package.json +++ b/components/log-viewer-webui/client/package.json @@ -26,16 +26,17 @@ "react-router": "^7.4.1", "react-syntax-highlighter": "^15.6.1", "socket.io-client": "^4.8.1", - "vite-tsconfig-paths": "^5.1.4", "zustand": "^5.0.3" }, "devDependencies": { + "@types/node": "^22.15.20", "@types/react": "^19.0.10", "@types/react-dom": "^19.0.4", "@types/react-syntax-highlighter": "^15.5.13", "@vitejs/plugin-react": "^4.3.4", "eslint-config-yscope": "latest", "typescript": "~5.6.2", - "vite": "^6.2.6" + "vite": "^6.2.6", + "vite-tsconfig-paths": "^5.1.4" } } diff --git a/components/log-viewer-webui/client/src/api/search/queryCancel.ts b/components/log-viewer-webui/client/src/api/search/queryCancel.ts new file mode 100644 index 0000000000..05e15aa790 --- /dev/null +++ b/components/log-viewer-webui/client/src/api/search/queryCancel.ts @@ -0,0 +1,31 @@ +import axios from "axios"; + +type QueryCancelArgs = { + searchJobId: string; + aggregationJobId: string; +}; + +const cancelQuery = async ({ + searchJobId, + aggregationJobId, +}: QueryCancelArgs) => { + const payload = { + searchJobId, + aggregationJobId, + }; + + console.log("Cancelling query with payload:", payload); + + try { + await axios.post("/api/search/cancel", payload); + } catch (error) { + if (axios.isAxiosError(error)) { + console.error("Axios error during query cancel:", error.message, error.response?.data); + } else { + console.error("Unknown error during query cancel:", error); + } + throw error; + } +}; + +export { cancelQuery }; diff --git a/components/log-viewer-webui/client/src/api/search/queryClear.ts b/components/log-viewer-webui/client/src/api/search/queryClear.ts new file mode 100644 index 0000000000..ef5f39196f --- /dev/null +++ b/components/log-viewer-webui/client/src/api/search/queryClear.ts @@ -0,0 +1,31 @@ +import axios from "axios"; + +type QueryClearArgs = { + searchJobId: string; + aggregationJobId: string; +}; + +const clearQueryResults = async ({ + searchJobId, + aggregationJobId, +}: QueryClearArgs) => { + const payload = { + searchJobId, + aggregationJobId, + }; + + console.log("Clearing query results with payload:", payload); + + try { + await axios.delete("/api/search/results", { data: payload }); + } catch (error) { + if (axios.isAxiosError(error)) { + console.error("Axios error during query clear:", error.message, error.response?.data); + } else { + console.error("Unknown error during query clear:", error); + } + throw error; + } +}; + +export { clearQueryResults }; diff --git a/components/log-viewer-webui/client/src/api/search/querySubmit.ts b/components/log-viewer-webui/client/src/api/search/querySubmit.ts new file mode 100644 index 0000000000..a0e2aa6eed --- /dev/null +++ b/components/log-viewer-webui/client/src/api/search/querySubmit.ts @@ -0,0 +1,49 @@ + +import axios from "axios"; + +type QueryArgs = { + timestampBegin: number; + timestampEnd: number; + ignoreCase: boolean; + timeRangeBucketSizeMillis: number; + queryString: string; +}; + +type QueryResponse = { + searchJobId: string; + aggregationJobId: string; +}; + +const submitQuery = async ({ + timestampBegin, + timestampEnd, + ignoreCase, + timeRangeBucketSizeMillis, + queryString, +}: QueryArgs) => { + const payload = { + timestampBegin, + timestampEnd, + ignoreCase, + timeRangeBucketSizeMillis, + queryString, + }; + + console.log("Submitting query with payload:", payload); + + try { + const response = + await axios.post("/api/search/query", payload); + const { searchJobId, aggregationJobId } = response.data; + return { searchJobId, aggregationJobId }; + } catch (error) { + if (axios.isAxiosError(error)) { + console.error("Axios error during query submission:", error.message, error.response?.data); + } else { + console.error("Unknown error during query submission:", error); + } + throw error; + } +}; + +export { submitQuery }; \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/api/socket/useCursor.tsx b/components/log-viewer-webui/client/src/api/socket/useCursor.tsx index 4242072831..f9cea19e3d 100644 --- a/components/log-viewer-webui/client/src/api/socket/useCursor.tsx +++ b/components/log-viewer-webui/client/src/api/socket/useCursor.tsx @@ -15,7 +15,7 @@ import {MongoCursorSocket} from "./MongoCursorSocket.js"; * @return Reactive array. */ const useCursor = ( - query: () => MongoCursorSocket, + query: () => MongoCursorSocket | null, dependencies: DependencyList = [] ): object[] => { const [data, setData] = useState([]); @@ -24,6 +24,10 @@ const useCursor = ( useEffect(() => { const cursor = query(); + if (!cursor) { + return; + } + // Flag to ignore updates after unmounting. let ignore = false; console.log("Subscribing to cursor"); diff --git a/components/log-viewer-webui/client/src/components/QueryBox/index.module.css b/components/log-viewer-webui/client/src/components/QueryBox/index.module.css index 97eb8d6eae..f78caf8f61 100644 --- a/components/log-viewer-webui/client/src/components/QueryBox/index.module.css +++ b/components/log-viewer-webui/client/src/components/QueryBox/index.module.css @@ -1,15 +1,9 @@ .queryBox { - width: 100%; -} - -.queryBoxPosition { - /* Defines a new positioning context for absolutely positioned descendants such as the wrapper and progress bar. */ position: relative; width: 100%; - height: 100%; } -.queryBoxWrapper { +.progressBarMask { /* Prevents the progress bar from obstructing the input box. */ pointer-events: none; width: 100%; @@ -17,11 +11,12 @@ position: absolute; top: 0; overflow: hidden; + /*Fix this eventaully*/ border-radius: var(--antd-border-radius); --antd-border-radius: 8px; } -.queryBoxProgress { +.progressBar { position: absolute; bottom: -8px; } diff --git a/components/log-viewer-webui/client/src/components/QueryBox/index.tsx b/components/log-viewer-webui/client/src/components/QueryBox/index.tsx index 9f8410a938..ae78772516 100644 --- a/components/log-viewer-webui/client/src/components/QueryBox/index.tsx +++ b/components/log-viewer-webui/client/src/components/QueryBox/index.tsx @@ -17,17 +17,16 @@ export interface QueryBoxProps extends InputProps { * * @param props * @param props.progress - * @param props.rest + * @param props.inputProps * @return */ -const QueryBox = ({progress, ...rest}: QueryBoxProps) => { +const QueryBox = ({progress, ...inputProps}: QueryBoxProps) => { return (
-
- -
+ +
{ "none" : "block"}}/>
-
); }; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx new file mode 100644 index 0000000000..d72ec469e9 --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx @@ -0,0 +1,75 @@ +import QueryBox from "../../../components/QueryBox"; +import useSearchStore from "../SearchState/index"; +import { isSearchSignalQuerying, SearchResultsMetadataDocument } from "@common/searchResultsMetadata.js"; +import { useResultsMetadata } from "../metadata"; + +import { + useEffect, + useRef, + useState, +} from "react"; + +// for pseudo progress bar +const PROGRESS_INCREMENT = 5; +const PROGRESS_INTERVAL_MILLIS = 100; + +/** + * Renders controls for submitting queries. + * + * @return + */ +const QueryInput = () => { + const queryString = useSearchStore((state) => state.queryString); + const updateQueryString = useSearchStore((state) => state.updateQueryString); + const resultsMetadata = useResultsMetadata(); + const [progress, setProgress] = useState(0); + const timerIntervalRef = useRef(null); + + console.log("QueryInput: resultsMetadata", resultsMetadata); + + useEffect(() => { + // I have idea to potentially just set the searchSignal here + // but will get back to later + if (Array.isArray(resultsMetadata) && resultsMetadata.length === 0) { + return; + } + let resultMetadata = resultsMetadata[0] as SearchResultsMetadataDocument; + console.log(resultsMetadata); + console.log("QueryInput: resultsMetadata.lastSignal", resultMetadata.lastSignal); + if (true === isSearchSignalQuerying(resultMetadata.lastSignal)) { + console.log("QueryInput: isSearchSignalQuerying"); + if (null === timerIntervalRef.current) { + timerIntervalRef.current = setInterval(() => { + setProgress((v) => { + if (v + PROGRESS_INCREMENT >= 100) { + clearInterval(timerIntervalRef.current!); + timerIntervalRef.current = null; + return 100; + } + return v + PROGRESS_INCREMENT; + }); + }, PROGRESS_INTERVAL_MILLIS); + } + } else { + if (null !== timerIntervalRef.current) { + clearInterval(timerIntervalRef.current); + timerIntervalRef.current = null; + } + setProgress(0); + } + }, [resultsMetadata]); + + return ( + { + updateQueryString(e.target.value); + }}/> + ); +}; + + +export default QueryInput; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton.tsx index be416a865b..0854ee897d 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton.tsx @@ -4,7 +4,8 @@ import { Tooltip, } from "antd"; -import useSearchStore, {SEARCH_STATE_DEFAULT} from "../SearchState"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../SearchState/index"; +import {handleQuerySubmit} from "../SearchState/query"; import styles from "./index.module.css"; @@ -31,7 +32,18 @@ const SearchButton = () => { icon={} size={"large"} type={"primary"} - > + onClick={() => { + handleQuerySubmit({ + queryString, + timestampBegin: 0, + timestampEnd: Date.now(), + ignoreCase: false, + timeRangeBucketSizeMillis: 150, + }).catch((error) => { + console.error("Error during query submission:", error); + } + ); + }}> Search diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx index 733626b368..591942acf9 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -1,8 +1,8 @@ -import QueryBox from "../../../components/QueryBox"; -import useSearchStore from "../SearchState"; + import styles from "./index.module.css"; import SearchButton from "./SearchButton"; import TimeRangeInput from "./TimeRangeInput"; +import QueryInput from "./QueryInput"; /** @@ -11,19 +11,10 @@ import TimeRangeInput from "./TimeRangeInput"; * @return */ const SearchControls = () => { - const queryString = useSearchStore((state) => state.queryString); - const updateQueryString = useSearchStore((state) => state.updateQueryString); return (
- { - updateQueryString(e.target.value); - }}/> +
diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx index f9456f4bab..0824ede78b 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx @@ -1,20 +1,21 @@ -import SyntaxHighlighter from "react-syntax-highlighter"; - -import {Typography} from "antd"; - -import LogViewerLink from "./LogViewerLink"; -import {highlighterCustomStyles} from "./utils"; - +import { useMemo } from "react"; +import hljs from "highlight.js/lib/core"; +import armasm from "highlight.js/lib/languages/armasm"; import "highlight.js/styles/intellij-light.css"; +import { Typography } from "antd"; +import LogViewerLink from "./LogViewerLink"; -const {Text} = Typography; +const { Paragraph } = Typography; interface MessageProps { message: string; filePath: string; } +// Register the language once (outside component is OK too) +hljs.registerLanguage("armasm", armasm); + /** * Renders a message with syntax highlighting and a file path link. * @@ -23,22 +24,48 @@ interface MessageProps { * @param props.filePath * @return */ -const Message = ({message, filePath}: MessageProps) => { +const Message = ({ message, filePath }: MessageProps) => { + const highlighted = useMemo(() => { + const html = hljs.highlight(message, { + language: "armasm", + ignoreIllegals: true, + }).value; + + // Convert newlines into inline spans with a space + return html.replace(/\n/g, " "); // or \n if needed + }, [message]); + return ( <> - {/* Parent `Text` component allows syntax highlighter to inherit AntD fonts. */} - - - {message} - - - + + + + + ); }; -export default Message; +export default Message; \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx index 77e6b441de..c014ec03da 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx @@ -5,35 +5,7 @@ import { searchResultsTableColumns, } from "./typings"; - -// eslint-disable-next-line no-warning-comments -// TODO: Replace with values from database once api implemented. -const DUMMY_RESULTS: SearchResult[] = [ - { - id: 1, - timestamp: "2023-01-01 12:00:00", - message: "INFO: User login successful for user 'john.doe'.", - filePath: "/var/logs/auth.log", - }, - { - id: 2, - timestamp: "2023-01-01 12:01:00", - message: "ERROR: Failed to connect to database 'logs_db'.", - filePath: "/var/logs/db.log", - }, - { - id: 3, - timestamp: "2023-01-01 12:02:00", - message: "WARN: Disk space running low on volume '/var/logs'.", - filePath: "/var/logs/system.log", - }, - { - id: 4, - timestamp: "2023-01-01 12:03:00", - message: "DEBUG: Processing request ID 12345.", - filePath: "/var/logs/app.log", - }, -]; +import {useSearchResults} from "../../../SearchPage/searchResults"; /** * Renders search results in a table. @@ -41,12 +13,17 @@ const DUMMY_RESULTS: SearchResult[] = [ * @return */ const SearchResultsTable = () => { + + let searchResults = useSearchResults(); + console.log("searchResults", searchResults); + return ( + scroll={{ y: 400 }} columns={searchResultsTableColumns} - dataSource={DUMMY_RESULTS} + dataSource={searchResults as SearchResult[]} pagination={false} - rowKey={(record) => record.id.toString()} + rowKey={(record) => record._id.toString()} virtual={true}/> ); }; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/typings.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/typings.tsx index af10344d45..e7b6b39093 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/typings.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/typings.tsx @@ -1,4 +1,5 @@ import {TableProps} from "antd"; +import dayjs from "dayjs"; import Message from "./Message"; @@ -7,12 +8,15 @@ import Message from "./Message"; * Structure of search results data displayed in the table. */ interface SearchResult { - id: number; - timestamp: string; + _id: string; + timestamp: number; message: string; filePath: string; + orig_file_path: string; + orig_file_id: string; + log_event_ix: number; } - +const DATETIME_FORMAT_TEMPLATE = "YYYY-MMM-DD HH:mm:ss"; /** * Columns configuration for the seach results table. */ @@ -23,13 +27,17 @@ const searchResultsTableColumns: NonNullable["columns"] sorter: true, title: "Timestamp", width: 15, + ellipsis: { + showTitle: false, + }, + render: (timestamp: number) => dayjs(timestamp).format(DATETIME_FORMAT_TEMPLATE), }, { dataIndex: "message", key: "message", render: (_, record) => ( ), title: "Message", diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState.tsx deleted file mode 100644 index 9d06fc7e17..0000000000 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import dayjs from "dayjs"; -import {create} from "zustand"; - -import { - DEFAULT_TIME_RANGE, - TIME_RANGE_OPTION_DAYJS_MAP, -} from "./SearchControls/TimeRangeInput/utils"; - - -/** - * Default values of the search state. - */ -const SEARCH_STATE_DEFAULT = Object.freeze({ - queryString: "", - timeRange: TIME_RANGE_OPTION_DAYJS_MAP[DEFAULT_TIME_RANGE], -}); - -interface SearchState { - queryString: string; - timeRange: [dayjs.Dayjs, dayjs.Dayjs]; - updateQueryString: (query: string) => void; - updateTimeRange: (range: [dayjs.Dayjs, dayjs.Dayjs]) => void; -} - -const useSearchStore = create((set) => ({ - queryString: SEARCH_STATE_DEFAULT.queryString, - timeRange: SEARCH_STATE_DEFAULT.timeRange, - updateQueryString: (query) => { - set({queryString: query}); - }, - updateTimeRange: (range) => { - set({timeRange: range}); - }, -})); - - -export {SEARCH_STATE_DEFAULT}; -export default useSearchStore; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/index.tsx new file mode 100644 index 0000000000..bea409848d --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/index.tsx @@ -0,0 +1,61 @@ +import dayjs from "dayjs"; +import {create} from "zustand"; + +import { + DEFAULT_TIME_RANGE, + TIME_RANGE_OPTION_DAYJS_MAP, +} from "../SearchControls/TimeRangeInput/utils"; + +import {SearchResultsMetadataDocument} from "@common/searchResultsMetadata.js"; + + +/** + * Default values of the search state. + */ +const SEARCH_STATE_DEFAULT = Object.freeze({ + queryString: "", + searchJobId: null, + aggregationJobId: null, + timeRange: TIME_RANGE_OPTION_DAYJS_MAP[DEFAULT_TIME_RANGE], + searchResultsMetadata: null, +}); + +interface SearchState { + queryString: string; + timeRange: [dayjs.Dayjs, dayjs.Dayjs]; + searchJobId: string | null; + aggregationJobId: string | null; + searchResultsMetadata: SearchResultsMetadataDocument | null; + updateQueryString: (query: string) => void; + updateTimeRange: (range: [dayjs.Dayjs, dayjs.Dayjs]) => void; + updateSearchJobId: (id: string | null) => void; + updateAggregationJobId: (id: string | null) => void; + updateSearchResultsMetadata: (metadata: SearchResultsMetadataDocument) => void; +} + +const useSearchStore = create((set) => ({ + queryString: SEARCH_STATE_DEFAULT.queryString, + timeRange: SEARCH_STATE_DEFAULT.timeRange, + searchJobId: SEARCH_STATE_DEFAULT.searchJobId, + aggregationJobId: SEARCH_STATE_DEFAULT.aggregationJobId, + searchResultsMetadata: SEARCH_STATE_DEFAULT.searchResultsMetadata, + updateQueryString: (query) => { + set({queryString: query}); + }, + updateTimeRange: (range) => { + set({timeRange: range}); + }, + updateSearchJobId: (id) => { + set({searchJobId: id}); + }, + updateAggregationJobId: (id) => { + set({aggregationJobId: id}); + }, + updateSearchResultsMetadata: (metadata) => { + set({searchResultsMetadata: metadata}); + }, +})); + + +export {SEARCH_STATE_DEFAULT}; +export default useSearchStore; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts new file mode 100644 index 0000000000..9ffe189649 --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts @@ -0,0 +1,80 @@ +import useSearchStore from "./index"; +import { clearQueryResults } from "../../../api/search/queryClear"; +import { submitQuery } from "../../../api/search/querySubmit"; +import { cancelQuery } from "../../../api/search/queryCancel"; + +export const handleClearResults = async () => { + const store = useSearchStore.getState(); + + const searchJobId = store.searchJobId; + const aggregationJobId = store.aggregationJobId; + + if (!searchJobId || !aggregationJobId) { + console.warn("No searchJobId or aggregationJobId to clear."); + return; + } + + try { + await clearQueryResults({ + searchJobId, + aggregationJobId, + }); + store.updateSearchJobId(null); + store.updateAggregationJobId(null); + console.log("Query results cleared successfully"); + } catch (error) { + console.error("Failed to clear query results:", error); + } +}; + +// Add handleQuerySubmit that uses submitQuery from the API +type QueryArgs = { + timestampBegin: number; + timestampEnd: number; + ignoreCase: boolean; + timeRangeBucketSizeMillis: number; + queryString: string; +}; + +export const handleQuerySubmit = async (args: QueryArgs) => { + const store = useSearchStore.getState(); + + const searchJobId = store.searchJobId; + + if (null !== searchJobId) { + // Clear result caches before starting a new query + await handleClearResults(); + } + + const result = await submitQuery(args); + + // Update store with new job IDs + store.updateSearchJobId(result.searchJobId); + store.updateAggregationJobId(result.aggregationJobId); + + return result; +}; + +export const handleQueryCancel = async () => { + const store = useSearchStore.getState(); + + const searchJobId = store.searchJobId; + const aggregationJobId = store.aggregationJobId; + + if (!searchJobId || !aggregationJobId) { + console.warn("No searchJobId or aggregationJobId to cancel."); + return; + } + + try { + await cancelQuery({ + searchJobId, + aggregationJobId, + }); + store.updateSearchJobId(null); + store.updateAggregationJobId(null); + console.log("Query cancelled successfully"); + } catch (error) { + console.error("Failed to cancel query:", error); + } +}; \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts b/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts new file mode 100644 index 0000000000..f372f7fcc3 --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts @@ -0,0 +1,30 @@ +import {useCursor} from "../../api/socket/useCursor"; +import MongoCollectionSocket from "../../api/socket/MongoCollectionSocket"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "./SearchState/index"; +import { SearchResultsMetadataDocument } from "@common/searchResultsMetadata"; + + +/** + * Custom hook to get results metadata for the current searchJobId. + */ +const useResultsMetadata = () => { + const searchJobId = useSearchStore((state) => state.searchJobId); + console.log(`this is search jobID ${searchJobId}`); + + const resultsMetadataCursor = useCursor( + () => { + if (!searchJobId || searchJobId === SEARCH_STATE_DEFAULT.searchJobId) { + return null; + } + const collection = new MongoCollectionSocket("results-metadata"); + return collection.find({ _id: searchJobId?.toString() }, { limit: 1 }); + }, + [searchJobId] + ); + + return resultsMetadataCursor as SearchResultsMetadataDocument[]; +}; + +export { useResultsMetadata }; + + diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/searchResults.ts b/components/log-viewer-webui/client/src/pages/SearchPage/searchResults.ts new file mode 100644 index 0000000000..06dedda32e --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/searchResults.ts @@ -0,0 +1,27 @@ +import {useCursor} from "../../api/socket/useCursor"; +import MongoCollectionSocket from "../../api/socket/MongoCollectionSocket"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "./SearchState"; + +/** + * Custom hook to get results metadata for the current searchJobId. + */ +const useSearchResults = () => { + const searchJobId = useSearchStore((state) => state.searchJobId); + console.log(`this is search jobID ${searchJobId}`); + + const resultsMetadataCursor = useCursor( + () => { + if (!searchJobId || searchJobId === SEARCH_STATE_DEFAULT.searchJobId) { + return null; + } + const collection = new MongoCollectionSocket(searchJobId.toString()); + return collection.find({}, {}); + }, + [searchJobId] + ); + + return resultsMetadataCursor; +}; + +export { useSearchResults }; + diff --git a/components/log-viewer-webui/client/tsconfig/tsconfig.app.json b/components/log-viewer-webui/client/tsconfig/tsconfig.app.json index 3f2fa4631b..62c40c6b66 100644 --- a/components/log-viewer-webui/client/tsconfig/tsconfig.app.json +++ b/components/log-viewer-webui/client/tsconfig/tsconfig.app.json @@ -18,7 +18,7 @@ "useDefineForClassFields": true, "baseUrl": "../", "paths": { - "@common/*": ["../common/*"], // Map imports from "@common/*" to the shared folder + "@common/*": ["../common/*"], }, /* Bundler mode */ diff --git a/components/log-viewer-webui/client/vite.config.ts b/components/log-viewer-webui/client/vite.config.ts index f3e6c5d260..6149e4b359 100644 --- a/components/log-viewer-webui/client/vite.config.ts +++ b/components/log-viewer-webui/client/vite.config.ts @@ -1,6 +1,6 @@ import react from "@vitejs/plugin-react"; import {defineConfig} from "vite"; -import tsconfigPaths from "vite-tsconfig-paths"; +import path from "node:path"; // https://vite.dev/config/ @@ -8,8 +8,12 @@ export default defineConfig({ base: "./", plugins: [ react(), - tsconfigPaths(), ], + resolve: { + alias: { + "@common": path.resolve(__dirname, "../common"), + } + }, publicDir: "public", server: { port: 8080, @@ -17,11 +21,17 @@ export default defineConfig({ "/query/": { // Below target should match the server's configuration in // `components/log-viewer-webui/server/.env` (or `.env.local` if overridden) - target: "http://localhost:3000/", + target: "http://localhost:3001/", + changeOrigin: true, + }, + "/api/": { + // Below target should match the server's configuration in + // `components/log-viewer-webui/server/.env` (or `.env.local` if overridden) + target: "http://localhost:3001/", changeOrigin: true, }, "/socket.io/": { - target: "ws://localhost:3000/", + target: "ws://localhost:3001/", changeOrigin: true, ws: true, }, diff --git a/components/log-viewer-webui/common/searchResultsMetadata.ts b/components/log-viewer-webui/common/searchResultsMetadata.ts new file mode 100644 index 0000000000..444f06cc5c --- /dev/null +++ b/components/log-viewer-webui/common/searchResultsMetadata.ts @@ -0,0 +1,70 @@ +/** + * Enum of search-related signals. + * + * This includes request and response signals for various search operations and their respective + * states. + */ +enum SEARCH_SIGNAL { + NONE = "none", + + REQ_CANCELLING = "req-cancelling", + REQ_CLEARING = "req-clearing", + REQ_QUERYING = "req-querying", + + RESP_DONE = "resp-done", + RESP_QUERYING = "resp-querying", +} +/** + * Checks if the given search signal is a search signal request. + * + * @param {SearchSignal} s + * @return {boolean} + */ +const isSearchSignalReq = (s: SEARCH_SIGNAL) => s.startsWith("req-"); + +/** + * Checks if the given search signal is a search signal response. + * + * @param {SearchSignal} s + * @return {boolean} + */ +const isSearchSignalResp = (s: SEARCH_SIGNAL) => s.startsWith("resp-"); + +/** + * Checks if the given search signal is a querying request / response. + * + * @param {SearchSignal} s + * @return {boolean} + */ +const isSearchSignalQuerying = (s: SEARCH_SIGNAL) => s.endsWith("-querying"); + +/** + * Checks if the given search signal is an operation in progress, which can be used as a + * condition to disable UI elements. + * + * @param {SearchSignal} s + * @return {boolean} + */ +const isOperationInProgress = (s: SEARCH_SIGNAL) => ( + (true === isSearchSignalReq(s)) || + (true === isSearchSignalQuerying(s)) +); + +/** + * MongoDB document for search results metadata. `numTotalResults` is optional + * since it is only set when the search job is completed. + */ +interface SearchResultsMetadataDocument { + _id: string; + errorMsg: string | null; + lastSignal: SEARCH_SIGNAL; + numTotalResults?: number; +} + +export type {SearchResultsMetadataDocument}; +export { + isSearchSignalQuerying, + isSearchSignalResp, + isOperationInProgress, + SEARCH_SIGNAL +}; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts index be824004f8..83c4d551c4 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts @@ -1,6 +1,6 @@ import type {MySQLPromisePool} from "@fastify/mysql"; import {encode} from "@msgpack/msgpack"; -import settings from "@settings" with { type: "json" }; +import settings from "../../../../../../settings.json" with { type: "json" }; import {FastifyInstance} from "fastify"; import fp from "fastify-plugin"; import {ResultSetHeader} from "mysql2"; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts index 1f68c41328..5305e3b465 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/SearchResultsMetadataCollection/index.ts @@ -1,4 +1,4 @@ -import settings from "@settings" with {type: "json"}; +import settings from "../../../../../../settings.json" with { type: "json" }; import {FastifyInstance} from "fastify"; import fp from "fastify-plugin"; diff --git a/components/log-viewer-webui/server/src/plugins/MongoSocketIoServer/index.ts b/components/log-viewer-webui/server/src/plugins/MongoSocketIoServer/index.ts index 87c3d7d52c..a63c6792a3 100644 --- a/components/log-viewer-webui/server/src/plugins/MongoSocketIoServer/index.ts +++ b/components/log-viewer-webui/server/src/plugins/MongoSocketIoServer/index.ts @@ -33,6 +33,8 @@ import { removeItemFromArray, } from "./utils.js"; +let globalQueryIdCounter = 0; + /** * Integrates MongoDB with Socket.IO to provide real-time updates for MongoDB queries. @@ -171,20 +173,24 @@ class MongoSocketIoServer { */ #getQueryId (queryParams: QueryParameters): number { const queryHash = getQueryHash(queryParams); + this.#fastify.log.info( + `QueryId:${globalQueryIdCounter} params:${queryParams} query hash:${queryHash}` + ); for (const [queryId, hash] of this.#queryIdToQueryHashMap.entries()) { + this.#fastify.log.info( + `QueryId:${queryId} hash:${hash} requested hash:${queryHash}` + ); if (hash === queryHash) { return queryId; } } + this.#fastify.log.info( + `QueryId ${globalQueryIdCounter} not found in query map. Creating a new one.` + ); - let queryId = 0; - if (0 === this.#queryIdToQueryHashMap.size) { - this.#queryIdToQueryHashMap.set(queryId, queryHash); - } else { - const maxKey = Math.max(...Array.from(this.#queryIdToQueryHashMap.keys())); - queryId = maxKey + 1; - this.#queryIdToQueryHashMap.set(queryId, queryHash); - } + let queryId = globalQueryIdCounter; + this.#queryIdToQueryHashMap.set(queryId, queryHash); + globalQueryIdCounter += 1; return queryId; } diff --git a/components/log-viewer-webui/server/src/plugins/MongoSocketIoServer/utils.ts b/components/log-viewer-webui/server/src/plugins/MongoSocketIoServer/utils.ts index ab12bbf12c..acdb8dd9ea 100644 --- a/components/log-viewer-webui/server/src/plugins/MongoSocketIoServer/utils.ts +++ b/components/log-viewer-webui/server/src/plugins/MongoSocketIoServer/utils.ts @@ -43,7 +43,7 @@ const convertQueryToChangeStreamFormat = ( const getQueryHash = ( queryParams: QueryParameters ): string => { - return JSON.stringify(queryParams, Object.keys(queryParams).sort()); + return JSON.stringify(queryParams); }; /** diff --git a/components/log-viewer-webui/server/tsconfig.json b/components/log-viewer-webui/server/tsconfig.json index 7eb501605f..b843124917 100644 --- a/components/log-viewer-webui/server/tsconfig.json +++ b/components/log-viewer-webui/server/tsconfig.json @@ -19,7 +19,6 @@ "paths": { // Map imports from "@common/*" to the common server-client folder "@common/*": ["../common/*"], - "@settings": ["settings.json"] }, "alwaysStrict": true, From 624b205aa46ec18a7f6ba11509163e47651a3f45 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 22 May 2025 21:35:27 +0000 Subject: [PATCH 22/55] latest --- .../SearchControls/SearchButton.tsx | 54 --------------- .../SearchButton/SearchButton.tsx | 47 +++++++++++++ .../SearchButton/SearchCancelButton.tsx | 22 ++++++ .../SearchButton/SearchSubmitButton.tsx | 35 ++++++++++ .../SearchButton/index.module.css | 13 ++++ .../SearchControls/index.module.css | 9 +-- .../pages/SearchPage/SearchControls/index.tsx | 2 +- .../SearchResultsTable/Message/index.tsx | 68 ++++++------------- .../app/search/QueryJobsDbManager/index.ts | 8 +++ .../src/fastify-v2/plugins/external/env.ts | 2 +- .../src/fastify-v2/routes/api/search/utils.ts | 6 ++ 11 files changed, 155 insertions(+), 111 deletions(-) delete mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton.tsx create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchCancelButton.tsx create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchSubmitButton.tsx create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/index.module.css diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton.tsx deleted file mode 100644 index 0854ee897d..0000000000 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton.tsx +++ /dev/null @@ -1,54 +0,0 @@ -import {SearchOutlined} from "@ant-design/icons"; -import { - Button, - Tooltip, -} from "antd"; - -import useSearchStore, {SEARCH_STATE_DEFAULT} from "../SearchState/index"; -import {handleQuerySubmit} from "../SearchState/query"; -import styles from "./index.module.css"; - - -/** - * Renders a button to submit the search query. - * - * @return - */ -const SearchButton = () => { - const queryString = useSearchStore((state) => state.queryString); - - const isQueryStringEmpty: boolean = - queryString === SEARCH_STATE_DEFAULT.queryString; - - return ( - - - - ); -}; - - -export default SearchButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx new file mode 100644 index 0000000000..f1a1bf9c2f --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx @@ -0,0 +1,47 @@ +import { Tooltip } from "antd"; +import useSearchStore, { SEARCH_STATE_DEFAULT } from "../../SearchState/index"; +import { useResultsMetadata } from "../../metadata"; +import { SEARCH_SIGNAL, SearchResultsMetadataDocument, isSearchSignalQuerying } from "@common/searchResultsMetadata"; +import SearchSubmitButton from "./SearchSubmitButton"; +import SearchCancelButton from "./SearchCancelButton"; +import styles from "./index.module.css"; + + +/** + * Renders a button to submit the search query. + * + * @return + */ +const SearchButton = () => { + const queryString = useSearchStore((state) => state.queryString); + const resultsMetadata = useResultsMetadata(); + let searchSignal; + + if (Array.isArray(resultsMetadata) && resultsMetadata.length === 0) { + searchSignal = SEARCH_SIGNAL.NONE; + } else { + const resultMetadata = resultsMetadata[0] as SearchResultsMetadataDocument; + searchSignal = resultMetadata.lastSignal; + } + + const isQueryStringEmpty: boolean = + queryString === SEARCH_STATE_DEFAULT.queryString; + + return ( + <> + {!isSearchSignalQuerying(searchSignal) && + + + + } + {isSearchSignalQuerying(searchSignal) && + + } + + ); +}; + +export default SearchButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchCancelButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchCancelButton.tsx new file mode 100644 index 0000000000..56b534619d --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchCancelButton.tsx @@ -0,0 +1,22 @@ +import { CloseOutlined } from "@ant-design/icons"; +import { Button } from "antd"; +import { handleQueryCancel } from "../../SearchState/query"; +import styles from "./index.module.css"; + +const SearchCancelButton = () => ( + +); + +export default SearchCancelButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchSubmitButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchSubmitButton.tsx new file mode 100644 index 0000000000..682b789194 --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchSubmitButton.tsx @@ -0,0 +1,35 @@ +import { SearchOutlined } from "@ant-design/icons"; +import { Button } from "antd"; +import { handleQuerySubmit } from "../../SearchState/query"; +import styles from "./index.module.css"; + +type Props = { + queryString: string; + isQueryStringEmpty: boolean; + className?: string; +}; + +const SearchSubmitButton = ({ queryString, isQueryStringEmpty}: Props) => ( + +); + +export default SearchSubmitButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/index.module.css b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/index.module.css new file mode 100644 index 0000000000..4993586a0d --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/index.module.css @@ -0,0 +1,13 @@ +.gradientButton { + background-image: linear-gradient(135deg, #6253e1, #04befe); + color: white; + transition: background 0.3s; + background-size: 200% auto; +} + +.gradientButtonCancel { + background-image: linear-gradient(135deg, #e15253, #08b990); + color: red; + transition: background 0.3s; + background-size: 200% auto; +} diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.module.css b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.module.css index 8119ea54bc..7a593efd55 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.module.css +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.module.css @@ -1,11 +1,4 @@ .searchControlsContainer { display: flex; gap: 10px; -} - -.gradientButton { - background-image: linear-gradient(135deg, #6253e1, #04befe); - color: white; - transition: background 0.3s; - background-size: 200% auto; -} +} \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx index 591942acf9..fa8915530b 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -1,6 +1,6 @@ import styles from "./index.module.css"; -import SearchButton from "./SearchButton"; +import SearchButton from "./SearchButton/SearchButton"; import TimeRangeInput from "./TimeRangeInput"; import QueryInput from "./QueryInput"; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx index 0824ede78b..335bf58e73 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/Message/index.tsx @@ -1,21 +1,20 @@ -import { useMemo } from "react"; -import hljs from "highlight.js/lib/core"; -import armasm from "highlight.js/lib/languages/armasm"; -import "highlight.js/styles/intellij-light.css"; +import SyntaxHighlighter from "react-syntax-highlighter"; + +import {Typography} from "antd"; -import { Typography } from "antd"; import LogViewerLink from "./LogViewerLink"; +import {highlighterCustomStyles} from "./utils"; + +import "highlight.js/styles/intellij-light.css"; -const { Paragraph } = Typography; + +const {Text} = Typography; interface MessageProps { message: string; filePath: string; } -// Register the language once (outside component is OK too) -hljs.registerLanguage("armasm", armasm); - /** * Renders a message with syntax highlighting and a file path link. * @@ -24,46 +23,21 @@ hljs.registerLanguage("armasm", armasm); * @param props.filePath * @return */ -const Message = ({ message, filePath }: MessageProps) => { - const highlighted = useMemo(() => { - const html = hljs.highlight(message, { - language: "armasm", - ignoreIllegals: true, - }).value; - - // Convert newlines into inline spans with a space - return html.replace(/\n/g, " "); // or \n if needed - }, [message]); - +const Message = ({message, filePath}: MessageProps) => { return ( <> - - - - - + {/* Parent `Text` component allows syntax highlighter to inherit AntD fonts. */} + + + {message} + + + ); }; diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts index 83c4d551c4..208bfc4290 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/app/search/QueryJobsDbManager/index.ts @@ -108,6 +108,10 @@ class QueryJobsDbManager { * cancelled, or if the job completed in an unexpected state. */ async awaitJobCompletion (jobId: number): Promise { + console.log( + "am i waiting for job completion?", + jobId, + ); // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition while (true) { let rows: QueryJob[]; @@ -150,6 +154,10 @@ class QueryJobsDbManager { } break; } + console.log() + console.log( + `Job ${jobId} is still in progress. Status: ${Object.keys(QUERY_JOB_STATUS)[status]}` + ); await setTimeout(JOB_COMPLETION_STATUS_POLL_INTERVAL_MILLIS); } diff --git a/components/log-viewer-webui/server/src/fastify-v2/plugins/external/env.ts b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/env.ts index e858483329..96e9fd4133 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/plugins/external/env.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/plugins/external/env.ts @@ -62,8 +62,8 @@ export const autoConfig = { // Needed to read .env in root folder dotenv: { path: [ - ".env.local", ".env", + ".env.local", ], }, diff --git a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts index 40d1fe8745..0aeafc4928 100644 --- a/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts +++ b/components/log-viewer-webui/server/src/fastify-v2/routes/api/search/utils.ts @@ -60,6 +60,8 @@ const updateSearchSignalWhenJobsFinish = async ({ }: UpdateSearchSignalWhenJobsFinishProps) => { let errorMsg: string | null = null; + logger.info("Waiting for job completion"); + try { await queryJobsDbManager.awaitJobCompletion(searchJobId); await queryJobsDbManager.awaitJobCompletion(aggregationJobId); @@ -69,6 +71,8 @@ const updateSearchSignalWhenJobsFinish = async ({ "Error while waiting for job completion"; } + logger.info("job finised"); + let numResultsInCollection: number; try { @@ -97,6 +101,8 @@ const updateSearchSignalWhenJobsFinish = async ({ logger: logger, searchResultsMetadataCollection: searchResultsMetadataCollection, }); + + logger.info("job updated"); }; /** From 5d77bb105097e3cc6ac07f1bb2ceffebb64ddbca Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 23 May 2025 19:28:03 +0000 Subject: [PATCH 23/55] latest --- .../log-viewer-webui/client/src/api/query.ts | 4 +- .../client/src/api/search/queryCancel.ts | 15 +-- .../client/src/api/search/queryClear.ts | 15 +-- .../client/src/api/search/querySubmit.ts | 22 +--- .../SearchButton/SearchButton.tsx | 7 +- .../pages/SearchPage/SearchState/index.tsx | 8 ++ .../pages/SearchPage/SearchState/metadata.ts | 0 .../src/pages/SearchPage/SearchState/query.ts | 101 ++++++++++++------ .../pages/SearchPage/SearchState/typings.ts | 9 ++ .../hooks/useUpdateSearchUiStateOnDone.ts | 15 +++ .../client/src/pages/SearchPage/index.tsx | 12 +++ .../client/src/pages/SearchPage/metadata.ts | 3 +- 12 files changed, 128 insertions(+), 83 deletions(-) create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState/metadata.ts create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState/typings.ts create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/hooks/useUpdateSearchUiStateOnDone.ts diff --git a/components/log-viewer-webui/client/src/api/query.ts b/components/log-viewer-webui/client/src/api/query.ts index c4e6d21087..ec6d262de1 100644 --- a/components/log-viewer-webui/client/src/api/query.ts +++ b/components/log-viewer-webui/client/src/api/query.ts @@ -19,13 +19,13 @@ import { * @param onUploadProgress Callback to handle upload progress events. * @return The API response. */ -const submitExtractStreamJob = async ( +const submitExtractStreamJob = ( extractJobType: QUERY_JOB_TYPE, streamId: string, logEventIdx: number, onUploadProgress: (progressEvent: AxiosProgressEvent) => void, ): Promise> => { - return await axios.post( + return axios.post( "/query/extract-stream", { extractJobType, diff --git a/components/log-viewer-webui/client/src/api/search/queryCancel.ts b/components/log-viewer-webui/client/src/api/search/queryCancel.ts index 05e15aa790..cd4c303bc2 100644 --- a/components/log-viewer-webui/client/src/api/search/queryCancel.ts +++ b/components/log-viewer-webui/client/src/api/search/queryCancel.ts @@ -5,10 +5,10 @@ type QueryCancelArgs = { aggregationJobId: string; }; -const cancelQuery = async ({ +const cancelQuery = ({ searchJobId, aggregationJobId, -}: QueryCancelArgs) => { +}: QueryCancelArgs): Promise> => { const payload = { searchJobId, aggregationJobId, @@ -16,16 +16,7 @@ const cancelQuery = async ({ console.log("Cancelling query with payload:", payload); - try { - await axios.post("/api/search/cancel", payload); - } catch (error) { - if (axios.isAxiosError(error)) { - console.error("Axios error during query cancel:", error.message, error.response?.data); - } else { - console.error("Unknown error during query cancel:", error); - } - throw error; - } + return axios.post("/api/search/cancel", payload); }; export { cancelQuery }; diff --git a/components/log-viewer-webui/client/src/api/search/queryClear.ts b/components/log-viewer-webui/client/src/api/search/queryClear.ts index ef5f39196f..9837163d48 100644 --- a/components/log-viewer-webui/client/src/api/search/queryClear.ts +++ b/components/log-viewer-webui/client/src/api/search/queryClear.ts @@ -5,10 +5,10 @@ type QueryClearArgs = { aggregationJobId: string; }; -const clearQueryResults = async ({ +const clearQueryResults = ({ searchJobId, aggregationJobId, -}: QueryClearArgs) => { +}: QueryClearArgs): Promise> => { const payload = { searchJobId, aggregationJobId, @@ -16,16 +16,7 @@ const clearQueryResults = async ({ console.log("Clearing query results with payload:", payload); - try { - await axios.delete("/api/search/results", { data: payload }); - } catch (error) { - if (axios.isAxiosError(error)) { - console.error("Axios error during query clear:", error.message, error.response?.data); - } else { - console.error("Unknown error during query clear:", error); - } - throw error; - } + return axios.delete("/api/search/results", { data: payload }); }; export { clearQueryResults }; diff --git a/components/log-viewer-webui/client/src/api/search/querySubmit.ts b/components/log-viewer-webui/client/src/api/search/querySubmit.ts index a0e2aa6eed..0e6233de57 100644 --- a/components/log-viewer-webui/client/src/api/search/querySubmit.ts +++ b/components/log-viewer-webui/client/src/api/search/querySubmit.ts @@ -1,5 +1,4 @@ - -import axios from "axios"; +import axios, {AxiosResponse} from "axios"; type QueryArgs = { timestampBegin: number; @@ -14,13 +13,13 @@ type QueryResponse = { aggregationJobId: string; }; -const submitQuery = async ({ +const submitQuery = ({ timestampBegin, timestampEnd, ignoreCase, timeRangeBucketSizeMillis, queryString, -}: QueryArgs) => { +}: QueryArgs): Promise => { const payload = { timestampBegin, timestampEnd, @@ -31,19 +30,8 @@ const submitQuery = async ({ console.log("Submitting query with payload:", payload); - try { - const response = - await axios.post("/api/search/query", payload); - const { searchJobId, aggregationJobId } = response.data; - return { searchJobId, aggregationJobId }; - } catch (error) { - if (axios.isAxiosError(error)) { - console.error("Axios error during query submission:", error.message, error.response?.data); - } else { - console.error("Unknown error during query submission:", error); - } - throw error; - } + return axios.post("/api/search/query", payload) + .then(response => response.data); }; export { submitQuery }; \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx index f1a1bf9c2f..d358b71e46 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx @@ -29,17 +29,16 @@ const SearchButton = () => { return ( <> - {!isSearchSignalQuerying(searchSignal) && + {!isSearchSignalQuerying(searchSignal) ? ( - } - {isSearchSignalQuerying(searchSignal) && + ) : ( - } + )} ); }; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/index.tsx index bea409848d..d7e5fface2 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/index.tsx @@ -5,6 +5,7 @@ import { DEFAULT_TIME_RANGE, TIME_RANGE_OPTION_DAYJS_MAP, } from "../SearchControls/TimeRangeInput/utils"; +import {SEARCH_UI_STATE} from "./typings"; import {SearchResultsMetadataDocument} from "@common/searchResultsMetadata.js"; @@ -16,6 +17,7 @@ const SEARCH_STATE_DEFAULT = Object.freeze({ queryString: "", searchJobId: null, aggregationJobId: null, + searchUiState: SEARCH_UI_STATE.DEFAULT, timeRange: TIME_RANGE_OPTION_DAYJS_MAP[DEFAULT_TIME_RANGE], searchResultsMetadata: null, }); @@ -25,11 +27,13 @@ interface SearchState { timeRange: [dayjs.Dayjs, dayjs.Dayjs]; searchJobId: string | null; aggregationJobId: string | null; + searchUiState: SEARCH_UI_STATE; searchResultsMetadata: SearchResultsMetadataDocument | null; updateQueryString: (query: string) => void; updateTimeRange: (range: [dayjs.Dayjs, dayjs.Dayjs]) => void; updateSearchJobId: (id: string | null) => void; updateAggregationJobId: (id: string | null) => void; + updateSearchUiState: (state: SEARCH_UI_STATE) => void; updateSearchResultsMetadata: (metadata: SearchResultsMetadataDocument) => void; } @@ -38,10 +42,14 @@ const useSearchStore = create((set) => ({ timeRange: SEARCH_STATE_DEFAULT.timeRange, searchJobId: SEARCH_STATE_DEFAULT.searchJobId, aggregationJobId: SEARCH_STATE_DEFAULT.aggregationJobId, + searchUiState: SEARCH_STATE_DEFAULT.searchUiState, searchResultsMetadata: SEARCH_STATE_DEFAULT.searchResultsMetadata, updateQueryString: (query) => { set({queryString: query}); }, + updateSearchUiState: (state) => { + set({searchUiState: state}); + }, updateTimeRange: (range) => { set({timeRange: range}); }, diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/metadata.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/metadata.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts index 9ffe189649..90dbbf8c4c 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts @@ -2,29 +2,29 @@ import useSearchStore from "./index"; import { clearQueryResults } from "../../../api/search/queryClear"; import { submitQuery } from "../../../api/search/querySubmit"; import { cancelQuery } from "../../../api/search/queryCancel"; +import { SEARCH_UI_STATE } from "./typings"; -export const handleClearResults = async () => { +const handleClearResults = () => { const store = useSearchStore.getState(); + if (store.searchUiState === SEARCH_UI_STATE.DEFAULT) { + return; + } + const searchJobId = store.searchJobId; const aggregationJobId = store.aggregationJobId; if (!searchJobId || !aggregationJobId) { - console.warn("No searchJobId or aggregationJobId to clear."); - return; + throw new Error("No searchJobId or aggregationJobId to clear."); } - try { - await clearQueryResults({ - searchJobId, - aggregationJobId, - }); - store.updateSearchJobId(null); - store.updateAggregationJobId(null); - console.log("Query results cleared successfully"); - } catch (error) { + clearQueryResults({ + searchJobId, + aggregationJobId, + }).catch((error) => { console.error("Failed to clear query results:", error); - } + }); + }; // Add handleQuerySubmit that uses submitQuery from the API @@ -36,45 +36,76 @@ type QueryArgs = { queryString: string; }; -export const handleQuerySubmit = async (args: QueryArgs) => { +const handleQuerySubmit = () => { + const store = useSearchStore.getState(); - const searchJobId = store.searchJobId; + if (store.searchUiState === SEARCH_UI_STATE.QUERY_SUBMITTED ) { + console.warn("Query already submitted, ignoring submit request."); + return; + } - if (null !== searchJobId) { - // Clear result caches before starting a new query - await handleClearResults(); + if (store.searchUiState !== SEARCH_UI_STATE.DEFAULT && + store.searchUiState !== SEARCH_UI_STATE.DONE + ) { + throw new Error("Cannot submit query when not in DEFAULT or DONE state."); } - const result = await submitQuery(args); + handleClearResults(); - // Update store with new job IDs - store.updateSearchJobId(result.searchJobId); - store.updateAggregationJobId(result.aggregationJobId); + const args: QueryArgs = { + queryString: store.queryString, + timestampBegin: store.timeRange[0].valueOf(), + timestampEnd: store.timeRange[1].valueOf(), + ignoreCase: false, + timeRangeBucketSizeMillis: 150, + }; - return result; + store.updateSearchUiState(SEARCH_UI_STATE.QUERY_SUBMITTED); + + submitQuery(args) + .then((result) => { + store.updateSearchJobId(result.searchJobId); + store.updateAggregationJobId(result.aggregationJobId); + store.updateSearchUiState(SEARCH_UI_STATE.QUERYING); + }) + .catch((error) => { + console.error("Failed to submit query:", error); + }); }; -export const handleQueryCancel = async () => { +const handleQueryCancel = () => { const store = useSearchStore.getState(); + if (store.searchUiState !== SEARCH_UI_STATE.QUERYING) { + throw new Error("Cannot cancel query when not in QUERYING state."); + } + const searchJobId = store.searchJobId; const aggregationJobId = store.aggregationJobId; if (!searchJobId || !aggregationJobId) { - console.warn("No searchJobId or aggregationJobId to cancel."); - return; + throw new Error("Cannot cancel query without jobIDs"); } - try { - await cancelQuery({ - searchJobId, - aggregationJobId, - }); - store.updateSearchJobId(null); - store.updateAggregationJobId(null); + cancelQuery({ + searchJobId, + aggregationJobId, + }).then(() => { console.log("Query cancelled successfully"); - } catch (error) { + }).catch((error) => { console.error("Failed to cancel query:", error); - } + }); + + // Reset the state. I dont need to wait for promise to resolve. Another query + // can be submitted before previous cancel finished + store.updateSearchJobId(null); + store.updateAggregationJobId(null); + store.updateSearchUiState(SEARCH_UI_STATE.DEFAULT); +}; + +export { + handleClearResults, + handleQuerySubmit, + handleQueryCancel, }; \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/typings.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/typings.ts new file mode 100644 index 0000000000..165b13bda6 --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/typings.ts @@ -0,0 +1,9 @@ + +enum SEARCH_UI_STATE { + DEFAULT, + QUERY_SUBMITTED, + QUERYING, + DONE, +} + +export {SEARCH_UI_STATE}; \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/hooks/useUpdateSearchUiStateOnDone.ts b/components/log-viewer-webui/client/src/pages/SearchPage/hooks/useUpdateSearchUiStateOnDone.ts new file mode 100644 index 0000000000..1acd33b6db --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/hooks/useUpdateSearchUiStateOnDone.ts @@ -0,0 +1,15 @@ +import { useEffect } from "react"; +import { SEARCH_SIGNAL, SearchResultsMetadataDocument } from "@common/searchResultsMetadata.js"; +import { SEARCH_UI_STATE } from "../SearchState/typings"; + +export function useUpdateSearchUiStateOnDone(resultsMetadata: any, updateSearchUiState: (state: SEARCH_UI_STATE) => void) { + useEffect(() => { + if (Array.isArray(resultsMetadata) && resultsMetadata.length === 0) { + return; + } + let resultMetadata = resultsMetadata[0] as SearchResultsMetadataDocument; + if (resultMetadata.lastSignal === SEARCH_SIGNAL.RESP_DONE) { + updateSearchUiState(SEARCH_UI_STATE.DONE); + } + }, [resultsMetadata, updateSearchUiState]); +} diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/index.tsx index d14124b4bf..bdd6214840 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/index.tsx @@ -2,6 +2,13 @@ import styles from "./index.module.css"; import SearchControls from "./SearchControls"; import SearchResultsTable from "./SearchResults/SearchResultsTable"; +import { useResultsMetadata } from "./metadata"; +import { isSearchSignalQuerying, SEARCH_SIGNAL, SearchResultsMetadataDocument } from "@common/searchResultsMetadata.js"; +import { useEffect, useRef, useState } from "react"; +import useSearchStore, { SEARCH_STATE_DEFAULT } from "./SearchState/index"; +import { SEARCH_UI_STATE } from "./SearchState/typings"; +import { useUpdateSearchUiStateOnDone } from "./hooks/useUpdateSearchUiStateOnDone"; + /** * Provides a search interface that allows users to query archives and visualize search results. @@ -9,6 +16,11 @@ import SearchResultsTable from "./SearchResults/SearchResultsTable"; * @return */ const SearchPage = () => { + const store = useSearchStore(); + const resultsMetadata = useResultsMetadata(); + + useUpdateSearchUiStateOnDone(resultsMetadata, store.updateSearchUiState); + return (
diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts b/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts index f372f7fcc3..5a710b8892 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts +++ b/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts @@ -27,4 +27,5 @@ const useResultsMetadata = () => { export { useResultsMetadata }; - +// we get the metadata. If the metadata is not empty +// We check the signal and set the UI state. From 7fdc18a02916c4afb822bb21db0f3cbe07dd736a Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 23 May 2025 20:21:24 +0000 Subject: [PATCH 24/55] latest --- .../client/src/api/socket/useCursor.tsx | 6 ++-- .../SearchPage/SearchControls/QueryInput.tsx | 21 +++-------- ...earchCancelButton.tsx => CancelButton.tsx} | 11 +++--- .../SearchButton/SearchButton.tsx | 36 +++++-------------- .../SearchButton/SearchSubmitButton.tsx | 35 ------------------ .../SearchButton/SubmitButton.tsx | 29 +++++++++++++++ .../SearchResultsTable/index.tsx | 2 +- .../mongo-queries/useResultsMetadata.ts | 34 ++++++++++++++++++ .../mongo-queries/useSearchResults.ts} | 6 ++-- .../SearchState/useUpdateStateWithMetadata.ts | 24 +++++++++++++ .../hooks/useUpdateSearchUiStateOnDone.ts | 15 -------- .../client/src/pages/SearchPage/index.tsx | 12 ++----- .../client/src/pages/SearchPage/metadata.ts | 31 ---------------- 13 files changed, 114 insertions(+), 148 deletions(-) rename components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/{SearchCancelButton.tsx => CancelButton.tsx} (61%) delete mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchSubmitButton.tsx create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useResultsMetadata.ts rename components/log-viewer-webui/client/src/pages/SearchPage/{searchResults.ts => SearchState/mongo-queries/useSearchResults.ts} (75%) create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState/useUpdateStateWithMetadata.ts delete mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/hooks/useUpdateSearchUiStateOnDone.ts delete mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts diff --git a/components/log-viewer-webui/client/src/api/socket/useCursor.tsx b/components/log-viewer-webui/client/src/api/socket/useCursor.tsx index f9cea19e3d..2f93768eb8 100644 --- a/components/log-viewer-webui/client/src/api/socket/useCursor.tsx +++ b/components/log-viewer-webui/client/src/api/socket/useCursor.tsx @@ -12,13 +12,13 @@ import {MongoCursorSocket} from "./MongoCursorSocket.js"; * * @param query Function which returns a `MongoCursorSocket` instance. * @param dependencies Array of dependencies for the query. - * @return Reactive array. + * @return Reactive array or null while the query is pending response. */ const useCursor = ( query: () => MongoCursorSocket | null, dependencies: DependencyList = [] -): object[] => { - const [data, setData] = useState([]); +): object[]| null => { + const [data, setData] = useState(null); useEffect(() => { diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx index d72ec469e9..cc3e01b3c0 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx @@ -1,13 +1,11 @@ import QueryBox from "../../../components/QueryBox"; import useSearchStore from "../SearchState/index"; -import { isSearchSignalQuerying, SearchResultsMetadataDocument } from "@common/searchResultsMetadata.js"; -import { useResultsMetadata } from "../metadata"; - import { useEffect, useRef, useState, } from "react"; +import { SEARCH_UI_STATE } from "../SearchState/typings"; // for pseudo progress bar const PROGRESS_INCREMENT = 5; @@ -21,23 +19,12 @@ const PROGRESS_INTERVAL_MILLIS = 100; const QueryInput = () => { const queryString = useSearchStore((state) => state.queryString); const updateQueryString = useSearchStore((state) => state.updateQueryString); - const resultsMetadata = useResultsMetadata(); + const searchUiState = useSearchStore((state) => state.searchUiState); const [progress, setProgress] = useState(0); const timerIntervalRef = useRef(null); - console.log("QueryInput: resultsMetadata", resultsMetadata); - useEffect(() => { - // I have idea to potentially just set the searchSignal here - // but will get back to later - if (Array.isArray(resultsMetadata) && resultsMetadata.length === 0) { - return; - } - let resultMetadata = resultsMetadata[0] as SearchResultsMetadataDocument; - console.log(resultsMetadata); - console.log("QueryInput: resultsMetadata.lastSignal", resultMetadata.lastSignal); - if (true === isSearchSignalQuerying(resultMetadata.lastSignal)) { - console.log("QueryInput: isSearchSignalQuerying"); + if (searchUiState === SEARCH_UI_STATE.QUERYING) { if (null === timerIntervalRef.current) { timerIntervalRef.current = setInterval(() => { setProgress((v) => { @@ -57,7 +44,7 @@ const QueryInput = () => { } setProgress(0); } - }, [resultsMetadata]); + }, [searchUiState]); return ( ( +const CancelButton = () => ( ); -export default SearchCancelButton; +export default CancelButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx index d358b71e46..c2605be09d 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx @@ -1,10 +1,7 @@ -import { Tooltip } from "antd"; -import useSearchStore, { SEARCH_STATE_DEFAULT } from "../../SearchState/index"; -import { useResultsMetadata } from "../../metadata"; -import { SEARCH_SIGNAL, SearchResultsMetadataDocument, isSearchSignalQuerying } from "@common/searchResultsMetadata"; -import SearchSubmitButton from "./SearchSubmitButton"; -import SearchCancelButton from "./SearchCancelButton"; -import styles from "./index.module.css"; +import useSearchStore from "../../SearchState/index"; +import SubmitButton from "./SubmitButton"; +import CancelButton from "./CancelButton"; +import { SEARCH_UI_STATE } from "../../SearchState/typings"; /** @@ -13,31 +10,14 @@ import styles from "./index.module.css"; * @return */ const SearchButton = () => { - const queryString = useSearchStore((state) => state.queryString); - const resultsMetadata = useResultsMetadata(); - let searchSignal; - - if (Array.isArray(resultsMetadata) && resultsMetadata.length === 0) { - searchSignal = SEARCH_SIGNAL.NONE; - } else { - const resultMetadata = resultsMetadata[0] as SearchResultsMetadataDocument; - searchSignal = resultMetadata.lastSignal; - } - - const isQueryStringEmpty: boolean = - queryString === SEARCH_STATE_DEFAULT.queryString; + const store = useSearchStore(); return ( <> - {!isSearchSignalQuerying(searchSignal) ? ( - - - + {(store.searchUiState === SEARCH_UI_STATE.QUERYING) ? ( + ) : ( - + )} ); diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchSubmitButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchSubmitButton.tsx deleted file mode 100644 index 682b789194..0000000000 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchSubmitButton.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { SearchOutlined } from "@ant-design/icons"; -import { Button } from "antd"; -import { handleQuerySubmit } from "../../SearchState/query"; -import styles from "./index.module.css"; - -type Props = { - queryString: string; - isQueryStringEmpty: boolean; - className?: string; -}; - -const SearchSubmitButton = ({ queryString, isQueryStringEmpty}: Props) => ( - -); - -export default SearchSubmitButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx new file mode 100644 index 0000000000..4f1e09379c --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx @@ -0,0 +1,29 @@ +import { SearchOutlined } from "@ant-design/icons"; +import { Button, Tooltip } from "antd"; +import { handleQuerySubmit } from "../../SearchState/query"; +import useSearchStore, { SEARCH_STATE_DEFAULT } from "../../SearchState/index"; +import styles from "./index.module.css"; + +const SubmitButton = () => { + const queryString = useSearchStore((state) => state.queryString); + const isQueryStringEmpty = queryString === SEARCH_STATE_DEFAULT.queryString; + + return ( + + + + ); +}; + +export default SubmitButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx index c014ec03da..137a8eb669 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx @@ -5,7 +5,7 @@ import { searchResultsTableColumns, } from "./typings"; -import {useSearchResults} from "../../../SearchPage/searchResults"; +import {useSearchResults} from "../../SearchState/mongo-queries/useSearchResults"; /** * Renders search results in a table. diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useResultsMetadata.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useResultsMetadata.ts new file mode 100644 index 0000000000..4c7247e0e4 --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useResultsMetadata.ts @@ -0,0 +1,34 @@ +import {useCursor} from "../../../../api/socket/useCursor"; +import MongoCollectionSocket from "../../../../api/socket/MongoCollectionSocket"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../index"; +import { SearchResultsMetadataDocument } from "@common/searchResultsMetadata"; +import { Nullable } from "../../../../typings/common"; + +import settings from "../../../../../settings.json" with { type: "json" }; + +/** + * Custom hook to get results metadata for the current searchJobId. + */ +const useResultsMetadata = () => { + const {searchJobId} = useSearchStore(); + + console.log(`Requesting metadata for searchJobID:${searchJobId}`); + const resultsMetadataCursor = useCursor( + () => { + // If there is no active search job, there is no metadata to fetch. The cursor will + // return null. + if (searchJobId === SEARCH_STATE_DEFAULT.searchJobId) { + return null; + } + const collection = new MongoCollectionSocket( + settings.MongoDbSearchResultsMetadataCollectionName + ); + return collection.find({ _id: searchJobId.toString() }, { limit: 1 }); + }, + [searchJobId] + ); + + return resultsMetadataCursor as Nullable; +}; + +export { useResultsMetadata }; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/searchResults.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useSearchResults.ts similarity index 75% rename from components/log-viewer-webui/client/src/pages/SearchPage/searchResults.ts rename to components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useSearchResults.ts index 06dedda32e..1e46063c58 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/searchResults.ts +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useSearchResults.ts @@ -1,6 +1,6 @@ -import {useCursor} from "../../api/socket/useCursor"; -import MongoCollectionSocket from "../../api/socket/MongoCollectionSocket"; -import useSearchStore, {SEARCH_STATE_DEFAULT} from "./SearchState"; +import {useCursor} from "../../../../api/socket/useCursor"; +import MongoCollectionSocket from "../../../../api/socket/MongoCollectionSocket"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../SearchState"; /** * Custom hook to get results metadata for the current searchJobId. diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/useUpdateStateWithMetadata.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/useUpdateStateWithMetadata.ts new file mode 100644 index 0000000000..e250ca8af3 --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/useUpdateStateWithMetadata.ts @@ -0,0 +1,24 @@ +import { useEffect } from "react"; +import { SEARCH_UI_STATE } from "./typings"; +import { SEARCH_SIGNAL, SearchResultsMetadataDocument } from "@common/searchResultsMetadata.js"; +import useSearchStore from "./index"; +import { Nullable } from "../../../typings/common"; + + +const useUpdateUiStateWithMetadata = (resultsMetadata: Nullable) => { + const {updateSearchUiState} = useSearchStore(); + useEffect(() => { + if (null === resultsMetadata || + (Array.isArray(resultsMetadata) && resultsMetadata.length === 0) + ) { + return; + } + + let resultMetadata = resultsMetadata[0] as SearchResultsMetadataDocument; + if (resultMetadata.lastSignal === SEARCH_SIGNAL.RESP_DONE) { + updateSearchUiState(SEARCH_UI_STATE.DONE); + } + }, [resultsMetadata]); +} + +export { useUpdateUiStateWithMetadata} ; \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/hooks/useUpdateSearchUiStateOnDone.ts b/components/log-viewer-webui/client/src/pages/SearchPage/hooks/useUpdateSearchUiStateOnDone.ts deleted file mode 100644 index 1acd33b6db..0000000000 --- a/components/log-viewer-webui/client/src/pages/SearchPage/hooks/useUpdateSearchUiStateOnDone.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { useEffect } from "react"; -import { SEARCH_SIGNAL, SearchResultsMetadataDocument } from "@common/searchResultsMetadata.js"; -import { SEARCH_UI_STATE } from "../SearchState/typings"; - -export function useUpdateSearchUiStateOnDone(resultsMetadata: any, updateSearchUiState: (state: SEARCH_UI_STATE) => void) { - useEffect(() => { - if (Array.isArray(resultsMetadata) && resultsMetadata.length === 0) { - return; - } - let resultMetadata = resultsMetadata[0] as SearchResultsMetadataDocument; - if (resultMetadata.lastSignal === SEARCH_SIGNAL.RESP_DONE) { - updateSearchUiState(SEARCH_UI_STATE.DONE); - } - }, [resultsMetadata, updateSearchUiState]); -} diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/index.tsx index bdd6214840..4e0acf9c2e 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/index.tsx @@ -2,12 +2,8 @@ import styles from "./index.module.css"; import SearchControls from "./SearchControls"; import SearchResultsTable from "./SearchResults/SearchResultsTable"; -import { useResultsMetadata } from "./metadata"; -import { isSearchSignalQuerying, SEARCH_SIGNAL, SearchResultsMetadataDocument } from "@common/searchResultsMetadata.js"; -import { useEffect, useRef, useState } from "react"; -import useSearchStore, { SEARCH_STATE_DEFAULT } from "./SearchState/index"; -import { SEARCH_UI_STATE } from "./SearchState/typings"; -import { useUpdateSearchUiStateOnDone } from "./hooks/useUpdateSearchUiStateOnDone"; +import { useResultsMetadata } from "./SearchState/mongo-queries/useResultsMetadata"; +import { useUpdateUiStateWithMetadata } from "./SearchState/useUpdateStateWithMetadata"; /** @@ -16,10 +12,8 @@ import { useUpdateSearchUiStateOnDone } from "./hooks/useUpdateSearchUiStateOnDo * @return */ const SearchPage = () => { - const store = useSearchStore(); const resultsMetadata = useResultsMetadata(); - - useUpdateSearchUiStateOnDone(resultsMetadata, store.updateSearchUiState); + useUpdateUiStateWithMetadata(resultsMetadata); return (
diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts b/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts deleted file mode 100644 index 5a710b8892..0000000000 --- a/components/log-viewer-webui/client/src/pages/SearchPage/metadata.ts +++ /dev/null @@ -1,31 +0,0 @@ -import {useCursor} from "../../api/socket/useCursor"; -import MongoCollectionSocket from "../../api/socket/MongoCollectionSocket"; -import useSearchStore, {SEARCH_STATE_DEFAULT} from "./SearchState/index"; -import { SearchResultsMetadataDocument } from "@common/searchResultsMetadata"; - - -/** - * Custom hook to get results metadata for the current searchJobId. - */ -const useResultsMetadata = () => { - const searchJobId = useSearchStore((state) => state.searchJobId); - console.log(`this is search jobID ${searchJobId}`); - - const resultsMetadataCursor = useCursor( - () => { - if (!searchJobId || searchJobId === SEARCH_STATE_DEFAULT.searchJobId) { - return null; - } - const collection = new MongoCollectionSocket("results-metadata"); - return collection.find({ _id: searchJobId?.toString() }, { limit: 1 }); - }, - [searchJobId] - ); - - return resultsMetadataCursor as SearchResultsMetadataDocument[]; -}; - -export { useResultsMetadata }; - -// we get the metadata. If the metadata is not empty -// We check the signal and set the UI state. From d585c9609f3476fe2223c7495d4b5df9f3fd5f17 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 23 May 2025 20:43:40 +0000 Subject: [PATCH 25/55] latest --- .../client/src/api/search/querySubmit.ts | 2 +- .../client/src/api/socket/useCursor.tsx | 9 +++++---- .../SearchPage/SearchControls/QueryInput.tsx | 14 +++++--------- .../SearchResults/SearchResultsTable/index.tsx | 2 +- .../mongo-queries/useSearchResults.ts | 16 +++++++++++----- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/components/log-viewer-webui/client/src/api/search/querySubmit.ts b/components/log-viewer-webui/client/src/api/search/querySubmit.ts index 0e6233de57..6a3b65346c 100644 --- a/components/log-viewer-webui/client/src/api/search/querySubmit.ts +++ b/components/log-viewer-webui/client/src/api/search/querySubmit.ts @@ -34,4 +34,4 @@ const submitQuery = ({ .then(response => response.data); }; -export { submitQuery }; \ No newline at end of file +export { submitQuery }; diff --git a/components/log-viewer-webui/client/src/api/socket/useCursor.tsx b/components/log-viewer-webui/client/src/api/socket/useCursor.tsx index 2f93768eb8..97b0413227 100644 --- a/components/log-viewer-webui/client/src/api/socket/useCursor.tsx +++ b/components/log-viewer-webui/client/src/api/socket/useCursor.tsx @@ -5,6 +5,7 @@ import { } from "react"; import {MongoCursorSocket} from "./MongoCursorSocket.js"; +import {Nullable} from "../../typings/common"; /** @@ -15,16 +16,16 @@ import {MongoCursorSocket} from "./MongoCursorSocket.js"; * @return Reactive array or null while the query is pending response. */ const useCursor = ( - query: () => MongoCursorSocket | null, + query: () => Nullable, dependencies: DependencyList = [] -): object[]| null => { - const [data, setData] = useState(null); +): Nullable => { + const [data, setData] = useState>(null); useEffect(() => { const cursor = query(); - if (!cursor) { + if (null === cursor) { return; } diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx index cc3e01b3c0..2fa5c6def4 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx @@ -17,21 +17,17 @@ const PROGRESS_INTERVAL_MILLIS = 100; * @return */ const QueryInput = () => { - const queryString = useSearchStore((state) => state.queryString); - const updateQueryString = useSearchStore((state) => state.updateQueryString); - const searchUiState = useSearchStore((state) => state.searchUiState); - const [progress, setProgress] = useState(0); - const timerIntervalRef = useRef(null); + const {queryString, updateQueryString, searchUiState}= useSearchStore(); + const [progress, setProgress] = useState(0); + const timerIntervalRef = useRef | null>(null); useEffect(() => { if (searchUiState === SEARCH_UI_STATE.QUERYING) { if (null === timerIntervalRef.current) { timerIntervalRef.current = setInterval(() => { setProgress((v) => { - if (v + PROGRESS_INCREMENT >= 100) { - clearInterval(timerIntervalRef.current!); - timerIntervalRef.current = null; - return 100; + if (v + PROGRESS_INCREMENT >= 95) { + return 95; } return v + PROGRESS_INCREMENT; }); diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx index 137a8eb669..fe1c584b71 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTable/index.tsx @@ -21,7 +21,7 @@ const SearchResultsTable = () => { scroll={{ y: 400 }} columns={searchResultsTableColumns} - dataSource={searchResults as SearchResult[]} + dataSource={searchResults ? searchResults : []} pagination={false} rowKey={(record) => record._id.toString()} virtual={true}/> diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useSearchResults.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useSearchResults.ts index 1e46063c58..569f324c99 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useSearchResults.ts +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/mongo-queries/useSearchResults.ts @@ -1,17 +1,23 @@ import {useCursor} from "../../../../api/socket/useCursor"; import MongoCollectionSocket from "../../../../api/socket/MongoCollectionSocket"; -import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../SearchState"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../index"; +import { + SearchResult, +} from "../../SearchResults/SearchResultsTable/typings"; +import { Nullable } from "../../../../typings/common"; /** * Custom hook to get results metadata for the current searchJobId. */ const useSearchResults = () => { - const searchJobId = useSearchStore((state) => state.searchJobId); + const {searchJobId} = useSearchStore(); console.log(`this is search jobID ${searchJobId}`); - const resultsMetadataCursor = useCursor( + const searchResultsCursor = useCursor( () => { - if (!searchJobId || searchJobId === SEARCH_STATE_DEFAULT.searchJobId) { + // If there is no active search job, there is no metadata to fetch. The cursor will + // return null. + if (searchJobId === SEARCH_STATE_DEFAULT.searchJobId) { return null; } const collection = new MongoCollectionSocket(searchJobId.toString()); @@ -20,7 +26,7 @@ const useSearchResults = () => { [searchJobId] ); - return resultsMetadataCursor; + return searchResultsCursor as Nullable; }; export { useSearchResults }; From 3cb8f92111b4324597c9a3bb286fe5cb944a12d5 Mon Sep 17 00:00:00 2001 From: Marco Date: Mon, 26 May 2025 13:55:58 +0000 Subject: [PATCH 26/55] saving --- .../log-viewer-webui/client/settings.json | 3 + .../SearchPage/SearchControls/QueryInput.tsx | 4 +- .../SearchButton/SubmitButton.tsx | 4 +- .../SearchControls/TimeRangeInput/utils.tsx | 6 +- .../pages/SearchPage/SearchState/datetime.ts | 198 ++++++++++++++++++ .../src/pages/SearchPage/SearchState/query.ts | 10 +- .../src/pages/SearchPage/SearchState/temp.ts | 46 ++++ .../src/plugins/MongoSocketIoServer/index.ts | 2 +- 8 files changed, 266 insertions(+), 7 deletions(-) create mode 100644 components/log-viewer-webui/client/settings.json create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState/datetime.ts create mode 100644 components/log-viewer-webui/client/src/pages/SearchPage/SearchState/temp.ts diff --git a/components/log-viewer-webui/client/settings.json b/components/log-viewer-webui/client/settings.json new file mode 100644 index 0000000000..cc1fe697c7 --- /dev/null +++ b/components/log-viewer-webui/client/settings.json @@ -0,0 +1,3 @@ +{ + "MongoDbSearchResultsMetadataCollectionName": "results-metadata" +} diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx index 2fa5c6def4..a3dedc3232 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput.tsx @@ -26,8 +26,8 @@ const QueryInput = () => { if (null === timerIntervalRef.current) { timerIntervalRef.current = setInterval(() => { setProgress((v) => { - if (v + PROGRESS_INCREMENT >= 95) { - return 95; + if (v + PROGRESS_INCREMENT >= 100) { + return 100; } return v + PROGRESS_INCREMENT; }); diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx index 4f1e09379c..d19f88f71a 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx @@ -3,16 +3,18 @@ import { Button, Tooltip } from "antd"; import { handleQuerySubmit } from "../../SearchState/query"; import useSearchStore, { SEARCH_STATE_DEFAULT } from "../../SearchState/index"; import styles from "./index.module.css"; +import { SEARCH_UI_STATE } from "../../SearchState/typings"; const SubmitButton = () => { const queryString = useSearchStore((state) => state.queryString); + const {searchUiState} = useSearchStore(); const isQueryStringEmpty = queryString === SEARCH_STATE_DEFAULT.queryString; return ( -); +const CancelButton = () => { + const {searchJobId, aggregationJobId} = useSearchStore(); + + return ( + + ) +}; export default CancelButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx index baf3de11a3..8d391c9a55 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx @@ -1,12 +1,13 @@ import { SearchOutlined } from "@ant-design/icons"; import { Button, Tooltip } from "antd"; -import { handleQuerySubmit } from "../../SearchState/query"; +import { handleQuerySubmit} from "../requests"; import useSearchStore, { SEARCH_STATE_DEFAULT } from "../../SearchState/index"; import styles from "./index.module.css"; import { SEARCH_UI_STATE } from "../../SearchState/typings"; +import { computeTimelineConfig } from "../../SearchResults/SearchResultsTimeline/utils"; const SubmitButton = () => { - const {searchUiState, queryString} = useSearchStore(); + const {searchUiState, timeRange, queryString} = useSearchStore(); const isQueryStringEmpty = queryString === SEARCH_STATE_DEFAULT.queryString; return ( @@ -18,7 +19,15 @@ const SubmitButton = () => { size="large" type="primary" onClick={() => { - handleQuerySubmit(); + const timelineConfig = computeTimelineConfig(timeRange[0].valueOf(), timeRange[1].valueOf()); + + handleQuerySubmit({ + queryString: queryString, + timestampBegin: timeRange[0].valueOf(), + timestampEnd: timeRange[1].valueOf(), + timeRangeBucketSizeMillis: timelineConfig.bucketDuration.asMilliseconds(), + ignoreCase: false, + }); }} > Search diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx index fa8915530b..fc4c84bc56 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -2,7 +2,7 @@ import styles from "./index.module.css"; import SearchButton from "./SearchButton/SearchButton"; import TimeRangeInput from "./TimeRangeInput"; -import QueryInput from "./QueryInput"; +import QueryInput from "./QueryInput/QueryInput"; /** diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/requests.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/requests.ts new file mode 100644 index 0000000000..6d44b4bb1c --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/requests.ts @@ -0,0 +1,76 @@ +import useSearchStore from "../SearchState/index"; +import { clearQueryResults } from "../../../api/search"; +import { submitQuery } from "../../../api/search"; +import { cancelQuery } from "../../../api/search"; +import { SEARCH_UI_STATE } from "../SearchState/typings"; + +import {QueryJobSchema, QueryJobCreationSchema} from "../../../api/search"; + +const handleClearResults = () => { + const {searchUiState, searchJobId, aggregationJobId} = useSearchStore.getState(); + + if (searchUiState === SEARCH_UI_STATE.DEFAULT) { + return; + } + + if (null === searchJobId || null === aggregationJobId) { + throw new Error("No searchJobId or aggregationJobId to clear."); + } + + clearQueryResults( + {searchJobId,aggregationJobId} + ).catch((error) => { + console.error("Failed to clear query results:", error); + }); + +}; + +const handleQuerySubmit = (payload: QueryJobCreationSchema) => { + const store = useSearchStore.getState(); + + if ( + store.searchUiState !== SEARCH_UI_STATE.DEFAULT && + store.searchUiState !== SEARCH_UI_STATE.DONE + ) { + throw new Error("Cannot submit query when not in DEFAULT or DONE state."); + } + + handleClearResults(); + + store.updateSearchUiState(SEARCH_UI_STATE.QUERY_ID_PENDING); + + submitQuery(payload) + .then((result) => { + store.updateSearchJobId(result.data.searchJobId); + store.updateAggregationJobId(result.data.aggregationJobId); + store.updateSearchUiState(SEARCH_UI_STATE.QUERYING); + console.log("Query ID Returned", result); + }) + .catch((error) => { + console.error("Failed to submit query:", error); + }); +}; + +const handleQueryCancel = (payload: QueryJobSchema) => { + const store = useSearchStore.getState(); + + if (store.searchUiState !== SEARCH_UI_STATE.QUERYING) { + throw new Error("Cannot cancel query when not in QUERYING state."); + } + + cancelQuery( + payload + ).then(() => { + console.log("Query cancelled successfully"); + }).catch((error) => { + console.error("Failed to cancel query:", error); + }); + + store.updateSearchUiState(SEARCH_UI_STATE.DONE); +}; + + +export { + handleQuerySubmit, + handleQueryCancel, +} \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx similarity index 80% rename from components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline.tsx rename to components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx index 88b2543349..60f64a2fe8 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/index.tsx @@ -1,13 +1,12 @@ import {Card} from "antd"; import dayjs from "dayjs"; -import { TimelineBucket } from "../../../components/ResultsTimeline/typings"; -import ResultsTimeline from "../../../components/ResultsTimeline/index"; -import {expandTimeRangeToDurationMultiple} from "../../../components/ResultsTimeline/datetime"; -import {TimeRange} from "../../../components/ResultsTimeline/datetime/typings"; -import {computeTimelineConfig} from "../../../components/ResultsTimeline/utils"; -import {TIME_RANGE_OPTION} from "../SearchControls/TimeRangeInput/utils"; -import useSearchStore from "../SearchState"; +import { TimelineBucket } from "../../../../components/ResultsTimeline/typings"; +import ResultsTimeline from "../../../../components/ResultsTimeline/index"; +import {expandTimeRangeToDurationMultiple, computeTimelineConfig } from "./utils"; +import {TimeRange} from "../../../../components/ResultsTimeline/datetime/typings"; +import {TIME_RANGE_OPTION} from "../../SearchControls/TimeRangeInput/utils"; +import useSearchStore from "../../SearchState"; // eslint-disable-next-line no-warning-comments // TODO: Replace with values from database once api implemented. diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/utils.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/utils.ts new file mode 100644 index 0000000000..e40736ac16 --- /dev/null +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/utils.ts @@ -0,0 +1,99 @@ + + + +import dayjs from "dayjs"; +import {DurationUnitType} from "dayjs/plugin/duration"; + +import { + MAX_DATA_POINTS_PER_TIMELINE, + TimelineConfig, +} from "../../../../components/ResultsTimeline/typings"; + +import DayjsDuration from "dayjs/plugin/duration"; +import {TimeRange} from "../../../../components/ResultsTimeline/datetime/typings"; + +dayjs.extend(DayjsDuration); + +/** + * Expands the time range so that both extremes are multiples of the given duration. + * + * @param duration + * @param timeRange The time range to be expanded. + * @param timeRange.begin + * @param timeRange.end + * @return The expanded time range. + */ +const expandTimeRangeToDurationMultiple = (duration: DayjsDuration.Duration, { + begin, + end, +}: TimeRange) => { + const adjustedBegin = begin.valueOf() - (begin.valueOf() % duration.asMilliseconds()); + const adjustedEnd = + Math.floor( + (end.valueOf() + duration.asMilliseconds() - 1) / duration.asMilliseconds() + ) * duration.asMilliseconds(); + + return {begin: dayjs.utc(adjustedBegin), end: dayjs.utc(adjustedEnd)}; +}; + +/** + * Computes the timestamp range and bucket duration necessary to render the bars in the timeline + * chart. + * + * @param timestampBeginUnixMillis + * @param timestampEndUnixMillis + * @return + */ +const computeTimelineConfig = ( + timestampBeginUnixMillis: number, + timestampEndUnixMillis: number +): TimelineConfig => { + const timeRangeMillis = timestampEndUnixMillis - timestampBeginUnixMillis; + const exactTimelineBucketMillis = timeRangeMillis / MAX_DATA_POINTS_PER_TIMELINE; + + // A list of predefined bucket durations, ordered from least to greatest so that the + // `durationSelections.find()` below can find the smallest bucket containing + // `exactTimelineBucketMillis`. + const durationSelections = [ + /* eslint-disable @stylistic/array-element-newline, no-magic-numbers */ + {unit: "second", values: [1, 2, 5, 10, 15, 30]}, + {unit: "minute", values: [1, 2, 5, 10, 15, 20, 30]}, + {unit: "hour", values: [1, 2, 3, 4, 8, 12]}, + {unit: "day", values: [1, 2, 5, 15]}, + {unit: "month", values: [1, 2, 3, 4, 6]}, + {unit: "year", values: [1]}, + /* eslint-enable @stylistic/array-element-newline, no-magic-numbers */ + ].flatMap( + ({ + unit, + values, + }) => values.map( + (value) => dayjs.duration(value, unit as DurationUnitType), + ), + ); + + const bucketDuration = + durationSelections.find( + (duration) => (exactTimelineBucketMillis <= duration.asMilliseconds()), + ) || + + // If the timeline bucket spans more than 1 year, calculate a custom multi-year duration. + dayjs.duration( + Math.ceil(exactTimelineBucketMillis / + dayjs.duration(1, "year").asMilliseconds()), + "year", + ); + + return { + range: expandTimeRangeToDurationMultiple(bucketDuration, { + begin: dayjs.utc(timestampBeginUnixMillis), + end: dayjs.utc(timestampEndUnixMillis), + }), + bucketDuration: bucketDuration, + }; +}; + +export { + computeTimelineConfig, + expandTimeRangeToDurationMultiple, +} \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts deleted file mode 100644 index 2d4ee12e9e..0000000000 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/query.ts +++ /dev/null @@ -1,115 +0,0 @@ -import useSearchStore from "./index"; -import { clearQueryResults } from "../../../api/search"; -import { submitQuery } from "../../../api/search"; -import { cancelQuery } from "../../../api/search"; -import { SEARCH_UI_STATE } from "./typings"; -//import { SEARCH_STATE_DEFAULT } from "./index"; - -import {computeTimelineConfig} from "../../../components/ResultsTimeline/utils"; - -const handleClearResults = () => { - const store = useSearchStore.getState(); - - if (store.searchUiState === SEARCH_UI_STATE.DEFAULT) { - return; - } - - const searchJobId = store.searchJobId; - const aggregationJobId = store.aggregationJobId; - - if (!searchJobId || !aggregationJobId) { - throw new Error("No searchJobId or aggregationJobId to clear."); - } - - //store.updateSearchJobId(SEARCH_STATE_DEFAULT.searchJobId); - //store.updateAggregationJobId(SEARCH_STATE_DEFAULT.aggregationJobId); - - clearQueryResults({ - searchJobId, - aggregationJobId, - }).catch((error) => { - console.error("Failed to clear query results:", error); - }); - -}; - -// Add handleQuerySubmit that uses submitQuery from the API -type QueryArgs = { - timestampBegin: number; - timestampEnd: number; - ignoreCase: boolean; - timeRangeBucketSizeMillis: number; - queryString: string; -}; - -const handleQuerySubmit = () => { - - const store = useSearchStore.getState(); - - if (store.searchUiState !== SEARCH_UI_STATE.DEFAULT && - store.searchUiState !== SEARCH_UI_STATE.DONE - ) { - throw new Error("Cannot submit query when not in DEFAULT or DONE state."); - } - - const newTimelineConfig = computeTimelineConfig( - store.timeRange[0].valueOf(), - store.timeRange[1].valueOf(), - ); - - //Fix time range later - const args: QueryArgs = { - queryString: store.queryString, - timestampBegin: store.timeRange[0].valueOf(), - timestampEnd: store.timeRange[1].valueOf(), - ignoreCase: false, - timeRangeBucketSizeMillis: newTimelineConfig.bucketDuration.asMilliseconds(), - }; - - store.updateSearchUiState(SEARCH_UI_STATE.QUERY_ID_PENDING); - - submitQuery(args) - .then((result) => { - store.updateSearchJobId(result.data.searchJobId); - store.updateAggregationJobId(result.data.aggregationJobId); - store.updateSearchUiState(SEARCH_UI_STATE.QUERYING); - console.log("Query ID Returned", result); - }) - .catch((error) => { - console.error("Failed to submit query:", error); - }); -}; - -const handleQueryCancel = () => { - const store = useSearchStore.getState(); - - if (store.searchUiState !== SEARCH_UI_STATE.QUERYING) { - throw new Error("Cannot cancel query when not in QUERYING state."); - } - - const searchJobId = store.searchJobId; - const aggregationJobId = store.aggregationJobId; - - if (!searchJobId || !aggregationJobId) { - throw new Error("Cannot cancel query without jobIDs"); - } - - cancelQuery({ - searchJobId, - aggregationJobId, - }).then(() => { - console.log("Query cancelled successfully"); - }).catch((error) => { - console.error("Failed to cancel query:", error); - }); - - // Reset the state. I dont need to wait for promise to resolve. Another query - // can be submitted before previous cancel finished - store.updateSearchUiState(SEARCH_UI_STATE.DONE); -}; - -export { - handleClearResults, - handleQuerySubmit, - handleQueryCancel, -}; \ No newline at end of file diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/reactive-mongo-queries/useSearchResults.ts b/components/log-viewer-webui/client/src/pages/SearchPage/reactive-mongo-queries/useSearchResults.ts index f574236919..054d8e7503 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/reactive-mongo-queries/useSearchResults.ts +++ b/components/log-viewer-webui/client/src/pages/SearchPage/reactive-mongo-queries/useSearchResults.ts @@ -31,12 +31,3 @@ const useSearchResults = () => { }; export { useSearchResults }; - - -// Call Clear, which will delete the mongo collection. -// But the searchResults are still monitoring it. -// We need to first unsubsribe. -// When the query Finishes, we should drop the cursor? - -// when the query is done. we stop getting updates. -// If we cancel \ No newline at end of file From 3a7690446c9434872f8a32e98b231151dccfa069 Mon Sep 17 00:00:00 2001 From: Marco Date: Tue, 27 May 2025 20:25:04 +0000 Subject: [PATCH 34/55] latest --- .../src/components/ResultsTimeline/index.tsx | 8 ++++---- .../pages/SearchPage/SearchControls/requests.ts | 13 ++++++++++++- .../SearchResults/SearchResultsTimeline/utils.ts | 2 +- .../src/pages/SearchPage/SearchState/typings.ts | 15 +++++++++++++++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/components/log-viewer-webui/client/src/components/ResultsTimeline/index.tsx b/components/log-viewer-webui/client/src/components/ResultsTimeline/index.tsx index e17d2c08f1..1dbe6bae91 100644 --- a/components/log-viewer-webui/client/src/components/ResultsTimeline/index.tsx +++ b/components/log-viewer-webui/client/src/components/ResultsTimeline/index.tsx @@ -181,10 +181,10 @@ const ResultsTimeline = ({ return; } const {min, max} = xAxis; - const newTimeRange = [ - convertZoomTimestampToUtcDatetime(min), - convertZoomTimestampToUtcDatetime(max), - ] + const newTimeRange = { + begin: convertZoomTimestampToUtcDatetime(min), + end: convertZoomTimestampToUtcDatetime(max), + }; onTimelineZoom(newTimeRange); }, diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/requests.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/requests.ts index 6d44b4bb1c..e465299050 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/requests.ts +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/requests.ts @@ -6,6 +6,9 @@ import { SEARCH_UI_STATE } from "../SearchState/typings"; import {QueryJobSchema, QueryJobCreationSchema} from "../../../api/search"; +/** + * Clears the current search and aggregation query results. + */ const handleClearResults = () => { const {searchUiState, searchJobId, aggregationJobId} = useSearchStore.getState(); @@ -25,6 +28,10 @@ const handleClearResults = () => { }; +/** + * Submits a new search query. + * @param payload + */ const handleQuerySubmit = (payload: QueryJobCreationSchema) => { const store = useSearchStore.getState(); @@ -51,6 +58,10 @@ const handleQuerySubmit = (payload: QueryJobCreationSchema) => { }); }; +/** + * Cancels an ongoing search query. + * @param payload + */ const handleQueryCancel = (payload: QueryJobSchema) => { const store = useSearchStore.getState(); @@ -73,4 +84,4 @@ const handleQueryCancel = (payload: QueryJobSchema) => { export { handleQuerySubmit, handleQueryCancel, -} \ No newline at end of file +} diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/utils.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/utils.ts index e40736ac16..e691d945bc 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/utils.ts +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchResults/SearchResultsTimeline/utils.ts @@ -96,4 +96,4 @@ const computeTimelineConfig = ( export { computeTimelineConfig, expandTimeRangeToDurationMultiple, -} \ No newline at end of file +} diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/typings.ts b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/typings.ts index dc6dc1877f..3b6ede7642 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/typings.ts +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchState/typings.ts @@ -1,8 +1,23 @@ +/** + * Enum representing the different states of the search UI. + */ enum SEARCH_UI_STATE { + /** + * Default state when client starts. + */ DEFAULT, + /** + * When query is submitted, but the query ID is not yet available. + */ QUERY_ID_PENDING, + /** + * When the query is being processed. + */ QUERYING, + /** + * When the query is finished or cancelled. + */ DONE, } From eb982875017d8a9f2c9f5e777e70c742fd2f6ec9 Mon Sep 17 00:00:00 2001 From: Marco Date: Tue, 27 May 2025 20:27:38 +0000 Subject: [PATCH 35/55] latest --- .../log-viewer-webui/client/src/api/search.ts | 57 +++++++++++++++++++ .../client/src/api/socket/useCursor.tsx | 13 +++-- .../client/src/components/QueryBox/index.tsx | 13 ++--- .../ResultsTimeline/datetime/index.ts | 1 + .../ResultsTimeline/datetime/typings.ts | 1 + .../src/components/ResultsTimeline/index.tsx | 6 +- .../src/components/ResultsTimeline/utils.ts | 5 +- .../SearchControls/QueryInput/QueryInput.tsx | 29 ++++++---- .../SearchButton/CancelButton.tsx | 23 ++++---- .../SearchButton/SearchButton.tsx | 16 +++--- .../SearchButton/SubmitButton.tsx | 32 +++++++---- .../SearchControls/TimeRangeInput/index.tsx | 13 ++--- .../SearchControls/TimeRangeInput/utils.tsx | 3 +- .../pages/SearchPage/SearchControls/index.tsx | 6 +- .../SearchPage/SearchControls/requests.ts | 31 +++++----- .../SearchResultsTable/index.tsx | 8 +-- .../SearchResultsTable/typings.tsx | 2 +- .../SearchResultsTimeline/index.tsx | 10 +++- .../SearchResultsTimeline/utils.ts | 10 +--- .../pages/SearchPage/SearchState/index.tsx | 9 ++- .../pages/SearchPage/SearchState/typings.ts | 6 +- .../SearchState/useUpdateStateWithMetadata.ts | 20 ++++--- .../client/src/pages/SearchPage/index.tsx | 3 +- .../useResultsMetadata.ts | 16 +++--- .../useSearchResults.ts | 13 ++--- .../client/src/typings/datetime.ts | 2 +- .../log-viewer-webui/client/vite.config.ts | 5 +- 27 files changed, 223 insertions(+), 130 deletions(-) create mode 100644 components/log-viewer-webui/client/src/api/search.ts diff --git a/components/log-viewer-webui/client/src/api/search.ts b/components/log-viewer-webui/client/src/api/search.ts new file mode 100644 index 0000000000..e6d2390d4d --- /dev/null +++ b/components/log-viewer-webui/client/src/api/search.ts @@ -0,0 +1,57 @@ +import axios, {AxiosResponse} from "axios"; + + +// eslint-disable-next-line no-warning-comments +// TODO: Replace with shared types from the `@common` directory once refactoring is completed. Currently, +// these server types (see schemas directory) require typebox dependency so they cannot be moved to +// the `@common` directory with current implementation. +type QueryJobSchema = { + searchJobId: string; + aggregationJobId: string; +}; + +type QueryJobCreationSchema = { + ignoreCase: boolean; + queryString: string; + timeRangeBucketSizeMillis: number; + timestampBegin: number; + timestampEnd: number; +}; + +/** + * + * @param payload + */ +const submitQuery = (payload: QueryJobCreationSchema): Promise> => { + console.log("Submitting query:", JSON.stringify(payload)); + + return axios.post("/api/search/query", payload); +}; + +/** + * + * @param payload + */ +const cancelQuery = (payload: QueryJobSchema): Promise> => { + console.log("Cancelling query:", JSON.stringify(payload)); + + return axios.post("/api/search/cancel", payload); +}; + +/** + * + * @param payload + */ +const clearQueryResults = (payload: QueryJobSchema): Promise> => { + console.log("Clearing query:", JSON.stringify(payload)); + + return axios.delete("/api/search/results", {data: payload}); +}; + +export { + cancelQuery, + clearQueryResults, + QueryJobCreationSchema, + QueryJobSchema, + submitQuery, +}; diff --git a/components/log-viewer-webui/client/src/api/socket/useCursor.tsx b/components/log-viewer-webui/client/src/api/socket/useCursor.tsx index f78b3c0495..deca7a8916 100644 --- a/components/log-viewer-webui/client/src/api/socket/useCursor.tsx +++ b/components/log-viewer-webui/client/src/api/socket/useCursor.tsx @@ -4,8 +4,8 @@ import { useState, } from "react"; -import {MongoCursorSocket} from "./MongoCursorSocket.js"; import {Nullable} from "../../typings/common"; +import {MongoCursorSocket} from "./MongoCursorSocket.js"; /** @@ -14,13 +14,13 @@ import {Nullable} from "../../typings/common"; * @template T The document type returned by the cursor. * @param query Function which returns a `MongoCursorSocket` instance or null. * @param dependencies Array of dependencies for the query. - * @returns + * @return * - If `query` returns a `MongoCursorSocket` instance: - * - null while the subscription is pending, and a reactive array of documents when - * the subscription is ready. + * - null while the subscription is pending, and a reactive array of documents when + * the subscription is ready. * - If `query` returns null, then the hook also returns null. */ -function useCursor( +function useCursor ( query: () => Nullable, dependencies: DependencyList = [] ): Nullable { @@ -31,6 +31,7 @@ function useCursor( if (null === cursor) { setData(null); + return; } @@ -73,6 +74,6 @@ function useCursor( }, dependencies); return data; -}; +} export {useCursor}; diff --git a/components/log-viewer-webui/client/src/components/QueryBox/index.tsx b/components/log-viewer-webui/client/src/components/QueryBox/index.tsx index 3a797038c6..897da760a0 100644 --- a/components/log-viewer-webui/client/src/components/QueryBox/index.tsx +++ b/components/log-viewer-webui/client/src/components/QueryBox/index.tsx @@ -11,7 +11,7 @@ import styles from "./index.module.css"; export interface QueryBoxProps extends InputProps { /** * The progress of the progress bar from `0` to `100`. Hides the bar if null. - */ + */ progress: Nullable; } @@ -24,24 +24,23 @@ export interface QueryBoxProps extends InputProps { * @return */ const QueryBox = ({progress, ...inputProps}: QueryBoxProps) => { - const { token } = theme.useToken(); + const {token} = theme.useToken(); return (
- {progress !== null && ( + {null !== progress && ( + status={"active"} + strokeLinecap={"butt"}/> )}
diff --git a/components/log-viewer-webui/client/src/components/ResultsTimeline/datetime/index.ts b/components/log-viewer-webui/client/src/components/ResultsTimeline/datetime/index.ts index 45caf235ef..2aa8f1b63d 100644 --- a/components/log-viewer-webui/client/src/components/ResultsTimeline/datetime/index.ts +++ b/components/log-viewer-webui/client/src/components/ResultsTimeline/datetime/index.ts @@ -2,6 +2,7 @@ import dayjs, {Dayjs} from "dayjs"; import DayjsTimezone from "dayjs/plugin/timezone"; import DayjsUtc from "dayjs/plugin/utc"; + dayjs.extend(DayjsUtc); dayjs.extend(DayjsTimezone); diff --git a/components/log-viewer-webui/client/src/components/ResultsTimeline/datetime/typings.ts b/components/log-viewer-webui/client/src/components/ResultsTimeline/datetime/typings.ts index 1380bc5b4c..78982a353b 100644 --- a/components/log-viewer-webui/client/src/components/ResultsTimeline/datetime/typings.ts +++ b/components/log-viewer-webui/client/src/components/ResultsTimeline/datetime/typings.ts @@ -1,5 +1,6 @@ import {Dayjs} from "dayjs"; + // eslint-disable-next-line no-warning-comments // TODO: replace `TimeRange` with [dayjs.Dayjs, dayjs.Dayjs] // to keep consistent with `TimeRangeInput`. diff --git a/components/log-viewer-webui/client/src/components/ResultsTimeline/index.tsx b/components/log-viewer-webui/client/src/components/ResultsTimeline/index.tsx index 1dbe6bae91..7126d620c8 100644 --- a/components/log-viewer-webui/client/src/components/ResultsTimeline/index.tsx +++ b/components/log-viewer-webui/client/src/components/ResultsTimeline/index.tsx @@ -16,14 +16,12 @@ import zoomPlugin from "chartjs-plugin-zoom"; import dayjs from "dayjs"; import {Nullable} from "../../typings/common"; +import {DATETIME_FORMAT_TEMPLATE} from "../../typings/datetime"; import { convertUtcDatetimeToSameLocalDate, convertZoomTimestampToUtcDatetime, } from "./datetime"; -import { - TimeRange, -} from "./datetime/typings"; -import {DATETIME_FORMAT_TEMPLATE} from "../../typings/datetime"; +import {TimeRange} from "./datetime/typings"; import styles from "./index.module.css"; import { TimelineBucket, diff --git a/components/log-viewer-webui/client/src/components/ResultsTimeline/utils.ts b/components/log-viewer-webui/client/src/components/ResultsTimeline/utils.ts index e26ed959d5..90e4c62fbd 100644 --- a/components/log-viewer-webui/client/src/components/ResultsTimeline/utils.ts +++ b/components/log-viewer-webui/client/src/components/ResultsTimeline/utils.ts @@ -1,6 +1,5 @@ -import { - TimelineBucket, -} from "./typings"; +import {TimelineBucket} from "./typings"; + /** * Converts an array of timeline buckets into an array of objects compatible with Chart.js. diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput/QueryInput.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput/QueryInput.tsx index 72cdcf4552..55725e801b 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput/QueryInput.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/QueryInput/QueryInput.tsx @@ -1,12 +1,17 @@ -import QueryBox from "../../../../components/QueryBox"; -import useSearchStore from "../../SearchState/index"; import { useEffect, useRef, useState, } from "react"; -import { SEARCH_UI_STATE } from "../../SearchState/typings"; -import { PROGRESS_INCREMENT, PROGRESS_INTERVAL_MILLIS } from "./typings"; + +import QueryBox from "../../../../components/QueryBox"; +import useSearchStore from "../../SearchState/index"; +import {SEARCH_UI_STATE} from "../../SearchState/typings"; +import { + PROGRESS_INCREMENT, + PROGRESS_INTERVAL_MILLIS, +} from "./typings"; + /** * Renders a query input and pseudo progress bar. @@ -14,27 +19,27 @@ import { PROGRESS_INCREMENT, PROGRESS_INTERVAL_MILLIS } from "./typings"; * @return */ const QueryInput = () => { - const {queryString, updateQueryString, searchUiState}= useSearchStore(); + const {queryString, updateQueryString, searchUiState} = useSearchStore(); const [pseudoProgress, setPseudoProgress] = useState(0); const intervalIdRef = useRef(0); useEffect(() => { - if (searchUiState === SEARCH_UI_STATE.QUERY_ID_PENDING) - { - if (intervalIdRef.current !== 0) { + if (searchUiState === SEARCH_UI_STATE.QUERY_ID_PENDING) { + if (0 !== intervalIdRef.current) { console.warn("Interval already set for submitted query"); + return; } intervalIdRef.current = window.setInterval(() => { setPseudoProgress((v) => { - if (v + PROGRESS_INCREMENT >= 100) { + if (100 <= v + PROGRESS_INCREMENT) { return 100; } + return v + PROGRESS_INCREMENT; }); }, PROGRESS_INTERVAL_MILLIS); - } - else if (searchUiState === SEARCH_UI_STATE.DONE) { + } else if (searchUiState === SEARCH_UI_STATE.DONE) { clearInterval(intervalIdRef.current); intervalIdRef.current = 0; setPseudoProgress(0); @@ -51,11 +56,11 @@ const QueryInput = () => { placeholder={"Enter your query"} progress={pseudoProgress} size={"large"} + value={queryString} disabled={ searchUiState === SEARCH_UI_STATE.QUERY_ID_PENDING || searchUiState === SEARCH_UI_STATE.QUERYING } - value={queryString} onChange={(e) => { updateQueryString(e.target.value); }}/> diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx index 0fea49e4b6..84e36a27a3 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/CancelButton.tsx @@ -1,7 +1,9 @@ -import { CloseOutlined } from "@ant-design/icons"; -import { Button } from "antd"; -import { handleQueryCancel } from "../requests"; +import {CloseOutlined} from "@ant-design/icons"; +import {Button} from "antd"; + import useSearchStore from "../../SearchState/index"; +import {handleQueryCancel} from "../requests"; + /** * Renders a button to cancel the search query. @@ -13,23 +15,24 @@ const CancelButton = () => { return ( - ) + ); }; export default CancelButton; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx index 135da2449a..ab29a22729 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SearchButton.tsx @@ -1,7 +1,7 @@ import useSearchStore from "../../SearchState/index"; -import SubmitButton from "./SubmitButton"; +import {SEARCH_UI_STATE} from "../../SearchState/typings"; import CancelButton from "./CancelButton"; -import { SEARCH_UI_STATE } from "../../SearchState/typings"; +import SubmitButton from "./SubmitButton"; /** @@ -14,11 +14,13 @@ const SearchButton = () => { return ( <> - {(store.searchUiState === SEARCH_UI_STATE.QUERYING) ? ( - - ) : ( - - )} + {(store.searchUiState === SEARCH_UI_STATE.QUERYING) ? + ( + + ) : + ( + + )} ); }; diff --git a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx index 8d391c9a55..979854fb77 100644 --- a/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx +++ b/components/log-viewer-webui/client/src/pages/SearchPage/SearchControls/SearchButton/SubmitButton.tsx @@ -1,23 +1,35 @@ -import { SearchOutlined } from "@ant-design/icons"; -import { Button, Tooltip } from "antd"; -import { handleQuerySubmit} from "../requests"; -import useSearchStore, { SEARCH_STATE_DEFAULT } from "../../SearchState/index"; +import {SearchOutlined} from "@ant-design/icons"; +import { + Button, + Tooltip, +} from "antd"; + +import {computeTimelineConfig} from "../../SearchResults/SearchResultsTimeline/utils"; +import useSearchStore, {SEARCH_STATE_DEFAULT} from "../../SearchState/index"; +import {SEARCH_UI_STATE} from "../../SearchState/typings"; +import {handleQuerySubmit} from "../requests"; import styles from "./index.module.css"; -import { SEARCH_UI_STATE } from "../../SearchState/typings"; -import { computeTimelineConfig } from "../../SearchResults/SearchResultsTimeline/utils"; + +/** + * + */ const SubmitButton = () => { const {searchUiState, timeRange, queryString} = useSearchStore(); const isQueryStringEmpty = queryString === SEARCH_STATE_DEFAULT.queryString; return ( - +