feat(new-webui): Integrate database queries for IngestPage dashboard components.

components/log-viewer-webui/client/src/pages/IngestPage/Details/Files.tsx

@@ -1,19 +1,17 @@
+import useIngestStore from "../IngestState";
 import DetailsCard from "./DetailsCard";
 
 
-// eslint-disable-next-line no-warning-comments
-// TODO: Replace with values from database once api implemented.
-const DUMMY_FILES = 124;
-
 /**
  * Renders the files statistic.
  *
  * @return
  */
 const Files = () => {
+    const files = useIngestStore((state) => state.numFiles);
     return (
         <DetailsCard
-            stat={DUMMY_FILES.toString()}
+            stat={files.toString()}
             title={"Files"}/>
     );
 };

components/log-viewer-webui/client/src/pages/IngestPage/Details/Messages.tsx

@@ -1,19 +1,17 @@
+import useIngestStore from "../IngestState";
 import DetailsCard from "./DetailsCard";
 
 
-// eslint-disable-next-line no-warning-comments
-// TODO: Replace with values from database once api implemented.
-const DUMMY_MESSAGES = 1235844;
-
 /**
  * Renders the messages statistic.
  *
  * @return
  */
 const Messages = () => {
+    const messages = useIngestStore((state) => state.numMessages);
     return (
         <DetailsCard
-            stat={DUMMY_MESSAGES.toString()}
+            stat={messages.toString()}
             title={"Messages"}/>
     );
 };

components/log-viewer-webui/client/src/pages/IngestPage/Details/TimeRange.tsx

@@ -1,13 +1,7 @@
-import dayjs from "dayjs";
-
+import useIngestStore from "../IngestState";
 import DetailsCard from "./DetailsCard";
 
 
-// eslint-disable-next-line no-warning-comments
-// TODO: Replace with values from database once api implemented.
-const DUMMY_START_DATE = "2021-12-14";
-const DUMMY_END_DATE = "2025-04-16";
-
 const DATE_FORMAT = "MMMM D, YYYY";
 
 /**
@@ -16,8 +10,9 @@ const DATE_FORMAT = "MMMM D, YYYY";
  * @return
  */
 const TimeRange = () => {
-    const formattedStat = `${dayjs(DUMMY_START_DATE).format(DATE_FORMAT)} -
-        ${dayjs(DUMMY_END_DATE).format(DATE_FORMAT)}`;
+    const [startDate, endDate] = useIngestStore((state) => state.timeRange);
+    const formattedStat = `${startDate.format(DATE_FORMAT)} -
+        ${endDate.format(DATE_FORMAT)}`;
 
     return (
         <DetailsCard

components/log-viewer-webui/client/src/pages/IngestPage/IngestState/QueryJobsResp.ts

@@ -0,0 +1,128 @@
+import dayjs from "dayjs";
+
+import {Nullable} from "../../../typings/common";
+import {JobData} from "../Jobs/typings";
+
+
+/**
+ * Enum of compression job statuses, matching the `CompressionJobStatus` class in
+ * `job_orchestration.scheduler.constants`.
+ *
+ * @enum {CompressionJobStatus}
+ */
+enum CompressionJobStatus {
+    PENDING = 0,
+    RUNNING = 1,
+    SUCCEEDED = 2,
+    FAILED = 3,
+}
+
+/**
+ * Convert JobStatus to Antd's badge string.
+ *
+ * @param status
+ * @return
+ * @throws an Error if status is not a CompressionJobStatus.
+ */
+const convertJobStatusToBadgeString = (status: CompressionJobStatus) => {
+    switch (status) {
+        case CompressionJobStatus.PENDING:
+            return "warning";
+        case CompressionJobStatus.RUNNING:
+            return "processing";
+        case CompressionJobStatus.SUCCEEDED:
+            return "success";
+        case CompressionJobStatus.FAILED:
+            return "error";
+        default:
+            throw new Error("Unexpected status value.");
+    }
+};
+
+const BYTES_PER_KIBIBYTE = 1024;
+
+/**
+ * Computes a human-readable representation of a size in bytes.
+ *
+ * @param num
+ * @return
+ */
+const computeHumanSize = (num: number) => {
+    const siPrefixes = ["",
+        "K",
+        "M",
+        "G",
+        "T",
+        "P",
+        "E",
+        "Z"];
+
+    for (let i = 0; i < siPrefixes.length; ++i) {
+        if (BYTES_PER_KIBIBYTE > Math.abs(num)) {
+            return `${Math.round(num)} ${siPrefixes[i]}B`;
+        }
+        num /= BYTES_PER_KIBIBYTE;
+    }
+
+    return `${Math.round(num)} B`;
+};
+
+interface QueryJobsItem {
+    compressed_size: number;
+    duration: Nullable<number>;
+    retrieval_time: number;
+    start_time: Nullable<string>;
+    status: number;
+    status_msg: string;
+    uncompressed_size: number;
+    update_time: string;
+    _id: number;
+}
+
+type QueryJobsResp = QueryJobsItem[];
+
+/**
+ * Convert a QuryJobsItem to JobData
+ *
+ * @param job
+ * @return
+ */
+const convertQueryJobsItemToJobData = (job: QueryJobsItem): JobData => {
+    let uncompressedSizeText = "";
+    let compressedSizeText = "";
+    let speedText = "";
+
+    if (null === job.duration && null !== job.start_time) {
+        job.duration = dayjs().unix() - dayjs(job.start_time).unix();
+    }
+
+    const uncompressedSize = Number(job.uncompressed_size);
+    if (false === isNaN(uncompressedSize) && 0 !== uncompressedSize) {
+        uncompressedSizeText = computeHumanSize(uncompressedSize);
+    }
+
+    const compressedSize = Number(job.compressed_size);
+    if (false === isNaN(compressedSize) && 0 !== compressedSize) {
+        compressedSizeText = computeHumanSize(compressedSize);
+    }
+
+    if (
+        false === isNaN(uncompressedSize) &&
+    0 !== uncompressedSize &&
+    0 < job.duration
+    ) {
+        speedText = `${computeHumanSize(job.uncompressed_size / job.duration)}/s`;
+    }
+
+    return {
+        compressedSize: compressedSizeText,
+        dataIngested: uncompressedSizeText,
+        jobId: String(job._id),
+        key: String(job._id),
+        speed: speedText,
+        status: convertJobStatusToBadgeString(job.status),
+    };
+};
+
+export type {QueryJobsResp};
+export {convertQueryJobsItemToJobData};

components/log-viewer-webui/client/src/pages/IngestPage/IngestState/index.ts

@@ -0,0 +1,100 @@
+import axios, {HttpStatusCode} from "axios";
+import dayjs from "dayjs";
+import {create} from "zustand";
+
+import {JobData} from "../Jobs/typings";
+import {
+    convertQueryJobsItemToJobData,
+    QueryJobsResp,
+} from "./QueryJobsResp";
+import {
+    getQueryJobsSql,
+    getQueryStatsSql,
+    QueryStatsResp,
+} from "./sql";
+
+
+interface IngestState {
+    compressedSize: number;
+    uncompressedSize: number;
+    timeRange: [dayjs.Dayjs, dayjs.Dayjs];
+    numMessages: number;
+    numFiles: number;
+    jobs: JobData[];
+    updateStats: () => Promise<void>;
+    updateJobs: () => Promise<void>;
+}
+
+/**
+ * Default values of the ingest state.
+ */
+const INGEST_STATE_DEFAULT = Object.freeze<IngestState>({
+    compressedSize: 0,
+    jobs: [],
+    numFiles: 0,
+    numMessages: 0,
+    timeRange: [dayjs(),
+        dayjs()],
+    uncompressedSize: 0,
+    updateJobs: async () => {
+    },
+    updateStats: async () => {
+    },
+});
+
+const JOB_DAYS_TO_SHOW: number = 30;
+
+const useIngestStore = create<IngestState>((set) => ({
+    ...INGEST_STATE_DEFAULT,
+    updateStats: async () => {
+        try {
+            const response = await axios.post<QueryStatsResp>("/query/sql", {
+                queryString: getQueryStatsSql(),
+            });
+
+            if (Number(HttpStatusCode.Ok) !== response.status) {
+                throw new Error();
+            }
+            const [resp] = response.data;
+            if ("undefined" === typeof resp) {
+                throw new Error();
+            }
+            set(() => ({
+                compressedSize: resp.total_compressed_size,
+                numFiles: resp.num_files,
+                numMessages: resp.num_messages,
+                timeRange: [dayjs(resp.begin_timestamp),
+                    dayjs(resp.end_timestamp)],
+                uncompressedSize: resp.total_uncompressed_size,
+            }));
+        } catch (error: unknown) {
+            console.error("An error occurred when fetching stats: ", error);
+        }
+    },
+    updateJobs: async () => {
+        try {
+            const beginTimestamp = dayjs().subtract(JOB_DAYS_TO_SHOW, "days")
+                .unix();
+            const response = await axios.post<QueryJobsResp>("/query/sql", {
+                queryString: getQueryJobsSql(beginTimestamp),
+            });
+
+            if (Number(HttpStatusCode.Ok) !== response.status) {
+                throw new Error();
+            }
+            const resp = response.data;
+            const jobs = resp
+                .map((item): JobData => convertQueryJobsItemToJobData(item))
+                .sort((a, b): number => Number(a.key) - Number(b.key));
+
+            set(() => ({
+                jobs: jobs,
+            }));
+        } catch (error: unknown) {
+            console.error("An error occurred when fetching stats: ", error);
+        }
+    },
+}));
+
+export {INGEST_STATE_DEFAULT};
+export default useIngestStore;