Allow multiple run id online under same project id

visualDust · Dec 12, 2023 · 38d7306 · 38d7306
2 parents f67d2b0 + e5e3abc
commit 38d7306
Show file tree

Hide file tree

Showing 26 changed files with 267 additions and 192 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -12,6 +12,7 @@ repos:
       - id: check-yaml
       - id: check-toml
       - id: check-added-large-files
+      - id: check-merge-conflict
   # For Python files
   - repo: https://github.com/psf/black.git
     rev: 23.9.1

diff --git a/README.md b/README.md
@@ -7,11 +7,11 @@ Python API, backend server, frontend, ALL IN ONE. A tool box for Logging/Debuggi
 [![wakatime](https://wakatime.com/badge/user/b93a26b6-8ea1-44ef-99ed-bcb6e2c732f1/project/8f99904d-dbb1-49e4-814d-8d18bf1e6d1c.svg)](https://wakatime.com/badge/user/b93a26b6-8ea1-44ef-99ed-bcb6e2c732f1/project/8f99904d-dbb1-49e4-814d-8d18bf1e6d1c) [![pytest](https://github.com/visualDust/neetbox/actions/workflows/poetry-pytest.yml/badge.svg)](https://github.com/visualDust/neetbox/actions/workflows/poetry-pytest.yml) ![GitHub Workflow Status (with event)](https://img.shields.io/github/actions/workflow/status/visualdust/neetbox/build-and-publish-pypi.yml) ![PyPI - Version](https://img.shields.io/pypi/v/neetbox)
  ![PyPI - Downloads](https://img.shields.io/pypi/dw/neetbox) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![codecov](https://codecov.io/gh/visualDust/neetbox/graph/badge.svg?token=WYWLQ4YKZJ)](https://codecov.io/gh/visualDust/neetbox)
 
-## screenshot
+## Screenshot
 
 ![](./docs/static/img/screenshot.jpg)
 
-## dev
+## Dev
 
 NEETBOX is under heavy development.
 
@@ -26,11 +26,11 @@ NEETBOX is under heavy development.
 - [ ] attach remote logging in command line cli
 - [ ] distinguish different runs
 
-## docs
+## Docs
 
 [neetbox.550w.host](https://neetbox.550w.host). (APIs are ready but we are not ready for the doc yet)
 
-## quick start
+## Quick Start
 
 install neetbox:
 ```bash
@@ -50,7 +50,7 @@ import neetbox
 
 run your code and visit https://localhost:20202 to see your dashboard.
 
-## usage examples
+## Usage Examples
 
 [how to guides](todo) provides easy examples of basic neetbox funcionalities.
 

diff --git a/frontend/src/components/dashboard/project/actions.tsx b/frontend/src/components/dashboard/project/actions.tsx
@@ -58,11 +58,11 @@ export const ActionItem = memo(({ name, actionOptions: options, blocking, setBlo
   );
   const [running, setCurrentBlocking] = useState(false);
   const [result, setResult] = useState<string | null>(null);
-  const { projectId, isOnlineRun } = useCurrentProject()!;
+  const { projectId, runId, isOnlineRun } = useCurrentProject()!;
   const handleRun = () => {
     if (options.blocking) setBlocking(true);
     setCurrentBlocking(true);
-    getProject(projectId).sendAction(name, args, ({ error: err, result: res }) => {
+    getProject(projectId).sendAction(runId!, name, args, ({ error: err, result: res }) => {
       if (options.blocking) setBlocking(false);
       setCurrentBlocking(false);
       setResult(err ? `error:\n${err}` : `result:\n${JSON.stringify(res)}`);

diff --git a/frontend/src/components/dashboard/project/hardware/index.tsx b/frontend/src/components/dashboard/project/hardware/index.tsx
@@ -6,17 +6,19 @@ import { TimeDataMapper } from "../../../../utils/timeDataMapper";
 import { CPUGraph } from "./cpugraph";
 import { GPUGraph } from "./gpugraph";
 import { RAMGraph } from "./ramgraph";
+import { fetchDataCount } from "./utils";
 
 export function Hardware() {
-  const { projectId } = useCurrentProject();
+  const { projectId, runId } = useCurrentProject();
   const data = useProjectData<HardwareInfo>({
     projectId,
+    runId,
     type: "hardware",
     transformHTTP: (x) => ({ timestamp: x.timestamp, ...x.metadata }),
     transformWS: (x) => ({ timestamp: x.timestamp, ...x.payload }),
-    limit: 240,
+    limit: fetchDataCount,
   });
-  return data ? (
+  return data?.length ? (
     <div>
       {data.every((x) => x.gpus?.length) ? (
         data[0].gpus.map((_, i) => <GPUGraph key={i} data={new TimeDataMapper(data, (x) => x.gpus[i])} />)

diff --git a/frontend/src/components/dashboard/project/hardware/utils.ts b/frontend/src/components/dashboard/project/hardware/utils.ts
@@ -1,9 +1,13 @@
 import { TimeDataMapper } from "../../../../utils/timeDataMapper";
 
+const viewRangeSeconds = 300;
+const dataInterval = 2;
+export const fetchDataCount = Math.ceil(viewRangeSeconds / dataInterval);
+
 export function getTimeAxisOptions(mapper: TimeDataMapper) {
   const latestTime = new Date(mapper.data[mapper.data.length - 1].timestamp).getTime();
   return {
-    min: latestTime - 120 * 1000,
+    min: latestTime - viewRangeSeconds * 1000,
     max: latestTime,
   };
 }
diff --git a/frontend/src/components/dashboard/project/runSelect.tsx b/frontend/src/components/dashboard/project/runSelect.tsx
@@ -6,10 +6,8 @@ import Loading from "../../loading";
 import { useCurrentProject, useProjectRunIds } from "../../../hooks/useProject";
 import { fetcher } from "../../../services/api";
 
-export const RunSelect = memo(({ setRunId }: { setRunId }) => {
-  const { projectId, runId, isOnlineRun, projectOnline } = useCurrentProject();
-  const { data: runIds, mutate: mutateRunIds } = useProjectRunIds(projectId);
-
+export const RunSelect = memo((props: any) => {
+  const { setRunId, runIds, mutateRunIds, projectId, runId, isOnlineRun } = props;
   const items = useMemo(
     () =>
       [...(runIds ?? [])]
@@ -47,12 +45,12 @@ export const RunSelect = memo(({ setRunId }: { setRunId }) => {
             <>
               {isOnlineRun ? (
                 <Tag color="green">Online</Tag>
-              ) : runId != items[0].id ? (
+              ) : runId != items[0].runid ? (
                 <Tag color="orange">History</Tag>
               ) : (
                 <Tag color="red">Offline</Tag>
               )}
-              {items.find((x) => x.id == p.value).timestamp}
+              {items.find((x) => x.runid == p.value).timestamp}
             </>
           )}
         >
@@ -62,12 +60,12 @@ export const RunSelect = memo(({ setRunId }: { setRunId }) => {
                 style={{ gap: "5px" }}
                 // workaround for semi-design select: won't update label unless key changed
                 // https://github.com/DouyinFE/semi-design/blob/c7982af07ad92e6cafe72d96604ed63a8ca595d6/packages/semi-ui/select/index.tsx#L640
-                key={item.id + "-" + item.metadata?.name + "-" + (i == 0 ? projectOnline : "")}
-                value={item.id}
+                key={item.runid + "-" + item.metadata?.name + "-" + item.online}
+                value={item.runid}
               >
                 <span style={{ fontFamily: "monospace", fontSize: 12 }}>{item.timestamp}</span>
                 <span style={{ fontFamily: "monospace", fontSize: 12 }}>
-                  ({item.metadata?.name ?? item.id})
+                  ({item.metadata?.name ?? item.runid})
                 </span>
                 <span style={{ flex: 1 }}></span>
                 <Button
@@ -80,7 +78,7 @@ export const RunSelect = memo(({ setRunId }: { setRunId }) => {
                     setEditing(item);
                   }}
                 />
-                {!(i == 0 && projectOnline) && (
+                {!item.online && (
                   <Button
                     type="danger"
                     icon={<IconDelete />}
@@ -90,11 +88,11 @@ export const RunSelect = memo(({ setRunId }: { setRunId }) => {
                       closeDropDown();
                       Modal.error({
                         title: "Are you sure?",
-                        content: `Deleting run ${item.timestamp} (${item.id})`,
+                        content: `Deleting run ${item.timestamp} (${item.runid})`,
                         centered: true,
                         onOk: async () => {
                           // await new Promise((r) => setTimeout(r, 1000));
-                          await fetcher(`/project/${projectId}/run/${item.id}`, { method: "DELETE" });
+                          await fetcher(`/project/${projectId}/run/${item.runid}`, { method: "DELETE" });
                           mutateRunIds();
                           Toast.success({
                             content: `Deleted ${item.timestamp}`,
@@ -104,7 +102,7 @@ export const RunSelect = memo(({ setRunId }: { setRunId }) => {
                     }}
                   />
                 )}
-                {i == 0 && (projectOnline ? <Tag color="green">Online</Tag> : <Tag color="red">Offline</Tag>)}
+                {item.online ? <Tag color="green">Online</Tag> : <Tag color="red">Offline</Tag>}
               </Select.Option>
             );
           })}
@@ -147,7 +145,7 @@ const RunEditor = memo((props: { data: any; onResult: (edited: boolean) => void
       onCancel={() => props.onResult(false)}
       onOk={async () => {
         const values = formRef.current.formApi.getValues();
-        await fetcher(`/project/${projectId}/run/${data.id}`, {
+        await fetcher(`/project/${projectId}/run/${data.runid}`, {
           method: "PUT",
           headers: { "content-type": "application/json" },
           body: JSON.stringify({
@@ -160,7 +158,7 @@ const RunEditor = memo((props: { data: any; onResult: (edited: boolean) => void
       centered
     >
       <Form initValues={data} ref={formRef as any}>
-        <Form.Input field="id" label="ID" disabled></Form.Input>
+        <Form.Input field="runid" label="ID" disabled></Form.Input>
         <Form.Input field="metadata.name" label="Name"></Form.Input>
       </Form>
     </Modal>

diff --git a/frontend/src/components/dashboard/project/scatters.tsx b/frontend/src/components/dashboard/project/scatters.tsx
@@ -27,8 +27,8 @@ export const ScatterViewer = memo(({ series }: { series: string }) => {
     projectId,
     runId,
     conditions: { series },
-    transformWS: (x) => ({ x: x.payload.x, y: x.payload.y }),
-    transformHTTP: (x) => ({ x: x.metadata.x, y: x.metadata.y }),
+    transformWS: (x) => ({ id: x.id, x: x.payload.x, y: x.payload.y }),
+    transformHTTP: (x) => ({ id: x.id, x: x.metadata.x, y: x.metadata.y }),
   });
   const [hadZoom, setHadZoom] = useState<string | null>(null);
   const dataZoomOption = (init = false) => {

diff --git a/frontend/src/hooks/useProject.ts b/frontend/src/hooks/useProject.ts
@@ -13,16 +13,14 @@ export const ProjectContext = createContext<{
   projectName?: string;
   runId?: string;
   isOnlineRun: boolean;
-  projectOnline: boolean;
 } | null>(null);
 
 export function useCurrentProject() {
   return useContext(ProjectContext)!;
 }
 
 export function useProjectStatus(id: string) {
-  const { data } = useAPI(`/project/${id}`, { refreshInterval: 5000 });
-  return data;
+  return useAPI(`/project/${id}`, { refreshInterval: 5000 });
 }
 
 export function useProjectRunIds(id: string) {

diff --git a/frontend/src/pages/console/projectDashboard.tsx b/frontend/src/pages/console/projectDashboard.tsx
@@ -24,7 +24,7 @@ function ProjectDashboard() {
   const [searchParams, setSearchParams] = useSearchParams();
   if (!projectId) throw new Error("projectId required");
 
-  const status = useProjectStatus(projectId);
+  const { data: status, mutate } = useProjectStatus(projectId);
   const projectName = status?.name ?? projectId;
 
   useEffect(() => {
@@ -33,26 +33,27 @@ function ProjectDashboard() {
     }
   }, [projectId, projectName]);
 
-  const { data: runIds } = useProjectRunIds(projectId);
-  const lastRunId = runIds ? runIds[runIds.length - 1].id : undefined;
+  const runIds = status?.runids;
+  const lastRunId = runIds ? runIds[runIds.length - 1] : undefined;
   const paramRun = searchParams.get("run");
-  const paramRunFound = runIds?.find((x) => x.id == paramRun)?.id;
-  const runId = paramRunFound ?? lastRunId;
-  const projectOnline = Boolean(status?.online);
-  const isOnlineRun = Boolean(runId && runId == lastRunId && status?.online);
+  const paramRunFound = runIds?.find((x) => x.runid == paramRun);
+  const runInfo = paramRunFound ?? lastRunId;
+  const runId = runInfo?.runid;
+  const isOnlineRun = Boolean(runInfo?.online);
 
   useEffect(() => {
-    if (paramRun && !paramRunFound) {
+    if (runIds && paramRun && !paramRunFound) {
       addNotice({
         type: "error",
         title: `Can not find run "${paramRun}"`,
         content: "Showing the latest run",
       });
     }
-  }, [paramRun, paramRunFound]);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [!!runIds, paramRun, paramRunFound]);
 
   const setRunId = (id: string) => {
-    if (id === lastRunId) {
+    if (id === lastRunId.runid) {
       setSearchParams({});
     } else {
       setSearchParams({ run: id });
@@ -65,9 +66,8 @@ function ProjectDashboard() {
       projectName,
       runId,
       isOnlineRun,
-      projectOnline,
     }),
-    [projectId, projectName, runId, isOnlineRun, projectOnline],
+    [projectId, projectName, runId, isOnlineRun],
   );
 
   return (
@@ -76,7 +76,7 @@ function ProjectDashboard() {
         <AppTitle
           extra={
             <ProjectContext.Provider key={projectId} value={projectContextData}>
-              <RunSelect setRunId={setRunId} />
+              <RunSelect {...{ setRunId, runIds, mutateRunIds: mutate, projectId, runId, isOnlineRun }} />
             </ProjectContext.Provider>
           }
         >

diff --git a/frontend/src/pages/console/sidebar.tsx b/frontend/src/pages/console/sidebar.tsx
@@ -3,6 +3,7 @@ import { IconHome, IconListView } from "@douyinfe/semi-icons";
 import { useLocation, useNavigate } from "react-router-dom";
 import { useAPI } from "../../services/api";
 import Loading from "../../components/loading";
+import "./sidebarStyleFix.css";
 
 export default function ConsoleNavBar() {
   const location = useLocation();
@@ -24,7 +25,7 @@ export default function ConsoleNavBar() {
           items: data
             ? data.map(({ projectid: id, name, online }) => ({
                 text: (
-                  <Space style={{ width: "100%" }}>
+                  <Space style={{ width: "100%", minWidth: "140px" }}>
                     <Typography.Text
                       type={online ? "primary" : "tertiary"}
                       style={{ flex: "1 1 50%" }}

diff --git a/frontend/src/pages/console/sidebarStyleFix.css b/frontend/src/pages/console/sidebarStyleFix.css
@@ -0,0 +1,4 @@
+.semi-navigation-sub-wrap .semi-navigation-sub-title-text,
+.semi-navigation-item-text {
+  flex: 1;
+}
diff --git a/frontend/src/services/projects.ts b/frontend/src/services/projects.ts
@@ -20,10 +20,16 @@ export class Project {
     checkLogForNotification(log, this);
   }
 
-  sendAction(action: string, args: Record<string, string>, onReply?: (result: { error; result }) => void) {
+  sendAction(
+    runId: string,
+    action: string,
+    args: Record<string, string>,
+    onReply?: (result: { error; result }) => void,
+  ) {
     this.wsClient.send(
       {
         "event-type": "action",
+        runid: runId,
         payload: {
           name: action,
           args,

diff --git a/neetbox/__init__.py b/neetbox/__init__.py
@@ -22,7 +22,24 @@ def _load_workspace(connect_daemon=True):
 else:
     _load_workspace(connect_daemon=True)
 
-from neetbox.client import action, add_image, add_scalar, listen, watch
+from neetbox.client import (
+    action,
+    add_figure,
+    add_hyperparams,
+    add_image,
+    add_scalar,
+    listen,
+    watch,
+)
 from neetbox.logging import logger
 
-__all__ = ["add_image", "add_scalar", "action", "logger", "watch", "listen"]
+__all__ = [
+    "add_image",
+    "add_figure",
+    "add_scalar",
+    "add_hyperparams",
+    "action",
+    "logger",
+    "watch",
+    "listen",
+]
diff --git a/neetbox/_daemon/_protocol.py b/neetbox/_daemon/_protocol.py
@@ -53,6 +53,7 @@ class EventMsg:
     who: str = None
     timestamp: str = get_timestamp()
     history_len: int = -1
+    id: int = None  # id in database
 
     def json(self):
         return {
@@ -65,6 +66,7 @@ def json(self):
             PAYLOAD_KEY: self.payload,
             TIMESTAMP_KEY: self.timestamp,
             HISTORY_LEN_KEY: self.history_len,
+            ID_KEY: self.id,
         }
 
     def dumps(self):
@@ -84,6 +86,7 @@ def loads(cls, src):
             event_id=src.get(EVENT_ID_KEY, -1),
             timestamp=src.get(TIMESTAMP_KEY, get_timestamp()),
             history_len=src.get(HISTORY_LEN_KEY, -1),
+            id=src.get(ID_KEY, None),
         )
 
     @classmethod