Add nfs storage info in gpu card page , fix the issue of blank load in job page, issue of prreemtible job total and training job submission error (microsoft#613)

LeoHongyi · YinYangOfDao · commit d382056105db · 2019-11-05T01:56:16.000Z
diff --git a/src/dashboard/src/pages/ClusterStatus/index.tsx b/src/dashboard/src/pages/ClusterStatus/index.tsx
@@ -65,8 +65,8 @@ const ClusterStatus: FC = () => {
     response['prometheus'] = prometheus;
     return response;
   }
-  const fetchClusterStatus = () => {
-    if (clusters) {
+  const fetchClusterStatus = (mount: boolean) => {
+    if (clusters && mount) {
       const params = new URLSearchParams({
         query:`count+(task_gpu_percent{vc_name="${selectedTeam}"}+==+0)+by+(username)`,
       });
@@ -172,6 +172,9 @@ const ClusterStatus: FC = () => {
               if (!mu.hasOwnProperty('idle')) {
                 mu['idle'] = "0";
               }
+              if (!mu.hasOwnProperty('preemptableGPU')) {
+                mu['preemptableGPU'] = "0";
+              }
             });
             let finalUserStatus = _.values(mergeTwoObjsByKey(tmpMerged,prometheusResp,'userName'));
             let totalRow: any = {};
@@ -180,8 +183,9 @@ const ClusterStatus: FC = () => {
             totalRow['idle'] = 0;
             totalRow['usedGPU'] = 0;
             totalRow['idleGPU'] = 0;
+            totalRow['preemptableGPU'] = 0;
             for (let us of finalUserStatus) {
-              console.log(us);
+              console.log(us['preemptableGPU']);
               totalRow['booked'] += parseInt(us['booked']);
               totalRow['idle'] += parseInt(us['idle']);
               totalRow['usedGPU'] += parseInt(us['usedGPU']);
@@ -210,21 +214,21 @@ const ClusterStatus: FC = () => {
     let mount = true;
     let timeout: any;
     if (mount) {
-      fetchClusterStatus()
-      timeout = setTimeout(() => {fetchClusterStatus()},30000)
+      fetchClusterStatus(mount)
+      timeout = setTimeout(() => {fetchClusterStatus(mount)}, 30000)
     }
 
     return () => {
       mount = false;
       clearTimeout(timeout)
     }
   },[clusters, selectedTeam])
-  const handleChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+  const handleChange = (event: React.ChangeEvent<HTMLInputElement>, mount: boolean) => {
     setSelectedValue(event.target.value);
     localStorage.setItem('selectedCluster', event.target.value);
     const filteredVCStatus: any = vcStatus.filter((vc)=>vc['ClusterName'] === event.target.value);
     console.log(vcStatus)
-    fetchClusterStatus()
+    fetchClusterStatus(mount)
     setNodeStatus(filteredVCStatus[0]['node_status']);
     setIframeUrl((filteredVCStatus[0]['GranaUrl']));
   }
@@ -238,7 +242,7 @@ const ClusterStatus: FC = () => {
           onChangeIndex={(value) => handleChangeIndex(value, setValue)}
         >
           <DLTSTabPanel value={value} index={0} dir={theme.direction} title={ClusterStatusTitles[value]}>
-            <TeamVirtualClusterStatus vcStatus={vcStatus} selectedValue={selectedValue} handleChange={handleChange}/>
+            <TeamVirtualClusterStatus vcStatus={vcStatus} selectedValue={selectedValue} handleChange={(event: React.ChangeEvent<HTMLInputElement>) => handleChange(event, true)}/>
           </DLTSTabPanel>
           <DLTSTabPanel value={value} index={1} dir={theme.direction} title={ClusterStatusTitles[value]}>
             <TeamVCUserStatus userStatus={userStatus} currentCluster={selectedValue} showCurrentUser={showCurrentUser} handleSwitch={handleSwitch}/>
diff --git a/src/dashboard/src/pages/CommonComponents/DLTSTabs.tsx b/src/dashboard/src/pages/CommonComponents/DLTSTabs.tsx
@@ -25,7 +25,7 @@ export const DLTSTabs = (props: TabsProps) => {
           {...other}
         >
           { titles && titles.map((title, index)=>(
-            <Tab label={title} />
+            <Tab label={title} key={index}/>
           )) }
           {children}
         </Tabs>
diff --git a/src/dashboard/src/pages/Home/GPUCard.tsx b/src/dashboard/src/pages/Home/GPUCard.tsx
@@ -1,30 +1,52 @@
 import React, {useEffect, useState} from "react";
 import { Link } from "react-router-dom";
 import useFetch from "use-http";
+import Table from '@material-ui/core/Table';
+import TableBody from '@material-ui/core/TableBody';
+import TableCell from '@material-ui/core/TableCell';
+import TableRow from '@material-ui/core/TableRow';
+import LinearProgress from "@material-ui/core/LinearProgress";
 import {
+  Box,
   Button,
   Card,
   CardActions,
   CardContent,
-  CardHeader,
+  CardHeader, createMuiTheme,
   Divider,
   IconButton,
   InputAdornment,
   Menu,
-  MenuItem,
+  MenuItem, MuiThemeProvider,
   TextField,
-  Tooltip
+  Tooltip, Typography, withStyles
 } from "@material-ui/core";
-import { makeStyles, createStyles, useTheme, Theme } from "@material-ui/core/styles";
+import {
+  makeStyles,
+  createStyles,
+  useTheme,
+  Theme,
+  lighten
+} from "@material-ui/core/styles";
 import { MoreVert, FileCopyRounded} from "@material-ui/icons";
 
 import {Cell, PieChart, Pie, ResponsiveContainer,Sector} from "recharts";
 import UserContext from "../../contexts/User";
 import TeamsContext from '../../contexts/Teams';
-import {green, lightGreen, deepOrange } from "@material-ui/core/colors";
+import {
+  green,
+  lightGreen,
+  deepOrange,
+  red,
+  yellow
+} from "@material-ui/core/colors";
 import copy from 'clipboard-copy'
 import {checkObjIsEmpty, sumValues} from "../../utlities/ObjUtlities";
 import {DLTSSnackbar} from "../CommonComponents/DLTSSnackbar";
+
+import _ from "lodash";
+import {type} from "os";
+import useCheckIsDesktop from "../../utlities/layoutUtlities";
 const useStyles = makeStyles((theme: Theme) => createStyles({
   avatar: {
     backgroundColor: theme.palette.secondary.main,
@@ -48,6 +70,14 @@ const useStyles = makeStyles((theme: Theme) => createStyles({
   },
   container: {
     margin: '0 auto',
+  },
+  tableTitle: {
+    display: "flex",
+    justifyContent: "center"
+  },
+  tableInfo: {
+    justifyContent: "space-between",
+    display: "flex"
   }
 }));
 
@@ -242,6 +272,7 @@ const GPUCard: React.FC<{ cluster: string }> = ({ cluster }) => {
     const name = typeof email === 'string' ?  email.split('@', 1)[0] : email;
     setDataStorage(data.dataStorage);
     setWorkStorage(`${data.workStorage}/${name}`);
+    return data;
   }
   const fetchClusterStatusUrl = `/api`;
   const requestClusterStatus = useFetch(fetchClusterStatusUrl);
@@ -250,9 +281,71 @@ const GPUCard: React.FC<{ cluster: string }> = ({ cluster }) => {
     const data = await requestClusterStatus.get(`/teams/${selectedTeam}/clusters/${cluster}`);
     return data;
   }
+  const [nfsStorage, setNfsStorage] = useState([]);
   useEffect(()=>{
-    fetchDirectories();
-    fetchClusterStatus().then((res)=>{
+    fetchDirectories().then((res) => {
+      let fetchStorage = [];
+      let freeBytesSubPath = '/prometheus/api/v1/query?query=node_filesystem_free_bytes%7Bfstype%3D%27nfs4%27%7D';
+      let sizeBytesSubPath = '/prometheus/api/v1/query?query=node_filesystem_size_bytes%7Bfstype%3D%27nfs4%27%7D';
+      fetchStorage.push(fetch(`${res['prometheus']}${freeBytesSubPath}`));
+      fetchStorage.push(fetch(`${res['prometheus']}${sizeBytesSubPath}`));
+      let storageRes: any = [];
+      let tmpStorage: any = [];
+      Promise.all(fetchStorage).then((responses) => {
+        responses.forEach(async (response: any) => {
+          const res = await response.json();
+          if (res['data']) {
+            for (let item of res['data']["result"]) {
+              let tmp = {} as any;
+              if (item['metric']['__name__'] == "node_filesystem_size_bytes") {
+                let mountpointName = item['metric']['mountpoint']
+                let val = Math.floor(item['value'][1] / (Math.pow(10, 9)))
+                tmp['mountpointName'] = mountpointName;
+                tmp['total'] = val;
+              }
+              let tmpUsed = {} as any;
+              //node_filesystem_free_bytes
+              if (item['metric']['__name__'] == "node_filesystem_free_bytes") {
+                let mountpointName = item['metric']['mountpoint']
+                let val = Math.floor(item['value'][1] / (Math.pow(10, 9)))
+                tmpUsed['mountpointName'] = mountpointName;
+                tmpUsed['Free'] = val;
+              }
+              tmpStorage.push(tmp)
+              tmpStorage.push(tmpUsed)
+            }
+          }
+          //({ mountpointName: key, users: value })
+          storageRes = tmpStorage.filter((store: any) => !checkObjIsEmpty(store));
+          let finalStorageRes: any = [];
+          if (storageRes && storageRes.length > 0) {
+            finalStorageRes = _.chain(storageRes).groupBy('mountpointName').map((value, key) => {
+              let tmpTotal: any = value.filter((item: any) => item.hasOwnProperty('total'));
+              let tmpFree: any = value.filter((item: any) => item.hasOwnProperty('Free'));
+              let total = 0;
+              let used = 0;
+              if (typeof tmpTotal[0] !== "undefined" && typeof  tmpFree[0] !== "undefined") {
+                total = tmpTotal[0]["total"];
+                used = tmpTotal[0]["total"] - tmpFree[0]["Free"]
+              }
+              return {
+                mountpointName: key, total:total, used: used
+              }
+            }).value();
+          }
+          finalStorageRes.forEach((item: any,i: number) => {
+            if(item["mountpointName"].indexOf("dlws/nfs") !== -1){
+              finalStorageRes.splice(i, 1);
+              finalStorageRes.unshift(item);
+            }
+          });
+          setNfsStorage(finalStorageRes.filter((store: any) => {
+            return store['mountpointName'].indexOf(selectedTeam) !== -1 || store['mountpointName'].indexOf("dlws/nfs") !== -1;
+          }));
+        });
+      });
+    });
+    fetchClusterStatus().then((res) => {
       const availableGpu = !checkObjIsEmpty(res['gpu_avaliable']) ? (Number)(sumValues(res['gpu_avaliable'])) : 0;
       setAvailable(availableGpu);
       const usedGpu = !checkObjIsEmpty(res['gpu_used']) ? (Number)(sumValues(res['gpu_used'])) : 0;
@@ -263,6 +356,50 @@ const GPUCard: React.FC<{ cluster: string }> = ({ cluster }) => {
       setActivate(true);
     })
   },[selectedTeam]);
+  const tableTheme = createMuiTheme({
+    overrides: {
+      MuiTableCell: {
+        root: {
+          paddingTop: 10,
+          paddingBottom: 10,
+          paddingLeft:2,
+          paddingRight:5,
+        }
+      }
+    }
+  });
+  const BorderLinearProgress = withStyles({
+    root: {
+      height: 10,
+      backgroundColor: lighten('#363636', 0.5),
+    },
+    bar: {
+      borderRadius: 20,
+      backgroundColor: green[400],
+    },
+  })(LinearProgress);
+  const GenernalLinerProgress = withStyles({
+    root: {
+      height: 10,
+      backgroundColor: lighten('#363636', 0.5),
+    },
+    bar: {
+      borderRadius: 20,
+      backgroundColor: yellow[800],
+    },
+  })(LinearProgress);
+  const FullBorderLinearProgress = withStyles({
+    root: {
+      height: 10,
+      backgroundColor: lighten('#363636', 0.5),
+    },
+    bar: {
+      borderRadius: 20,
+      backgroundColor: red[400],
+    },
+  })(LinearProgress);
+  const theme = useTheme();
+
   return (
     <Card>
       <CardHeader
@@ -278,6 +415,41 @@ const GPUCard: React.FC<{ cluster: string }> = ({ cluster }) => {
       />
       <CardContent className={styles.chart}>
         <Chart available={available} used={used} reserved={reversed} isActive={activate} />
+        <Divider />
+        <Typography  variant="h6" id="tableTitle" className={styles.tableTitle}>
+          {"Storage (GB)"}
+        </Typography>
+        <Box minHeight={100} style={{ overflow: 'auto' }}>
+          <MuiThemeProvider theme={tableTheme}>
+            <Table>
+              <TableBody>
+                {
+                  nfsStorage.map((nfs: any, index: number) => {
+                    let nfsMountNames = nfs['mountpointName'].split("/");
+                    let mounName = "";
+                    if (nfs['mountpointName'].indexOf("dlws") !== -1) {
+                      mounName = "/data";
+                    } else {
+                      nfsMountNames.splice(0, nfsMountNames.length - 1);
+                      mounName = "/" + nfsMountNames.join('/');
+                    }
+                    let value = nfs['total'] == 0 ? 0 : (nfs['used'] / nfs['total']) * 100;
+                    return (
+                      <TableRow key={index}>
+                        <TableCell>
+                          {
+                            value < 80 ? <BorderLinearProgress value={value} variant={"determinate"}/> : value >= 80 && value < 90 ? <GenernalLinerProgress value={value} variant={"determinate"}/> : <FullBorderLinearProgress value={value} variant={"determinate"}/>
+                          }
+                          <div className={styles.tableInfo}><span>{`${mounName}`}</span><span>{`(${nfs['used']}/${nfs['total']}) ${Math.floor(value)}% used`}</span></div>
+                        </TableCell>
+                      </TableRow>
+                    )
+                  })
+                }
+              </TableBody>
+            </Table>
+          </MuiThemeProvider>
+        </Box>
       </CardContent>
       <CardActions>
         <Button component={Link}
diff --git a/src/dashboard/src/pages/Jobs/index.tsx b/src/dashboard/src/pages/Jobs/index.tsx
@@ -96,19 +96,6 @@ const Jobs: React.FC = (props: any) => {
   const classes = useStyles();
   const [value, setValue] = React.useState(0);
   const [refresh, setRefresh] = React.useState(window.navigator.userAgent.indexOf('Edge') == -1);
-  useEffect(()=>{
-    let mount = true;
-    let timeout: any;
-    if (window.navigator.userAgent.indexOf('Edge') != -1) {
-      timeout = setTimeout(()=>{
-        setRefresh(true);
-      },1000);
-    }
-    return () => {
-      mount = false;
-      clearTimeout(timeout)
-    }
-  },[])
   const[open, setOpen] = React.useState(false);
   const[openApprove, setOpenApprove] = React.useState(false);
   const[openPause, setOpenPause] = React.useState(false);
@@ -123,6 +110,19 @@ const Jobs: React.FC = (props: any) => {
   const [currentJob, setCurrentJob] = React.useState({jobId:'',cluster:'',priority: 100});
   const deleteUrl = `/api/clusters/`;
   const requestDelete =  useFetch(deleteUrl);
+  useEffect(() => {
+    let mount = true;
+    let timeout: any;
+    if (window.navigator.userAgent.indexOf('Edge') != -1) {
+      timeout = setTimeout(()=>{
+        setRefresh(true);
+      }, 1000);
+    }
+    return () => {
+      mount = false;
+      clearTimeout(timeout)
+    }
+  }, [])
   const killJob = async () => {
     const body = {"status":"killing"};
     const data = await requestDelete.put(`${currentJob.cluster}/jobs/${currentJob.jobId}/status/`,body);
@@ -303,7 +303,7 @@ const Jobs: React.FC = (props: any) => {
           disabled={!isAdmin}
           key={rowData['jobId']}
           type="number"
-          id={rowData.tableData.id}
+          id={rowData.tableData.id.toString()}
           defaultValue={rowData.priority}
           onKeyPress={(event) => handlePriorityKeyPress(rowData, event)}
           onChange={(event)=>handleChangePriority(rowData, event)}
diff --git a/src/dashboard/src/pages/Submission/Training.tsx b/src/dashboard/src/pages/Submission/Training.tsx

Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,7 @@ export const DLTSTabs = (props: TabsProps) => {`
`25`	`25`	`{...other}`
`26`	`26`	`>`
`27`	`27`	`{ titles && titles.map((title, index)=>(`
`28`		`- <Tab label={title} />`
	`28`	`+ <Tab label={title} key={index}/>`
`29`	`29`	`)) }`
`30`	`30`	`{children}`
`31`	`31`	`</Tabs>`