Merge pull request #564 from alicevision/dev/stats

Visual interface for node statistics

meshroom/core/node.py

@@ -285,6 +285,7 @@ def process(self, forceCompute=False):
             # ask and wait for the stats thread to stop
             self.statThread.stopRequest()
             self.statThread.join()
+            self.statistics = stats.Statistics()
             del runningProcesses[self.name]
 
         self.upgradeStatusTo(Status.SUCCESS)

meshroom/core/stats.py

@@ -1,8 +1,18 @@
 from collections import defaultdict
+import subprocess
 import logging
 import psutil
 import time
 import threading
+import platform
+import os
+import sys
+
+if sys.version_info[0] == 2:
+    # On Python 2 use C implementation for performance and to avoid lots of warnings
+    from xml.etree import cElementTree as ET
+else:
+    import xml.etree.ElementTree as ET
 
 
 def bytes2human(n):
@@ -25,15 +35,58 @@ def bytes2human(n):
 
 class ComputerStatistics:
     def __init__(self):
-        # TODO: init
         self.nbCores = 0
         self.cpuFreq = 0
+        self.ramTotal = 0
         self.ramAvailable = 0  # GB
         self.vramAvailable = 0  # GB
         self.swapAvailable = 0
-
+        self.gpuMemoryTotal = 0
+        self.gpuName = ''
         self.curves = defaultdict(list)
 
+        self._isInit = False
+
+    def initOnFirstTime(self):
+        if self._isInit:
+            return
+        self._isInit = True
+
+        self.cpuFreq = psutil.cpu_freq().max
+        self.ramTotal = psutil.virtual_memory().total / 1024/1024/1024
+
+        if platform.system() == "Windows":
+            from distutils import spawn
+            # If the platform is Windows and nvidia-smi
+            # could not be found from the environment path,
+            # try to find it from system drive with default installation path
+            self.nvidia_smi = spawn.find_executable('nvidia-smi')
+            if self.nvidia_smi is None:
+                self.nvidia_smi = "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" % os.environ['systemdrive']
+        else:
+            self.nvidia_smi = "nvidia-smi"
+
+        try:
+            p = subprocess.Popen([self.nvidia_smi, "-q", "-x"], stdout=subprocess.PIPE)
+            xmlGpu, stdError = p.communicate()
+
+            smiTree = ET.fromstring(xmlGpu)
+            gpuTree = smiTree.find('gpu')
+
+            try:
+                self.gpuMemoryTotal = gpuTree.find('fb_memory_usage').find('total').text.split(" ")[0]
+            except Exception as e:
+                logging.debug('Failed to get gpuMemoryTotal: "{}".'.format(str(e)))
+                pass
+            try:
+                self.gpuName = gpuTree.find('product_name').text
+            except Exception as e:
+                logging.debug('Failed to get gpuName: "{}".'.format(str(e)))
+                pass
+
+        except Exception as e:
+            logging.debug('Failed to get information from nvidia_smi at init: "{}".'.format(str(e)))
+
     def _addKV(self, k, v):
         if isinstance(v, tuple):
             for ki, vi in v._asdict().items():
@@ -45,11 +98,41 @@ def _addKV(self, k, v):
             self.curves[k].append(v)
 
     def update(self):
+        self.initOnFirstTime()
         self._addKV('cpuUsage', psutil.cpu_percent(percpu=True)) # interval=None => non-blocking (percentage since last call)
         self._addKV('ramUsage', psutil.virtual_memory().percent)
         self._addKV('swapUsage', psutil.swap_memory().percent)
         self._addKV('vramUsage', 0)
         self._addKV('ioCounters', psutil.disk_io_counters())
+        self.updateGpu()
+
+    def updateGpu(self):
+        try:
+            p = subprocess.Popen([self.nvidia_smi, "-q", "-x"], stdout=subprocess.PIPE)
+            xmlGpu, stdError = p.communicate()
+
+            smiTree = ET.fromstring(xmlGpu)
+            gpuTree = smiTree.find('gpu')
+
+            try:
+                self._addKV('gpuMemoryUsed', gpuTree.find('fb_memory_usage').find('used').text.split(" ")[0])
+            except Exception as e:
+                logging.debug('Failed to get gpuMemoryUsed: "{}".'.format(str(e)))
+                pass
+            try:
+                self._addKV('gpuUsed', gpuTree.find('utilization').find('gpu_util').text.split(" ")[0])
+            except Exception as e:
+                logging.debug('Failed to get gpuUsed: "{}".'.format(str(e)))
+                pass
+            try:
+                self._addKV('gpuTemperature', gpuTree.find('temperature').find('gpu_temp').text.split(" ")[0])
+            except Exception as e:
+                logging.debug('Failed to get gpuTemperature: "{}".'.format(str(e)))
+                pass
+
+        except Exception as e:
+            logging.debug('Failed to get information from nvidia_smi: "{}".'.format(str(e)))
+            return
 
     def toDict(self):
         return self.__dict__
@@ -145,12 +228,13 @@ def fromDict(self, d):
 class Statistics:
     """
     """
-    fileVersion = 1.0
+    fileVersion = 2.0
 
     def __init__(self):
         self.computer = ComputerStatistics()
         self.process = ProcStatistics()
         self.times = []
+        self.interval = 5
 
     def update(self, proc):
         '''
@@ -169,19 +253,28 @@ def toDict(self):
             'computer': self.computer.toDict(),
             'process': self.process.toDict(),
             'times': self.times,
+            'interval': self.interval
             }
 
     def fromDict(self, d):
-        version = d.get('fileVersion', 1.0)
+        version = d.get('fileVersion', 0.0)
         if version != self.fileVersion:
-            logging.info('Cannot load statistics, version was {} and we only support {}.'.format(version, self.fileVersion))
-            self.computer = {}
-            self.process = {}
-            self.times = []
-            return
-        self.computer.fromDict(d.get('computer', {}))
-        self.process.fromDict(d.get('process', {}))
-        self.times = d.get('times', [])
+            logging.debug('Statistics: file version was {} and the current version is {}.'.format(version, self.fileVersion))
+        self.computer = {}
+        self.process = {}
+        self.times = []
+        try:
+            self.computer.fromDict(d.get('computer', {}))
+        except Exception as e:
+            logging.debug('Failed while loading statistics: computer: "{}".'.format(str(e)))
+        try:
+            self.process.fromDict(d.get('process', {}))
+        except Exception as e:
+            logging.debug('Failed while loading statistics: process: "{}".'.format(str(e)))
+        try:
+            self.times = d.get('times', [])
+        except Exception as e:
+            logging.debug('Failed while loading statistics: times: "{}".'.format(str(e)))
 
 
 bytesPerGiga = 1024. * 1024. * 1024.
@@ -204,7 +297,7 @@ def run(self):
         try:
             while True:
                 self.updateStats()
-                if self._stopFlag.wait(60):
+                if self._stopFlag.wait(self.statistics.interval):
                     # stopFlag has been set
                     # update stats one last time and exit main loop
                     if self.proc.is_running():

meshroom/ui/qml/Charts/ChartViewCheckBox.qml

@@ -0,0 +1,34 @@
+import QtQuick 2.9
+import QtQuick.Controls 2.3
+
+
+/**
+ * A custom CheckBox designed to be used in ChartView's legend.
+ */
+CheckBox {
+    id: root
+
+    property color color
+
+    leftPadding: 0
+    font.pointSize: 8
+
+    indicator: Rectangle {
+        width: 11
+        height: width
+        border.width: 1
+        border.color: root.color
+        color: "transparent"
+        anchors.verticalCenter: parent.verticalCenter
+
+        Rectangle {
+            anchors.fill: parent
+            anchors.margins: parent.border.width + 1
+            visible: parent.parent.checkState != Qt.Unchecked
+            anchors.topMargin: parent.parent.checkState === Qt.PartiallyChecked ? 5 : 2
+            anchors.bottomMargin: anchors.topMargin
+            color: parent.border.color
+            anchors.centerIn: parent
+        }
+    }
+}

meshroom/ui/qml/Charts/ChartViewLegend.qml

@@ -0,0 +1,105 @@
+import QtQuick 2.9
+import QtQuick.Controls 2.9
+import QtCharts 2.3
+
+
+/**
+ * ChartViewLegend is an interactive legend component for ChartViews.
+ * It provides a CheckBox for each series that can control its visibility,
+ * and highlight on hovering.
+ */
+Flow {
+    id: root
+
+    // The ChartView to create the legend for
+    property ChartView chartView
+    // Currently hovered series
+    property var hoveredSeries: null
+
+   readonly property ButtonGroup buttonGroup: ButtonGroup {
+        id: legendGroup
+        exclusive: false
+    }
+
+    /// Shortcut function to clear legend
+    function clear() {
+        seriesModel.clear();
+    }
+
+    // Update internal ListModel when ChartView's series change
+    Connections {
+        target: chartView
+        onSeriesAdded: seriesModel.append({"series": series})
+        onSeriesRemoved: {
+            for(var i = 0; i < seriesModel.count; ++i)
+            {
+                if(seriesModel.get(i)["series"] === series)
+                {
+                    seriesModel.remove(i);
+                    return;
+                }
+            }
+        }
+    }
+
+    onChartViewChanged: {
+        clear();
+        for(var i = 0; i < chartView.count; ++i)
+            seriesModel.append({"series": chartView.series(i)});
+    }
+
+    Repeater {
+
+        // ChartView series can't be accessed directly as a model.
+        // Use an intermediate ListModel populated with those series.
+        model: ListModel {
+            id: seriesModel
+        }
+
+        ChartViewCheckBox {
+            ButtonGroup.group: legendGroup
+
+            checked: series.visible
+            text: series.name
+            color: series.color
+
+            onHoveredChanged: {
+                if(hovered && series.visible)
+                    root.hoveredSeries = series;
+                else
+                    root.hoveredSeries = null;
+            }
+
+            // hovered serie properties override
+            states: [
+                State {
+                    when: series && root.hoveredSeries === series
+                    PropertyChanges { target: series; width: 5.0 }
+                },
+                State {
+                    when: series && root.hoveredSeries && root.hoveredSeries !== series
+                    PropertyChanges { target: series; width: 0.2 }
+                }
+            ]
+
+            MouseArea {
+                anchors.fill: parent
+                onClicked: {
+                    if(mouse.modifiers & Qt.ControlModifier)
+                        root.soloSeries(index);
+                    else
+                        series.visible = !series.visible;
+                }
+            }
+        }
+    }
+
+    /// Hide all series but the one at index 'idx'
+    function soloSeries(idx) {
+        for(var i = 0; i < seriesModel.count; i++) {
+            chartView.series(i).visible = false;
+        }
+        chartView.series(idx).visible = true;
+    }
+
+}

meshroom/ui/qml/Charts/qmldir

@@ -0,0 +1,4 @@
+module Charts
+
+ChartViewLegend 1.0 ChartViewLegend.qml
+ChartViewCheckBox 1.0 ChartViewCheckBox.qml

meshroom/ui/qml/GraphEditor/NodeLog.qml

@@ -89,8 +89,10 @@ FocusScope {
                     // only set text file viewer source when ListView is fully ready
                     // (either empty or fully populated with a valid currentChunk)
                     // to avoid going through an empty url when switching between two nodes
+
                     if(!chunksLV.count || chunksLV.currentChunk)
-                        textFileViewer.source = Filepath.stringToUrl(currentFile);
+                        logComponentLoader.source = Filepath.stringToUrl(currentFile);
+
                 }
 
                 TabButton {
@@ -111,12 +113,35 @@ FocusScope {
                 }
             }
 
-            TextFileViewer {
-                id: textFileViewer
+            Loader {
+                id: logComponentLoader
+                clip: true
                 Layout.fillWidth: true
                 Layout.fillHeight: true
-                autoReload: chunksLV.currentChunk !== undefined && chunksLV.currentChunk.statusName === "RUNNING"
-                // source is set in fileSelector
+                property url source
+                sourceComponent: fileSelector.currentItem.fileProperty === "statisticsFile" ? statViewerComponent : textFileViewerComponent
+            }
+
+            Component {
+                id: textFileViewerComponent
+                TextFileViewer {
+                    id: textFileViewer
+                    source: logComponentLoader.source
+                    Layout.fillWidth: true
+                    Layout.fillHeight: true
+                    autoReload: chunksLV.currentChunk !== undefined && chunksLV.currentChunk.statusName === "RUNNING"
+                    // source is set in fileSelector
+                }
+            }
+
+            Component {
+                id: statViewerComponent
+                StatViewer {
+                    id: statViewer
+                    Layout.fillWidth: true
+                    Layout.fillHeight: true
+                    source: logComponentLoader.source
+                }
             }
         }
     }