statistics fixes and improvement to support latest devel branch

backend/globaleaks/handlers/public.py

@@ -421,6 +421,7 @@ def serialize_field(session, tid, field, language, data=None, serialize_template
         'fieldgroup_id': field.fieldgroup_id if field.fieldgroup_id else '',
         'multi_entry': field.multi_entry,
         'required': field.required,
+        'statistics': field.statistics,
         'attrs': attrs,
         'x': field.x,
         'y': field.y,

backend/globaleaks/models/__init__.py

@@ -423,6 +423,7 @@ class _Field(Model):
     description = Column(JSON, default=dict, nullable=False)
     hint = Column(JSON, default=dict, nullable=False)
     placeholder = Column(JSON, default=dict, nullable=False)
+    statistics = Column(Boolean, default=False, nullable=False)
     required = Column(Boolean, default=False, nullable=False)
     multi_entry = Column(Boolean, default=False, nullable=False)
     triggered_by_score = Column(Integer, default=0, nullable=False)
@@ -445,7 +446,7 @@ def __table_args__(self):
     unicode_keys = ['type', 'instance', 'key']
     int_keys = ['x', 'y', 'width', 'triggered_by_score']
     localized_keys = ['label', 'description', 'hint', 'placeholder']
-    bool_keys = ['multi_entry', 'required']
+    bool_keys = ['multi_entry', 'required', 'statistics']
     optional_references = ['template_id', 'step_id', 'fieldgroup_id', 'template_override_id']
 
 

backend/globaleaks/rest/requests.py

@@ -310,6 +310,7 @@ def get_multilang_request_format(request_format, localized_strings):
     'fieldgroup_id': key_regexp_or_empty,
     'label': str,
     'description': str,
+    'statistics': bool,
     'hint': str,
     'placeholder': str,
     'multi_entry': bool,

client/Gruntfile.js

@@ -40,6 +40,8 @@ module.exports = function(grunt) {
           { dest: "app/lib/js/", cwd: "./node_modules/", src: ["@flowjs/flow.js/dist/flow.js"], expand: true, flatten: true },
           { dest: "app/lib/js/", cwd: "./node_modules/", src: ["@flowjs/ng-flow/dist/ng-flow.js"], expand: true, flatten: true },
           { dest: "app/lib/js/", cwd: "./node_modules/", src: ["angular/angular.js"], expand: true, flatten: true },
+          { dest: "app/lib/js/", cwd: "./node_modules/", src: ["chart.js/dist/chart.min.js"], expand: true, flatten: true },
+          { dest: "app/lib/js/", cwd: "./node_modules/", src: ["chartjs-plugin-datalabels/dist/chartjs-plugin-datalabels.min.js"], expand: true, flatten: true },
           { dest: "app/lib/js/", cwd: "./node_modules/", src: ["angular-aria/angular-aria.js"], expand: true, flatten: true },
           { dest: "app/lib/js/", cwd: "./node_modules/", src: ["angular-dynamic-locale/dist/tmhDynamicLocale.js"], expand: true, flatten: true },
           { dest: "app/lib/js/", cwd: "./node_modules/", src: ["angular-filter/dist/angular-filter.js"], expand: true, flatten: true },

client/app/index.html

@@ -194,6 +194,8 @@ <h1>Error!</h1><br /><br />
     <script src="js/controllers/actions/forcedpasswordchange.js"></script>
     <script src="js/controllers/actions/forcedtwofactor.js"></script>
     <script src="js/controllers/passwordreset.js"></script>
+    <script src="lib/js/chart.min.js"></script>
+    <script src="lib/js/chartjs-plugin-datalabels.min.js"></script>
     <!-- endbuild -->
 
     <noscript>

client/app/js/app.js

@@ -202,6 +202,16 @@ var GL = angular.module("GL", [
           resources: fetchResources("receiver", ["preferences", "rtips"])
         }
       }).
+      when("/recipient/statistics", {
+        templateUrl: "views/recipient/statistics.html",
+        controller: "StatisticsCtrl",
+        header_title: "Statistics",
+        sidebar: "views/recipient/sidebar.html",
+        resolve: {
+          access: requireAuth("receiver"),
+          resources: fetchResources("receiver", ["rtips","preferences"])
+        }
+      }).
       when("/admin/home", {
         templateUrl: "views/admin/home.html",
         controller: "HomeCtrl",

client/app/js/controllers/recipient.js

@@ -215,4 +215,299 @@ GL.controller("ReceiverTipsCtrl", ["$scope",  "$filter", "$http", "$location", "
       })(i);
     }
   };
-}]);
+}])
+.controller("StatisticsCtrl", ["$scope", "RTip", "Statistics",
+  function ($scope, RTip, Statistics) {
+
+     var statsModel = Statistics.getStatisticsModel();
+
+    $scope.flush = function () {
+      $scope.reportingChannel = [];
+      $scope.startDatePickerOpen = false;
+      $scope.endDatePickerOpen = false;
+      $scope.staticData = [];
+
+     statsModel = Statistics.getStatisticsModel();
+     answerArray = [];
+     dropdownOptions = [];
+
+    };
+
+    $scope.openStartDatePicker = function () {
+      $scope.startDatePickerOpen = true;
+    };
+
+    $scope.openEndDatePicker = function () {
+      $scope.endDatePickerOpen = true;
+    };
+
+    $scope.initializeTips = function () {
+      let promises = [];
+
+      for (let tip of $scope.resources.rtips) {
+        tip.context = $scope.contexts_by_id[tip.context_id];
+
+        if ($scope.reportingChannel.indexOf(tip.context.name) === -1) {
+          $scope.reportingChannel.push(tip.context.name);
+        }
+
+        let creationDate = new Date(tip.creation_date);
+        let expirationDate = new Date(tip.expiration_date);
+
+        if (($scope.channel && tip.context.name !== $scope.channel) ||
+          ($scope.startDate && $scope.startDate > creationDate) ||
+          ($scope.endDate && $scope.endDate < expirationDate)) {
+          continue;
+        }
+
+        statsModel.totalReports += 1;
+        statsModel.reports.push(tip);
+
+        tip.submissionStatusStr = $scope.Utils.getSubmissionStatusText(tip.status, tip.substatus, $scope.submission_statuses);
+
+        if (tip.status !== "new") {
+          let promise = new Promise(function(resolve, reject) {
+            RTip({ id: tip.id }, function (tip) {
+              $scope.tip = tip
+              if (tip.comments.length > 0) {
+                let lastComment = tip.comments[tip.comments.length - 1];
+
+                if (lastComment.type === "whistleblower") {
+                  statsModel.unansweredTipsCount += 1;
+                }
+
+                if (lastComment.type === "receiver" && (tip.label.length > 0 || tip.wbfiles.length > 0 || tip.status !== "opened")) {
+                  statsModel.receiverCount++
+                }
+              }
+
+              statsModel.recipients.push({
+                reportId: tip.progressive,
+                recipients: tip.receivers.map(receiver => receiver.name).join(" - ")
+              });
+
+              $scope.parseAnswers(tip);
+              $scope.initializeStaticData();
+              resolve();
+            });
+          });
+
+          promises.push(promise);
+        }
+
+        let lastAccessDate = new Date(tip.last_access);
+
+        statsModel.statusLabelCount[tip.submissionStatusStr] = (statsModel.statusLabelCount[tip.submissionStatusStr] || 0) + 1;
+
+        const creationDateObj = new Date(tip.creation_date);
+        const monthYear = `${creationDateObj.toLocaleString("default", { month: "long" })} ${creationDateObj.getFullYear()}`;
+        const reportCreationDate = new Date(tip.creation_date);
+        const reportUpdateDate = new Date(tip.update_date);
+
+        const closureTime = reportUpdateDate - reportCreationDate;
+        statsModel.reportCountPerMonth[monthYear] = (statsModel.reportCountPerMonth[monthYear] || 0) + 1;
+
+        if (lastAccessDate.getTime() !== creationDate.getTime())
+            statsModel.reciprocatingWhistleBlower++;
+
+        if (tip.tor)
+            statsModel.torCount++;
+
+        if (tip.submissionStatusStr === "Closed") {
+          statsModel.averageClosureTime += closureTime;
+          statsModel.totalClosedTips++;
+        }
+
+        const label = tip.label;
+        if (label) {
+          statsModel.allLabeledCount+=1
+          const words = label.split(" ").filter(word => word.length > 0);
+          const labeledWords = words.filter(word => word[0] === "$");
+
+          if (labeledWords.length == 0){
+            statsModel.unlabeledCount += 1;
+          }
+          statsModel.labeledCountDefault += labeledWords.length;
+
+          labeledWords.forEach(word => {
+            statsModel.labelCounts[word] = (statsModel.labelCounts[word] || 0) + 1;
+          });
+        } else {
+          statsModel.unlabeledCount++;
+          statsModel.unlabeledCountDefault++;
+        }
+      }
+
+      Promise.all(promises).then(function() {
+        $scope.generateAnswersGraph();
+      });
+    };
+
+    $scope.generateGeneralGraph = function () {
+      let statusPercentages = Object.keys(statsModel.statusLabelCount).map(status => {
+        const count = statsModel.statusLabelCount[status];
+        const percentage = (statsModel.totalReports !== 0) ? ((count / statsModel.totalReports) * 100).toFixed(2) : 0;
+        return { status, count, percentage };
+      });
+
+      statusPercentages.sort((a, b) => a.status.toLowerCase().localeCompare(b.status.toLowerCase()));
+      statusPercentages.unshift({
+        status: "Total Reports",
+        count: statsModel.totalReports,
+        percentage: statsModel.totalReports !== 0 ? "100" : "0.00"
+      });
+
+      const labels = statusPercentages.map(item => `${item.status} | ${item.percentage} %`);
+      const data = statusPercentages.map(item => item.count);
+
+      if ($scope.statusBarChart) {
+        $scope.statusBarChart.data.labels = labels;
+        $scope.statusBarChart.data.datasets[0].data = data;
+        $scope.statusBarChart.update();
+      } else {
+        $scope.statusBarChart = Statistics.generateBarGraph("statusBarChart", labels, "General Statistics", data, "Number of Reports", "Status");
+      }
+    };
+
+    $scope.generateInteractionLineGraph = function () {
+      statsModel.averageClosureTime = (statsModel.averageClosureTime !== 0) ? ((statsModel.averageClosureTime / statsModel.totalClosedTips) / (1000 * 60 * 60 * 24)).toFixed(3) : 0;
+
+      const labels = Object.keys(statsModel.reportCountPerMonth);
+      const reportData = Object.values(statsModel.reportCountPerMonth);
+
+      if ($scope.perMonthLineGraph) {
+        $scope.perMonthLineGraph.data.labels = labels;
+        $scope.perMonthLineGraph.data.datasets[0].data = reportData;
+        $scope.perMonthLineGraph.update();
+      } else {
+        $scope.perMonthLineGraph = Statistics.generateLineGraph("perMonthLineGraph", labels, "Interaction Statistics", reportData, "Month", "Reports");
+      }
+    };
+
+    $scope.generateLabelGraph = function () {
+      let totalItemCount = statsModel.totalReports;
+
+      angular.forEach(statsModel.labelCounts, function (count, label) {
+        let percentage = (count / totalItemCount) * 100;
+        statsModel.labelCounts[label] = {
+          count: count,
+          percentage: percentage.toFixed(2) + "%"
+        };
+      });
+
+      let unlabeledPercentage = (statsModel.unlabeledCount / totalItemCount) * 100;
+      statsModel.unlabeledCount = {
+        count: statsModel.unlabeledCount,
+        percentage: unlabeledPercentage.toFixed(2) + "%"
+      };
+
+      let labelCountsData = Object.values(statsModel.labelCounts).map(function (label) {
+        return label.count;
+      });
+
+      let unlabeledCountData = statsModel.unlabeledCount.count;
+      let labels = ["Total Reports", ...Object.keys(statsModel.labelCounts), "Unlabeled"];
+      let data = [statsModel.totalReports, ...labelCountsData, unlabeledCountData];
+
+      if ($scope.labelCountsChart) {
+        $scope.labelCountsChart.data.labels = labels;
+        $scope.labelCountsChart.data.datasets[0].data = data;
+        $scope.labelCountsChart.update();
+      } else {
+        $scope.labelCountsChart = Statistics.generateBarGraph("labelCountsChart", labels, "Labels Statistics", data, "Number of Reports", "Label");
+      }
+    };
+
+    $scope.generateAnswersGraph = function () {
+      const sortedOptions = dropdownOptions.slice().sort((a, b) => a.optionLabel.localeCompare(b.optionLabel));
+      const labels = ["Total Reports", ...sortedOptions.map(entry => entry.optionLabel)];
+      const tooltip = ["Total Report", ...sortedOptions.map(entry => entry.question)];
+      const data = [statsModel.totalReports, ...sortedOptions.map(entry => entry.count)];
+
+      if ($scope.channelCountsChart) {
+        $scope.channelCountsChart.data.labels = labels;
+        $scope.channelCountsChart.data.datasets[0].data = data;
+        $scope.channelCountsChart.options.plugins.tooltip = {
+          callbacks: {
+            title: function (context) {
+              const dataIndex = context[0].dataIndex;
+              return tooltip[dataIndex];
+            }
+          }
+        };
+        $scope.channelCountsChart.update();
+      } else {
+        $scope.channelCountsChart = Statistics.generateBarGraph("dropdownOptionsChart", labels, "Statistics", data, "Number of Reports", "DropdownOptions");
+      }
+    };
+
+    $scope.initializeStaticData = function () {
+
+      const totalReports = statsModel.totalReports;
+      const statPercentageCalculator = (value, totalvalue) => (!totalvalue ? 0 : ((value / totalvalue) * 100).toFixed(2) + " %");
+
+      const submissionStatus = {
+        label: ["Total", "New", "Opened", "Closed", "Labeled", "Unlabeled"],
+        data: [
+          totalReports,
+          statPercentageCalculator(statsModel.statusLabelCount["New"], totalReports),
+          statPercentageCalculator(statsModel.statusLabelCount["Opened"], totalReports),
+          statPercentageCalculator(statsModel.statusLabelCount["Closed"], totalReports),
+          statPercentageCalculator(statsModel.allLabeledCount, totalReports),
+          statPercentageCalculator(statsModel.unlabeledCountDefault, totalReports)
+        ]
+      };
+
+      const interactionStatus = {
+        label: ["Total Report", "Average closure time (Days)", "Total Unanswered Tips", "Number of interactions", "Tor Connections", "Reciprocating whistle blower"],
+        data: [
+          totalReports,
+          statsModel.averageClosureTime,
+          statsModel.unansweredTipsCount,
+          statsModel.receiverCount,
+          statsModel.torCount,
+          statsModel.reciprocatingWhistleBlower
+        ]
+      };
+
+      $scope.staticData = {
+        submissionStatus: submissionStatus,
+        interactionStatus: interactionStatus
+      };
+    };
+
+    $scope.initialize = function () {
+      $scope.flush();
+
+      $scope.initializeTips();
+
+      $scope.initializeStaticData();
+      $scope.generateGeneralGraph();
+      $scope.generateInteractionLineGraph();
+      $scope.generateLabelGraph();
+    };
+
+    $scope.parseAnswers = function (tip) {
+
+      tip.questionnaires.forEach((item) => {
+        item.steps.forEach((step) => {
+          step.children.forEach((children) => {
+            if (children.statistics === true) {
+              if (["selectbox", "multichoice", "checkbox"].includes(children.type)) {
+                Statistics.parseStaticAnswers(tip, item, children);
+              } else if (["textarea", "inputbox", "date", "daterange"].includes(children.type)) {
+                Statistics.parseTextualAnswers(tip, item, children);
+              }
+            }
+          });
+        });
+      });
+    };
+
+    $scope.export = function () {
+      Statistics.export(answerArray, statsModel.reports, statsModel.recipients);
+    };
+
+    $scope.initialize();
+  }
+]);