Ar2rsawseen/feature/tracking

api/api.js

@@ -16,6 +16,7 @@ const {WriteBatcher, ReadBatcher, InsertBatcher} = require('./parts/data/batcher
 const pack = require('../package.json');
 const versionInfo = require('../frontend/express/version.info.js');
 const moment = require("moment");
+const tracker = require('./parts/mgmt/tracker.js');
 
 var t = ["countly:", "api"];
 common.processRequest = processRequest;
@@ -38,6 +39,9 @@ else {
 process.title = t.join(' ');
 
 plugins.connectToAllDatabases().then(function() {
+    plugins.loadConfigs(common.db, function() {
+        tracker.enable();
+    });
     common.writeBatcher = new WriteBatcher(common.db);
     common.readBatcher = new ReadBatcher(common.db);
     common.insertBatcher = new InsertBatcher(common.db);
@@ -142,6 +146,41 @@ plugins.connectToAllDatabases().then(function() {
         require('./utils/log.js').ipcHandler(msg);
     });
 
+    /**
+    * Set tracking config
+    */
+    plugins.setConfigs("tracking", {
+        self_tracking_app: "",
+        self_tracking_url: "",
+        self_tracking_app_key: "",
+        self_tracking_id_policy: "_id",
+        self_tracking_sessions: true,
+        self_tracking_events: true,
+        self_tracking_views: true,
+        self_tracking_feedback: true,
+        self_tracking_user_details: true,
+        server_sessions: true,
+        server_events: true,
+        server_crashes: true,
+        server_views: true,
+        server_feedback: true,
+        server_user_details: true,
+        /*user_sessions: true,
+        user_events: true,
+        user_crashes: true,
+        user_views: true,
+        user_feedback: true,
+        user_details: true*/
+    });
+
+    /*plugins.setUserConfigs("tracking", {
+        user_sessions: false,
+        user_events: false,
+        user_crashes: false,
+        user_views: false,
+        user_feedback: false
+    });*/
+
     /**
     * Initialize Plugins
     */
@@ -309,7 +348,7 @@ plugins.connectToAllDatabases().then(function() {
         // Allow configs to load & scanner to find all jobs classes
         setTimeout(() => {
             jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day');
-            jobs.job('api:ping').replace().schedule('every 1 day');
+            jobs.job('api:ping').replace().schedule('at 00:01 am ' + 'every 1 day');
             jobs.job('api:clear').replace().schedule('every 1 day');
             jobs.job('api:clearTokens').replace().schedule('every 1 day');
             jobs.job('api:clearAutoTasks').replace().schedule('every 1 day');

api/jobs/ping.js

@@ -1,12 +1,8 @@
 'use strict';
 
 const job = require('../parts/jobs/job.js'),
-    log = require('../utils/log.js')('job:ping'),
-    countlyConfig = require("../../frontend/express/config.js"),
-    versionInfo = require('../../frontend/express/version.info'),
     plugins = require('../../plugins/pluginManager.js'),
-    request = require('countly-request')(plugins.getConfig("security"));
-
+    tracker = require('../parts/mgmt/tracker.js');
 
 /** Class for the job of pinging servers **/
 class PingJob extends job.Job {
@@ -16,86 +12,114 @@ class PingJob extends job.Job {
      * @param {done} done callback
      */
     run(db, done) {
-        request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {});
-        var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig));
-        var url = "https://count.ly/configurations/ce/tracking";
-        if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") {
-            url = "https://count.ly/configurations/ee/tracking";
-        }
-        plugins.loadConfigs(db, function() {
+        plugins.loadConfigs(db, async function() {
             const offlineMode = plugins.getConfig("api").offline_mode;
-            const { countly_tracking } = plugins.getConfig('frontend');
             if (!offlineMode) {
-                request(url, function(err, response, body) {
-                    if (typeof body === "string") {
-                        try {
-                            body = JSON.parse(body);
-                        }
-                        catch (ex) {
-                            body = null;
-                        }
+                var server = tracker.getBulkServer();
+                var user = tracker.getBulkUser(server);
+                if (!user) {
+                    return done();
+                }
+
+                try {
+                    var custom = await tracker.getAllData();
+                    if (Object.keys(custom).length) {
+                        user.user_details({"custom": custom });
                     }
-                    if (body) {
-                        if (countlyConfigOrig.web.use_intercom && typeof body.intercom !== "undefined") {
-                            countlyConfig.web.use_intercom = body.intercom;
-                        }
-                        if (typeof countlyConfigOrig.web.track === "undefined" && typeof body.stats !== "undefined") {
-                            if (body.stats) {
-                                countlyConfig.web.track = null;
-                            }
-                            else {
-                                countlyConfig.web.track = "none";
-                            }
-                        }
+                }
+                catch (ex) {
+                    console.log("Error collecting server data:", ex);
+                }
+                var days = 90;
+                var current_sync = Date.now();
+
+                // Atomically retrieve old last_sync value and set new one
+                var syncResult = await db.collection("plugins").findOneAndUpdate(
+                    {_id: "version"},
+                    {$set: {last_sync: current_sync}},
+                    {
+                        upsert: true,
+                        returnDocument: 'before',
+                        projection: {last_sync: 1}
                     }
-                    log.d(err, body, countlyConfigOrig, countlyConfig);
-                    if (countly_tracking) {
-                        db.collection("members").findOne({global_admin: true}, function(err2, member) {
-                            if (!err2 && member) {
-                                var date = new Date();
-                                let domain = plugins.getConfig('api').domain;
+                );
 
-                                try {
-                                    // try to extract hostname from full domain url
-                                    const urlObj = new URL(domain);
-                                    domain = urlObj.hostname;
-                                }
-                                catch (_) {
-                                    // do nothing, domain from config will be used as is
-                                }
+                var last_sync = syncResult.value ? syncResult.value.last_sync : null;
+                if (last_sync) {
+                    days = Math.floor((new Date().getTime() - last_sync) / (1000 * 60 * 60 * 24));
+                }
 
-                                request({
-                                    uri: "https://stats.count.ly/i",
-                                    method: "GET",
-                                    timeout: 4E3,
-                                    qs: {
-                                        device_id: domain,
-                                        app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f",
-                                        timestamp: Math.floor(date.getTime() / 1000),
-                                        hour: date.getHours(),
-                                        dow: date.getDay(),
-                                        no_meta: true,
-                                        events: JSON.stringify([
-                                            {
-                                                key: "PING",
-                                                count: 1
-                                            }
-                                        ])
+                if (days > 0) {
+                    //calculate seconds timestamp of days before today
+                    var startTs = Math.round((new Date().getTime() - (30 * 24 * 60 * 60 * 1000)) / 1000);
+
+                    //sync server events - use aggregation pipeline to group by day and action on MongoDB side
+                    var aggregationPipeline = [
+                        // Match documents with timestamp greater than startTs and valid action
+                        {
+                            $match: {
+                                ts: { $gt: startTs }
+                            }
+                        },
+                        // Add calculated fields for day grouping
+                        {
+                            $addFields: {
+                                // Convert timestamp to date and set to noon (12:00:00)
+                                dayDate: {
+                                    $dateFromParts: {
+                                        year: { $year: { $toDate: { $multiply: ["$ts", 1000] } } },
+                                        month: { $month: { $toDate: { $multiply: ["$ts", 1000] } } },
+                                        day: { $dayOfMonth: { $toDate: { $multiply: ["$ts", 1000] } } },
+                                        hour: 12,
+                                        minute: 0,
+                                        second: 0
                                     }
-                                }, function(a/*, c, b*/) {
-                                    log.d('Done running ping job: %j', a);
-                                    done();
-                                });
+                                }
+                            }
+                        },
+                        // Convert back to timestamp in seconds
+                        {
+                            $addFields: {
+                                noonTimestamp: {
+                                    $divide: [{ $toLong: "$dayDate" }, 1000]
+                                }
                             }
-                            else {
-                                done();
+                        },
+                        // Group by day and action
+                        {
+                            $group: {
+                                _id: {
+                                    day: "$noonTimestamp",
+                                    action: "$a"
+                                },
+                                count: { $sum: 1 }
                             }
-                        });
+                        },
+                        // Project to final format
+                        {
+                            $project: {
+                                _id: 0,
+                                action: "$_id.action",
+                                timestamp: "$_id.day",
+                                count: 1
+                            }
+                        }
+                    ];
+
+                    var cursor = db.collection("systemlogs").aggregate(aggregationPipeline);
+
+                    while (cursor && await cursor.hasNext()) {
+                        let eventData = await cursor.next();
+                        user.add_event({key: eventData.action, count: eventData.count, timestamp: eventData.timestamp});
                     }
-                    else {
+                    server.start(function() {
+                        server.stop();
                         done();
-                    }
-                });
+                    });
+                }
+                else {
+                    done();
+                }
             }
             else {
                 done();
@@ -104,4 +128,4 @@ class PingJob extends job.Job {
     }
 }
 
-module.exports = PingJob;
+module.exports = PingJob;