From 38597db351ba6d335571d011deb001b2737f579f Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Thu, 4 Sep 2025 16:07:14 +0300 Subject: [PATCH 01/21] Initial tracking settings --- api/api.js | 35 ++++++++++++ .../public/javascripts/countly.views.js | 25 +++++++-- .../public/localization/plugins.properties | 53 +++++++++++++++++-- 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/api/api.js b/api/api.js index 843deb94f1b..e1b206a8ea5 100644 --- a/api/api.js +++ b/api/api.js @@ -142,6 +142,41 @@ plugins.connectToAllDatabases().then(function() { require('./utils/log.js').ipcHandler(msg); }); + /** + * Set tracking config + */ + plugins.setConfigs("tracking", { + self_tracking_app: "", + self_tracking_url: "", + self_tracking_app_key: "", + self_tracking_id_policy: "_id", + self_tracking_sessions: true, + self_tracking_events: true, + self_tracking_views: true, + self_tracking_feedback: true, + self_tracking_user_details: true, + server_sessions: true, + server_events: true, + server_crashes: true, + server_views: true, + server_feedback: true, + server_user_details: true, + user_sessions: true, + user_events: true, + user_crashes: true, + user_views: true, + user_feedback: true, + user_details: true + }); + + plugins.setUserConfigs("tracking", { + user_sessions: false, + user_events: false, + user_crashes: false, + user_views: false, + user_feedback: false + }); + /** * Initialize Plugins */ diff --git a/plugins/plugins/frontend/public/javascripts/countly.views.js b/plugins/plugins/frontend/public/javascripts/countly.views.js index 27bae3d1761..197ba4feaad 100644 --- a/plugins/plugins/frontend/public/javascripts/countly.views.js +++ b/plugins/plugins/frontend/public/javascripts/countly.views.js @@ -298,7 +298,7 @@ back: this.$route.params.namespace === "search", configsData: {}, configsList: [], - coreDefaults: ['api', 'frontend', 'logs', 'security'], + coreDefaults: ['api', 'frontend', 'logs', 'security', 'tracking'], diff: [], diff_: {}, selectedConfig: this.$route.params.namespace || "api", @@ -513,7 +513,7 @@ }, getLabelName: function(id, ns) { ns = ns || this.selectedConfig; - if (ns !== "frontend" && ns !== "api" && ns !== "apps" && ns !== "logs" && ns !== "security" && ns !== "feedback" && countlyGlobal.plugins.indexOf(ns) === -1) { + if (this.coreDefaults.indexOf(ns) === -1 && ns !== "feedback" && countlyGlobal.plugins.indexOf(ns) === -1) { return null; } @@ -1182,17 +1182,25 @@ } }); - var appList = [{value: "", label: jQuery.i18n.map["configs.frontend-self_tracking.none"]}]; + var appList = [{value: "", label: jQuery.i18n.map["configs.tracking.self_tracking.none"]}]; for (var a in countlyGlobal.apps) { appList.push({value: countlyGlobal.apps[a].key, label: countlyGlobal.apps[a].name}); } - app.configurationsView.registerInput("frontend.self_tracking", { + app.configurationsView.registerInput("tracking.self_tracking_app", { input: "el-select", attrs: {}, list: appList }); + var idList = [{value: "_id", label: "_id"}, {value: "email", label: "email"}]; + + app.configurationsView.registerInput("tracking.self_tracking_id_policy", { + input: "el-select", + attrs: {}, + list: idList + }); + app.configurationsView.registerStructure("api", { description: "configs.api.description", groups: [ @@ -1203,6 +1211,15 @@ ] }); + app.configurationsView.registerStructure("tracking", { + description: "configs.tracking.description", + groups: [ + {label: "configs.tracking.self_tracking", list: ["self_tracking_app", "self_tracking_url", "self_tracking_app_key", "self_tracking_id_policy", "self_tracking_sessions", "self_tracking_events", "self_tracking_crashes", "self_tracking_views", "self_tracking_feedback", "self_tracking_user_details"]}, + {label: "configs.tracking.user", list: ["user_sessions", "user_events", "user_crashes", "user_views", "user_feedback", "user_details"]}, + {label: "configs.tracking.server", list: ["server_sessions", "server_events", "server_crashes", "server_views", "server_feedback", "server_user_details"]}, + ] + }); + app.configurationsView.registerStructure("logs", { description: "", groups: [ diff --git a/plugins/plugins/frontend/public/localization/plugins.properties b/plugins/plugins/frontend/public/localization/plugins.properties index eb10290d9c6..aecc967e390 100644 --- a/plugins/plugins/frontend/public/localization/plugins.properties +++ b/plugins/plugins/frontend/public/localization/plugins.properties @@ -64,8 +64,6 @@ configs.no-theme = Default Theme configs.frontend-code = Show Code Generator for SDK integration configs.frontend-offline_mode = Offline mode configs.frontend-countly_tracking = Countly -configs.frontend-self_tracking = Self-tracking using Countly -configs.frontend-self_tracking.none = -- Not tracked -- configs.security-login_tries = Allowed login attempts configs.security-login_wait = Incorrect login block time increment configs.security-dashboard_additional_headers = Additional Dashboard HTTP Response headers @@ -179,7 +177,6 @@ configs.help.frontend-countly_tracking = When enabled, Countly will be activated configs.help.frontend-production = Initial load of dashboard should be faster, due to smaller files and smaller file amount, but when developing a plugin, you need to regenerate them to see changes configs.help.frontend-theme = Selected theme will be available server-wide, for all apps and users configs.help.frontend-session_timeout = User will be forced to logout after session timeout (in minutes) of inactivity. If you want to disable force logout, set to 0. -configs.help.frontend-self_tracking = If you want to track usage of this server and users that are using the dashboard, select an app where to collect this data. Make sure to create a new app specifically for this purpose, or else you would merge collected data with existing. Data will only be stored on this server and will not be sent anywhere else. By selecting an app, you will enabling tracking of this server and it will count towards your datapoint quota. The scale of datapoints will depend on your user count and often usage of this dashboard. configs.help.security-login_tries = Account will be blocked for some time after provided number of incorrect login attempts. See below for time increments. configs.help.security-login_wait = Incremental period of time account is blocked after provided number of incorrect login attempts (in seconds) configs.help.security-password_rotation = Amount of previous passwords user should not be able to reuse @@ -252,3 +249,53 @@ configs.tooltip.server-performance = Performance systemlogs.action.change_configs = Setting Changed systemlogs.action.change_plugins = Plugins Changed + +configs.tracking = Tracking +configs.tracking.description = Control the tracking settings of this server +configs.tracking.server = Server level tracking that helps Countly improve their product. All data is collected on the server level and cannot be tied to any specific user +configs.tracking.user = User level tracking that is enabled only with consent from the user +configs.tracking.self_tracking = Self-tracking using Countly +configs.tracking.self_tracking.none = -- Not tracked -- +configs.tracking.self_tracking_app = Track using this same server by selecting an app +configs.tracking.self_tracking_url = Track using other server, provide Server URL +configs.tracking.self_tracking_app_key = Tracking using other server, provide App Key +configs.tracking.self_tracking_id_policy = What user identifier to use for self tracking +configs.tracking.self_tracking_sessions = Allow tracking self tracking sessions +configs.tracking.self_tracking_events = Allow tracking self tracking events +configs.tracking.self_tracking_crashes = Allow tracking self tracking crashes +configs.tracking.self_tracking_views = Allow tracking self tracking views +configs.tracking.self_tracking_feedback = Allow tracking self tracking feedback +configs.tracking.self_tracking_user_details = Allow tracking self tracking user details +configs.tracking.server_sessions = Allow tracking server uptime +configs.tracking.server_events = Allow tracking feature usage +configs.tracking.server_crashes = Allow tracking server errors +configs.tracking.server_views = Allow tracking visited sections +configs.tracking.server_feedback = Allow us to ask for feedback periodically +configs.tracking.server_user_details = Allow collecting server specific properties +configs.tracking.user_sessions = Allow tracking user sessions +configs.tracking.user_events = Allow tracking feature usage on user level +configs.tracking.user_crashes = Allow tracking errors +configs.tracking.user_views = Allow tracking visited sections of the dashboard on user level +configs.tracking.user_feedback = Allow asking users for feedback +configs.tracking.user_details = Allow tracking ownership of features +configs.help.tracking-self_tracking_app = If you want to track usage of this server and users that are using the dashboard, select an app where to collect this data. Make sure to create a new app specifically for this purpose, or else you would merge collected data with existing. Data will only be stored on this server and will not be sent anywhere else. By selecting an app, you will enabling tracking of this server and it will count towards your datapoint quota. The scale of datapoints will depend on your user count and often usage of this dashboard. +configs.help.tracking.self_tracking_url = If you want to report self tracking to some other server, instead of selecting an app, you can provide Countly server URL here. Make sure to also provide App Key below. Data will only be stored on the provided server and will not be sent anywhere else. By providing a URL, you will enabling tracking of this server and it will count towards your datapoint quota on the provided server. The scale of datapoints will depend on your user count and often usage of this dashboard. +configs.help.tracking.self_tracking_app_key = If you want to report self tracking to some other server, instead of selecting an app, you can provide Countly App Key here. Make sure to also provide Server URL above. Data will only be stored on the provided server and will not be sent anywhere else. By providing an App Key, you will enabling tracking of this server and it will count towards your datapoint quota on the provided server. The scale of datapoints will depend on your user count and often usage of this dashboard. +configs.help.tracking.self_tracking_id_policy = You can choose which user property to use to identify users. Selecting _id would provide more anonymous tracking, while selecting email address would allow you to see which of your users are using the dashboard most often. No personal data will be collected or sent anywhere else, but if you are using email address, it will be stored on this server and would be visible to your server administrator. +configs.help.tracking.self_tracking_sessions = Tracking how often users use dashboard +configs.help.tracking.self_tracking_events = Tracking which features users use in the dashboard +configs.help.tracking.self_tracking_views = Tracking which sections users visit during the time the use dashboard +configs.help.tracking.self_tracking_feedback = Displaying NPS, ratings and Surveys targeted to specific users +configs.help.tracking.self_tracking_user_details = Populating user properties with information we know about users +configs.help.tracking-server_sessions = Tracking server uptime and restarts. Also collects OS, OS version +configs.help.tracking-server_events = Tracking feature usability. We collect aggregated data per day on how many actions were performed on the server, like creating, editing and deleting something +configs.help.tracking-server_crashes = Tracking server errors. If any error happens on the backend that may impact server usability, we would see the stack trace of that problem and time when it happened. +configs.help.tracking-server_views = Tracking which sections of the dashboard are visited and how often in aggregate from all users +configs.help.tracking-server_feedback = Receiving some server wide periodic feedback checks from us, like asking you to rate your experience or notify of issues. Feedback will be collected anonymously and in aggregate +configs.help.tracking-server_user_details = Allow tracking some properties that define how server is used. How many apps are on the server, how many users using the server (only numbers), when somebody logged in to server last time, and when was the last time server received any data +configs.help.tracking-user_sessions = Tracking how often users use dashboard +configs.help.tracking-user_events = Tracking which features users use in the dashboard +configs.help.tracking-user_crashes = Tracking errors in user interaction with UI +configs.help.tracking-user_views = Tracking which sections users visit during the time the use dashboard +configs.help.tracking-user_feedback = Displaying NPS, ratings and Surveys targeted to specific users +configs.help.tracking-user_details = Tracking how many cohorts, funnels, journeys, surveys, NPS, ratings, alerts, dashboards, widgets, email_reports, hooks, formulas, drill queries users have created \ No newline at end of file From 5b387ac5e61453aba63ba1607a5dccd58438c0fd Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Thu, 4 Sep 2025 17:45:58 +0300 Subject: [PATCH 02/21] Simplify tracking --- api/parts/mgmt/tracker.js | 285 +++++-------------------- frontend/express/app.js | 8 +- frontend/express/libs/members.js | 102 --------- plugins/server-stats/api/jobs/stats.js | 128 +++-------- 4 files changed, 83 insertions(+), 440 deletions(-) mode change 100755 => 100644 frontend/express/libs/members.js diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index d878ab7188f..fa416fa5f4e 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -10,76 +10,42 @@ var tracker = {}, logger = require('../../utils/log.js'), //log = logger("tracker:server"), Countly = require('countly-sdk-nodejs'), - countlyConfig = require("../../../frontend/express/config.js"), - versionInfo = require('../../../frontend/express/version.info'), - ip = require('./ip.js'), - cluster = require('cluster'), - os = require('os'), fs = require('fs'), - asyncjs = require('async'), - trial = "79199134e635edb05fc137e8cd202bb8640fb0eb", - app = "e70ec21cbe19e799472dfaee0adb9223516d238f", - server = "e0693b48a5513cb60c112c21aede3cab809d52d0", + path = require('path'), + versionInfo = require('../../../frontend/express/version.info'), + server = "9c28c347849f2c03caf1b091ec7be8def435e85e", + user = "fa6e9ae7b410cb6d756e8088c5f3936bf1fab5f3", url = "https://stats.count.ly", plugins = require('../../../plugins/pluginManager.js'), - request = require('countly-request')(plugins.getConfig("security")), - offlineMode = plugins.getConfig("api").offline_mode, domain = plugins.getConfig("api").domain; +var IS_FLEX = false; -//update configs -var cache = {}; -if (countlyConfig.web && countlyConfig.web.track === "all") { - countlyConfig.web.track = null; -} -var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig)); -var url_check = "https://count.ly/configurations/ce/tracking"; -if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") { - url_check = "https://count.ly/configurations/ee/tracking"; -} - -if (!offlineMode) { - request(url_check, function(err, response, body) { - if (typeof body === "string") { - try { - body = JSON.parse(body); - } - catch (ex) { - body = null; - } - } - if (body) { - if (countlyConfigOrig.web.use_intercom && typeof body.intercom !== "undefined") { - countlyConfig.web.use_intercom = body.intercom; - } - if (typeof countlyConfigOrig.web.track === "undefined" && typeof body.stats !== "undefined") { - if (body.stats) { - countlyConfig.web.track = null; - } - else { - countlyConfig.web.track = "none"; - } - } - if (typeof countlyConfigOrig.web.server_track === "undefined" && typeof body.server !== "undefined") { - if (body.server) { - countlyConfig.web.server_track = null; - } - else { - countlyConfig.web.server_track = "none"; - } - } +if (fs.existsSync(path.resolve('/opt/deployment_env.json'))) { + var deploymentConf = fs.readFileSync('/opt/deployment_env.json', 'utf8'); + try { + if (JSON.parse(deploymentConf).DEPLOYMENT_ID) { + IS_FLEX = true; } - }); + } + catch (e) { + IS_FLEX = false; + } } +//update configs var isEnabled = false; /** * Enable tracking for this server **/ tracker.enable = function() { + if (isEnabled) { + return; + } + var config = { - app_key: (versionInfo.trial) ? trial : server, + app_key: server, url: url, app_version: versionInfo.version, storage_path: "../../../.sdk/", @@ -102,59 +68,22 @@ tracker.enable = function() { else if (!domain) { checkDomain(); } - isEnabled = true; - if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") { - Countly.track_errors(); - } - if (cluster.isMaster) { + Countly.user_details({"name": config.device_id }); + plugins.loadConfigs(common.db, function() { + if (plugins.getConfig("tracking").server_sessions) { + Countly.begin_session(true); + } + if (plugins.getConfig("tracking").server_crashes) { + Countly.track_errors(); + } setTimeout(function() { - if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") { - Countly.begin_session(true); - setTimeout(function() { - collectServerStats(); - collectServerData(); - }, 20000); + if (plugins.getConfig("tracking").server_user_details) { + collectServerStats(); } - }, 1000); - //report app start trace - if (Countly.report_app_start) { - Countly.report_app_start(); - } - } -}; - -/** -* Enable tracking for dashboard process -**/ -tracker.enableDashboard = function() { - var config = { - app_key: (versionInfo.trial) ? trial : server, - url: url, - app_version: versionInfo.version, - storage_path: "../../../.sdk/", - interval: 60000, - fail_timeout: 600, - session_update: 120, - debug: (logger.getLevel("tracker:server") === "debug") - }; - //set static device id if domain is defined - if (domain) { - config.device_id = stripTrailingSlash((domain + "").split("://").pop()); - } - Countly.init(config); - - //change device id if is it not domain - if (domain && Countly.get_device_id() !== domain) { - Countly.change_id(stripTrailingSlash((domain + "").split("://").pop()), true); - } - else if (!domain) { - checkDomain(); - } - isEnabled = true; - if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") { - Countly.track_errors(); - } + collectServerData(); + }, 20000); + }); }; @@ -163,7 +92,7 @@ tracker.enableDashboard = function() { * @param {object} event - event object **/ tracker.reportEvent = function(event) { - if (isEnabled && countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") { + if (isEnabled && plugins.getConfig("tracking").server_events) { Countly.add_event(event); } }; @@ -172,12 +101,11 @@ tracker.reportEvent = function(event) { * Report user level event * @param {string} id - id of the device * @param {object} event - event object -* @param {string} level - tracking level **/ -tracker.reportUserEvent = function(id, event, level) { - if (isEnabled && countlyConfig.web.track !== "none" && (!level || countlyConfig.web.track === level) && countlyConfig.web.server_track !== "none") { +tracker.reportUserEvent = function(id, event) { + if (isEnabled && plugins.getConfig("tracking").user_events) { Countly.request({ - app_key: app, + app_key: user, device_id: id, events: JSON.stringify([event]) }); @@ -186,11 +114,10 @@ tracker.reportUserEvent = function(id, event, level) { /** * Check if tracking enabled -* @param {boolean|string} level - level of tracking * @returns {boolean} if enabled **/ -tracker.isEnabled = function(level) { - return (isEnabled && countlyConfig.web.track !== "none" && (!level || countlyConfig.web.track === level) && countlyConfig.web.server_track !== "none"); +tracker.isEnabled = function() { + return isEnabled; }; /** @@ -235,133 +162,21 @@ function collectServerStats() { // eslint-disable-line no-unused-vars * Get server data **/ function collectServerData() { + Countly.userData.set("trial", versionInfo.trial ? true : false); Countly.userData.set("plugins", plugins.getPlugins()); - var cpus = os.cpus(); - if (cpus && cpus.length) { - Countly.userData.set("cores", cpus.length); - } - Countly.userData.set("nodejs_version", process.version); - if (common.db.build && common.db.build.version) { - Countly.userData.set("db_version", common.db.build.version); - } - common.db.command({ serverStatus: 1 }, function(errCmd, res) { - if (res && res.storageEngine && res.storageEngine.name) { - Countly.userData.set("db_engine", res.storageEngine.name); - } - getDomain(function(err, domainname) { - if (!err) { - Countly.userData.set("domain", domainname); - Countly.user_details({"name": stripTrailingSlash((domainname + "").split("://").pop())}); - } - getDistro(function(err2, distro) { - if (!err2) { - Countly.userData.set("distro", distro); - } - getHosting(function(err3, hosting) { - if (!err3) { - Countly.userData.set("hosting", hosting); - } - Countly.userData.save(); - }); - }); - }); - }); -} - -/** -* Get server domain or ip -* @param {function} callback - callback to get results -**/ -function getDomain(callback) { - if (cache.domain) { - callback(false, cache.domain); + Countly.userData.set("nodejs", process.version); + var edition = "Lite"; + if (IS_FLEX) { + edition = "Flex"; } - else { - ip.getHost(function(err, host) { - cache.domain = host; - callback(err, host); - }); + else if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") { + edition = "Enterprise"; } -} - -/** -* Get server hosting provider -* @param {function} callback - callback to get results -**/ -function getHosting(callback) { - if (cache.hosting) { - callback(cache.hosting.length === 0, cache.hosting); - } - else { - var hostings = { - "Digital Ocean": "http://169.254.169.254/metadata/v1/hostname", - "Google Cloud": "http://metadata.google.internal", - "AWS": "http://169.254.169.254/latest/dynamic/instance-identity/" - }; - asyncjs.eachSeries(Object.keys(hostings), function(host, done) { - request(hostings[host], function(err, response) { - if (response && response.statusCode >= 200 && response.statusCode < 300) { - cache.hosting = host; - callback(false, cache.hosting); - done(true); - } - else { - done(); - } - }); - }, function() { - if (!cache.hosting) { - callback(true, cache.hosting); - } - }); - } -} - -/** -* Get OS distro -* @param {function} callback - callback to get results -**/ -function getDistro(callback) { - if (cache.distro) { - callback(cache.distro.length === 0, cache.distro); - } - else { - var oses = {"win32": "Windows", "darwin": "macOS"}; - var osName = os.platform(); - // Linux is a special case. - if (osName !== 'linux') { - cache.distro = oses[osName] ? oses[osName] : osName; - cache.distro += " " + os.release(); - callback(false, cache.distro); - } - else { - var distros = { - "/etc/lsb-release": {name: "Ubuntu", regex: /distrib_release=(.*)/i}, - "/etc/redhat-release": {name: "RHEL/Centos", regex: /release ([^ ]+)/i} - }; - asyncjs.eachSeries(Object.keys(distros), function(distro, done) { - //check ubuntu - fs.readFile(distro, 'utf8', (err, data) => { - if (!err && data) { - cache.distro = distros[distro].name; - var match = data.match(distros[distro].regex); - if (match[1]) { - cache.distro += " " + match[1]; - } - callback(false, cache.distro); - done(true); - } - else { - done(null); - } - }); - }, function() { - if (!cache.distro) { - callback(true, cache.distro); - } - }); - } + Countly.userData.set("edition", edition); + if (common.db.build && common.db.build.version) { + Countly.userData.set("mongodb", common.db.build.version); } + Countly.userData.save(); } /** diff --git a/frontend/express/app.js b/frontend/express/app.js index 9680f48bb00..1d721680d67 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -136,8 +136,7 @@ plugins.setConfigs("frontend", { session_timeout: 30, use_google: true, code: true, - offline_mode: false, - self_tracking: "", + offline_mode: false }); if (!plugins.isPluginEnabled('tracker')) { @@ -834,11 +833,6 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ res.send(plugins.getConfig("security").robotstxt); }); - app.get(countlyConfig.path + '/configs', function(req, res) { - membersUtility.recheckConfigs(countlyConfigOrig, countlyConfig); - res.send("Success"); - }); - app.get(countlyConfig.path + '/session', function(req, res, next) { if (req.session.auth_token) { authorize.verify_return({ diff --git a/frontend/express/libs/members.js b/frontend/express/libs/members.js old mode 100755 new mode 100644 index a9c42ad2f83..ec99041c7ff --- a/frontend/express/libs/members.js +++ b/frontend/express/libs/members.js @@ -12,18 +12,12 @@ var authorize = require('./../../../api/utils/authorizer.js'); //for token validations var common = require('./../../../api/utils/common.js'); var plugins = require('./../../../plugins/pluginManager.js'); -var { getUserApps } = require('./../../../api/utils/rights.js'); var configs = require('./../config', 'dont-enclose'); var countlyMail = require('./../../../api/parts/mgmt/mail.js'); -var countlyStats = require('./../../../api/parts/data/stats.js'); -var request = require('countly-request')(plugins.getConfig("security")); var url = require('url'); var crypto = require('crypto'); var argon2 = require('argon2'); -var versionInfo = require('./../version.info'), - COUNTLY_TYPE = versionInfo.type; - /** @lends module:frontend/express/libs/members */ var membersUtility = { }; //Helper functions @@ -55,44 +49,6 @@ membersUtility.emptyPermission = { } }; -/** Checks remote configuration and sets variables to configuration object - * @param {object} countlyConfigOrig - configuration settings object. Original(ar read from file) - * @param {object} countlyConfig - contiguration. Changes if are done on this object. -*/ -membersUtility.recheckConfigs = function(countlyConfigOrig, countlyConfig) { - var checkUrl = "https://count.ly/configurations/ce/tracking"; - if (COUNTLY_TYPE !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") { - checkUrl = "https://count.ly/configurations/ee/tracking"; - } - if (!plugins.getConfig("api").offline_mode) { - request(checkUrl, function(error, response, body) { - if (typeof body === "string") { - try { - body = JSON.parse(body); - } - catch (ex) { - body = null; - } - } - if (body) { - if (countlyConfigOrig.web.use_intercom && typeof body.intercom !== "undefined") { - countlyConfig.web.use_intercom = body.intercom; - } - if (typeof countlyConfigOrig.web.track === "undefined" && typeof body.stats !== "undefined") { - if (body.stats) { - countlyConfig.web.track = null; - } - else { - countlyConfig.web.track = "none"; - } - } - } - }); - } -}; -var origConf = JSON.parse(JSON.stringify(membersUtility.countlyConfig)); -membersUtility.recheckConfigs(origConf, membersUtility.countlyConfig); - /** * Is hashed string argon2? * @param {string} hashedStr | argon2 hashed string @@ -334,58 +290,6 @@ membersUtility.verifyCredentials = function(username, password, callback) { } }; -/** -* Update Stats for member. -* -* @param {object} member - member properties -* @example -* membersUtility.updateStats(member ); -**/ -membersUtility.updateStats = function(member) { - if (plugins.getConfig('frontend').countly_tracking && !plugins.getConfig("api").offline_mode) { - countlyStats.getUser(membersUtility.db, member, function(statsObj) { - const userApps = getUserApps(member); - var custom = { - apps: (userApps) ? userApps.length : 0, - platforms: {"$addToSet": statsObj["total-platforms"]}, - events: statsObj["total-events"], - pushes: statsObj["total-msg-sent"], - crashes: statsObj["total-crash-groups"], - users: statsObj["total-users"] - }; - var date = new Date(); - let domain = plugins.getConfig('api').domain; - - try { - // try to extract hostname from full domain url - const urlObj = new URL(domain); - domain = urlObj.hostname; - } - catch (_) { - // do nothing, domain from config will be used as is - } - - request({ - uri: "https://stats.count.ly/i", - method: "GET", - timeout: 4E3, - qs: { - device_id: domain, - app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f", - timestamp: Math.round(date.getTime() / 1000), - hour: date.getHours(), - dow: date.getDay(), - user_details: JSON.stringify( - { - custom: custom - } - ) - } - }, function() {}); - }); - } -}; - /** * Tries to log in user based passed userame and password. Calls "plugins" * methods to notify successful and unsucessful logging in attempts. If @@ -422,9 +326,6 @@ membersUtility.login = function(req, res, callback) { else { plugins.callMethod("loginSuccessful", {req: req, data: member}); - // update stats - membersUtility.updateStats(member); - req.session.regenerate(function() { // will have a new session here @@ -495,9 +396,6 @@ membersUtility.loginWithExternalAuthentication = function(req, res, callback) { else { plugins.callMethod("loginSuccessful", {req: req, data: member}); - // update stats - membersUtility.updateStats(member); - req.session.regenerate(function() { // will have a new session here diff --git a/plugins/server-stats/api/jobs/stats.js b/plugins/server-stats/api/jobs/stats.js index 894ca43e416..92454a72deb 100644 --- a/plugins/server-stats/api/jobs/stats.js +++ b/plugins/server-stats/api/jobs/stats.js @@ -3,11 +3,10 @@ const job = require('../../../../api/parts/jobs/job.js'), tracker = require('../../../../api/parts/mgmt/tracker.js'), log = require('../../../../api/utils/log.js')('job:stats'), - config = require('../../../../frontend/express/config.js'), pluginManager = require('../../../pluginManager.js'), serverStats = require('../parts/stats.js'), - moment = require('moment-timezone'), - request = require('countly-request')(pluginManager.getConfig('security')); + common = require('../../../../api/utils/common.js'), + moment = require('moment-timezone'); let drill; try { @@ -18,14 +17,6 @@ catch (ex) { drill = null; } -const promisedLoadConfigs = function(db) { - return new Promise((resolve) => { - pluginManager.loadConfigs(db, () => { - resolve(); - }); - }); -}; - /** Representing a StatsJob. Inherits api/parts/jobs/job.js (job.Job) */ class StatsJob extends job.Job { /** @@ -128,94 +119,39 @@ class StatsJob extends job.Job { * @returns {undefined} Returns nothing, only callback **/ run(db, done) { - if (config.web.track !== 'none') { - db.collection('members').find({global_admin: true}).toArray(async(err, members) => { - if (!err && members.length > 0) { - let license = {}; - if (drill) { - try { - license = await drill.loadLicense(undefined, db); - } - catch (error) { - log.e(error); - // do nothing, most likely there is no license - } - } - - const options = { - monthlyBreakdown: true, - license_hosting: license.license_hosting, - }; - - serverStats.fetchDatapoints(db, {}, options, async(allData) => { - const dataSummary = StatsJob.generateDataSummary(allData); - - let date = new Date(); - const usersData = []; - - await promisedLoadConfigs(db); - - let domain = ''; - - try { - // try to extract hostname from full domain url - const urlObj = new URL(pluginManager.getConfig('api').domain); - domain = urlObj.hostname; - } - catch (_) { - // do nothing, domain from config will be used as is - } - - usersData.push({ - device_id: domain, - timestamp: Math.floor(date.getTime() / 1000), - hour: date.getHours(), - dow: date.getDay(), - user_details: JSON.stringify({ - custom: { - dataPointsAll: dataSummary.all, - dataPointsMonthlyAvg: dataSummary.avg, - dataPointsLast3Months: dataSummary.month3, - }, - }), - }); - - var formData = { - app_key: 'e70ec21cbe19e799472dfaee0adb9223516d238f', - requests: JSON.stringify(usersData) - }; - - request.post({ - url: 'https://stats.count.ly/i/bulk', - json: formData - }, function(a) { - log.d('Done running stats job: %j', a); - done(); - }); - - if (tracker.isEnabled()) { - const dataMonthly = StatsJob.generateDataMonthly(allData); - - const Countly = tracker.getSDK(); - Countly.user_details({ - 'custom': dataMonthly, - }); - - Countly.userData.save(); - } - }); + pluginManager.loadConfigs(db, async function() { + let license = {}; + if (drill) { + try { + license = await drill.loadLicense(undefined, db); } - else { - done(); + catch (error) { + log.e(error); + // do nothing, most likely there is no license } + } + + const options = { + monthlyBreakdown: true, + license_hosting: license.license_hosting, + }; + + serverStats.fetchDatapoints(db, {}, options, async(allData) => { + const dataSummary = StatsJob.generateDataSummary(allData); + const dataMonthly = StatsJob.generateDataMonthly(allData); + dataMonthly.dataPointsAll = dataSummary.all; + dataMonthly.dataPointsMonthlyAvg = dataSummary.avg; + dataMonthly.dataPointsLast3Months = dataSummary.month3; + common.db = db; // ensure common.db is set + tracker.enable(); // ensure tracking is enabled + const Countly = tracker.getSDK(); + Countly.user_details({ + 'custom': dataMonthly, + }); + Countly.userData.save(); + done(); }); - } - else { - db.collection('plugins').updateOne({_id: 'plugins'}, {$unset: {remoteConfig: 1}}).catch(dbe => { - log.e('Db error', dbe); - }); - done(); - } + }); } } From 9c52770e92d7af15482745323664c06ba2bc6ad9 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Fri, 5 Sep 2025 22:00:45 +0300 Subject: [PATCH 03/21] New server side tracking --- api/api.js | 2 +- api/jobs/ping.js | 163 +++++++++++++----------- api/parts/mgmt/tracker.js | 33 +++++ frontend/express/app.js | 4 - plugins/server-stats/api/api.js | 2 +- plugins/server-stats/api/jobs/stats.js | 91 ++++++++++--- plugins/server-stats/api/parts/stats.js | 17 +++ 7 files changed, 214 insertions(+), 98 deletions(-) diff --git a/api/api.js b/api/api.js index e1b206a8ea5..86b4c7cb951 100644 --- a/api/api.js +++ b/api/api.js @@ -344,7 +344,7 @@ plugins.connectToAllDatabases().then(function() { // Allow configs to load & scanner to find all jobs classes setTimeout(() => { jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day'); - jobs.job('api:ping').replace().schedule('every 1 day'); + jobs.job('api:ping').replace().schedule('at 00:01 am ' + 'every 1 day'); jobs.job('api:clear').replace().schedule('every 1 day'); jobs.job('api:clearTokens').replace().schedule('every 1 day'); jobs.job('api:clearAutoTasks').replace().schedule('every 1 day'); diff --git a/api/jobs/ping.js b/api/jobs/ping.js index 984e82c566c..ffc4665d698 100644 --- a/api/jobs/ping.js +++ b/api/jobs/ping.js @@ -1,12 +1,8 @@ 'use strict'; const job = require('../parts/jobs/job.js'), - log = require('../utils/log.js')('job:ping'), - countlyConfig = require("../../frontend/express/config.js"), - versionInfo = require('../../frontend/express/version.info'), plugins = require('../../plugins/pluginManager.js'), - request = require('countly-request')(plugins.getConfig("security")); - + tracker = require('../parts/mgmt/tracker.js'); /** Class for the job of pinging servers **/ class PingJob extends job.Job { @@ -16,86 +12,101 @@ class PingJob extends job.Job { * @param {done} done callback */ run(db, done) { - request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {}); - var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig)); - var url = "https://count.ly/configurations/ce/tracking"; - if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") { - url = "https://count.ly/configurations/ee/tracking"; - } - plugins.loadConfigs(db, function() { + plugins.loadConfigs(db, async function() { const offlineMode = plugins.getConfig("api").offline_mode; - const { countly_tracking } = plugins.getConfig('frontend'); if (!offlineMode) { - request(url, function(err, response, body) { - if (typeof body === "string") { - try { - body = JSON.parse(body); - } - catch (ex) { - body = null; - } - } - if (body) { - if (countlyConfigOrig.web.use_intercom && typeof body.intercom !== "undefined") { - countlyConfig.web.use_intercom = body.intercom; - } - if (typeof countlyConfigOrig.web.track === "undefined" && typeof body.stats !== "undefined") { - if (body.stats) { - countlyConfig.web.track = null; - } - else { - countlyConfig.web.track = "none"; - } - } + var server = tracker.getBulkServer(); + var user = tracker.getBulkUser(server); + var days = 30; + var current_sync = Date.now(); + + // Atomically retrieve old last_sync value and set new one + var syncResult = await db.collection("plugins").findOneAndUpdate( + {_id: "version"}, + {$set: {last_sync: current_sync}}, + { + upsert: true, + returnDocument: 'before', + projection: {last_sync: 1} } - log.d(err, body, countlyConfigOrig, countlyConfig); - if (countly_tracking) { - db.collection("members").findOne({global_admin: true}, function(err2, member) { - if (!err2 && member) { - var date = new Date(); - let domain = plugins.getConfig('api').domain; + ); - try { - // try to extract hostname from full domain url - const urlObj = new URL(domain); - domain = urlObj.hostname; - } - catch (_) { - // do nothing, domain from config will be used as is - } + var last_sync = syncResult.value ? syncResult.value.last_sync : null; + if (last_sync) { + days = Math.floor((new Date().getTime() - last_sync) / (1000 * 60 * 60 * 24)); + } - request({ - uri: "https://stats.count.ly/i", - method: "GET", - timeout: 4E3, - qs: { - device_id: domain, - app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f", - timestamp: Math.floor(date.getTime() / 1000), - hour: date.getHours(), - dow: date.getDay(), - no_meta: true, - events: JSON.stringify([ - { - key: "PING", - count: 1 - } - ]) + if (days > 0) { + //calculate seconds timestamp of days before today + var startTs = Math.round((new Date().getTime() - (30 * 24 * 60 * 60 * 1000)) / 1000); + + //sync server events - use aggregation pipeline to group by day and action on MongoDB side + var aggregationPipeline = [ + // Match documents with timestamp greater than startTs and valid action + { + $match: { + ts: { $gt: startTs } + } + }, + // Add calculated fields for day grouping + { + $addFields: { + // Convert timestamp to date and set to noon (12:00:00) + dayDate: { + $dateFromParts: { + year: { $year: { $toDate: { $multiply: ["$ts", 1000] } } }, + month: { $month: { $toDate: { $multiply: ["$ts", 1000] } } }, + day: { $dayOfMonth: { $toDate: { $multiply: ["$ts", 1000] } } }, + hour: 12, + minute: 0, + second: 0 } - }, function(a/*, c, b*/) { - log.d('Done running ping job: %j', a); - done(); - }); + } + } + }, + // Convert back to timestamp in seconds + { + $addFields: { + noonTimestamp: { + $divide: [{ $toLong: "$dayDate" }, 1000] + } } - else { - done(); + }, + // Group by day and action + { + $group: { + _id: { + day: "$noonTimestamp", + action: "$a" + }, + count: { $sum: 1 } } - }); + }, + // Project to final format + { + $project: { + _id: 0, + action: "$_id.action", + timestamp: "$_id.day", + count: 1 + } + } + ]; + + var cursor = db.collection("systemlogs").aggregate(aggregationPipeline); + + while (cursor && await cursor.hasNext()) { + let eventData = await cursor.next(); + user.add_event({key: eventData.action, count: eventData.count, timestamp: eventData.timestamp}); } - else { + server.start(function() { + server.stop(); done(); - } - }); + }); + } + else { + done(); + } } else { done(); @@ -104,4 +115,4 @@ class PingJob extends job.Job { } } -module.exports = PingJob; +module.exports = PingJob; \ No newline at end of file diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index fa416fa5f4e..4bd50b9b036 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -86,6 +86,25 @@ tracker.enable = function() { }); }; +/** + * Get bulk server instance + * @returns {Object} Countly Bulk instance + */ +tracker.getBulkServer = function() { + return new Countly.Bulk({ + app_key: server, + url: url + }); +}; + +/** + * Get bulk user instance + * @param {Object} serverInstance - Countly Bulk server instance + * @returns {Object} Countly Bulk User instance + */ +tracker.getBulkUser = function(serverInstance) { + return serverInstance.add_user({device_id: stripTrailingSlash((plugins.getConfig("api").domain + "").split("://").pop())}); +}; /** * Report server level event @@ -97,6 +116,20 @@ tracker.reportEvent = function(event) { } }; +/** +* Report server level event in bulk +* @param {Array} events - array of event objects +**/ +tracker.reportEventBulk = function(events) { + if (isEnabled && plugins.getConfig("tracking").server_events) { + Countly.request({ + app_key: server, + device_id: Countly.get_device_id(), + events: JSON.stringify(events) + }); + } +}; + /** * Report user level event * @param {string} id - id of the device diff --git a/frontend/express/app.js b/frontend/express/app.js index 1d721680d67..747d48ef94b 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -194,16 +194,12 @@ if (countlyConfig.web && countlyConfig.web.track === "all") { countlyConfig.web.track = null; } -var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig)); - Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_fs")]).then(function(dbs) { var countlyDb = dbs[0]; //reference for consistency between app and api processes membersUtility.db = common.db = countlyDb; countlyFs.setHandler(dbs[1]); - //checking remote configuration - membersUtility.recheckConfigs(countlyConfigOrig, countlyConfig); /** * Create sha1 hash string * @param {string} str - string to hash diff --git a/plugins/server-stats/api/api.js b/plugins/server-stats/api/api.js index abd498dd544..09ca36ce3ea 100644 --- a/plugins/server-stats/api/api.js +++ b/plugins/server-stats/api/api.js @@ -21,7 +21,7 @@ const internalEventsSkipped = ["[CLY]_orientation"]; plugins.register("/master", function() { // Allow configs to load & scanner to find all jobs classes setTimeout(() => { - require('../../../api/parts/jobs').job('server-stats:stats').replace().schedule('every 1 day'); + require('../../../api/parts/jobs').job('server-stats:stats').replace().schedule('at 00:01 am ' + 'every 1 day'); }, 10000); }); diff --git a/plugins/server-stats/api/jobs/stats.js b/plugins/server-stats/api/jobs/stats.js index 92454a72deb..358cb1b0130 100644 --- a/plugins/server-stats/api/jobs/stats.js +++ b/plugins/server-stats/api/jobs/stats.js @@ -5,7 +5,6 @@ const job = require('../../../../api/parts/jobs/job.js'), log = require('../../../../api/utils/log.js')('job:stats'), pluginManager = require('../../../pluginManager.js'), serverStats = require('../parts/stats.js'), - common = require('../../../../api/utils/common.js'), moment = require('moment-timezone'); let drill; @@ -64,9 +63,19 @@ class StatsJob extends job.Job { const ids6 = {}; const ids0 = {}; const order = []; + var thisMonth = ""; + var lastMonth = ""; + var thisMonthDP = 0; + var lastMonthDP = 0; for (let i = 0; i < 12; i++) { order.push(utcMoment.format('MMM YYYY')); + if (i === 0) { + thisMonth = utcMoment.format('YYYY:M'); + } + if (i === 1) { + lastMonth = utcMoment.format('YYYY:M'); + } ids[utcMoment.format('YYYY:M')] = utcMoment.format('MMM YYYY'); if (i < 7) { ids6[utcMoment.format('YYYY:M')] = utcMoment.format('MMM YYYY'); @@ -96,6 +105,12 @@ class StatsJob extends job.Job { avg6monthDP += DP[ids[key]]; avg6++; } + if (key === thisMonth) { + thisMonthDP = value; + } + if (key === lastMonth) { + lastMonthDP = value; + } } } @@ -103,10 +118,16 @@ class StatsJob extends job.Job { data.DP.push((i < 9 ? '0' + (i + 1) : i + 1) + '. ' + order[i] + ': ' + ((DP[order[i]] || 0).toLocaleString())); } if (avg12) { - data['Last 12 months'] = Math.round(avg12monthDP / avg12).toLocaleString(); + data['Last 12 months avg'] = Math.round(avg12monthDP / avg12); } if (avg6) { - data['Last 6 months'] = Math.round(avg6monthDP / avg6).toLocaleString(); + data['Last 6 months avg'] = Math.round(avg6monthDP / avg6); + } + if (lastMonthDP) { + data['Last month'] = lastMonthDP; + } + if (thisMonthDP) { + data['This month'] = thisMonthDP; } return data; @@ -131,26 +152,64 @@ class StatsJob extends job.Job { } } + var server = tracker.getBulkServer(); + var user = tracker.getBulkUser(server); + var days = 30; + var current_sync = Date.now(); + + //generate dates in YYYY:M:D format for dates from "days" variable back up to today + const specificDates = []; + const utcMoment = moment.utc(); + for (let i = 0; i < days; i++) { + specificDates.push(utcMoment.format('YYYY:M:D')); + utcMoment.subtract(1, 'days'); + } + const options = { + dailyDates: specificDates, monthlyBreakdown: true, license_hosting: license.license_hosting, }; - serverStats.fetchDatapoints(db, {}, options, async(allData) => { - const dataSummary = StatsJob.generateDataSummary(allData); - const dataMonthly = StatsJob.generateDataMonthly(allData); - dataMonthly.dataPointsAll = dataSummary.all; - dataMonthly.dataPointsMonthlyAvg = dataSummary.avg; - dataMonthly.dataPointsLast3Months = dataSummary.month3; - common.db = db; // ensure common.db is set - tracker.enable(); // ensure tracking is enabled - const Countly = tracker.getSDK(); - Countly.user_details({ - 'custom': dataMonthly, + // Atomically retrieve old last_sync value and set new one + var syncResult = await db.collection("plugins").findOneAndUpdate( + {_id: "version"}, + {$set: {last_dp_sync: current_sync}}, + { + upsert: true, + returnDocument: 'before', + projection: {last_dp_sync: 1} + } + ); + + var last_dp_sync = syncResult.value ? syncResult.value.last_dp_sync : null; + if (last_dp_sync) { + days = Math.floor((new Date().getTime() - last_dp_sync) / (1000 * 60 * 60 * 24)); + } + + if (days > 0) { + serverStats.fetchDatapoints(db, {}, options, async(allData) => { + const dataMonthly = StatsJob.generateDataMonthly(allData); + + if (allData.daily) { + for (const key in allData.daily) { + var parts = key.split(':'); + //convert date in YYYY:M:D format to timestamp for noon (12:00:00) of that day in UTC + const timestamp = moment.tz(parts[0] + '-' + parts[1] + '-' + parts[2] + ' 12:00:00', 'YYYY-M-D HH:mm:ss', 'UTC').valueOf(); + //send datapoint event with timestamp for noon of that day + user.add_event({key: "DP", count: allData.daily[key], timestamp: timestamp}); + } + } + user.user_details({'custom': dataMonthly}); + server.start(function() { + server.stop(); + done(); + }); }); - Countly.userData.save(); + } + else { done(); - }); + } }); } } diff --git a/plugins/server-stats/api/parts/stats.js b/plugins/server-stats/api/parts/stats.js index da95217a233..1cc51278f04 100644 --- a/plugins/server-stats/api/parts/stats.js +++ b/plugins/server-stats/api/parts/stats.js @@ -236,6 +236,7 @@ function punchCard(db, filter, options) { * @param {object} options - array with periods * @param {boolean} options.monthlyBreakdown - if true, will calculate monthly data points breakdown for all apps (used to get license metric) * @param {string} options.license_hosting - client hosting type, could be countly hosted or self hosted. This will determine how consolidated data points should be added to total data points + * @param {boolean} options.dailyDates - array of dates in YYYY:M:D format for daily data points (used to get data points for last 30 days) * @param {function} callback - callback */ function fetchDatapoints(db, filter, options, callback) { @@ -274,6 +275,22 @@ function fetchDatapoints(db, filter, options, callback) { acc[current.m] = dp; } + if (options.dailyDates && options.dailyDates.length && current.m && (!/^\[CLY\]_consolidated/.test(current._id) || options.license_hosting === 'Countly-Hosted')) { + if (!acc.daily) { + acc.daily = {}; + } + options.dailyDates.forEach(date => { + if (date.startsWith(current.m)) { + var day = date.split(":")[2]; + if (current.d && current.d[day] && Object.keys(current.d[day]).length) { + for (var hour in current.d[day]) { + acc.daily[date] = (acc.daily[date] || 0) + (current.d[day][hour].e || 0) + (current.d[day][hour].s || 0); + } + } + } + }); + } + return acc; }, {}); From 304c054f488a33fc241a7c568b7dd0fab38f3485 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Mon, 8 Sep 2025 15:12:55 +0300 Subject: [PATCH 04/21] Track server by default --- api/api.js | 2 ++ frontend/express/app.js | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/api/api.js b/api/api.js index 86b4c7cb951..ebc5ec38c0e 100644 --- a/api/api.js +++ b/api/api.js @@ -16,6 +16,7 @@ const {WriteBatcher, ReadBatcher, InsertBatcher} = require('./parts/data/batcher const pack = require('../package.json'); const versionInfo = require('../frontend/express/version.info.js'); const moment = require("moment"); +const tracker = require('./parts/mgmt/tracker.js'); var t = ["countly:", "api"]; common.processRequest = processRequest; @@ -38,6 +39,7 @@ else { process.title = t.join(' '); plugins.connectToAllDatabases().then(function() { + tracker.enable(); common.writeBatcher = new WriteBatcher(common.db); common.readBatcher = new ReadBatcher(common.db); common.insertBatcher = new InsertBatcher(common.db); diff --git a/frontend/express/app.js b/frontend/express/app.js index 747d48ef94b..267dd4b92d3 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -72,7 +72,8 @@ var versionInfo = require('./version.info'), argon2 = require('argon2'), countlyCommon = require('../../api/lib/countly.common.js'), timezones = require('../../api/utils/timezones.js').getTimeZones, - { validateCreate } = require('../../api/utils/rights.js'); + { validateCreate } = require('../../api/utils/rights.js'), + tracker = require('../../api/parts/mgmt/tracker.js'); console.log("Starting Countly", "version", versionInfo.version, "package", pack.version); @@ -199,6 +200,7 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ //reference for consistency between app and api processes membersUtility.db = common.db = countlyDb; countlyFs.setHandler(dbs[1]); + tracker.enable(); /** * Create sha1 hash string From 320f15e953fad3fe175e2852e3a46b7ce9785175 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Mon, 8 Sep 2025 15:21:26 +0300 Subject: [PATCH 05/21] Expose tracking settings --- frontend/express/app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/express/app.js b/frontend/express/app.js index 267dd4b92d3..01ccde76e18 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -1000,6 +1000,7 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ member: member, config: req.config, security: plugins.getConfig("security"), + tracking: plugins.getConfig("tracking"), plugins: plugins.getPlugins(), pluginsFull: plugins.getPlugins(true), path: countlyConfig.path || "", From ee53f775d0c5e9ed49b7201bc699e11771633a8a Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Fri, 12 Sep 2025 11:19:04 +0300 Subject: [PATCH 06/21] Add correct keys --- api/jobs/ping.js | 2 +- frontend/express/app.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/jobs/ping.js b/api/jobs/ping.js index ffc4665d698..d66946942bb 100644 --- a/api/jobs/ping.js +++ b/api/jobs/ping.js @@ -17,7 +17,7 @@ class PingJob extends job.Job { if (!offlineMode) { var server = tracker.getBulkServer(); var user = tracker.getBulkUser(server); - var days = 30; + var days = 90; var current_sync = Date.now(); // Atomically retrieve old last_sync value and set new one diff --git a/frontend/express/app.js b/frontend/express/app.js index 01ccde76e18..ec9d67eba12 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -1015,7 +1015,7 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ countlyTypeCE: COUNTLY_TYPE_CE, countly_tracking, countly_domain, - frontend_app: versionInfo.frontend_app || 'e70ec21cbe19e799472dfaee0adb9223516d238f', + frontend_app: versionInfo.frontend_app || "9c28c347849f2c03caf1b091ec7be8def435e85e", frontend_server: versionInfo.frontend_server || 'https://stats.count.ly/', usermenu: { feedbackLink: COUNTLY_FEEDBACK_LINK, From 5e95cdfe3d268a0f1d9ad65ceb026227a83fd1b4 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Fri, 12 Sep 2025 16:29:01 +0300 Subject: [PATCH 07/21] Track only servers with domain, no local developments --- api/api.js | 4 +- api/jobs/ping.js | 3 ++ api/parts/mgmt/tracker.js | 52 ++++++++++---------------- frontend/express/app.js | 1 + plugins/server-stats/api/jobs/stats.js | 3 ++ plugins/system-utility/api/api.js | 6 +-- 6 files changed, 33 insertions(+), 36 deletions(-) diff --git a/api/api.js b/api/api.js index ebc5ec38c0e..c9c858fcfb5 100644 --- a/api/api.js +++ b/api/api.js @@ -39,7 +39,9 @@ else { process.title = t.join(' '); plugins.connectToAllDatabases().then(function() { - tracker.enable(); + plugins.loadConfigs(common.db, function() { + tracker.enable(); + }); common.writeBatcher = new WriteBatcher(common.db); common.readBatcher = new ReadBatcher(common.db); common.insertBatcher = new InsertBatcher(common.db); diff --git a/api/jobs/ping.js b/api/jobs/ping.js index d66946942bb..dc43ab30bf2 100644 --- a/api/jobs/ping.js +++ b/api/jobs/ping.js @@ -17,6 +17,9 @@ class PingJob extends job.Job { if (!offlineMode) { var server = tracker.getBulkServer(); var user = tracker.getBulkUser(server); + if (!user) { + return done(); + } var days = 90; var current_sync = Date.now(); diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index 4bd50b9b036..2bd89138b43 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -16,8 +16,7 @@ var tracker = {}, server = "9c28c347849f2c03caf1b091ec7be8def435e85e", user = "fa6e9ae7b410cb6d756e8088c5f3936bf1fab5f3", url = "https://stats.count.ly", - plugins = require('../../../plugins/pluginManager.js'), - domain = plugins.getConfig("api").domain; + plugins = require('../../../plugins/pluginManager.js'); var IS_FLEX = false; @@ -51,26 +50,28 @@ tracker.enable = function() { storage_path: "../../../.sdk/", interval: 10000, fail_timeout: 600, - session_update: 120, + session_update: 60 * 60 * 12, remote_config: true, debug: (logger.getLevel("tracker:server") === "debug") }; + + var domain = plugins.getConfig("api").domain; + //set static device id if domain is defined if (domain) { config.device_id = stripTrailingSlash((domain + "").split("://").pop()); } - Countly.init(config); - //change device id if is it not domain - if (domain && Countly.get_device_id() !== domain) { - Countly.change_id(stripTrailingSlash((domain + "").split("://").pop()), true); - } - else if (!domain) { - checkDomain(); - } - isEnabled = true; - Countly.user_details({"name": config.device_id }); - plugins.loadConfigs(common.db, function() { + if (config.device_id && config.device_id !== "localhost") { + Countly.init(config); + + //change device id if is it not domain + if (Countly.get_device_id() !== domain) { + Countly.change_id(stripTrailingSlash((domain + "").split("://").pop()), true); + } + + isEnabled = true; + Countly.user_details({"name": config.device_id }); if (plugins.getConfig("tracking").server_sessions) { Countly.begin_session(true); } @@ -83,7 +84,7 @@ tracker.enable = function() { } collectServerData(); }, 20000); - }); + } }; /** @@ -103,7 +104,10 @@ tracker.getBulkServer = function() { * @returns {Object} Countly Bulk User instance */ tracker.getBulkUser = function(serverInstance) { - return serverInstance.add_user({device_id: stripTrailingSlash((plugins.getConfig("api").domain + "").split("://").pop())}); + var domain = stripTrailingSlash((plugins.getConfig("api").domain + "").split("://").pop()); + if (domain && domain !== "localhost") { + return serverInstance.add_user({device_id: domain}); + } }; /** @@ -224,20 +228,4 @@ function stripTrailingSlash(str) { return str; } -//check every hour if domain was provided -var checkDomain = function() { - if (!domain && domain !== plugins.getConfig("api").domain) { - domain = plugins.getConfig("api").domain; - if (Countly && isEnabled) { - Countly.change_id(stripTrailingSlash((domain + "").split("://").pop()), true); - Countly.userData.set("domain", domain); - Countly.user_details({"name": stripTrailingSlash((domain + "").split("://").pop())}); - Countly.userData.save(); - } - } - else if (!domain) { - setTimeout(checkDomain, 3600000); - } -}; - module.exports = tracker; \ No newline at end of file diff --git a/frontend/express/app.js b/frontend/express/app.js index ec9d67eba12..bc59023ac1f 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -414,6 +414,7 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ }; plugins.loadConfigs(countlyDb, function() { + tracker.enable(); curTheme = plugins.getConfig("frontend").theme; app.loadThemeFiles(curTheme); app.dashboard_headers = plugins.getConfig("security").dashboard_additional_headers; diff --git a/plugins/server-stats/api/jobs/stats.js b/plugins/server-stats/api/jobs/stats.js index 358cb1b0130..539a572f6b5 100644 --- a/plugins/server-stats/api/jobs/stats.js +++ b/plugins/server-stats/api/jobs/stats.js @@ -154,6 +154,9 @@ class StatsJob extends job.Job { var server = tracker.getBulkServer(); var user = tracker.getBulkUser(server); + if (!user) { + return done(); + } var days = 30; var current_sync = Date.now(); diff --git a/plugins/system-utility/api/api.js b/plugins/system-utility/api/api.js index 436ea2af9f9..ef04f854d40 100644 --- a/plugins/system-utility/api/api.js +++ b/plugins/system-utility/api/api.js @@ -1,6 +1,6 @@ var plugin = {}, common = require('../../../api/utils/common.js'), - tracker = require('../../../api/parts/mgmt/tracker.js'), + //tracker = require('../../../api/parts/mgmt/tracker.js'), plugins = require('../../pluginManager.js'), systemUtility = require('./system.utility'), log = common.log('system-utility:api'), @@ -261,7 +261,7 @@ function stopWithTimeout(type, fromTimeout = false) { }); - plugins.register("/master", function() { + /*plugins.register("/master", function() { //ping server stats periodically if (tracker.isEnabled()) { var timeout = 1000 * 60 * 15; @@ -326,7 +326,7 @@ function stopWithTimeout(type, fromTimeout = false) { }; getServerStats(); } - }); + });*/ }(plugin)); module.exports = plugin; \ No newline at end of file From c09b9ed905ea2338dd26b5680551352f17ae4e91 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Fri, 12 Sep 2025 16:42:39 +0300 Subject: [PATCH 08/21] Add frontend tracking --- frontend/express/views/dashboard.html | 229 ++++++++++---------------- 1 file changed, 90 insertions(+), 139 deletions(-) diff --git a/frontend/express/views/dashboard.html b/frontend/express/views/dashboard.html index 1c6ee63a39c..3b4cb7c32e5 100644 --- a/frontend/express/views/dashboard.html +++ b/frontend/express/views/dashboard.html @@ -1886,187 +1886,138 @@

{{this.nam <% } %> - <% if (!offline_mode) { %> - - - + - <% } %> + } + From 264ada6385c76b6e501c33787d447183bacd9585 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Fri, 12 Sep 2025 18:37:55 +0300 Subject: [PATCH 09/21] Remove user level tracking --- api/api.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/api.js b/api/api.js index c9c858fcfb5..efb4f061c65 100644 --- a/api/api.js +++ b/api/api.js @@ -165,21 +165,21 @@ plugins.connectToAllDatabases().then(function() { server_views: true, server_feedback: true, server_user_details: true, - user_sessions: true, + /*user_sessions: true, user_events: true, user_crashes: true, user_views: true, user_feedback: true, - user_details: true + user_details: true*/ }); - plugins.setUserConfigs("tracking", { + /*plugins.setUserConfigs("tracking", { user_sessions: false, user_events: false, user_crashes: false, user_views: false, user_feedback: false - }); + });*/ /** * Initialize Plugins From e71eee31874e073da3ee9832bb86624c6f88bc8b Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Mon, 15 Sep 2025 16:46:06 +0300 Subject: [PATCH 10/21] Output for test --- plugins/server-stats/tests/job.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/server-stats/tests/job.js b/plugins/server-stats/tests/job.js index 0657d113fda..da7760e2ced 100644 --- a/plugins/server-stats/tests/job.js +++ b/plugins/server-stats/tests/job.js @@ -15,6 +15,11 @@ for (let count = 0; count < 12; count += 1) { describe('Stats job', () => { it('Generates data summary', () => { const { all, avg, month3 } = StatsJob.generateDataSummary(allData); + console.log('All Data:', allData); + console.log('Data Summary:'); + console.log(`- All: ${all}`); + console.log(`- Avg: ${avg}`); + console.log(`- Month 3: ${month3}`); should(all).equal(12000); should(avg).equal(1000); From f7b62e43d5766375977aba2d456919533232504c Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Mon, 15 Sep 2025 16:57:21 +0300 Subject: [PATCH 11/21] More output for tests --- plugins/server-stats/tests/job.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/server-stats/tests/job.js b/plugins/server-stats/tests/job.js index da7760e2ced..04770961b88 100644 --- a/plugins/server-stats/tests/job.js +++ b/plugins/server-stats/tests/job.js @@ -37,9 +37,11 @@ describe('Stats job', () => { it('Generates data monthly', () => { const monthlyData = StatsJob.generateDataMonthly(allData); + console.log('All Data:', allData); + console.log(monthlyData); - should(monthlyData['Last 6 months']).equal((1000).toLocaleString()); - should(monthlyData['Last 12 months']).equal((1000).toLocaleString()); + should(monthlyData['Last 6 months']).equal(1000); + should(monthlyData['Last 12 months']).equal(1000); const expectedDP = []; for (let count = 0; count < 12; count += 1) { From 15660077dac3c3a70f5331a000e0f1626dc70a2d Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Mon, 15 Sep 2025 17:05:44 +0300 Subject: [PATCH 12/21] Fix tests --- plugins/server-stats/tests/job.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/server-stats/tests/job.js b/plugins/server-stats/tests/job.js index 04770961b88..366081d7e9d 100644 --- a/plugins/server-stats/tests/job.js +++ b/plugins/server-stats/tests/job.js @@ -40,8 +40,8 @@ describe('Stats job', () => { console.log('All Data:', allData); console.log(monthlyData); - should(monthlyData['Last 6 months']).equal(1000); - should(monthlyData['Last 12 months']).equal(1000); + should(monthlyData['Last 6 months avg']).equal(1000); + should(monthlyData['Last 12 months avg']).equal(1000); const expectedDP = []; for (let count = 0; count < 12; count += 1) { From 55620bf14ae4e3488c1ac2d2c9d861997e2ac0c0 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Wed, 1 Oct 2025 15:14:24 +0300 Subject: [PATCH 13/21] Sync properties on job too DP property naming convention --- api/jobs/ping.js | 5 +++ api/parts/mgmt/tracker.js | 55 +++++++++++++++++--------- plugins/server-stats/api/jobs/stats.js | 8 ++-- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/api/jobs/ping.js b/api/jobs/ping.js index dc43ab30bf2..ea20a9d5720 100644 --- a/api/jobs/ping.js +++ b/api/jobs/ping.js @@ -20,6 +20,11 @@ class PingJob extends job.Job { if (!user) { return done(); } + + var custom = tracker.getAllData(); + if (Object.keys(custom).length) { + user.user_details({"custom": custom }); + } var days = 90; var current_sync = Date.now(); diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index 2bd89138b43..2687c1d3409 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -79,10 +79,10 @@ tracker.enable = function() { Countly.track_errors(); } setTimeout(function() { - if (plugins.getConfig("tracking").server_user_details) { - collectServerStats(); + var custom = tracker.getAllData(); + if (Object.keys(custom).length) { + Countly.user_details({"custom": custom }); } - collectServerData(); }, 20000); } }; @@ -168,40 +168,44 @@ tracker.getSDK = function() { /** * Get server stats **/ -function collectServerStats() { // eslint-disable-line no-unused-vars +tracker.collectServerStats = function() { + var props = {}; stats.getServer(common.db, function(data) { common.db.collection("apps").aggregate([{$project: {last_data: 1}}, {$sort: {"last_data": -1}}, {$limit: 1}], {allowDiskUse: true}, function(errApps, resApps) { common.db.collection("members").aggregate([{$project: {last_login: 1}}, {$sort: {"last_login": -1}}, {$limit: 1}], {allowDiskUse: true}, function(errLogin, resLogin) { if (resApps && resApps[0]) { - Countly.userData.set("last_data", resApps[0].last_data || 0); + props.last_data = resApps[0].last_data || 0; } if (resLogin && resLogin[0]) { - Countly.userData.set("last_login", resLogin[0].last_login || 0); + props.last_login = resLogin[0].last_login || 0; } if (data) { if (data.app_users) { - Countly.userData.set("app_users", data.app_users); + props.app_users = data.app_users; } if (data.apps) { - Countly.userData.set("apps", data.apps); + props.apps = data.apps; } if (data.users) { - Countly.userData.set("users", data.users); + props.users = data.users; } } - Countly.userData.save(); + return props; }); }); }); -} +}; /** * Get server data +* @returns {Object} server data **/ -function collectServerData() { - Countly.userData.set("trial", versionInfo.trial ? true : false); - Countly.userData.set("plugins", plugins.getPlugins()); - Countly.userData.set("nodejs", process.version); +tracker.collectServerData = function() { + var props = {}; + props.trial = versionInfo.trial ? true : false; + props.plugins = plugins.getPlugins(); + props.nodejs = process.version; + props.countly = versionInfo.version; var edition = "Lite"; if (IS_FLEX) { edition = "Flex"; @@ -209,12 +213,25 @@ function collectServerData() { else if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") { edition = "Enterprise"; } - Countly.userData.set("edition", edition); + props.edition = edition; if (common.db.build && common.db.build.version) { - Countly.userData.set("mongodb", common.db.build.version); + props.mongodb = common.db.build.version; } - Countly.userData.save(); -} + return props; +}; + +/** + * Get all eligible data + * @returns {Object} all eligible data + */ +tracker.getAllData = function() { + var props = {}; + if (plugins.getConfig("tracking").server_user_details) { + Object.assign(props, tracker.collectServerStats()); + } + Object.assign(props, tracker.collectServerData()); + return props; +}; /** * Strip traling slashes from url diff --git a/plugins/server-stats/api/jobs/stats.js b/plugins/server-stats/api/jobs/stats.js index 539a572f6b5..0477012b1ba 100644 --- a/plugins/server-stats/api/jobs/stats.js +++ b/plugins/server-stats/api/jobs/stats.js @@ -118,16 +118,16 @@ class StatsJob extends job.Job { data.DP.push((i < 9 ? '0' + (i + 1) : i + 1) + '. ' + order[i] + ': ' + ((DP[order[i]] || 0).toLocaleString())); } if (avg12) { - data['Last 12 months avg'] = Math.round(avg12monthDP / avg12); + data.DPAvg12months = Math.round(avg12monthDP / avg12); } if (avg6) { - data['Last 6 months avg'] = Math.round(avg6monthDP / avg6); + data.DPAvg6months = Math.round(avg6monthDP / avg6); } if (lastMonthDP) { - data['Last month'] = lastMonthDP; + data.DPLastMonth = lastMonthDP; } if (thisMonthDP) { - data['This month'] = thisMonthDP; + data.DPThisMonth = thisMonthDP; } return data; From 3f74636f13eaad0bef39a7533fd8604f6442b7d4 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Wed, 1 Oct 2025 16:25:49 +0300 Subject: [PATCH 14/21] Added SDK and docker --- api/jobs/ping.js | 12 +- api/parts/mgmt/tracker.js | 265 +++++++++++++++++++++++++++--- plugins/server-stats/tests/job.js | 4 +- 3 files changed, 250 insertions(+), 31 deletions(-) diff --git a/api/jobs/ping.js b/api/jobs/ping.js index ea20a9d5720..4dce0a3196d 100644 --- a/api/jobs/ping.js +++ b/api/jobs/ping.js @@ -21,9 +21,15 @@ class PingJob extends job.Job { return done(); } - var custom = tracker.getAllData(); - if (Object.keys(custom).length) { - user.user_details({"custom": custom }); + try { + var custom = await tracker.getAllData(); + console.log("custom", custom); + if (Object.keys(custom).length) { + user.user_details({"custom": custom }); + } + } + catch (ex) { + console.log("Error collecting server data:", ex); } var days = 90; var current_sync = Date.now(); diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index 2687c1d3409..ee357f4182b 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -79,10 +79,11 @@ tracker.enable = function() { Countly.track_errors(); } setTimeout(function() { - var custom = tracker.getAllData(); - if (Object.keys(custom).length) { - Countly.user_details({"custom": custom }); - } + tracker.getAllData().then((custom) => { + if (Object.keys(custom).length) { + Countly.user_details({"custom": custom }); + } + }); }, 20000); } }; @@ -167,30 +168,33 @@ tracker.getSDK = function() { /** * Get server stats +* @returns {Promise} server stats **/ tracker.collectServerStats = function() { var props = {}; - stats.getServer(common.db, function(data) { - common.db.collection("apps").aggregate([{$project: {last_data: 1}}, {$sort: {"last_data": -1}}, {$limit: 1}], {allowDiskUse: true}, function(errApps, resApps) { - common.db.collection("members").aggregate([{$project: {last_login: 1}}, {$sort: {"last_login": -1}}, {$limit: 1}], {allowDiskUse: true}, function(errLogin, resLogin) { - if (resApps && resApps[0]) { - props.last_data = resApps[0].last_data || 0; - } - if (resLogin && resLogin[0]) { - props.last_login = resLogin[0].last_login || 0; - } - if (data) { - if (data.app_users) { - props.app_users = data.app_users; + return new Promise((resolve) => { + stats.getServer(common.db, function(data) { + common.db.collection("apps").aggregate([{$project: {last_data: 1}}, {$sort: {"last_data": -1}}, {$limit: 1}], {allowDiskUse: true}, function(errApps, resApps) { + common.db.collection("members").aggregate([{$project: {last_login: 1}}, {$sort: {"last_login": -1}}, {$limit: 1}], {allowDiskUse: true}, function(errLogin, resLogin) { + if (resApps && resApps[0]) { + props.last_data = resApps[0].last_data || 0; } - if (data.apps) { - props.apps = data.apps; + if (resLogin && resLogin[0]) { + props.last_login = resLogin[0].last_login || 0; } - if (data.users) { - props.users = data.users; + if (data) { + if (data.app_users) { + props.app_users = data.app_users; + } + if (data.apps) { + props.apps = data.apps; + } + if (data.users) { + props.users = data.users; + } } - } - return props; + resolve(props); + }); }); }); }); @@ -200,12 +204,13 @@ tracker.collectServerStats = function() { * Get server data * @returns {Object} server data **/ -tracker.collectServerData = function() { +tracker.collectServerData = async function() { var props = {}; props.trial = versionInfo.trial ? true : false; props.plugins = plugins.getPlugins(); props.nodejs = process.version; props.countly = versionInfo.version; + props.docker = hasDockerEnv() || hasDockerCGroup() || hasDockerMountInfo(); var edition = "Lite"; if (IS_FLEX) { edition = "Flex"; @@ -217,6 +222,14 @@ tracker.collectServerData = function() { if (common.db.build && common.db.build.version) { props.mongodb = common.db.build.version; } + const sdkData = await tracker.getSDKData(); + if (sdkData && sdkData.sdk_versions && Object.keys(sdkData.sdk_versions).length) { + props.sdks = Object.keys(sdkData.sdk_versions); + for (const [key, value] of Object.entries(sdkData.sdk_versions)) { + props[key] = value; + } + } + return props; }; @@ -224,15 +237,215 @@ tracker.collectServerData = function() { * Get all eligible data * @returns {Object} all eligible data */ -tracker.getAllData = function() { +tracker.getAllData = async function() { var props = {}; if (plugins.getConfig("tracking").server_user_details) { - Object.assign(props, tracker.collectServerStats()); + Object.assign(props, await tracker.collectServerStats()); } - Object.assign(props, tracker.collectServerData()); + Object.assign(props, await tracker.collectServerData()); return props; }; +/** + * Query sdks collection for current and previous year (month 0) and combine meta_v2 data + * @returns {Promise} Combined meta_v2 data from all matching documents + */ +tracker.getSDKData = async function() { + var currentYear = new Date().getFullYear(); + var previousYear = currentYear - 1; + + // Build regex pattern to match: appid_YYYY:0_shard + // Matches any app ID, year (current or previous), month 0, and any shard number + var yearPattern = `(${currentYear}|${previousYear})`; + var pattern = new RegExp(`^[a-f0-9]{24}_${yearPattern}:0_\\d+$`); + + try { + // Use aggregation pipeline to combine meta_v2 data on MongoDB side + var pipeline = [ + // Match documents for current and previous year, month 0, any shard + { + $match: { + _id: pattern + } + }, + // Project only meta_v2 field and convert to array of key-value pairs + { + $project: { + meta_v2: { $objectToArray: "$meta_v2" } + } + }, + // Unwind meta_v2 array to process each meta key separately + { + $unwind: "$meta_v2" + }, + // Convert nested objects to arrays for merging + { + $project: { + metaKey: "$meta_v2.k", + metaValue: { $objectToArray: "$meta_v2.v" } + } + }, + // Unwind nested values + { + $unwind: "$metaValue" + }, + // Group by meta key and inner key to collect all unique combinations + { + $group: { + _id: { + metaKey: "$metaKey", + innerKey: "$metaValue.k" + }, + value: { $first: "$metaValue.v" } + } + }, + // Group by meta key to rebuild nested structure + { + $group: { + _id: "$_id.metaKey", + values: { + $push: { + k: "$_id.innerKey", + v: "$value" + } + } + } + }, + // Convert arrays back to objects + { + $project: { + _id: 0, + k: "$_id", + v: { $arrayToObject: "$values" } + } + }, + // Group all into single document + { + $group: { + _id: null, + meta_v2: { + $push: { + k: "$k", + v: "$v" + } + } + } + }, + // Convert final array to object + { + $project: { + _id: 0, + meta_v2: { $arrayToObject: "$meta_v2" } + } + } + ]; + + var result = await common.db.collection("sdks").aggregate(pipeline).toArray(); + + // Extract combined meta_v2 or return empty object if no results + var combinedMeta = (result && result[0] && result[0].meta_v2) ? result[0].meta_v2 : {}; + + // Process sdk_version to extract highest version per SDK + var sdkVersions = {}; + if (combinedMeta.sdk_version) { + for (var versionKey in combinedMeta.sdk_version) { + // Parse SDK version format: [sdk_name]_major:minor:patch + var match = versionKey.match(/^\[([^\]]+)\]_(\d+):(\d+):(\d+)$/); + if (match) { + var sdkName = match[1]; + var major = parseInt(match[2], 10); + var minor = parseInt(match[3], 10); + var patch = parseInt(match[4], 10); + + // Check if this SDK exists and compare versions + if (!sdkVersions[sdkName]) { + sdkVersions[sdkName] = { + version: `${major}.${minor}.${patch}`, + major: major, + minor: minor, + patch: patch + }; + } + else { + var current = sdkVersions[sdkName]; + // Compare versions (major.minor.patch) + if (major > current.major || + (major === current.major && minor > current.minor) || + (major === current.major && minor === current.minor && patch > current.patch)) { + sdkVersions[sdkName] = { + version: `${major}.${minor}.${patch}`, + major: major, + minor: minor, + patch: patch + }; + } + } + } + } + } + + // Convert to simple object with just SDK name -> version string + var simpleSdkVersions = {}; + for (var sdk in sdkVersions) { + simpleSdkVersions[`sdk_${sdk}`] = sdkVersions[sdk].version; + } + + return { + meta_v2: combinedMeta, + sdk_versions: simpleSdkVersions, + years: [previousYear, currentYear], + month: 0 + }; + } + catch (error) { + logger("tracker:server").error("Error querying SDK data:", error); + return { + meta_v2: {}, + error: error.message + }; + } +}; + +/** + * Check if running in Docker environment + * @returns {boolean} if running in docker + */ +function hasDockerEnv() { + try { + fs.statSync('/.dockerenv'); + return true; + } + catch { + return false; + } +} + +/** + * Check if running in Docker by inspecting cgroup info + * @returns {boolean} if running in docker + */ +function hasDockerCGroup() { + try { + return fs.readFileSync('/proc/self/cgroup', 'utf8').includes('docker'); + } + catch { + return false; + } +} + +/** + * Check if running in Docker by inspecting mountinfo + * @returns {boolean} if running in docker + */ +function hasDockerMountInfo() { + try { + return fs.readFileSync('/proc/self/mountinfo', 'utf8').includes('/docker/containers/'); + } + catch { + return false; + } +} + /** * Strip traling slashes from url * @param {string} str - url to strip diff --git a/plugins/server-stats/tests/job.js b/plugins/server-stats/tests/job.js index 366081d7e9d..179160b32c1 100644 --- a/plugins/server-stats/tests/job.js +++ b/plugins/server-stats/tests/job.js @@ -40,8 +40,8 @@ describe('Stats job', () => { console.log('All Data:', allData); console.log(monthlyData); - should(monthlyData['Last 6 months avg']).equal(1000); - should(monthlyData['Last 12 months avg']).equal(1000); + should(monthlyData['DPAvg6months']).equal(1000); + should(monthlyData['DPAvg12months']).equal(1000); const expectedDP = []; for (let count = 0; count < 12; count += 1) { From c51e4f5ecb3ed80eb64e38e01d274650fbcf4314 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Wed, 1 Oct 2025 16:31:40 +0300 Subject: [PATCH 15/21] Remove log --- api/jobs/ping.js | 1 - 1 file changed, 1 deletion(-) diff --git a/api/jobs/ping.js b/api/jobs/ping.js index 4dce0a3196d..efc4e78ceea 100644 --- a/api/jobs/ping.js +++ b/api/jobs/ping.js @@ -23,7 +23,6 @@ class PingJob extends job.Job { try { var custom = await tracker.getAllData(); - console.log("custom", custom); if (Object.keys(custom).length) { user.user_details({"custom": custom }); } From 0c390405c5c8c4d4a0d6ad008d578a162ed16e3f Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Wed, 8 Oct 2025 12:54:23 +0300 Subject: [PATCH 16/21] Add total events --- api/parts/mgmt/tracker.js | 44 ++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index ee357f4182b..4245c9cc8b6 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -176,24 +176,38 @@ tracker.collectServerStats = function() { stats.getServer(common.db, function(data) { common.db.collection("apps").aggregate([{$project: {last_data: 1}}, {$sort: {"last_data": -1}}, {$limit: 1}], {allowDiskUse: true}, function(errApps, resApps) { common.db.collection("members").aggregate([{$project: {last_login: 1}}, {$sort: {"last_login": -1}}, {$limit: 1}], {allowDiskUse: true}, function(errLogin, resLogin) { - if (resApps && resApps[0]) { - props.last_data = resApps[0].last_data || 0; - } - if (resLogin && resLogin[0]) { - props.last_login = resLogin[0].last_login || 0; - } - if (data) { - if (data.app_users) { - props.app_users = data.app_users; + // Aggregate total list lengths across all documents in events collection + common.db.collection("events").aggregate([ + { + $group: { + _id: null, + totalListLength: { $sum: { $size: "$list" } } + } + } + ], {allowDiskUse: true}, function(errEvents, resEvents) { + + if (resApps && resApps[0]) { + props.last_data = resApps[0].last_data || 0; } - if (data.apps) { - props.apps = data.apps; + if (resLogin && resLogin[0]) { + props.last_login = resLogin[0].last_login || 0; } - if (data.users) { - props.users = data.users; + if (resEvents && resEvents[0]) { + props.total_events = resEvents[0].totalListLength || 0; } - } - resolve(props); + if (data) { + if (data.app_users) { + props.app_users = data.app_users; + } + if (data.apps) { + props.apps = data.apps; + } + if (data.users) { + props.users = data.users; + } + } + resolve(props); + }); }); }); }); From 900d47aa0b98661c9d4d1e721d606f242ecfd62f Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Wed, 8 Oct 2025 12:57:51 +0300 Subject: [PATCH 17/21] rename events --- api/parts/mgmt/tracker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index 4245c9cc8b6..f7179317dab 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -193,7 +193,7 @@ tracker.collectServerStats = function() { props.last_login = resLogin[0].last_login || 0; } if (resEvents && resEvents[0]) { - props.total_events = resEvents[0].totalListLength || 0; + props.events = resEvents[0].totalListLength || 0; } if (data) { if (data.app_users) { From 6f3ea01a747eecd72da5d5336e2fe44953c8f19d Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Wed, 8 Oct 2025 13:41:29 +0300 Subject: [PATCH 18/21] Removing countly_tracking --- frontend/express/app.js | 13 -- .../onboarding/javascripts/countly.views.js | 140 +----------------- .../public/javascripts/countly.views.js | 5 +- plugins/server-stats/frontend/app.js | 86 ----------- .../manage/configurations/configurations.js | 15 -- ui-tests/cypress/support/constants.js | 1 - 6 files changed, 6 insertions(+), 254 deletions(-) diff --git a/frontend/express/app.js b/frontend/express/app.js index bc59023ac1f..5fe426f0cb4 100644 --- a/frontend/express/app.js +++ b/frontend/express/app.js @@ -140,17 +140,6 @@ plugins.setConfigs("frontend", { offline_mode: false }); -if (!plugins.isPluginEnabled('tracker')) { - plugins.setConfigs('frontend', { - countly_tracking: null, - }); -} -else { - plugins.setConfigs('frontend', { - countly_tracking: true, - }); -} - plugins.setUserConfigs("frontend", { production: false, theme: false, @@ -924,7 +913,6 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ **/ function renderDashboard(req, res, next, member, adminOfApps, userOfApps, countlyGlobalApps, countlyGlobalAdminApps) { var configs = plugins.getConfig("frontend", member.settings), - countly_tracking = plugins.isPluginEnabled('tracker') ? true : plugins.getConfig('frontend').countly_tracking, countly_domain = plugins.getConfig('api').domain, licenseNotification, licenseError; var isLocked = false; @@ -1014,7 +1002,6 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_ countlyTypeName: overriddenCountlyNamedType, countlyTypeTrack: COUNTLY_TRACK_TYPE, countlyTypeCE: COUNTLY_TYPE_CE, - countly_tracking, countly_domain, frontend_app: versionInfo.frontend_app || "9c28c347849f2c03caf1b091ec7be8def435e85e", frontend_server: versionInfo.frontend_server || 'https://stats.count.ly/', diff --git a/frontend/express/public/core/onboarding/javascripts/countly.views.js b/frontend/express/public/core/onboarding/javascripts/countly.views.js index e8a147cd5b7..3e9739f208f 100644 --- a/frontend/express/public/core/onboarding/javascripts/countly.views.js +++ b/frontend/express/public/core/onboarding/javascripts/countly.views.js @@ -179,9 +179,8 @@ template: CV.T('/core/onboarding/templates/consent.html'), data: function() { return { - isCountlyHosted: countlyGlobal.plugins.includes('tracker'), + isCountlyHosted: true, newConsent: { - countly_tracking: true, countly_newsletter: true, }, }; @@ -226,43 +225,8 @@ }; countlyPlugins.updateConfigs(configs); - var domain = countlyGlobal.countly_domain || window.location.origin; - - try { - // try to extract hostname from full domain url - var urlObj = new URL(domain); - domain = urlObj.hostname; - } - catch (_) { - // do nothing, domain from config will be used as is - } - - var statsUrl = 'https://stats.count.ly/i'; - - try { - var uObj = new URL(countlyGlobal.frontend_server); - uObj.pathname = '/i'; - statsUrl = uObj.href; - } - catch (_) { - // do nothing, statsUrl will be used as is - } - - CV.$.ajax({ - type: 'GET', - url: statsUrl, - data: { - consent: JSON.stringify({countly_tracking: doc.countly_tracking}), - app_key: countlyGlobal.frontend_app, - device_id: (window.Countly && window.Countly.device_id) || domain, - }, - dataType: 'json', - complete: function() { - // go home - window.location.href = '#/home'; - window.location.reload(); - } - }); + window.location.href = '#/home'; + window.location.reload(); }, } }); @@ -271,7 +235,7 @@ template: CV.T('/core/onboarding/templates/consent.html'), data: function() { return { - isCountlyHosted: countlyGlobal.plugins.includes('tracker'), + isCountlyHosted: true, newConsent: { countly_newsletter: true, }, @@ -323,88 +287,6 @@ } }); - var notRespondedConsentView = CV.views.create({ - template: CV.T('/core/onboarding/templates/consent.html'), - data: function() { - return { - isCountlyHosted: countlyGlobal.plugins.includes('tracker'), - newConsent: { - countly_tracking: null, - }, - }; - }, - mounted: function() { - this.$store.dispatch('countlyOnboarding/fetchConsentItems'); - }, - computed: { - consentItems: function() { - return this.$store.getters['countlyOnboarding/consentItems'] - .filter(function(item) { - return item.type === 'tracking'; - }); - }, - }, - methods: { - decodeHtmlEntities: function(inp) { - var el = document.createElement('p'); - el.innerHTML = inp; - - var result = el.textContent || el.innerText; - el = null; - - return result; - }, - handleSubmit: function(doc) { - var configs = { - frontend: doc, - }; - - if (this.consentItems.length === 0) { - configs.frontend.countly_tracking = false; - } - - countlyPlugins.updateConfigs(configs); - var domain = countlyGlobal.countly_domain || window.location.origin; - - try { - // try to extract hostname from full domain url - var urlObj = new URL(domain); - domain = urlObj.hostname; - } - catch (_) { - // do nothing, domain from config will be used as is - } - - var statsUrl = 'https://stats.count.ly/i'; - - try { - var uObj = new URL(countlyGlobal.frontend_server); - uObj.pathname = '/i'; - statsUrl = uObj.href; - } - catch (_) { - // do nothing, statsUrl will be used as is - } - - CV.$.ajax({ - type: 'GET', - url: statsUrl, - data: { - consent: JSON.stringify({countly_tracking: doc.countly_tracking}), - app_key: countlyGlobal.frontend_app, - device_id: (window.Countly && window.Countly.device_id) || domain, - }, - dataType: 'json', - complete: function() { - // go home - window.location.href = '#/home'; - window.location.reload(); - } - }); - }, - } - }); - app.route('/initial-setup', 'initial-setup', function() { this.renderWhenReady(new CV.views.BackboneWrapper({ component: appSetupView, @@ -419,13 +301,6 @@ })); }); - app.route('/not-responded-consent', 'not-responded-consent', function() { - this.renderWhenReady(new CV.views.BackboneWrapper({ - component: notRespondedConsentView, - vuex: [{ clyModel: countlyOnboarding }], - })); - }); - var hasNewsLetter = typeof countlyGlobal.newsletter === "undefined" ? true : countlyGlobal.newsletter; app.route('/not-subscribed-newsletter', 'not-subscribed-newsletter', function() { @@ -470,12 +345,7 @@ } }); - if (typeof countlyGlobal.countly_tracking !== 'boolean' && isGlobalAdmin && !countlyGlobal.plugins.includes('tracker')) { - if (Backbone.history.fragment !== '/not-responded-consent' && !/initial-setup|initial-consent/.test(window.location.hash)) { - app.navigate("/not-responded-consent", true); - } - } - else if (hasNewsLetter && (typeof countlyGlobal.member.subscribe_newsletter !== 'boolean' && !store.get('disable_newsletter_prompt') && (countlyGlobal.member.login_count === 3 || moment().dayOfYear() % 90 === 0))) { + if (hasNewsLetter && (typeof countlyGlobal.member.subscribe_newsletter !== 'boolean' && !store.get('disable_newsletter_prompt') && (countlyGlobal.member.login_count === 3 || moment().dayOfYear() % 90 === 0))) { if (Backbone.history.fragment !== '/not-subscribed-newsletter' && !/initial-setup|initial-consent/.test(window.location.hash)) { app.navigate("/not-subscribed-newsletter", true); } diff --git a/plugins/plugins/frontend/public/javascripts/countly.views.js b/plugins/plugins/frontend/public/javascripts/countly.views.js index 197ba4feaad..9bf0f63590d 100644 --- a/plugins/plugins/frontend/public/javascripts/countly.views.js +++ b/plugins/plugins/frontend/public/javascripts/countly.views.js @@ -344,10 +344,7 @@ app.configurationsView.registerInput("frontend.__user", {input: "el-select", attrs: {multiple: true}, list: list}); } - if (self.configsData.frontend && countlyGlobal.plugins.includes('tracker')) { - // disable countly tracking config for countly hosted instances - delete self.configsData.frontend.countly_tracking; - } + delete self.configsData.frontend.countly_tracking; self.configsList.push({ "label": self.getLabel("core"), diff --git a/plugins/server-stats/frontend/app.js b/plugins/server-stats/frontend/app.js index 32a27465b01..e69de29bb2d 100644 --- a/plugins/server-stats/frontend/app.js +++ b/plugins/server-stats/frontend/app.js @@ -1,86 +0,0 @@ -var pluginExported = {}; -var versionInfo = require('../../../frontend/express/version.info'); -var moment = require('moment'); -const plugins = require('../../pluginManager.js'); -const request = require('countly-request')(plugins.getConfig("security")); -const { getUserApps } = require('../../../api/utils/rights'); - -(function(plugin) { - plugin.init = function(app, countlyDb) { - plugin.loginSuccessful = function(ob) { - var member = ob.data; - if (plugins.getConfig('frontend').countly_tracking) { - var match = {}; - if (versionInfo.trial) { - match.a = {$in: getUserApps(member) || []}; - } - countlyDb.collection("server_stats_data_points").aggregate([ - { - $match: match - }, - { - $group: { - _id: "$m", - e: { $sum: "$e"}, - s: { $sum: "$s"} - } - } - ], { allowDiskUse: true }, function(error, allData) { - var custom = {}; - if (!error && allData && allData.length) { - var data = {}; - data.all = 0; - data.month3 = []; - var utcMoment = moment.utc(); - var months = {}; - for (let i = 0; i < 3; i++) { - months[utcMoment.format("YYYY:M")] = true; - utcMoment.subtract(1, 'months'); - } - for (let i = 0; i < allData.length; i++) { - data.all += allData[i].e + allData[i].s; - if (months[allData[i]._id]) { - data.month3.push(allData[i]._id + " - " + (allData[i].e + allData[i].s)); - } - } - data.avg = Math.round((data.all / allData.length) * 100) / 100; - custom.dataPointsAll = data.all; - custom.dataPointsMonthlyAvg = data.avg; - custom.dataPointsLast3Months = data.month3; - var date = new Date(); - let domain = plugins.getConfig('api').domain; - - try { - // try to extract hostname from full domain url - const urlObj = new URL(domain); - domain = urlObj.hostname; - } - catch (_) { - // do nothing, domain from config will be used as is - } - - request({ - uri: "https://stats.count.ly/i", - method: "GET", - timeout: 4E3, - qs: { - device_id: domain, - app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f", - timestamp: Math.round(date.getTime() / 1000), - hour: date.getHours(), - dow: date.getDay(), - user_details: JSON.stringify( - { - custom: custom - } - ) - } - }, function(/*error, response, body*/) {}); - } - }); - } - }; - }; -}(pluginExported)); - -module.exports = pluginExported; \ No newline at end of file diff --git a/ui-tests/cypress/lib/dashboard/manage/configurations/configurations.js b/ui-tests/cypress/lib/dashboard/manage/configurations/configurations.js index e19040fea5a..92655d9bc7e 100755 --- a/ui-tests/cypress/lib/dashboard/manage/configurations/configurations.js +++ b/ui-tests/cypress/lib/dashboard/manage/configurations/configurations.js @@ -514,21 +514,6 @@ const verifyPageElements = () => { isChecked: true }); - cy.verifyElement({ - labelElement: configurationsListBoxElements({ subFeature: SETTINGS.FRONTED.COUNTLY_TRACKING }).SELECTED_SUBFEATURE_TITLE, - labelText: "Countly", - }); - - cy.verifyElement({ - labelElement: configurationsListBoxElements({ subFeature: SETTINGS.FRONTED.COUNTLY_TRACKING }).SELECTED_SUBFEATURE_DESCRIPTION, - labelText: "When enabled, Countly will be activated on this server to perform server-level analytics and gather user feedback to aid us in continuous product improvement. Personal user data/details or the data you process using this server will never be collected or analyzed. All data is sent exclusively to our dedicated Countly server located in Europe.", - }); - - cy.verifyElement({ - element: configurationsListBoxElements({ subFeature: SETTINGS.FRONTED.COUNTLY_TRACKING }).SELECTED_SUBFEATURE_CHECKBOX, - //isChecked: true //TODO: if empty data, it should be false - }); - cy.verifyElement({ labelElement: configurationsListBoxElements({ subFeature: SETTINGS.FRONTED.OFFLINE_MODE }).SELECTED_SUBFEATURE_TITLE, labelText: "Offline mode", diff --git a/ui-tests/cypress/support/constants.js b/ui-tests/cypress/support/constants.js index 59d1d2721a1..a1377dfbf45 100644 --- a/ui-tests/cypress/support/constants.js +++ b/ui-tests/cypress/support/constants.js @@ -162,7 +162,6 @@ module.exports.SETTINGS = { FRONTED: { CODE: 'Code', - COUNTLY_TRACKING: 'Countly Tracking', OFFLINE_MODE: 'Offline Mode', PRODUCTION: 'Production', SESSION_TIMEOUT: 'Session Timeout', From b4ca0dc614def2ad6d55072b988335c861b82cb3 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Wed, 8 Oct 2025 14:16:57 +0300 Subject: [PATCH 19/21] Remove tracking consent tests --- .../cypress/lib/onboarding/initialConsent.js | 36 ------------------- .../elements/onboarding/initialConsent.js | 6 ---- 2 files changed, 42 deletions(-) diff --git a/ui-tests/cypress/lib/onboarding/initialConsent.js b/ui-tests/cypress/lib/onboarding/initialConsent.js index 57afc8a10f1..c1b02f5cf45 100644 --- a/ui-tests/cypress/lib/onboarding/initialConsent.js +++ b/ui-tests/cypress/lib/onboarding/initialConsent.js @@ -1,13 +1,5 @@ import initialConsentPageElements from "../../support/elements/onboarding/initialConsent"; -const enableTracking = () => { - cy.clickElement(initialConsentPageElements.ENABLE_TRACKING_RADIO_BUTTON); -}; - -const dontEnableTracking = () => { - cy.clickElement(initialConsentPageElements.DONT_ENABLE_TRACKING_RADIO_BUTTON); -}; - const subscribeToNewsletter = () => { cy.clickElement(initialConsentPageElements.ENABLE_NEWSLETTER_RADIO_BUTTON); }; @@ -33,16 +25,6 @@ const verifyDefaultPageElements = () => { labelText: "Before we start..." }); - cy.verifyElement({ - labelElement: initialConsentPageElements.PAGE_DESC_TRACKING, - labelText: "We utilize Countly to understand user interactions and collect feedback, helping us enhance our product continuously. However, your privacy remains our priority. This analysis is done on the server level, so we won't see or collect any individual details or any data you record. The data is reported back only to our dedicated Countly server based in Europe. Please note, you can change your mind at any time in the settings." - }); - - cy.verifyElement({ - labelElement: initialConsentPageElements.PAGE_QUESTION_TRACKING, - labelText: "Considering our commitment to maintaining your privacy and the potential benefits for product enhancement, would you be comfortable enabling Countly on this server?" - }); - cy.verifyElement({ labelElement: initialConsentPageElements.PAGE_DESC_NEWSLETTER, labelText: "We offer a newsletter brimming with recent updates about our product, news from Countly, and information on product analytics. We assure you - our aim is to provide value and insights, not clutter your inbox with unwanted emails." @@ -53,20 +35,6 @@ const verifyDefaultPageElements = () => { labelText: "Would you be interested in subscribing to our newsletter?" }); - cy.verifyElement({ - element: initialConsentPageElements.ENABLE_TRACKING_RADIO_BUTTON, - isChecked: true, - labelElement: initialConsentPageElements.ENABLE_TRACKING_RADIO_BUTTON_LABEL, - labelText: "Yes, enable tracking on this server" - }); - - cy.verifyElement({ - element: initialConsentPageElements.DONT_ENABLE_TRACKING_RADIO_BUTTON, - isChecked: false, - labelElement: initialConsentPageElements.DONT_ENABLE_TRACKING_RADIO_BUTTON_LABEL, - labelText: "No, maybe later" - }); - cy.verifyElement({ element: initialConsentPageElements.ENABLE_NEWSLETTER_RADIO_BUTTON, //isChecked: true, @@ -88,17 +56,13 @@ const verifyDefaultPageElements = () => { }; const completeOnboardingInitialConsent = ({ - isEnableTacking, isSubscribeToNewsletter, }) => { - isEnableTacking ? enableTracking() : dontEnableTracking(); isSubscribeToNewsletter ? subscribeToNewsletter() : dontSubscribeToNewsletter(); clickContinue(); }; module.exports = { - enableTracking, - dontEnableTracking, subscribeToNewsletter, dontSubscribeToNewsletter, verifyDefaultPageElements, diff --git a/ui-tests/cypress/support/elements/onboarding/initialConsent.js b/ui-tests/cypress/support/elements/onboarding/initialConsent.js index 6bfb97831a9..69ad6656332 100644 --- a/ui-tests/cypress/support/elements/onboarding/initialConsent.js +++ b/ui-tests/cypress/support/elements/onboarding/initialConsent.js @@ -1,12 +1,6 @@ export default { LOGO: 'countly-logo', PAGE_TITLE: 'page-title', - PAGE_DESC_TRACKING: 'page-description-tracking', - PAGE_QUESTION_TRACKING: 'page-question-tracking', - ENABLE_TRACKING_RADIO_BUTTON_LABEL: 'enable-tracking-el-radio-label', - ENABLE_TRACKING_RADIO_BUTTON: 'enable-tracking-el-radio', - DONT_ENABLE_TRACKING_RADIO_BUTTON_LABEL: 'dont-enable-tracking-el-radio-label', - DONT_ENABLE_TRACKING_RADIO_BUTTON: 'dont-enable-tracking-el-radio', PAGE_DESC_NEWSLETTER: 'page-description-newsletter', PAGE_QUESTION_NEWSLETTER: 'page-question-newsletter', ENABLE_NEWSLETTER_RADIO_BUTTON_LABEL: 'enable-newsletter-el-radio-label', From 4dda4da441994b8e9e53b18fce6abd27096c507e Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Wed, 8 Oct 2025 17:29:38 +0300 Subject: [PATCH 20/21] Upload profile pics of servers --- api/parts/mgmt/tracker.js | 124 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js index f7179317dab..d75dce31e1a 100644 --- a/api/parts/mgmt/tracker.js +++ b/api/parts/mgmt/tracker.js @@ -8,10 +8,15 @@ var tracker = {}, stats = require('../data/stats.js'), common = require('../../utils/common.js'), logger = require('../../utils/log.js'), + countlyFs = require('../../utils/countlyFs.js'), //log = logger("tracker:server"), Countly = require('countly-sdk-nodejs'), fs = require('fs'), path = require('path'), + https = require('https'), + http = require('http'), + FormData = require('form-data'), + { Readable } = require('node:stream'), versionInfo = require('../../../frontend/express/version.info'), server = "9c28c347849f2c03caf1b091ec7be8def435e85e", user = "fa6e9ae7b410cb6d756e8088c5f3936bf1fab5f3", @@ -72,6 +77,17 @@ tracker.enable = function() { isEnabled = true; Countly.user_details({"name": config.device_id }); + if (plugins.getConfig("white-labeling") && (plugins.getConfig("white-labeling").favicon || plugins.getConfig("white-labeling").stopleftlogo || plugins.getConfig("white-labeling").prelogo)) { + var id = plugins.getConfig("white-labeling").favicon || plugins.getConfig("white-labeling").stopleftlogo || plugins.getConfig("white-labeling").prelogo; + countlyFs.gridfs.getDataById("white-labeling", id, function(errWhitelabel, data) { + if (!errWhitelabel && data) { + tracker.uploadBase64FileFromGridFS(data).catch(() => {}); + } + }); + } + else { + Countly.user_details({"picture": "./images/favicon.png" }); + } if (plugins.getConfig("tracking").server_sessions) { Countly.begin_session(true); } @@ -420,6 +436,114 @@ tracker.getSDKData = async function() { } }; +/** + * Upload a base64-encoded file from GridFS to the stats server + * This function handles files stored in GridFS as base64 strings (e.g., data URIs) + * and decodes them before uploading + * + * @param {Object} base64String - Picture data + * @returns {Promise} Upload result + */ +tracker.uploadBase64FileFromGridFS = function(base64String) { + return new Promise((resolve, reject) => { + var domain = stripTrailingSlash((plugins.getConfig("api").domain + "").split("://").pop()); + if (domain && domain !== "localhost") { + try { + let mimeType = "image/png"; + // Strip data URI prefix if present and stripDataURI is true + if (base64String.includes('base64,')) { + // Extract MIME type from data URI if not provided + const dataURIMatch = base64String.match(/^data:([^;]+);base64,/); + if (dataURIMatch) { + mimeType = dataURIMatch[1]; + } + // Remove data URI prefix + base64String = base64String.split('base64,')[1]; + } + + // Decode base64 to binary buffer + const binaryBuffer = Buffer.from(base64String, 'base64'); + + // Create a readable stream from the decoded buffer + const decodedStream = Readable.from(binaryBuffer); + + // Parse the URL + const statsUrl = new URL(url); + const protocol = statsUrl.protocol === 'https:' ? https : http; + + // Build query parameters + const queryParams = new URLSearchParams({ + device_id: domain, + app_key: server, + user_details: "" + }); + + // Create form data + const form = new FormData(); + + // Prepare form options with MIME type if available + const formOptions = { filename: "profile" }; + if (mimeType) { + formOptions.contentType = mimeType; + } + + form.append('file', decodedStream, formOptions); + + // Prepare request options + const requestOptions = { + hostname: statsUrl.hostname, + port: statsUrl.port || (statsUrl.protocol === 'https:' ? 443 : 80), + path: `/i?${queryParams.toString()}`, + method: 'POST', + headers: form.getHeaders() + }; + + // Make the request + const req = protocol.request(requestOptions, (res) => { + let data = ''; + + res.on('data', (chunk) => { + data += chunk; + }); + + res.on('end', () => { + if (res.statusCode >= 200 && res.statusCode < 300) { + try { + const result = JSON.parse(data); + resolve({ + success: true, + statusCode: res.statusCode, + data: result + }); + } + catch (e) { + resolve({ + success: true, + statusCode: res.statusCode, + data: data + }); + } + } + else { + reject(new Error(`Upload failed with status ${res.statusCode}: ${data}`)); + } + }); + }); + + req.on('error', (error) => { + reject(error); + }); + + // Pipe the form data to the request + form.pipe(req); + } + catch (error) { + reject(error); + } + } + }); +}; + /** * Check if running in Docker environment * @returns {boolean} if running in docker From 9418a6112469e335c2cd5594c82aa26df1dc04e0 Mon Sep 17 00:00:00 2001 From: Arturs Sosins Date: Thu, 9 Oct 2025 14:20:42 +0300 Subject: [PATCH 21/21] [server-stats] fixed when license not provided --- plugins/server-stats/api/jobs/stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/server-stats/api/jobs/stats.js b/plugins/server-stats/api/jobs/stats.js index 0477012b1ba..95ecf8cb88e 100644 --- a/plugins/server-stats/api/jobs/stats.js +++ b/plugins/server-stats/api/jobs/stats.js @@ -171,7 +171,7 @@ class StatsJob extends job.Job { const options = { dailyDates: specificDates, monthlyBreakdown: true, - license_hosting: license.license_hosting, + license_hosting: license?.license_hosting, }; // Atomically retrieve old last_sync value and set new one