Merge branch 'fix/SDK-4239_Low-performance-on-Recents-excluding-sensitive-nodes' into 'release/v7.8.0'

SDK-4239. Low performance on Recents excluding sensitive nodes (release/v7.8.0)

See merge request sdk/sdk!5761

Author: vmgaGH

include/mega/db.h

@@ -148,6 +148,9 @@ class MEGA_API DBTableNodes
                                 std::set<std::string>& tags,
                                 CancelToken cancelFlag) = 0;
 
+    virtual bool getRecentNodes(unsigned maxcount,
+                                m_time_t since,
+                                std::vector<std::pair<NodeHandle, NodeSerialized>>& nodes) = 0;
     virtual bool getNodesByFingerprint(const std::string& fingerprint, std::vector<std::pair<NodeHandle, NodeSerialized>>& nodes) = 0;
     virtual bool getNodeByFingerprint(const std::string& fingerprint, mega::NodeSerialized& node, NodeHandle& handle) = 0;
     virtual bool getRootNodes(std::vector<std::pair<NodeHandle, NodeSerialized>>& nodes) = 0;

include/mega/db/sqlite.h

@@ -90,6 +90,11 @@ class MEGA_API SqliteAccountState : public SqliteDbTable, public DBTableNodes
     bool getNodeByFingerprint(const std::string& fingerprint,
                               mega::NodeSerialized& node,
                               NodeHandle& handle) override;
+    // getRecentNodes() is for the deprecated method MegaClient::getRecentActions without
+    // excludeSensitives flag
+    bool getRecentNodes(unsigned maxcount,
+                        m_time_t since,
+                        std::vector<std::pair<NodeHandle, NodeSerialized>>& nodes) override;
     bool getFavouritesHandles(NodeHandle node, uint32_t count, std::vector<mega::NodeHandle>& nodes) override;
     bool childNodeByNameType(NodeHandle parentHanlde, const std::string& name, nodetype_t nodeType, std::pair<NodeHandle, NodeSerialized>& node) override;
     bool getNodeSizeTypeAndFlags(NodeHandle node, m_off_t& size, nodetype_t& nodeType, uint64_t &oldFlags) override;
@@ -156,6 +161,7 @@ class MEGA_API SqliteAccountState : public SqliteDbTable, public DBTableNodes
     sqlite3_stmt* mStmtChildNode = nullptr;
     sqlite3_stmt* mStmtIsAncestor = nullptr;
     sqlite3_stmt* mStmtNumChild = nullptr;
+    sqlite3_stmt* mStmtRecents = nullptr; // For getRecentNodes()
     sqlite3_stmt* mStmtFavourites = nullptr;
 
     // how many SQLite instructions will be executed between callbacks to the progress handler

include/mega/megaclient.h

@@ -1911,7 +1911,15 @@ class MEGA_API MegaClient
     std::shared_ptr<Node> nodebyfingerprint(LocalNode*);
 #endif /* ENABLE_SYNC */
 
+private:
+    // Private helper method for getRecentActions
+    recentactions_vector getRecentActionsFromSharedNodeVector(sharedNode_vector&& v);
+public:
     // get a vector of recent actions in the account
+    recentactions_vector getRecentActions(unsigned maxcount,
+                                          m_time_t since); // Old getRecentActions behavior without
+                                                           // newer excludeSensitive functionality
+
     recentactions_vector getRecentActions(unsigned maxcount,
                                           m_time_t since,
                                           bool excludeSensitives);

include/mega/nodemanager.h

@@ -173,6 +173,11 @@ class MEGA_API NodeManager
 
     sharedNode_vector getChildren(const NodeSearchFilter& filter, int order, CancelToken cancelFlag, const NodeSearchPage& page);
 
+    // get up to "maxcount" nodes, not older than "since", ordered by creation time
+    // Note: nodes are read from DB and loaded in memory
+    // (This is used by deprecated MegaClient::getRecentNodes without exclude sensitive filter)
+    sharedNode_vector getRecentNodes(unsigned maxcount, m_time_t since);
+
     sharedNode_vector searchNodes(const NodeSearchFilter& filter, int order, CancelToken cancelFlag, const NodeSearchPage& page);
 
     std::set<std::string> getAllNodeTags(const char* searchString, CancelToken cancelFlag);
@@ -403,6 +408,8 @@ class MEGA_API NodeManager
     sharedNode_vector searchNodes_internal(const NodeSearchFilter& filter, int order, CancelToken cancelFlag, const NodeSearchPage& page);
     sharedNode_vector processUnserializedNodes(const std::vector<std::pair<NodeHandle, NodeSerialized>>& nodesFromTable, CancelToken cancelFlag);
     sharedNode_vector getChildren_internal(const NodeSearchFilter& filter, int order, CancelToken cancelFlag, const NodeSearchPage& page);
+    sharedNode_vector getRecentNodes_internal(unsigned maxcount,
+                                              m_time_t since); // Old getRecentNodes functionality
 
     std::set<std::string> getAllNodeTags_internal(const char* searchString, CancelToken cancelFlag);
 

include/megaapi.h

@@ -4098,17 +4098,17 @@ class MegaUserAlertList
     virtual void clear();
 };
 
-
 /**
-* @brief Represents a set of files uploaded or updated in MEGA.
-* These are used to display the recent changes to an account.
-*
-* Objects of this class aren't live, they are snapshots of the state
-* in MEGA when the object is created, they are immutable.
-*
-* MegaRecentActionBuckets can be retrieved with MegaApi::getRecentActions
-*
-*/
+ * @brief Represents a set of files uploaded or updated in MEGA.
+ * These are used to display the recent changes to an account.
+ *
+ * Objects of this class aren't live, they are snapshots of the state
+ * in MEGA when the object is created, they are immutable.
+ *
+ * MegaRecentActionBuckets can be retrieved with MegaApi::getRecentActions
+ * and MegaApi::getRecentActionsAsync.
+ *
+ */
 class MegaRecentActionBucket
 {
 public:
@@ -4178,17 +4178,19 @@ class MegaRecentActionBucket
 };
 
 /**
-* @brief List of MegaRecentActionBucket objects
-*
-* A MegaRecentActionBucketList has the ownership of the MegaRecentActionBucket objects that it contains, so they will be
-* only valid until the MegaRecentActionBucketList is deleted. If you want to retain a MegaRecentActionBucket returned by
-* a MegaRecentActionBucketList, use MegaRecentActionBucket::copy.
-*
-* Objects of this class are immutable.
-*
-* @see MegaApi::getRecentActions
-*
-*/
+ * @brief List of MegaRecentActionBucket objects
+ *
+ * A MegaRecentActionBucketList has the ownership of the MegaRecentActionBucket objects that it
+ * contains, so they will be only valid until the MegaRecentActionBucketList is deleted. If you want
+ * to retain a MegaRecentActionBucket returned by a MegaRecentActionBucketList, use
+ * MegaRecentActionBucket::copy.
+ *
+ * Objects of this class are immutable.
+ *
+ * @see MegaApi::getRecentActions
+ * @see MegaApi::getRecentActionsAsync
+ *
+ */
 class MegaRecentActionBucketList
 {
 public:
@@ -4974,6 +4976,8 @@ class MegaRequest
          * - MegaApi::dismissBanner - Returns the id of the banner
          * - MegaApi::sendBackupHeartbeat - Returns the number of backup files uploaded
          * - MegaApi::getRecentActions - Returns the maximum number of nodes
+         * - MegaApi::getRecentActionsAsync - Returns the maximum number of nodes
+         * - MegaApi::importPasswordsFromFile - Returns source of the file provided as an argument
          *
          * @return Type of parameter related to the request
          */
@@ -5028,8 +5032,10 @@ class MegaRequest
          * - MegaApi::moveTransferToLast - Returns MegaTransfer::MOVE_TYPE_BOTTOM
          * - MegaApi::moveTransferToLastByTag - Returns MegaTransfer::MOVE_TYPE_BOTTOM
          * - MegaApi::moveTransferBefore - Returns the tag of the transfer with the target position
-         * - MegaApi::moveTransferBeforeByTag - Returns the tag of the transfer with the target position
-         * - MegaApi::setScheduledCopy - Returns the period between backups in deciseconds (-1 if cron time used)
+         * - MegaApi::moveTransferBeforeByTag - Returns the tag of the transfer with the target
+         * position
+         * - MegaApi::setScheduledCopy - Returns the period between backups in deciseconds (-1 if
+         * cron time used)
          * - MegaApi::abortCurrentScheduledCopy - Returns the tag of the aborted backup
          * - MegaApi::removeScheduledCopy - Returns the tag of the deleted backup
          * - MegaApi::startTimer - Returns the selected period
@@ -5046,6 +5052,8 @@ class MegaRequest
          * - MegaApi::getPaymentMethods - Returns a bitfield with the available payment methods
          * - MegaApi::getCloudStorageUsed - Returns the sum of the sizes of file cloud nodes.
          * - MegaApi::getRecentActions - Returns the number of days since nodes will be considerated
+         * - MegaApi::getRecentActionsAsync - Returns the number of days since nodes will be
+         * considered
          *
          * @return Number related to this request
          */
@@ -5087,7 +5095,7 @@ class MegaRequest
          * - MegaApi::getVisibleWelcomeDialog - Returns true if the Welcome dialog is visible
          * - MegaApi::getVisibleTermsOfService - Returns true if the Terms of Service need to be
          * displayed
-         * - MegaApi::getRecentActions - Returns true if exclude sensitives
+         * - MegaApi::getRecentActionsAsync - Returns true if exclude sensitives
          *
          * This value is valid for these request in onRequestFinish when the
          * error code is MegaError::API_OK:
@@ -18538,8 +18546,8 @@ class MegaApi
          *
          * Each bucket contains files that were added/modified in a set, by a single user.
          *
-         * Note: Nodes sensitives are excluded by default. Nodes are considered
-         * sensitive if they have that property set, or one of their ancestors has it
+         * Note: Nodes sensitives are NOT excluded by default. Nodes are considered
+         * sensitive if they have that property set, or one of their ancestors has it.
          *
          * @deprecated use getRecentActionsAsync
          *
@@ -18561,8 +18569,8 @@ class MegaApi
          *
          * You take the ownership of the returned value.
          *
-         * Note: Nodes sensitives are excluded by default. Nodes are considered
-         * sensitive if they have that property set, or one of their ancestors has it
+         * Note: Nodes sensitives are NOT excluded by default. Nodes are considered
+         * sensitive if they have that property set, or one of their ancestors has it.
          *
          * @deprecated use getRecentActionsAsync
          *
@@ -18579,7 +18587,6 @@ class MegaApi
          * Valid data in the MegaRequest object received on callbacks:
          * - MegaRequest::getNumber - Returns the number of days since nodes will be considerated
          * - MegaRequest::getParamType - Returns the maximun number of nodes
-         * - MegaRequest::getFlag - Returns true if sensitives are excluded
          *
          * The associated request type with this request is MegaRequest::TYPE_GET_RECENT_ACTIONS
          * Valid data in the MegaRequest object received in onRequestFinish when the error code
@@ -18590,8 +18597,10 @@ class MegaApi
          * The recommended values for the following parameters are to consider
          * interactions during the last 30 days and maximum 500 nodes.
          *
-         * Note: Nodes sensitives are excluded by default. Nodes are considered
-         * sensitive if they have that property set, or one of their ancestors has it
+         * Note: Nodes sensitives are NOT excluded by default. Nodes are considered
+         * sensitive if they have that property set, or one of their ancestors has it.
+         * Use getRecentActionsAsync with explicit excludeSensitives flag
+         * to search for sensitives and filter them depending on the flag value
          *
          * @param days Age of actions since added/modified nodes will be considered (in days)
          * @param maxnodes Maximum amount of nodes to be considered

include/megaapi_impl.h

@@ -3592,9 +3592,16 @@ class MegaApiImpl : public MegaApp
 
         long long getBandwidthOverquotaDelay();
 
-        MegaRecentActionBucketList* getRecentActions(unsigned days = 90,
-                                                     unsigned maxnodes = 500,
-                                                     bool excludeSensitives = true);
+    private:
+        void getRecentActionsAsyncInternal(unsigned days,
+                                           unsigned maxnodes,
+                                           bool* optExcludeSensitives,
+                                           MegaRequestListener* listener = NULL);
+    public:
+        MegaRecentActionBucketList* getRecentActions(unsigned days = 90, unsigned maxnodes = 500);
+        void getRecentActionsAsync(unsigned days,
+                                   unsigned maxnodes,
+                                   MegaRequestListener* listener = NULL);
         void getRecentActionsAsync(unsigned days,
                                    unsigned maxnodes,
                                    bool excludeSensitives,

src/db/sqlite.cpp

@@ -1200,6 +1200,9 @@ void SqliteAccountState::finalise()
     sqlite3_finalize(mStmtNumChild);
     mStmtNumChild = nullptr;
 
+    sqlite3_finalize(mStmtRecents);
+    mStmtRecents = nullptr;
+
     sqlite3_finalize(mStmtFavourites);
     mStmtFavourites = nullptr;
 }
@@ -1892,6 +1895,56 @@ bool SqliteAccountState::getNodeByFingerprint(const std::string &fingerprint, me
     return result;
 }
 
+bool SqliteAccountState::getRecentNodes(unsigned maxcount,
+                                        m_time_t since,
+                                        std::vector<std::pair<NodeHandle, NodeSerialized>>& nodes)
+{
+    if (!db)
+    {
+        return false;
+    }
+
+    const std::string filenode = std::to_string(FILENODE);
+    uint64_t excludeFlags = (1 << Node::FLAGS_IS_VERSION | 1 << Node::FLAGS_IS_IN_RUBBISH);
+    std::string sqlQuery = "SELECT n1.nodehandle, n1.counter, n1.node "
+                           "FROM nodes n1 "
+                           "WHERE n1.flags & " +
+                           std::to_string(excludeFlags) +
+                           " = 0 AND n1.ctime >= ? AND n1.type = " + filenode +
+                           " "
+                           "ORDER BY n1.ctime DESC LIMIT ?";
+
+    int sqlResult = SQLITE_OK;
+    if (!mStmtRecents)
+    {
+        sqlResult = sqlite3_prepare_v2(db, sqlQuery.c_str(), -1, &mStmtRecents, NULL);
+    }
+
+    bool stepResult = false;
+    if (sqlResult == SQLITE_OK)
+    {
+        if (sqlResult == sqlite3_bind_int64(mStmtRecents, 1, since))
+        {
+            // LIMIT expression evaluates to a negative value, then there is no upper bound on the
+            // number of rows returned
+            int64_t nodeCount = (maxcount > 0) ? static_cast<int64_t>(maxcount) : -1;
+            if (sqlResult == sqlite3_bind_int64(mStmtRecents, 2, nodeCount))
+            {
+                stepResult = processSqlQueryNodes(mStmtRecents, nodes);
+            }
+        }
+    }
+
+    if (sqlResult != SQLITE_OK)
+    {
+        errorHandler(sqlResult, "Get recent nodes", false);
+    }
+
+    sqlite3_reset(mStmtRecents);
+
+    return stepResult;
+}
+
 bool SqliteAccountState::getFavouritesHandles(NodeHandle node, uint32_t count, std::vector<mega::NodeHandle> &nodes)
 {
     if (!db)

src/megaapi.cpp

@@ -4138,7 +4138,7 @@ MegaRecentActionBucketList* MegaApi::getRecentActions()
 
 void MegaApi::getRecentActionsAsync(unsigned days, unsigned maxnodes, MegaRequestListener *listener)
 {
-    pImpl->getRecentActionsAsync(days, maxnodes, true /*excludeSensitives*/, listener);
+    pImpl->getRecentActionsAsync(days, maxnodes, listener);
 }
 
 void MegaApi::getRecentActionsAsync(unsigned days,