Use FetchContactsDeltaQuery for contact sync

This has a number of benefits: - Most of the time the contact sync reply will be empty - We can do contact sync more frequently (It's 5 mins now, was 30) - Figuring out what contacts were added or removed is much simpler and less likely to get things wrong. - Non-friends are no longer accidentally removed because there's no need to compare contact lists - On accounts with lots of friends this gets rid of one source of CPU usage spikes - Less load for facebook's servers (lol)
bitlbee · Jan 8, 2017 · 82e6bcf · 82e6bcf
1 parent e5e8c89
commit 82e6bcf
Show file tree

Hide file tree

Showing 3 changed files with 181 additions and 30 deletions.
diff --git a/facebook/facebook-api.c b/facebook/facebook-api.c
@@ -61,6 +61,7 @@ struct _FbApiPrivate
     gboolean invisible;
     guint unread;
     FbId lastmid;
+    gchar *contacts_delta;
 };
 
 struct _FbApiData
@@ -81,6 +82,9 @@ fb_api_message_send(FbApi *api, FbApiMessage *msg);
 static void
 fb_api_sticker(FbApi *api, FbId sid, FbApiMessage *msg);
 
+void
+fb_api_contacts_delta(FbApi *api, const gchar *delta_cursor);
+
 G_DEFINE_TYPE(FbApi, fb_api, G_TYPE_OBJECT);
 
 static void
@@ -174,6 +178,7 @@ fb_api_dispose(GObject *obj)
     g_free(priv->did);
     g_free(priv->stoken);
     g_free(priv->token);
+    g_free(priv->contacts_delta);
 }
 
 static void
@@ -335,6 +340,23 @@ fb_api_class_init(FbApiClass *klass)
                  G_TYPE_NONE,
                  2, G_TYPE_POINTER, G_TYPE_BOOLEAN);
 
+    /**
+     * FbApi::contacts-delta:
+     * @api: The #FbApi.
+     * @added: The #GSList of added #FbApiUser's.
+     * @removed: The #GSList of strings with removed user ids.
+     *
+     * Like 'contacts', but only the deltas.
+     */
+    g_signal_new("contacts-delta",
+                 G_TYPE_FROM_CLASS(klass),
+                 G_SIGNAL_ACTION,
+                 0,
+                 NULL, NULL,
+                 fb_marshal_VOID__POINTER_POINTER,
+                 G_TYPE_NONE,
+                 2, G_TYPE_POINTER, G_TYPE_POINTER);
+
     /**
      * FbApi::error:
      * @api: The #FbApi.
@@ -2023,8 +2045,10 @@ fb_api_cb_contacts_nodes(FbApi *api, JsonNode *root, GSList *users)
                        "$.structured_name.text");
     fb_json_values_add(values, FB_JSON_TYPE_STR, FALSE,
                        "$.hugePictureUrl.uri");
-    fb_json_values_set_array(values, FALSE, "$.viewer.messenger_contacts"
-                                             ".nodes");
+
+    if (JSON_NODE_TYPE(root) == JSON_NODE_ARRAY) {
+        fb_json_values_set_array(values, FALSE, "$");
+    }
 
     while (fb_json_values_update(values, &err)) {
         str = fb_json_values_next_str(values, "0");
@@ -2045,44 +2069,126 @@ fb_api_cb_contacts_nodes(FbApi *api, JsonNode *root, GSList *users)
         user->csum = fb_api_user_icon_checksum(user->icon);
 
         users = g_slist_prepend(users, user);
+
+        if (JSON_NODE_TYPE(root) != JSON_NODE_ARRAY) {
+            break;
+        }
     }
 
     g_object_unref(values);
 
     return users;
 }
 
+/* base64(contact:<our id>:<their id>:<whatever>) */
+static GSList *
+fb_api_cb_contacts_parse_removed(FbApi *api, JsonNode *node, GSList *users)
+{
+    gsize len;
+    char **split;
+    guchar *decoded = g_base64_decode(json_node_get_string(node), &len);
+
+    g_return_val_if_fail(decoded[len] == '\0', users);
+    g_return_val_if_fail(len == strlen(decoded), users);
+    g_return_val_if_fail(g_str_has_prefix(decoded, "contact:"), users);
+
+    split = g_strsplit_set(decoded, ":", 4);
+
+    g_return_val_if_fail(g_strv_length(split) == 4, users);
+
+    users = g_slist_prepend(users, g_strdup(split[2]));
+
+    g_strfreev(split);
+    g_free(decoded);
+
+    return users;
+}
+
 static void
 fb_api_cb_contacts(FbHttpRequest *req, gpointer data)
 {
     const gchar *cursor;
+    const gchar *delta_cursor;
     const gchar *str;
     FbApi *api = data;
     FbApiPrivate *priv = api->priv;
     FbApiUser *user;
     FbId uid;
     FbJsonValues *values;
     gboolean complete;
+    gboolean is_delta;
     GError *err = NULL;
+    GList *l;
     GSList *users = NULL;
     JsonNode *root;
+    JsonNode *croot;
+    JsonNode *node;
 
     if (!fb_api_http_chk(api, req, &root)) {
         return;
     }
 
-    users = fb_api_cb_contacts_nodes(api, root, users);
+    croot = fb_json_node_get(root, "$.viewer.messenger_contacts.deltas", NULL);
+    is_delta = (croot != NULL);
 
-    values = fb_json_values_new(root);
+    if (!is_delta) {
+        croot = fb_json_node_get(root, "$.viewer.messenger_contacts", NULL);
+        node = fb_json_node_get(croot, "$.nodes", NULL);
+        users = fb_api_cb_contacts_nodes(api, node, users);
+        json_node_free(node);
+
+    } else {
+        GSList *added = NULL;
+        GSList *removed = NULL;
+        JsonArray *arr = fb_json_node_get_arr(croot, "$.nodes", NULL);
+        GList *elms = json_array_get_elements(arr);
+
+        for (l = elms; l != NULL; l = l->next) {
+            if (node = fb_json_node_get(l->data, "$.added", NULL)) {
+                added = fb_api_cb_contacts_nodes(api, node, added);
+                json_node_free(node);
+            }
+
+            if (node = fb_json_node_get(l->data, "$.removed", NULL)) {
+                removed = fb_api_cb_contacts_parse_removed(api, node, removed);
+                json_node_free(node);
+            }
+        }
+
+        g_signal_emit_by_name(api, "contacts-delta", added, removed);
+
+        g_slist_free_full(added, (GDestroyNotify) fb_api_user_free);
+        g_slist_free_full(removed, (GDestroyNotify) g_free);
+
+        g_list_free(elms);
+        json_array_unref(arr);
+    }
+
+    values = fb_json_values_new(croot);
+    fb_json_values_add(values, FB_JSON_TYPE_BOOL, FALSE,
+                       "$.page_info.has_next_page");
     fb_json_values_add(values, FB_JSON_TYPE_STR, FALSE,
-                       "$.viewer.messenger_contacts.page_info.end_cursor");
+                       "$.page_info.delta_cursor");
+    fb_json_values_add(values, FB_JSON_TYPE_STR, FALSE,
+                       "$.page_info.end_cursor");
     fb_json_values_update(values, NULL);
 
+    complete = !fb_json_values_next_bool(values, FALSE);
+
+    delta_cursor = fb_json_values_next_str(values, NULL);
+
     cursor = fb_json_values_next_str(values, NULL);
 
     if (G_UNLIKELY(err == NULL)) {
-        complete = (cursor == NULL);
-        g_signal_emit_by_name(api, "contacts", users, complete);
+
+        if (is_delta || complete) {
+            g_free(priv->contacts_delta);
+            priv->contacts_delta = g_strdup(is_delta ? cursor : delta_cursor);
+        }
+
+        if (users) {
+            g_signal_emit_by_name(api, "contacts", users, complete);
+        }
 
         if (!complete) {
             fb_api_contacts_after(api, cursor);
@@ -2093,14 +2199,25 @@ fb_api_cb_contacts(FbHttpRequest *req, gpointer data)
 
     g_slist_free_full(users, (GDestroyNotify) fb_api_user_free);
     g_object_unref(values);
+
+    json_node_free(croot);
     json_node_free(root);
 }
 
 void
 fb_api_contacts(FbApi *api)
 {
+    FbApiPrivate *priv;
     JsonBuilder *bldr;
 
+    g_return_if_fail(FB_IS_API(api));
+    priv = api->priv;
+
+    if (priv->contacts_delta) {
+        fb_api_contacts_delta(api, priv->contacts_delta);
+        return;
+    }
+
     bldr = fb_json_bldr_new(JSON_NODE_OBJECT);
     fb_json_bldr_arr_begin(bldr, "0");
     fb_json_bldr_add_str(bldr, NULL, "user");
@@ -2127,6 +2244,24 @@ fb_api_contacts_after(FbApi *api, const gchar *cursor)
                       fb_api_cb_contacts);
 }
 
+void
+fb_api_contacts_delta(FbApi *api, const gchar *delta_cursor)
+{
+    JsonBuilder *bldr;
+
+    bldr = fb_json_bldr_new(JSON_NODE_OBJECT);
+
+    fb_json_bldr_add_str(bldr, "0", delta_cursor);
+
+    fb_json_bldr_arr_begin(bldr, "1");
+    fb_json_bldr_add_str(bldr, NULL, "user");
+    fb_json_bldr_arr_end(bldr);
+
+    fb_json_bldr_add_str(bldr, "2", G_STRINGIFY(FB_API_CONTACTS_COUNT));
+    fb_api_http_query(api, FB_API_QUERY_CONTACTS_DELTA, bldr,
+                      fb_api_cb_contacts);
+}
+
 void
 fb_api_connect(FbApi *api, gboolean invisible)
 {

diff --git a/facebook/facebook.c b/facebook/facebook.c
@@ -207,9 +207,9 @@ fb_sync_contacts_add_timeout(FbData *fata)
 
     sync = set_getint(&acct->set, "sync_interval");
 
-    if (sync < 5) {
-        set_setint(&acct->set, "sync_interval", 5);
-        sync = 5;
+    if (sync < 1) {
+        set_setint(&acct->set, "sync_interval", 1);
+        sync = 1;
     }
 
     sync *= 60 * 1000;
@@ -220,7 +220,6 @@ fb_sync_contacts_add_timeout(FbData *fata)
 static void
 fb_cb_api_contacts(FbApi *api, GSList *users, gboolean complete, gpointer data)
 {
-    bee_user_t *bu;
     FbApiUser *user;
     FbData *fata = data;
     FbId muid;
@@ -247,35 +246,47 @@ fb_cb_api_contacts(FbApi *api, GSList *users, gboolean complete, gpointer data)
         imcb_add_buddy(ic, uid, NULL);
         imcb_buddy_nick_hint(ic, uid, user->name);
         imcb_rename_buddy(ic, uid, user->name);
-
-        bu = imcb_buddy_by_handle(ic, uid);
-        FB_UTIL_PTRBIT_SET(bu->data, FB_PTRBIT_NEW_BUDDY, TRUE);
     }
 
     if (!complete) {
         return;
     }
 
-    l = ic->bee->users;
+    if (!(ic->flags & OPT_LOGGED_IN)) {
+        imcb_log(ic, "Connecting");
+        fb_api_connect(api, FALSE);
+    }
 
-    while (l != NULL) {
-        bu = l->data;
-        l = l->next;
+    fb_sync_contacts_add_timeout(fata);
+}
 
-        if (bu->ic != ic) {
-            continue;
-        }
+static void
+fb_cb_api_contacts_delta(FbApi *api, GSList *added, GSList *removed, gpointer data)
+{
+    bee_user_t *bu;
+    FbApiUser *user;
+    FbData *fata = data;
+    gchar uid[FB_ID_STRMAX];
+    GSList *l;
+    struct im_connection *ic;
 
-        if (FB_UTIL_PTRBIT_GET(bu->data, FB_PTRBIT_NEW_BUDDY)) {
-            FB_UTIL_PTRBIT_SET(bu->data, FB_PTRBIT_NEW_BUDDY, FALSE);
-        } else {
-            imcb_remove_buddy(ic, bu->handle, NULL);
-        }
+    ic = fb_data_get_connection(fata);
+
+    for (l = added; l != NULL; l = l->next) {
+        user = l->data;
+        FB_ID_TO_STR(user->uid, uid);
+
+        imcb_add_buddy(ic, uid, NULL);
+        imcb_buddy_nick_hint(ic, uid, user->name);
+        imcb_rename_buddy(ic, uid, user->name);
     }
 
-    if (!(ic->flags & OPT_LOGGED_IN)) {
-        imcb_log(ic, "Connecting");
-        fb_api_connect(api, FALSE);
+    for (l = removed; l != NULL; l = l->next) {
+        bu = imcb_buddy_by_handle(ic, l->data);
+
+        if (bu) {
+            imcb_remove_buddy(ic, bu->handle, NULL);
+        }
     }
 
     fb_sync_contacts_add_timeout(fata);
@@ -722,7 +733,7 @@ fb_init(account_t *acct)
     set_add(&acct->set, "mark_read", "false", fb_eval_mark_read, acct);
     set_add(&acct->set, "mark_read_reply", "false", set_eval_bool, acct);
     set_add(&acct->set, "show_unread", "false", set_eval_bool, acct);
-    set_add(&acct->set, "sync_interval", "30", set_eval_int, acct);
+    set_add(&acct->set, "sync_interval", "5", set_eval_int, acct);
 }
 
 static void
@@ -753,6 +764,10 @@ fb_login(account_t *acc)
                      "contacts",
                      G_CALLBACK(fb_cb_api_contacts),
                      fata);
+    g_signal_connect(api,
+                     "contacts-delta",
+                     G_CALLBACK(fb_cb_api_contacts_delta),
+                     fata);
     g_signal_connect(api,
                      "error",
                      G_CALLBACK(fb_cb_api_error),

diff --git a/facebook/marshaller.list b/facebook/marshaller.list
@@ -4,3 +4,4 @@ VOID:POINTER
 VOID:POINTER,BOOLEAN
 VOID:STRING,BOXED
 VOID:VOID
+VOID:POINTER,POINTER