Refactor tags list

davidmerfield · Feb 12, 2025 · b0d70f0 · b0d70f0
1 parent 3c393d2
commit b0d70f0
Show file tree

Hide file tree

Showing 2 changed files with 54 additions and 13 deletions.
diff --git a/app/models/tags/list.js b/app/models/tags/list.js
@@ -3,19 +3,34 @@ const ensure = require("helper/ensure");
 const key = require("./key");
 const clfdate = require("helper/clfdate");
 
+// Helper function to scan a Redis set using SSCAN
+async function scanSet(key, pattern = "*") {
+  let cursor = "0";
+  let results = [];
+
+  do {
+    const [nextCursor, chunk] = await new Promise((resolve, reject) => {
+      client.sscan(key, cursor, "MATCH", pattern, "COUNT", 100, (err, res) => {
+        if (err) return reject(err);
+        resolve(res);
+      });
+    });
+
+    cursor = nextCursor;
+    results = results.concat(chunk);
+  } while (cursor !== "0");
+
+  return results;
+}
+
 module.exports = async function getAll(blogID, callback) {
   try {
     ensure(blogID, "string").and(callback, "function");
 
     console.log(clfdate(), "Fetching all tags for", blogID);
 
-    // Fetch all tags using SMEMBERS
-    const allTags = await new Promise((resolve, reject) => {
-      client.smembers(key.all(blogID), (err, result) => {
-        if (err) return reject(err);
-        resolve(result || []);
-      });
-    });
+    // Fetch all tags using SSCAN
+    const allTags = await scanSet(key.all(blogID));
 
     console.log(clfdate(), "Found", allTags.length, "tags for", blogID);
 
@@ -28,12 +43,7 @@ module.exports = async function getAll(blogID, callback) {
     for (const tag of allTags) {
       console.log(clfdate(), "Fetching tag", tag, "for", blogID);
       const [entries, name] = await Promise.all([
-        new Promise((resolve, reject) => {
-          client.smembers(key.tag(blogID, tag), (err, result) => {
-            if (err) return reject(err);
-            resolve(result || []);
-          });
-        }),
+        scanSet(key.tag(blogID, tag)), // Use SSCAN to fetch entries of the tag
         new Promise((resolve, reject) => {
           client.get(key.name(blogID, tag), (err, result) => {
             if (err) return reject(err);

diff --git a/app/models/tags/tests/list.js b/app/models/tags/tests/list.js
@@ -6,6 +6,37 @@ describe("tags.list", function () {
     // Create a test user and blog before each spec
     global.test.blog();
 
+    it("it is fast for thousands of tags and entries", async function (done) {
+        const tags = [];
+        for (let i = 0; i < 1000; i++) {
+            tags.push({
+                id: `tag${i}`,
+            });
+        }
+
+        for (let i = 0; i < 1000; i++) {
+            // guarantee at least one tag per entry, and each tag is used at least once
+            const entryTags = [{id: `tag${i}`}];
+            const numberOfTags = Math.floor(Math.random() * 100) + 1;
+            for (let j = 0; j < numberOfTags; j++) {
+                const randomIndex = Math.floor(Math.random() * tags.length);
+                entryTags.push(tags[randomIndex]);
+            }
+            await this.blog.write({path: `/entry${i}.txt`, content: `Tags: ${entryTags.map(tag => tag.id).join(", ")}\n\nContent ${i}`});
+        }
+
+        await this.blog.rebuild();
+
+        const start = Date.now();
+
+        list(this.blog.id, function (err, tags) {
+            expect(err).toBeNull();
+            expect(tags.length).toEqual(1000);
+            console.log("Time taken: ", (Date.now() - start) + "ms");
+            done();
+        });
+    }, 30000);
+
     it("can be invoked without error", function (done) {
         const blogID = this.blog.id;
         const entry1 = {