Rewrite Google Drive client

app/clients/google-drive/README

@@ -0,0 +1,46 @@
+# Google Drive Client for Blot
+
+This repository contains the code for Blot’s Google Drive Client, designed to handle all interactions with Google Drive for the Blot blogging platform. The client is responsible for managing shared folders, syncing changes, and facilitating smooth integration between Blot and Google Drive.
+
+Right now we ask the user for their Google Drive email.
+
+The service account will automatically accept sharing requests sent to it.
+
+How is storage usage affected by 'service accounts'?
+- it seems like each service account gets its own storage quota of 15GB!?
+
+Do multiple service accounts share the same storage quota? If we upgrade the storage size and have this affect all service accounts?
+- No they do not share the same quota
+
+## How to add a new service account
+
+Service accounts are robot google drive accounts which sync the user's folder to Blot's server. The user shares their site folder through Google Drive with the service account, which in turn calls Blot.
+
+1. Create a Google Account and open the cloud console:
+
+https://console.cloud.google.com
+
+2. Go to 'Enabled APIs & services' and enable:
+
+Google Drive API 	
+
+3. Create a service account
+
+Service account name: Blot
+Service account ID: drive-a
+
+Do not grant any permissions (skip, skip)
+
+4. Create a key
+
+Add key > Create new key > JSON
+
+Download the JSON and then run:
+
+node scripts/google-drive/convertCredentialJSON <path to credentials.json>
+
+This script produces a new env variable. Copy/paste the output into the env file:
+- in development: ~/Projects/blot/.env
+- in production: /etc/blot/secrets.env
+
+5. Reload the server

app/clients/google-drive/database.js

@@ -0,0 +1,283 @@
+const promisify = require("util").promisify;
+const client = require("models/client");
+const set = promisify(client.set).bind(client);
+const get = promisify(client.get).bind(client);
+const del = promisify(client.del).bind(client);
+const hset = promisify(client.hset).bind(client);
+const hget = promisify(client.hget).bind(client);
+const hdel = promisify(client.hdel).bind(client);
+const hscan = promisify(client.hscan).bind(client);
+const sadd = promisify(client.sadd).bind(client);
+const smembers = promisify(client.smembers).bind(client);
+
+const keys = {
+  // Used to renew webhooks for all connected Google Drives
+  allAccounts: "clients:google-drive:all-accounts",
+
+  account: function (blogID) {
+    return "blog:" + blogID + ":google-drive:account";
+  },
+
+  // These are the service accounts that users share their site folders with
+  serviceAccount: function (serviceAccountId) {
+    return `clients:google-drive:${serviceAccountId}:service-account`;
+  },
+
+  // We need to list all accounts to refresh the storage usage for each
+  // service account. They are typically allocated 15GB and it remains
+  // unclear if this can be increased.
+  allServiceAccounts: "clients:google-drive:all-service-accounts",
+};
+
+const allAccounts = function (callback) {
+  client.smembers(keys.allAccounts, (err, blogIDs) => {
+    if (err) return callback(err);
+    if (!blogIDs || !blogIDs.length) return callback(null, []);
+    client.mget(blogIDs.map(keys.account), (err, accounts) => {
+      if (err) return callback(err);
+      if (!accounts || !accounts.length) return callback(null, []);
+      accounts = accounts
+        .map((serializedAccount, index) => {
+          if (!serializedAccount) return null;
+          try {
+            const account = JSON.parse(serializedAccount);
+            account.blogID = blogIDs[index];
+            return account;
+          } catch (e) {}
+          return null;
+        })
+        .filter((account) => !!account);
+
+      callback(null, accounts);
+    });
+  });
+};
+
+const getAccount = function (blogID, callback) {
+  const key = keys.account(blogID);
+  client.get(key, (err, account) => {
+    if (err) {
+      return callback(err);
+    }
+
+    if (!account) {
+      return callback(null, null);
+    }
+
+    try {
+      account = JSON.parse(account);
+    } catch (e) {
+      return callback(e);
+    }
+
+    callback(null, account);
+  });
+}
+
+const setAccount = function (blogID, changes, callback) {
+  const key = keys.account(blogID);
+
+  getAccount(blogID, (err, account) => {
+    account = account || {};
+
+    const multi = client.multi();
+
+    for (var i in changes) {
+      account[i] = changes[i];
+    }
+
+    multi
+      .sadd(keys.allAccounts, blogID)
+      .set(key, JSON.stringify(account));
+
+    multi.exec(callback);
+  });
+}
+
+
+const dropAccount = function (blogID, callback) {
+  getAccount(blogID, (err, account) => {
+    const multi = client.multi();
+
+    multi.del(keys.account(blogID)).srem(keys.allAccounts, blogID);
+
+    if (account && account.folderId) {
+      multi
+        .del(folder(account.folderId).key)
+        .del(folder(account.folderId).tokenKey);
+    }
+
+    multi.exec(callback);
+  });
+}
+
+
+const serviceAccount = {
+  get: async function (client_id) {
+    const account = await get(keys.serviceAccount(client_id));
+    if (!account) return null;
+    return JSON.parse(account);
+  },
+  set: async function (client_id, changes) {
+    const serviceAccountString = await get(keys.serviceAccount(client_id));
+    const serviceAccount = serviceAccountString ? JSON.parse(serviceAccountString) : {};
+
+    for (var i in changes) {
+      serviceAccount[i] = changes[i];
+    }
+
+    await set(
+      keys.serviceAccount(client_id),
+      JSON.stringify(serviceAccount)
+    );
+
+    await sadd(keys.allServiceAccounts, client_id);
+  },
+  all: async function () {
+    const client_ids = await smembers(keys.allServiceAccounts);
+
+    if (!client_ids || !client_ids.length) return [];
+
+    const serviceAccounts = await Promise.all(
+      client_ids.map(async (id) => {
+        const account = await serviceAccount.get(id);
+        return { ...account, client_id: id }; // Ensure client_id is included
+      })
+    );
+
+    return serviceAccounts;
+  }
+};
+
+function folder (folderId) {
+  this.key = `clients:google-drive:${folderId}:folder`;
+  this.tokenKey = `clients:google-drive:${folderId}:pageToken`;
+
+  this.set = async (id, path) => {
+    await hset(this.key, id, path);
+  };
+
+  this.get = async (id) => {
+    if (id === undefined || id === null) return null;
+    return await hget(this.key, id);
+  };
+
+  this.getByPath = async (path) => {
+    const START_CURSOR = "0";
+    let cursor = START_CURSOR;
+    let fileId, results;
+
+    const match = (el, index) =>
+      index % 2 === 0 && results[index + 1] === path;
+
+    do {
+      [cursor, results] = await hscan(this.key, cursor);
+      fileId = results.find(match);
+    } while (!fileId && cursor !== START_CURSOR);
+
+    return fileId || null;
+  };
+
+  this.move = async (id, to) => {
+    const START_CURSOR = "0";
+    const from = await this.get(id);
+
+    let movedPaths = [];
+
+    if (from === "/" || to === "/")
+      throw new Error("Attempt to move to/from root");
+
+    let [cursor, results] = await hscan(this.key, START_CURSOR);
+
+    do {
+      const changes = results
+        .map((el, i) => {
+          if (i % 2 !== 0) return null;
+          const path = results[i + 1];
+          const modifiedPath =
+            path === from || path.indexOf(from + "/") === 0
+              ? to + path.slice(from.length)
+              : path;
+          if (path === modifiedPath) return null;
+          movedPaths.push(path);
+          movedPaths.push(modifiedPath);
+          return { id: el, path: modifiedPath };
+        })
+        .filter((i) => !!i);
+
+      for (const { id, path } of changes) await this.set(id, path);
+
+      [cursor, results] = await hscan(this.key, cursor);
+    } while (cursor !== START_CURSOR);
+
+    return movedPaths;
+  };
+
+  this.remove = async (id) => {
+    const START_CURSOR = "0";
+    const from = await this.get(id);
+
+    if (from === null || from === undefined) return [];
+
+    let removedPaths = [];
+
+    let [cursor, results] = await hscan(this.key, START_CURSOR);
+
+    do {
+      const ids = results
+        .map((el, i) => {
+          if (i % 2 !== 0) return null;
+
+          const path = results[i + 1];
+
+          if (path === from) {
+            removedPaths.push(path);
+            return el;
+          }
+
+          if (from === "/") {
+            removedPaths.push(path);
+            return el;
+          }
+
+          if (path.indexOf(from + "/") === 0) {
+            removedPaths.push(path);
+            return el;
+          }
+
+          return null;
+        })
+        .filter((i) => i !== null);
+
+      await hdel(this.key, ids);
+
+      [cursor, results] = await hscan(this.key, cursor);
+    } while (cursor !== START_CURSOR);
+
+    return removedPaths;
+  };
+
+  this.reset = async () => {
+    await del(this.key);
+    await del(this.tokenKey);
+  };
+
+  this.setPageToken = async (token) => {
+    await set(this.tokenKey, token);
+  };
+
+  this.getPageToken = async () => {
+    return await get(this.tokenKey);
+  };
+
+  return this;
+};
+
+module.exports = {
+  serviceAccount: serviceAccount,
+  folder: folder,
+  getAccount: promisify(getAccount),
+  setAccount: promisify(setAccount),
+  dropAccount: promisify(dropAccount),
+  allAccounts: promisify(allAccounts),
+}

app/clients/google-drive/disconnect.js

@@ -0,0 +1,39 @@
+const Blog = require("models/blog");
+const establishSyncLock = require("./util/establishSyncLock");
+const database = require("./database");
+const createDriveClient = require("./util/createDriveClient");
+const debug = require("debug")("blot:clients:google-drive");
+
+module.exports = async (blogID, callback) => {
+    let done;
+
+    try {
+        let lock = await establishSyncLock(blogID);
+        done = lock.done;
+    } catch (err) {
+        return callback(err);
+    }
+
+    const account = await database.getAccount(blogID);
+
+    if (account && account.channel) {
+        try {
+            const drive = await createDriveClient(blogID);
+            debug("attempting to stop listening to webhooks");
+            await drive.channels.stop({
+              requestBody: account.channel,
+            });
+            debug("stop listening to webhooks successfully");
+          } catch (e) {
+            debug("failed to stop webhooks but no big deal:", e.message);
+            debug("it will expire automatically");
+          }
+    }
+
+    await database.dropAccount(blogID);
+
+    Blog.set(blogID, { client: "" }, async function (err) {
+        await done(err);
+        callback();
+    });
+};

app/clients/google-drive/index.js

@@ -0,0 +1,11 @@
+module.exports = {
+  display_name: "Google Drive",
+  description: "A file storage and synchronization service",
+  disconnect: require("./disconnect"),
+  resync: require('./sync/reset-to-blot'),
+  remove: require("./remove"),
+  write: require("./write"),
+  site_routes: require("./routes/site"),
+  dashboard_routes: require("./routes/dashboard"),
+  init: require('./init')
+};