diff --git a/app/clients/google-drive-2/README b/app/clients/google-drive-2/README deleted file mode 100644 index f73ff74e404..00000000000 --- a/app/clients/google-drive-2/README +++ /dev/null @@ -1,25 +0,0 @@ -# Google Drive Client for Blot - -This repository contains the code for Blot’s Google Drive Client, designed to handle all interactions with Google Drive for the Blot blogging platform. The client is responsible for managing shared folders, syncing changes, and facilitating smooth integration between Blot and Google Drive. - -There are two 'apps', one external which requests the user's email address through the OAUTH flow. -The other is a 'service account' which has access to a Drive folder created by contact@blot.im - -The service account creates a new site folder inside the Sites folder and then shares it with the user's email. I created - -## How It Works - 1. User Authentication: -Users log in via Blot’s external app, which captures their Google email address. - 2. Folder Creation and Sharing: -The internal Google Drive Client uses the captured email to: - • Create a folder in Blot’s Google Drive account. - • Share the folder with the user (role=editor). - 3. Change Tracking and Syncing: - • Monitor the shared folder for changes using the Google Drive API. - • Sync updates, including file uploads, edits, and deletions, to Blot’s server. - ---- - -SHould we ask the user to share a folder with our email? This was it's a 'true' folder and doesn't show up in 'shared with me'. - -How is storage usage affected by 'service accounts'? \ No newline at end of file diff --git a/app/clients/google-drive-2/index.js b/app/clients/google-drive-2/index.js deleted file mode 100644 index 7f9e93ad565..00000000000 --- a/app/clients/google-drive-2/index.js +++ /dev/null @@ -1,12 +0,0 @@ -module.exports = { - display_name: "Google Drive", - description: "A file storage and synchronization service", - disconnect: require("./disconnect"), - resync: require('./sync/reset-to-blot'), - remove: require("./remove"), - write: require("./write"), - site_routes: require("./routes/site"), - dashboard_routes: require("./routes/dashboard"), - init: require('./init') - }; - \ No newline at end of file diff --git a/app/clients/google-drive-2/test.js b/app/clients/google-drive-2/test.js deleted file mode 100644 index d15fbeafa9e..00000000000 --- a/app/clients/google-drive-2/test.js +++ /dev/null @@ -1,64 +0,0 @@ -const { google } = require('googleapis'); -const fs = require('fs'); -const path = require('path'); - -// Path to your service account credentials JSON file -const CREDENTIALS_PATH = path.join(__dirname, 'data', 'service-account-creds.json'); - -// The ID of the "Sites" folder in Blot's Google Drive -const SITES_FOLDER_ID = process.env.BLOT_GOOGLEDRIVE_FOLDER_ID; - -/** - * Authenticate with Google Drive using the service account credentials. - */ -async function authenticateWithServiceAccount() { - // Load service account credentials from file - const credentials = JSON.parse(fs.readFileSync(CREDENTIALS_PATH, 'utf8')); - - // Create a JWT client - const auth = new google.auth.GoogleAuth({ - credentials, - scopes: ['https://www.googleapis.com/auth/drive'], - }); - - return google.drive({ version: 'v3', auth }); -} - -/** - * List the contents of root folder. - */ -async function listFolderContents(drive, folderId) { - try { - const res = await drive.files.list({ - q: `trashed = false and '${folderId}' in parents`, - fields: 'files(id, name, mimeType)', - }); - - const files = res.data.files || []; - console.log(`Contents of folder (${folderId}):`); - files.forEach((file) => { - console.log(`- ${file.name} (${file.id}, ${file.mimeType})`); - }); - } catch (error) { - console.error('Error listing folder contents:', error.message); - } -} - -/** - * Main function to execute the operations. - */ -async function main() { - try { - // Authenticate with the service account - const drive = await authenticateWithServiceAccount(); - - - // List the contents of the "Sites" folder - await listFolderContents(drive, SITES_FOLDER_ID); - } catch (error) { - console.error('Error:', error.message); - } -} - -// Run the script -main(); \ No newline at end of file diff --git a/app/clients/google-drive/README b/app/clients/google-drive/README index 9a0d06fec57..f73ff74e404 100644 --- a/app/clients/google-drive/README +++ b/app/clients/google-drive/README @@ -1,57 +1,25 @@ -Fix, I have confirmed that the uploader of the file, regardless of the owner of the folder into which it is uploaded, pays the storage 'cost' for the file. To demonstate this I used two google drive accounts (dmerfield@gmail.com and david@blot.im) +# Google Drive Client for Blot -I created a shared folder in dmerfield@gmail.com, then shared it with david@blot.im. I uploaded a file into it using david@blot.im's account. The storage usage for david@blot.im went up, but dmerfield@gmail.com's storage usage did not. +This repository contains the code for Blot’s Google Drive Client, designed to handle all interactions with Google Drive for the Blot blogging platform. The client is responsible for managing shared folders, syncing changes, and facilitating smooth integration between Blot and Google Drive. ---- +There are two 'apps', one external which requests the user's email address through the OAUTH flow. +The other is a 'service account' which has access to a Drive folder created by contact@blot.im +The service account creates a new site folder inside the Sites folder and then shares it with the user's email. I created -Add new redirect URIs here: -https://console.cloud.google.com/apis/credentials?project=quickstart-1585441405190 +## How It Works + 1. User Authentication: +Users log in via Blot’s external app, which captures their Google email address. + 2. Folder Creation and Sharing: +The internal Google Drive Client uses the captured email to: + • Create a folder in Blot’s Google Drive account. + • Share the folder with the user (role=editor). + 3. Change Tracking and Syncing: + • Monitor the shared folder for changes using the Google Drive API. + • Sync updates, including file uploads, edits, and deletions, to Blot’s server. --- -Questions -- is there an overhead to creating the oauth2 client each time? can we safely reuse it? -- is it possible to restrict Blot's access to a single folder in the google drive? - - it doesn't seem like it is - -I believe we will need this API for drive changes: -https://developers.google.com/drive/activity/v2 - -Since the changes.list api doesn't return rename events, it seems. - -The webhooks are now tunnelled through webhooks.blot.im using clients/webhooks.js - -Test server -``` -http://localhost:8822/clients/googledrive/authenticate -``` - -Resources: - -[Verify domain ownership](https://search.google.com/search-console/settings?resource_id=sc-domain%3Ablot.im) - -[Domain verification for API use](https://console.cloud.google.com/apis/credentials/domainverification?organizationId=683828060430&project=quickstart-1585441405190) - -[Console for redirect URIs](https://console.cloud.google.com/apis/credentials/oauthclient/32772360147-pnntpgr8pjnlem4m6s1perkju3ghce3b.apps.googleusercontent.com?project=quickstart-1585441405190&pli=1) - -[Console for managing API permissions](https://console.developers.google.com/apis/credentials/oauthclient/32772360147-pnntpgr8pjnlem4m6s1perkju3ghce3b.apps.googleusercontent.com?project=quickstart-1585441405190) - -[Downloading files google drive](https://stackoverflow.com/questions/62476413/google-drive-api-downloading-file-nodejs) - -[Downloading files google drive](https://developers.google.com/drive/api/v3/manage-downloads) - -[Link to remove connection during testing](https://myaccount.google.com/permissions?pli=1) - -Documentation: - -[nodejs-client authentication-and-authorization](https://github.com/googleapis/google-api-nodejs-client#authentication-and-authorization) - -[quickstart](https://developers.google.com/drive/api/v3/quickstart/nodejs) +SHould we ask the user to share a folder with our email? This was it's a 'true' folder and doesn't show up in 'shared with me'. -Actions to test: -- Mass rename -> mass move -- Mass move -> mass rename -- Delete -> Restore -> Delete -- Move file from outside folder into folder -- Move file from inside folder out of folder \ No newline at end of file +How is storage usage affected by 'service accounts'? \ No newline at end of file diff --git a/app/clients/google-drive/database.js b/app/clients/google-drive/database.js index 242d66e01fc..486b97b7e0f 100644 --- a/app/clients/google-drive/database.js +++ b/app/clients/google-drive/database.js @@ -1,5 +1,6 @@ -const client = require("models/client"); const promisify = require("util").promisify; + +const client = require("models/client"); const set = promisify(client.set).bind(client); const get = promisify(client.get).bind(client); const del = promisify(client.del).bind(client); @@ -8,20 +9,11 @@ const hget = promisify(client.hget).bind(client); const hdel = promisify(client.hdel).bind(client); const hscan = promisify(client.hscan).bind(client); -const INVALID_ACCOUNT_STRING = - "Google Drive client: Error decoding JSON for account of blog "; - const database = { keys: { // Used to renew webhooks for all connected Google Drives allAccounts: "clients:google-drive:all-accounts", - // Used to determine if another blog is connected to a - // given Google Drive account during the disconnection process - allBlogs: function (permissionId) { - return "clients:google-drive:" + permissionId + ":blogs"; - }, - account: function (blogID) { return "blog:" + blogID + ":google-drive:account"; }, @@ -65,32 +57,13 @@ const database = { try { account = JSON.parse(account); } catch (e) { - return callback(new Error(INVALID_ACCOUNT_STRING + blogID)); + return callback(e); } callback(null, account); }); }, - // During account disconnection from Google Drive - // on Blot's folder settings page we need to determine - // whether or not to revoke the credentials, which - // unfortunately has global effects and would tank - // other blogs also connected to this Google Drive account. - canRevoke: function (permissionId, callback) { - let canRevokeCredentials; - - if (!permissionId) { - canRevokeCredentials = true; - return callback(null, canRevokeCredentials); - } - - client.smembers(this.keys.allBlogs(permissionId), (err, blogs) => { - canRevokeCredentials = !blogs || blogs.length < 2; - callback(null, canRevokeCredentials); - }); - }, - setAccount: function (blogID, changes, callback) { const key = this.keys.account(blogID); @@ -99,18 +72,6 @@ const database = { const multi = client.multi(); - if (changes.permissionId) - multi.sadd(this.keys.allBlogs(changes.permissionId), blogID); - - // Clean up if the permissionId for a blog changes - if ( - changes.permissionId && - account.permissionId && - changes.permissionId !== account.permissionId - ) { - multi.srem(this.keys.allBlogs(account.permissionId), blogID); - } - for (var i in changes) { account[i] = changes[i]; } @@ -129,10 +90,6 @@ const database = { multi.del(this.keys.account(blogID)).srem(this.keys.allAccounts, blogID); - if (account && account.permissionId) { - multi.srem(this.keys.allBlogs(account.permissionId), blogID); - } - if (account && account.folderId) { multi .del(this.folder(account.folderId).key) diff --git a/app/clients/google-drive/disconnect.js b/app/clients/google-drive/disconnect.js index 6901d2916f9..8bef00fb2ac 100644 --- a/app/clients/google-drive/disconnect.js +++ b/app/clients/google-drive/disconnect.js @@ -1,73 +1,39 @@ -const config = require("config"); -const database = require("./database"); -const google = require("googleapis").google; const Blog = require("models/blog"); const establishSyncLock = require("./util/establishSyncLock"); +const database = require("./database"); +const createDriveClient = require("./util/createDriveClient"); const debug = require("debug")("blot:clients:google-drive"); -async function disconnect(blogID, callback) { - let done; - - try { - let lock = await establishSyncLock(blogID); - done = lock.done; - } catch (err) { - return callback(err); - } - - debug("getting account info"); - const account = await database.getAccount(blogID); - - if (account && account.access_token && account.refresh_token) { - const canRevoke = await database.canRevoke(account.permissionId); - const auth = new google.auth.OAuth2( - config.google.drive.key, - config.google.drive.secret - ); - - auth.setCredentials({ - refresh_token: account.refresh_token, - access_token: account.access_token, - }); - - if (account.channel) { - try { - debug("attempting to stop listening to webhooks"); - const drive = google.drive({ version: "v3", auth }); - await drive.channels.stop({ - requestBody: account.channel, - }); - debug("stop listening to webhooks successfully"); - } catch (e) { - debug("failed to stop webhooks but no big deal:", e.message); - debug("it will expire automatically"); - } - } - - // We need to preserve Blot's access to this Google - // Drive account if another blog uses it. Unfortunately - // it seems impossible to simple revoke one blog's access - // other blogs connected to the account lose access too. - if (canRevoke) { - try { - debug("Trying to revoke Google API credentials"); - // destroys the oauth2Client's active - // refresh_token and access_token - await auth.revokeCredentials(); - } catch (e) { - debug("Failed to revoke but token should expire naturally", e.message); - } - } - } - - debug("dropping blog from database"); - await database.dropAccount(blogID); - - debug("resetting client setting"); - Blog.set(blogID, { client: "" }, async function (err) { - await done(err); - callback(); - }); -} - -module.exports = disconnect; +module.exports = async (blogID, callback) => { + let done; + + try { + let lock = await establishSyncLock(blogID); + done = lock.done; + } catch (err) { + return callback(err); + } + + const account = await database.getAccount(blogID); + + if (account && account.channel) { + const drive = await createDriveClient(blogID); + try { + debug("attempting to stop listening to webhooks"); + await drive.channels.stop({ + requestBody: account.channel, + }); + debug("stop listening to webhooks successfully"); + } catch (e) { + debug("failed to stop webhooks but no big deal:", e.message); + debug("it will expire automatically"); + } + } + + await database.dropAccount(blogID); + + Blog.set(blogID, { client: "" }, async function (err) { + await done(err); + callback(); + }); +}; \ No newline at end of file diff --git a/app/clients/google-drive/remove.js b/app/clients/google-drive/remove.js index 128cd4cbc8b..97f72f9d9dd 100644 --- a/app/clients/google-drive/remove.js +++ b/app/clients/google-drive/remove.js @@ -7,7 +7,8 @@ const database = require("./database"); module.exports = async function remove(blogID, path, callback) { const prefix = () => clfdate() + " Google Drive:"; try { - const { drive, account } = await createDriveClient(blogID); + const drive = await createDriveClient(blogID); + const account = await database.getAccount(blogID); const { getByPath } = database.folder(account.folderId); console.log(prefix(), "Looking up fileId for", path); diff --git a/app/clients/google-drive/routes/dashboard.js b/app/clients/google-drive/routes/dashboard.js index 83d84fd6e20..669968e485e 100644 --- a/app/clients/google-drive/routes/dashboard.js +++ b/app/clients/google-drive/routes/dashboard.js @@ -1,34 +1,21 @@ -const config = require("config"); -const google = require("googleapis").google; const clfdate = require("helper/clfdate"); const database = require("../database"); const disconnect = require("../disconnect"); -const resetFromBlot = require("../sync/reset-from-blot"); -const createDriveClient = require("../util/createDriveClient"); -const setupWebhook = require("../util/setupWebhook"); const express = require("express"); const dashboard = new express.Router(); const establishSyncLock = require("../util/establishSyncLock"); +const createDriveClient = require("../util/createDriveClient"); +const setupWebhook = require("../util/setupWebhook"); +const resetFromBlot = require("../sync/reset-from-blot"); +const parseBody = require("body-parser").urlencoded({ extended: false }); const VIEWS = require("path").resolve(__dirname + "/../views") + "/"; -const REDIRECT_URL = - config.environment === "development" - ? `https://${config.webhooks.relay_host}/clients/google-drive/authenticate` - : `https://${config.host}/clients/google-drive/authenticate`; - dashboard.use(async function (req, res, next) { const account = await database.getAccount(req.blog.id); - - if (account && account.folderPath) - account.folderParents = account.folderPath - .split("/") - .slice(1) - .map((name, i, arr) => { - return { name, last: arr.length - 1 === i }; - }); - + res.locals.blot_googledrive_client_email = process.env.BLOT_GOOGLEDRIVE_CLIENT_EMAIL; res.locals.account = account; + next(); }); @@ -36,6 +23,37 @@ dashboard.get("/", function (req, res) { res.render(VIEWS + "index"); }); +dashboard + .route("/disconnect") + .get(function (req, res) { + res.render(VIEWS + "disconnect"); + }) + .post(function (req, res, next) { + disconnect(req.blog.id, next); + }); + + +dashboard.route("/set-up-folder") + .post(parseBody, async function (req, res, next) { + console.log('req.body', req.body); + + if (req.body.cancel){ + return disconnect(req.blog.id, next); + } + + if (req.body.email) { + await database.setAccount(req.blog.id, { + email: req.body.email, + preparing: true + }); + + setUpBlogFolder(req.blog, req.body.email); + } + + console.log(clfdate(), "Google Drive Client", "Setting up folder"); + res.redirect(req.baseUrl); + }); + dashboard .route("/set-up-folder/cancel") .all(function (req, res, next) { @@ -50,146 +68,14 @@ dashboard error: null, channel: null, folderId: null, - folderPath: null, + folderName: null, preparing: null }); res.message(req.baseUrl, "Cancelled the creation of your new folder"); }); -dashboard - .route("/disconnect") - .get(function (req, res) { - res.render(VIEWS + "disconnect"); - }) - .post(function (req, res, next) { - disconnect(req.blog.id, next); - }); - -dashboard.get("/redirect", function (req, res) { - const oauth2Client = new google.auth.OAuth2( - config.google.drive.key, - config.google.drive.secret, - REDIRECT_URL - ); - - // It's important that sameSite is set to false so the - // cookie is exposed to us when OAUTH redirect occurs - res.cookie("blogToAuthenticate", req.blog.handle, { - domain: "", - path: "/", - secure: true, - httpOnly: true, - maxAge: 15 * 60 * 1000, // 15 minutes - sameSite: "Lax" // otherwise we will not see it - }); - - res.redirect( - oauth2Client.generateAuthUrl({ - access_type: "offline", - scope: "https://www.googleapis.com/auth/drive", - // Prompt: consent forces us to revisit the consent - // screen even if we had previously authorized Blot. - // This is neccessary to connect multiple blogs under - // one google account to Drive. More discussion here: - // https://github.com/googleapis/google-api-python-client/issues/213 - prompt: "consent" - }) - ); -}); - -dashboard.get("/authenticate", function (req, res) { - if (!req.query.code) { - return res.message( - req.baseUrl, - new Error("Please authorize Blot to access your Google Drive") - ); - } - - const oauth2Client = new google.auth.OAuth2( - config.google.drive.key, - config.google.drive.secret, - REDIRECT_URL - ); - - // This will provide an object with the access_token and refresh_token. - // Save these somewhere safe so they can be used at a later time. - oauth2Client.getToken(req.query.code, async function (err, credentials) { - if (err) { - return res.message(req.baseUrl, err); - } - - const { refresh_token, access_token, expiry_date } = credentials; - - if (!refresh_token) { - return res.message( - req.baseUrl, - new Error( - "Missing refresh_token from Google Drive. This probably happened because you had previously connected Blot with Google Drive. Please remove Blot on your Google Account permissions page." - ) - ); - } - - await database.setAccount(req.blog.id, { - refresh_token, - access_token, - expiry_date - }); - - try { - const { drive } = await createDriveClient(req.blog.id); - let email; - let permissionId; - const response = await drive.about.get({ fields: "*" }); - - email = response.data.user.emailAddress; - - // The user's ID as visible in the permissions collection. - // https://developers.google.com/drive/api/v2/reference/about#resource - // we use this to work out if another blog is connected - // to this google drive account during disconnection so we - // can determine whether or not to revoke the refresh_token - // which happens globally and would affect other blogs. We could - // use the email address but it seems like the ID is more robust - // since I suppose the user could change their email address... - permissionId = response.data.user.permissionId; - - // If we are re-authenticating because of an error - // then remove the error message! - await database.setAccount(req.blog.id, { - email, - permissionId, - error: "" - }); - } catch (e) { - console.log("Error getting info"); - await database.setAccount(req.blog.id, { error: e.message }); - return res.redirect(req.baseUrl); - } - - const account = await database.getAccount(req.blog.id); - res.message(req.baseUrl, "Connected to Google Drive"); - - try { - if (!account.folderId) { - await database.setAccount(req.blog.id, { preparing: true }); - await setUpBlogFolder(req.blog); - } else { - await resetFromBlot(req.blog.id); - await setupWebhook(req.blog.id); - } - } catch (e) { - await database.setAccount(req.blog.id, { - error: "Could not set up webhooks", - channel: null, - folderId: null, - folderPath: null - }); - } - }); -}); - -const setUpBlogFolder = async function (blog) { +const setUpBlogFolder = async function (blog, email) { let releaseLock; try { const checkWeCanContinue = async () => { @@ -202,27 +88,26 @@ const setUpBlogFolder = async function (blog) { const publish = folder.status; publish("Establishing connection to Google Drive"); - const { drive } = await createDriveClient(blog.id); + const drive = await createDriveClient(blog.id); - var fileMetadata = { - name: blog.title.split("/").join("").trim(), - mimeType: "application/vnd.google-apps.folder" - }; + // var fileMetadata = { + // name: blog.title.split("/").join("").trim(), + // mimeType: "application/vnd.google-apps.folder" + // }; await checkWeCanContinue(); - publish("Creating new folder"); - const blogFolder = await drive.files.create({ - resource: fileMetadata, - fields: "id, name" - }); + publish("Waiting for folder to be created"); - const folderId = blogFolder.data.id; - const folderPath = "/My Drive/" + blogFolder.data.name; + const {folderId, folderName} = await waitForSharedFolder(drive, email); + + // const blogFolder = await drive.files.create({ + // resource: fileMetadata, + // fields: "id, name" + // }); - await database.setAccount(blog.id, { - folderId: folderId, - folderPath: folderPath - }); + // const folderId = blogFolder.data.id; + + await database.setAccount(blog.id, { folderId, folderName }); await checkWeCanContinue(); publish("Ensuring new folder is in sync"); @@ -249,11 +134,37 @@ const setUpBlogFolder = async function (blog) { preparing: null, channel: null, folderId: null, - folderPath: null + folderName: null }); if (releaseLock) releaseLock(); } }; +/** + * List the contents of root folder. + */ +async function waitForSharedFolder(drive, email) { + try { + console.log('Listing root folder contents...' + email); + const res = await drive.files.list({ + supportsAllDrives: true, + includeItemsFromAllDrives: true, + q: `'${email}' in owners and trashed = false and mimeType = 'application/vnd.google-apps.folder'`, + }); + + if (res.data.files.length === 0) { + console.log('No shared folder found.... waiting 3 seconds and trying again ' + email); + await new Promise(resolve => setTimeout(resolve, 3000)); + return waitForSharedFolder(drive, email); + } + + const folderId = res.data.files[0].id; + console.log(`Shared folder found with ID: ${folderId}`); + return { folderId, folderName: res.data.files[0].name }; + } catch (error) { + console.error('Error listing folder contents:', error.message); + } + } + module.exports = dashboard; diff --git a/app/clients/google-drive/routes/site.js b/app/clients/google-drive/routes/site.js index 289045a5c00..2293ce1254d 100644 --- a/app/clients/google-drive/routes/site.js +++ b/app/clients/google-drive/routes/site.js @@ -3,64 +3,12 @@ const moment = require("moment"); const config = require("config"); const querystring = require("querystring"); const hash = require("helper/hash"); -const sync = require("../sync"); const clfdate = require("helper/clfdate"); -const database = require("../database"); const express = require("express"); const site = new express.Router(); -const cookieParser = require("cookie-parser"); -// Customers are sent back to: -// blot.im/clients/google-drive/authenticate -// when they have authorized (or declined to authorize) -// Blot's access to their folder. This is a public-facing -// route without access to the customer's session by default. -// We need to work out which blog they were -// authenticating based on a value stored in their session -// before they were sent out to Google Drive. Unfortunately we -// can't pass a blog username in the URL, since it needs to -// be the same URL every time, e.g. this would not work: -// blot.im/clients/google-drive/authenticate?handle=david - -// Additionally, in development mode, we are: -// first sent back to: -// tunnel.blot.im/clients/google-drive/authenticate -// are then redirected to: -// blot.development/clients/google-drive/authenticate -// and finally redirected to: -// blot.development/dashboard/*/client/google-drive/authenticate -site.get("/authenticate", cookieParser(), function (req, res) { - // This means we hit the public routes on Blot's site - if (req.cookies.blogToAuthenticate) { - const redirect = - "/sites/" + - req.cookies.blogToAuthenticate + - "/client/google-drive/authenticate?" + - querystring.stringify(req.query); - - res.clearCookie("blogToAuthenticate"); - res.send(` - - - - - - - -`); - // This means we hit the public routes on Blot's webhook - // forwarding host (e.g. tunnel.blot.im) we don't have access - // to the session info yet so we redirect to the public routes - // on Blot's site, which will be able to access the session. - } else { - const url = - config.protocol + - config.host + - "/clients/google-drive/authenticate?" + - querystring.stringify(req.query); - res.redirect(url); - } -}); +const sync = require("clients/google-drive/sync"); +const database = require("clients/google-drive/database"); site .route("/webhook") @@ -101,25 +49,14 @@ site // We can't call drive.stop on the stale channel since the // refresh_token likely changed, just let it expire instead. if (!account || !_.isEqual(channel, account.channel)) { - return res.send("OK"); + console.log(prefix(), blogID, "Stale channel, ignoring"); + return res.send("OK"); } res.send("OK"); - try { - await sync(blogID); - } catch (err) { - console.error(prefix(), blogID, "Error:", err); - // folder.log("Error:", err.message); - // try { - // await reset(blogID); - // const blog = await getBlog({ id: blogID }); - // await fix(blog); - // } catch (e) { - // folder.log("Error verifying folder:", e.message); - // return done(e, callback); - // } - } + console.log(prefix(), blogID, "Received webhook begin sync"); + sync(blogID); }); module.exports = site; diff --git a/app/clients/google-drive-2/share.js b/app/clients/google-drive/share.js similarity index 86% rename from app/clients/google-drive-2/share.js rename to app/clients/google-drive/share.js index 09ffba3bfe4..853baa815b5 100644 --- a/app/clients/google-drive-2/share.js +++ b/app/clients/google-drive/share.js @@ -79,12 +79,16 @@ async function main(userEmail) { // Authenticate with the service account const drive = await authenticateWithServiceAccount(); - // Create a subfolder in the "Sites" folder - const subFolderName = `UserFolder-${Date.now()}`; - const subFolderId = await createSubFolder(drive, SITES_FOLDER_ID, subFolderName); + // // Create a subfolder in the "Sites" folder + // const subFolderName = `UserFolder-${Date.now()}`; + // const subFolderId = await createSubFolder(drive, SITES_FOLDER_ID, subFolderName); - // Share the subfolder with the user - await shareFolderWithUser(drive, subFolderId, userEmail); + const res = await drive.drives.list(); + + console.log(res.data); + + // // Share the subfolder with the user + // await shareFolderWithUser(drive, subFolderId, userEmail); } catch (error) { console.error('Error:', error.message); } diff --git a/app/clients/google-drive/sync/index.js b/app/clients/google-drive/sync/index.js index 3bd578358a3..52c36500123 100644 --- a/app/clients/google-drive/sync/index.js +++ b/app/clients/google-drive/sync/index.js @@ -5,17 +5,29 @@ const database = require("../database"); const download = require("../util/download"); const { promisify } = require("util"); const createDriveClient = require("../util/createDriveClient"); -const determinePathToFolder = require("../util/determinePathToFolder"); const establishSyncLock = require("../util/establishSyncLock"); const getBlog = promisify(require("models/blog").get); const fix = promisify(require("sync/fix")); const reset = require("./reset-to-blot"); +// const determinePathToFolder = require("../util/determinePathToFolder"); + async function sync(blogID) { + + console.log('SYNCING', blogID); + const blog = await getBlog({ id: blogID }); + + console.log('Got blog', blog); + + console.log("waiting for lock"); const { done, folder } = await establishSyncLock(blogID); - const { drive, account } = await createDriveClient(blogID); + console.log("got lock"); + + + const drive = await createDriveClient(blogID); + const account = await database.getAccount(blogID); const { folderId, error } = account; if (error) { @@ -64,8 +76,8 @@ async function sync(blogID) { return done(); } else if (id === folderId) { folder.status("You moved your folder on Google Drive"); - const folderPath = await determinePathToFolder(drive, id); - await database.setAccount(blogID, { folderPath }); + // const folderPath = await determinePathToFolder(drive, id); + // await database.setAccount(blogID, { folderPath }); } else if ((trashed || movedOutsideFolder) && storedPathForId) { folder.status("Removing", storedPathForId); const removedPaths = await db.remove(id); diff --git a/app/clients/google-drive/sync/reset-from-blot.js b/app/clients/google-drive/sync/reset-from-blot.js index f960810533f..630bf595499 100644 --- a/app/clients/google-drive/sync/reset-from-blot.js +++ b/app/clients/google-drive/sync/reset-from-blot.js @@ -12,7 +12,8 @@ module.exports = async (blogID, publish) => { console.log(clfdate() + " Google Drive:", args.join(" ")); }; - const { drive, account } = await createDriveClient(blogID); + const drive = await createDriveClient(blogID); + const account = await database.getAccount(blogID); const { folderId } = account; const checkWeCanContinue = async () => { diff --git a/app/clients/google-drive/sync/reset-to-blot.js b/app/clients/google-drive/sync/reset-to-blot.js index 9545f804f46..cfb493081c7 100644 --- a/app/clients/google-drive/sync/reset-to-blot.js +++ b/app/clients/google-drive/sync/reset-to-blot.js @@ -13,7 +13,8 @@ module.exports = async (blogID, publish, update) => { console.log(clfdate() + " Google Drive:", args.join(" ")); }; - const { drive, account } = await createDriveClient(blogID); + const drive = await createDriveClient(blogID); + const account = await database.getAccount(blogID); const { reset, get, set, remove } = database.folder(account.folderId); // resets pageToken and folderState diff --git a/app/clients/google-drive/test.js b/app/clients/google-drive/test.js new file mode 100644 index 00000000000..5a0b46ae6d3 --- /dev/null +++ b/app/clients/google-drive/test.js @@ -0,0 +1,129 @@ +const { google } = require('googleapis'); +const fs = require('fs'); +const path = require('path'); +const config = require('config'); +const guid = require("helper/guid"); +const hash = require("helper/hash"); +const querystring = require("querystring"); + +/** + * Authenticate with Google Drive using the service account credentials. + */ +async function authenticateWithServiceAccount() { + + const credentials = { + "type": "service_account", + "project_id": process.env.BLOT_GOOGLEDRIVE_PROJECT_ID, + "private_key_id": process.env.BLOT_GOOGLEDRIVE_PRIVATE_KEY_ID, + "private_key": Buffer.from(process.env.BLOT_GOOGLEDRIVE_PRIVATE_KEY_BASE64, 'base64').toString(), + "client_email": process.env.BLOT_GOOGLEDRIVE_CLIENT_EMAIL, + "client_id": process.env.BLOT_GOOGLEDRIVE_CLIENT_ID, + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": `https://www.googleapis.com/robot/v1/metadata/x509/${encodeURIComponent(process.env.BLOT_GOOGLEDRIVE_CLIENT_EMAIL)}`, + "universe_domain": "googleapis.com" + }; + + const auth = new google.auth.GoogleAuth({ + credentials, + scopes: ['https://www.googleapis.com/auth/drive'], + }); + + return google.drive({ version: 'v3', auth }); +} + +/** + * List the contents of root folder. + */ +async function waitForSharedFolder(drive, email) { + try { + console.log('Listing root folder contents...' + email); + const res = await drive.files.list({ + supportsAllDrives: true, + includeItemsFromAllDrives: true, + q: `'${email}' in owners and trashed = false and mimeType = 'application/vnd.google-apps.folder'`, + }); + + if (res.data.files.length === 0) { + console.log('No shared folder found.... waiting 3 seconds and trying again ' + email); + await new Promise(resolve => setTimeout(resolve, 3000)); + return waitForSharedFolder(drive, email); + } + + const folderId = res.data.files[0].id; + console.log(`Shared folder found with ID: ${folderId}`); + return folderId; + } catch (error) { + console.error('Error listing folder contents:', error.message); + } +} + +/** + * Main function to execute the operations. + */ +async function main(email) { + if (!email) { + console.error('Please provide an email address to share the folder with.'); + return; + } + + try { + + + // Authenticate with the service account + const drive = await authenticateWithServiceAccount(); + + + // List the contents of the "Sites" folder + const sharedFolderID = await waitForSharedFolder(drive, email); + + console.log('Shared folder ID:', sharedFolderID); + + const { data: { startPageToken} } = await drive.changes.getStartPageToken({ + supportsAllDrives: true, + includeDeleted: true, + includeCorpusRemovals: true, + includeItemsFromAllDrives: true, + }); + + console.log('Start page token:', startPageToken); + + const listChanges = async (pageToken) => { + const {data} = await drive.changes.list({ + supportsAllDrives: true, + includeDeleted: true, + includeCorpusRemovals: true, + includeItemsFromAllDrives: true, + fields: [ + "nextPageToken", + "newStartPageToken", + "changes/file/id", + "changes/file/name", + "changes/file/mimeType", + "changes/file/trashed", + "changes/file/parents", + "changes/file/modifiedTime", + "changes/file/md5Checksum", + ].join(","), + pageToken + }); + + console.log('Changes:', data); + + setTimeout(() => listChanges(data.newStartPageToken || pageToken), 5000); + }; + + listChanges(startPageToken); + + // attempt to set up a webhook + + console.log('done!'); + + } catch (error) { + console.error('Error:', error.message); + } +} + +// Run the script +main(process.argv[2]); \ No newline at end of file diff --git a/app/clients/google-drive/tests/database.js b/app/clients/google-drive/tests/database.js deleted file mode 100644 index 8eddd0136ec..00000000000 --- a/app/clients/google-drive/tests/database.js +++ /dev/null @@ -1,200 +0,0 @@ -describe("google drive client: database", function () { - const database = require("../database"); - const redisKeys = require("util").promisify(require("helper/redisKeys")); - - global.test.blog(); - - beforeEach(function () { - this.db = database.folder(this.blog.id); - }); - - // afterEach(async function () { - // await this.folder.print(); - // }); - - it("can store and retrieve account information", async function () { - const blogId = "blog_" + Date.now().toString(); - await database.setAccount(blogId, { foo: "bar" }); - const account = await database.getAccount(blogId); - expect(account).toEqual({ foo: "bar" }); - }); - - it("can determine whether its safe to revoke credentials", async function () { - const permissionId = "permission_" + Date.now().toString(); - const secondPermissionId = "permission_" + (Date.now() + 1).toString(); - const blogId = "blog_" + Date.now().toString(); - const secondBlogId = "blog_" + (Date.now() + 1).toString(); - - expect(await database.canRevoke(permissionId)).toEqual(true); - await database.setAccount(blogId, { permissionId }); - expect(await database.canRevoke(permissionId)).toEqual(true); - await database.setAccount(secondBlogId, { permissionId }); - expect(await database.canRevoke(permissionId)).toEqual(false); - await database.setAccount(secondBlogId, { - permissionId: secondPermissionId, - }); - expect(await database.canRevoke(permissionId)).toEqual(true); - await database.dropAccount(secondBlogId); - expect(await database.canRevoke(permissionId)).toEqual(true); - }); - - it("deletes the permissionId keys when dropping an account", async function () { - const blogId = "blog_" + Date.now().toString(); - const permissionId = "permission_" + Date.now().toString(); - - await database.setAccount(blogId, { permissionId }); - await database.dropAccount(blogId); - expect(await redisKeys("*" + permissionId + "*")).toEqual([]); - }); - - it("changing the permissionId removes the corresponding key", async function () { - const blogId = "blog_" + Date.now().toString(); - const permissionId = "permission_" + Date.now().toString(); - const newPermissionId = "permission_" + (Date.now() + 1).toString(); - - await database.setAccount(blogId, { permissionId }); - await database.setAccount(blogId, { permissionId: newPermissionId }); - expect(await redisKeys("*" + permissionId + "*")).toEqual([]); - }); - - it("deletes the folder keys when dropping an account", async function () { - const blogId = "blog_" + Date.now().toString(); - const folderId = "folder_" + Date.now().toString(); - const fileId = "file_" + Date.now().toString(); - - await database.setAccount(blogId, { folderId }); - const { set } = database.folder(folderId); - await set(fileId, "/Hello.txt"); - await database.dropAccount(blogId); - expect(await redisKeys("*" + folderId + "*")).toEqual([]); - }); - - it("moves all files in a folder", async function () { - const { set, move, get } = this.db; - - await set("000", "/foo"); - await set("123", "/foo/bar.txt"); - await set("456", "/foo/too.txt"); - - await move("000", "/bar"); - - expect(await get("123")).toEqual("/bar/bar.txt"); - expect(await get("456")).toEqual("/bar/too.txt"); - }); - - it("del removes all children", async function () { - const { set, remove, get } = this.db; - - await set("000", "/foo"); - await set("123", "/foo/bar.txt"); - await set("456", "/foo/too.txt"); - await set("789", "/bar.txt"); - - await remove("000"); - - expect(await get("123")).toEqual(null); - expect(await get("456")).toEqual(null); - expect(await get("789")).toEqual("/bar.txt"); - }); - - it("del removes all children for root", async function () { - const { set, remove, get } = this.db; - - await set("000", "/"); - await set("123", "/foo.txt"); - await set("456", "/foo/too.txt"); - - await remove("000"); - - expect(await get("123")).toEqual(null); - expect(await get("456")).toEqual(null); - }); - - it("remove returns a list of dropped paths", async function () { - const { set, remove } = this.db; - - await set("0", "/"); - await set("1", "/bar.txt"); - await set("2", "/foo.txt"); - - await set("3", "/foo"); - await set("4", "/foo/too.txt"); - - expect((await remove("3")).sort()).toEqual(["/foo", "/foo/too.txt"]); - expect((await remove("0")).sort()).toEqual(["/", "/bar.txt", "/foo.txt"]); - }); - - it("move handles a single file", async function () { - const { set, move, get } = this.db; - await set("123", "/bar.txt"); - await move("123", "/baz.txt"); - expect(await get("123")).toEqual("/baz.txt"); - }); - - it("move returns a list of affected paths", async function () { - const { set, move } = this.db; - await set("1", "/bar.txt"); - await set("2", "/bar"); - await set("3", "/bar/foo.txt"); - - expect((await move("1", "/baz.txt")).sort()).toEqual([ - "/bar.txt", - "/baz.txt", - ]); - - expect((await move("2", "/baz")).sort()).toEqual([ - "/bar", - "/bar/foo.txt", - "/baz", - "/baz/foo.txt", - ]); - }); - - it("move wont clobber a similar file", async function () { - const { set, move, get } = this.db; - await set("123", "/bar (1).txt"); - await set("456", "/bar"); - await set("789", "/bar/foo.txt"); - - await move("456", "/foo"); - - expect(await get("123")).toEqual("/bar (1).txt"); - expect(await get("456")).toEqual("/foo"); - expect(await get("789")).toEqual("/foo/foo.txt"); - }); - - it("you can lookup an ID by path", async function () { - const { set, getByPath } = this.db; - await set("123", "/bar (1).txt"); - await set("789", "/bar/foo.txt"); - - expect(await getByPath("/bar (1).txt")).toEqual("123"); - expect(await getByPath("/bar/foo.txt")).toEqual("789"); - expect(await getByPath("/bar")).toEqual(null); - }); - - it("del wont clobber a similar file", async function () { - const { set, remove, get } = this.db; - await set("123", "/bar (1).txt"); - await set("456", "/bar"); - await set("789", "/bar/foo.txt"); - - await remove("456"); - - expect(await get("123")).toEqual("/bar (1).txt"); - expect(await get("456")).toEqual(null); - expect(await get("789")).toEqual(null); - }); - - it("move handles folder children", async function () { - const { set, move, get } = this.db; - await set("000", "/foo"); - await set("123", "/foo/bar.txt"); - - await move("000", "/bar"); - expect(await get("123")).toEqual("/bar/bar.txt"); - - await move("123", "/bar/foo.txt"); - expect(await get("123")).toEqual("/bar/foo.txt"); - }); -}); diff --git a/app/clients/google-drive/util/createDriveClient.js b/app/clients/google-drive/util/createDriveClient.js index 59f7fc0870a..a346e7bde52 100644 --- a/app/clients/google-drive/util/createDriveClient.js +++ b/app/clients/google-drive/util/createDriveClient.js @@ -1,21 +1,28 @@ -const config = require("config"); -const google = require("googleapis").google; -const database = require("../database"); -const debug = require("debug")("blot:clients:google-drive"); +const { google } = require('googleapis'); -module.exports = async function createDriveClient(blogID) { - debug("Blog", blogID, "creating Drive client"); - const account = await database.getAccount(blogID); - const oauth2Client = new google.auth.OAuth2( - config.google.drive.key, - config.google.drive.secret - ); - oauth2Client.setCredentials({ - refresh_token: account.refresh_token, - access_token: account.access_token, - expiry_date: account.expiry_date, - }); - const drive = google.drive({ version: "v3", auth: oauth2Client }); - debug("Blog", blogID, "created Drive client"); - return { drive, account }; -}; +/** + * Authenticate with Google Drive using the service account credentials. + */ +module.exports = async function authenticateWithServiceAccount() { + + const credentials = { + "type": "service_account", + "project_id": process.env.BLOT_GOOGLEDRIVE_PROJECT_ID, + "private_key_id": process.env.BLOT_GOOGLEDRIVE_PRIVATE_KEY_ID, + "private_key": Buffer.from(process.env.BLOT_GOOGLEDRIVE_PRIVATE_KEY_BASE64, 'base64').toString(), + "client_email": process.env.BLOT_GOOGLEDRIVE_CLIENT_EMAIL, + "client_id": process.env.BLOT_GOOGLEDRIVE_CLIENT_ID, + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": `https://www.googleapis.com/robot/v1/metadata/x509/${encodeURIComponent(process.env.BLOT_GOOGLEDRIVE_CLIENT_EMAIL)}`, + "universe_domain": "googleapis.com" + }; + + const auth = new google.auth.GoogleAuth({ + credentials, + scopes: ['https://www.googleapis.com/auth/drive'], + }); + + return google.drive({ version: 'v3', auth }); +} \ No newline at end of file diff --git a/app/clients/google-drive/util/determinePathToFolder.js b/app/clients/google-drive/util/determinePathToFolder.js deleted file mode 100644 index 0f938ac74da..00000000000 --- a/app/clients/google-drive/util/determinePathToFolder.js +++ /dev/null @@ -1,25 +0,0 @@ -const join = require("path").join; - -const determinePathToFolder = async (drive, folderId) => { - let { data } = await drive.files.get({ - fileId: folderId, - fields: "name, parents", - }); - - const path = [data.name]; - - while (data.parents && data.parents.length) { - data = ( - await drive.files.get({ - fileId: data.parents[0], - fields: "name, parents", - }) - ).data; - path.unshift(data.name); - } - - console.log("/" + join(path.join("/"))); - return "/" + join(path.join("/")); -}; - -module.exports = determinePathToFolder; diff --git a/app/clients/google-drive/util/setupWebhook.js b/app/clients/google-drive/util/setupWebhook.js index f6e96d7a4fc..5b35442e4ae 100644 --- a/app/clients/google-drive/util/setupWebhook.js +++ b/app/clients/google-drive/util/setupWebhook.js @@ -1,62 +1,73 @@ +const config = require('config'); const guid = require("helper/guid"); const hash = require("helper/hash"); const querystring = require("querystring"); -const database = require("../database"); -const config = require("config"); -const createDriveClient = require("./createDriveClient"); +const createDriveClient = require('./createDriveClient'); +const database = require('../database'); +const file = require('../../../dashboard/site/folder/file'); const debug = require("debug")("blot:clients:google-drive"); module.exports = async (blogID) => { - const { drive, account } = await createDriveClient(blogID); - - if (account.channel) { - try { - debug("attempting to stop listening to webhooks"); - await drive.channels.stop({ - requestBody: account.channel, - }); - debug("stop listening to webhooks successfully"); - } catch (e) { - debug("failed to stop webhooks but no big deal:", e.message); - debug("it will expire automatically"); - } - } - - const { - data: { startPageToken }, - } = await drive.changes.getStartPageToken({ - supportsAllDrives: true, - }); - const id = guid(); - const expectedSignatureInput = blogID + id + config.session.secret; - const expectedSignature = hash(expectedSignatureInput); - - const response = await drive.changes.watch({ - // Whether the user is acknowledging the risk of downloading known malware or other abusive files. - // The ID for the file in question. - supportsAllDrives: true, - includeDeleted: true, - includeCorpusRemovals: true, - includeItemsFromAllDrives: true, - pageToken: startPageToken, - // Request body metadata - requestBody: { - id: id, - resourceId: account.folderId, - type: "web_hook", - token: querystring.stringify({ - blogID: blogID, - signature: expectedSignature, - }), + + const drive = await createDriveClient(blogID); + const account = await database.getAccount(blogID); + + if (account.channel) { + try { + debug("attempting to stop listening to webhooks"); + await drive.channels.stop({ + requestBody: account.channel, + }); + debug("stop listening to webhooks successfully"); + } catch (e) { + debug("failed to stop webhooks but no big deal:", e.message); + debug("it will expire automatically"); + } + } + + const {folderId} = account; + + const id = guid(); + const expectedSignatureInput = blogID + id + config.session.secret; + const expectedSignature = hash(expectedSignatureInput); + + const { data: { startPageToken } } = await drive.changes.getStartPageToken({ + supportsAllDrives: true, + }); + + // attempt to set up a webhook + + console.log('setting up webhook...'); + + const payload = { + supportsAllDrives: true, + includeDeleted: true, + acknowledgeAbuse: true, + includeCorpusRemovals: true, + includeItemsFromAllDrives: true, + restrictToMyDrive: false, + fileId: folderId, + // Request body metadata + requestBody: { + id: id, + resourceId: folderId, + type: "web_hook", + token: querystring.stringify({ + blogID, + signature: expectedSignature, + }), kind: "api#channel", address: `https://${ - config.environment === "development" + config.environment === "development" ? config.webhooks.relay_host : config.host }/clients/google-drive/webhook`, - }, - }); + }, + }; + console.log('payload:', payload); + const response = await drive.files.watch(payload); + console.log('response:', response); + await database.folder(account.folderId).setPageToken(startPageToken); + await database.setAccount(blogID, { channel: response.data }); - await database.folder(account.folderId).setPageToken(startPageToken); - await database.setAccount(blogID, { channel: response.data }); -}; +} \ No newline at end of file diff --git a/app/clients/google-drive/views/index.html b/app/clients/google-drive/views/index.html index c2446de8844..661b71d514d 100644 --- a/app/clients/google-drive/views/index.html +++ b/app/clients/google-drive/views/index.html @@ -15,11 +15,9 @@
Location - - - {{#folderParents}} - {{name}} {{^last}}{{/last}} - {{/folderParents}} + {{#folderName}} + {{folderName}} + {{/folderName}} @@ -36,6 +34,15 @@
+ {{/folderId}} + + {{^folderId}} +
+ Location + Please create a new folder in your Google Drive and share the new folder with the following: + {{blot_googledrive_client_email}} + +
{{/folderId}}
@@ -57,16 +64,19 @@

Sync folder using Google Drive

-
+ + +
- Connect to Google Drive - + +
+
{{/account}} \ No newline at end of file diff --git a/app/clients/google-drive/write.js b/app/clients/google-drive/write.js index e4a12c20d1c..d9ec2aa3c49 100644 --- a/app/clients/google-drive/write.js +++ b/app/clients/google-drive/write.js @@ -49,7 +49,8 @@ module.exports = async function write(blogID, path, input, callback) { return callback(null); } - const { drive, account } = await createDriveClient(blogID); + const drive = await createDriveClient(blogID); + const account = await database.getAccount(blogID); if (account.folderId) { console.log(prefix(), "will save remote file"); diff --git a/app/clients/index.js b/app/clients/index.js index 7059e25ecbd..b4c29f2329a 100644 --- a/app/clients/index.js +++ b/app/clients/index.js @@ -20,7 +20,7 @@ if ( // If we have the require creds to run // the google drive app -if (config.google.drive.key && config.google.drive.secret) { +if (process.env.BLOT_GOOGLEDRIVE_PROJECT_ID) { clients['google-drive'] = require("./google-drive"); }