diff --git a/app/client/gitbook-algolia-lambda.js b/app/client/gitbook-algolia-lambda.js index 993644ee0982..b64502aa345a 100644 --- a/app/client/gitbook-algolia-lambda.js +++ b/app/client/gitbook-algolia-lambda.js @@ -1,8 +1,27 @@ -var https = require("https"); -const algoliasearch = require('algoliasearch'); +const https = require("https"); +const algoliasearch = require("algoliasearch"); +const aws = require("aws-sdk"); -const client = algoliasearch('AZ2Z9CJSJ0', 'e92300d4e8dbaf2cbaa9ebbbeb4e06e6'); -const index = client.initIndex('test_appsmith'); +const SSM = new aws.SSM(); + +const DOCS_VERSION = "v1.2.1"; + +const orderArr = [{ + path: "master/core-concepts/building-the-ui", + order: 0, +}, { + path: "master/core-concepts/connecting-to-databases", + order: 1 +}, { + path: "master/core-concepts/apis", + order: 2 +}, { + path: "master/core-concepts/connecting-ui-and-logic", + order: 3 +}, { + path: "master/core-concepts/building-the-ui/calling-apis-from-widgets#sending-data-to-apis-queries", + order: 4 +}]; var options = { headers: { @@ -17,7 +36,9 @@ console.log("Loading function"); function getPage(pageId) { return new Promise((resolve, reject) => { - https.get(`https://api-beta.gitbook.com/v1/spaces/-Lzuzdhj8LjrQPaeyCxr/content/v/master/id/${pageId}?format=markdown`, options, (res) => { + const url = `https://api-beta.gitbook.com/v1/spaces/-Lzuzdhj8LjrQPaeyCxr/content/v/${DOCS_VERSION}/id/${pageId}?format=markdown`; + console.log("Getting URL", url); + https.get(url, options, (res) => { res.setEncoding('utf8'); let rawData = ''; res.on('data', (chunk) => { rawData += chunk; }); @@ -26,21 +47,22 @@ function getPage(pageId) { const parsedData = JSON.parse(rawData); resolve(parsedData); } catch (e) { - reject(e) console.error(e.message); + console.error("Full response body:", rawData); + reject(e); } }); - }) + }); }); } -const pages = [] +const pages = []; function pushChildPages(masterPage) { if (masterPage.pages) { masterPage.pages.forEach(page => { - page.path = masterPage.path + "/" + page.path; + page.path = (masterPage.path || masterPage.ref) + "/" + page.path; pushChildPages(page); page.pages = undefined; pages.push(page); @@ -48,95 +70,121 @@ function pushChildPages(masterPage) { } } -const orderArr = [{ - path: "master/quick-start", - order: 0, -}, { - path: "master/core-concepts/building-the-ui", - order: 1 -}, { - path: "master/core-concepts/building-the-ui/displaying-api-data", - order: 2 -}, { - path: "master/core-concepts/apis", - order: 3 -}, { - path: "master/core-concepts/apis/taking-inputs-from-widgets", - order: 4 -}] - function swap(arr, index1, index2) { - let x = arr[index1] - arr[index1] = arr[index2] - arr[index2] = x; + let x = arr[index1]; + arr[index1] = arr[index2]; + arr[index2] = x; } -exports.handler = async (event) => { - const response = await new Promise((resolve, reject) => { - const req = https.get("https://api-beta.gitbook.com/v1/spaces/-Lzuzdhj8LjrQPaeyCxr/content", options, (res) => { +exports.handler = async (event, context, callback) => { + const parameters = await loadParametersFromStore("/" + process.env.ENV + "/algolia"); + console.log('Received event:', JSON.stringify(event, null, 2)); + + const client = algoliasearch(parameters.application_id, parameters.api_key); + const algoliaIndex = client.initIndex("test_appsmith"); + + return await new Promise((resolve, reject) => { + https.get("https://api-beta.gitbook.com/v1/spaces/-Lzuzdhj8LjrQPaeyCxr/content", options, (res) => { + console.log("Setting up response handlers for GitBook API request"); res.setEncoding('utf8'); let rawData = ''; res.on('data', (chunk) => { rawData += chunk; }); res.on('end', () => { try { const parsedData = JSON.parse(rawData); - let masterPage = parsedData.variants[0].page; + let requiredIndex = parsedData.variants.findIndex(varaint => varaint.uid === DOCS_VERSION); + let masterPage = parsedData.variants[requiredIndex].page; pushChildPages(masterPage); - masterPage.pages = undefined; + delete masterPage.pages; pages.push(masterPage); let promises = pages.map(page => page.uid).map(getPage); + Promise.all(promises).then(updatedPages => { + updatedPages.forEach((page, index) => { page.path = pages[index].path; - page.pages = undefined; + delete page.pages; page.objectID = page.uid; - if (page.path === "master/changelog") { - page.document = undefined + delete page.uid; + if(page.path.endsWith("/changelog")) { + delete page.document; } }); orderArr.forEach(order => { - let index = updatedPages.findIndex(i => i.path === order.path) - if (index !== -1) { - swap(updatedPages, index, order.order) - } - }) + let index = updatedPages.findIndex(i => i.path === order.path); + if(index !== -1) { + swap(updatedPages, index, order.order); + } + }); + + updatedPages = updatedPages.map((item, index) => {return {...item, defaultOrder: index}}); - updatedPages = updatedPages.map((item, index) => { return { ...item, defaultOrder: index } }) + // Truncate large docs. + updatedPages.filter(page => page.document).forEach(page => { + const size = JSON.stringify(page).length; + if (size < 10000) { + return; + } + console.log("Truncating page", page); + page.document = page.document.substr(0, page.document.length - (JSON.stringify(page).length - 9900)); + });//*/ + console.log("Pages:", updatedPages.map(page => ({objectID: page.objectID, size: JSON.stringify(page).length, title: page.title, path: page.path}))); // resolve({ // statusCode: 200, // body: JSON.stringify(updatedPages) // }) - index.replaceAllObjects(updatedPages, { + algoliaIndex.replaceAllObjects(updatedPages, { autoGenerateObjectIDIfNotExist: true }).then(({ objectIDs }) => { - console.log(objectIDs); + console.log("Algolia upload finished", objectIDs); + callback(null, 'Finished'); resolve({ statusCode: 200, body: JSON.stringify(updatedPages) - }) + }); }).catch(e => { + console.error("Algolia upload failed", e); reject({ statusCode: 500, body: 'Algolia upload failed.' - }) - }) + }); + }); }); } catch (e) { reject({ - statusCode: 500, - body: 'Most probably gitbook getPage apis failed' - }) + statusCode: 500, + body: 'Most probably gitbook getPage apis failed' + }); } }); + }).on("error", (e) => { + console.error(e); + reject({ + statusCode: 500, + body: "Error executing GitBook API request", + error: e + }); }); }); - return response; }; + +async function loadParametersFromStore(prefix) { + const parametersResponse = await SSM.getParametersByPath({Path: prefix, WithDecryption: true}).promise(); + console.log("parametersResponse", parametersResponse); + + const parameters = {}; + for (const paramObject of parametersResponse.Parameters) { + parameters[paramObject.Name.replace(prefix + "/", "")] = paramObject.Value; + } + + console.log("Parameters", parameters); + return parameters; +}