Skip to content

ci: Fix large documents breaking upload to Algolia #6815

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sharat87 merged 2 commits into from
Aug 26, 2021
Merged

ci: Fix large documents breaking upload to Algolia #6815

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
38a9691
Fix large documents breaking upload to Algolia
sharat87 Aug 24, 2021
67b6e64
Load secrets from AWS SSM Parameter Store
sharat87 Aug 25, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 100 additions & 52 deletions app/client/gitbook-algolia-lambda.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,27 @@
var https = require("https");
const algoliasearch = require('algoliasearch');
const https = require("https");
const algoliasearch = require("algoliasearch");
const aws = require("aws-sdk");

const client = algoliasearch('AZ2Z9CJSJ0', 'e92300d4e8dbaf2cbaa9ebbbeb4e06e6');
const index = client.initIndex('test_appsmith');
const SSM = new aws.SSM();

const DOCS_VERSION = "v1.2.1";

const orderArr = [{
path: "master/core-concepts/building-the-ui",
order: 0,
}, {
path: "master/core-concepts/connecting-to-databases",
order: 1
}, {
path: "master/core-concepts/apis",
order: 2
}, {
path: "master/core-concepts/connecting-ui-and-logic",
order: 3
}, {
path: "master/core-concepts/building-the-ui/calling-apis-from-widgets#sending-data-to-apis-queries",
order: 4
}];

var options = {
headers: {
Expand All @@ -17,7 +36,9 @@ console.log("Loading function");

function getPage(pageId) {
return new Promise((resolve, reject) => {
https.get(`https://api-beta.gitbook.com/v1/spaces/-Lzuzdhj8LjrQPaeyCxr/content/v/master/id/${pageId}?format=markdown`, options, (res) => {
const url = `https://api-beta.gitbook.com/v1/spaces/-Lzuzdhj8LjrQPaeyCxr/content/v/${DOCS_VERSION}/id/${pageId}?format=markdown`;
console.log("Getting URL", url);
https.get(url, options, (res) => {
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => { rawData += chunk; });
Expand All @@ -26,117 +47,144 @@ function getPage(pageId) {
const parsedData = JSON.parse(rawData);
resolve(parsedData);
} catch (e) {
reject(e)
console.error(e.message);
console.error("Full response body:", rawData);
reject(e);
}
});
})
});
});
}

const pages = []
const pages = [];


function pushChildPages(masterPage) {
if (masterPage.pages) {
masterPage.pages.forEach(page => {
page.path = masterPage.path + "/" + page.path;
page.path = (masterPage.path || masterPage.ref) + "/" + page.path;
pushChildPages(page);
page.pages = undefined;
pages.push(page);
});
}
}

const orderArr = [{
path: "master/quick-start",
order: 0,
}, {
path: "master/core-concepts/building-the-ui",
order: 1
}, {
path: "master/core-concepts/building-the-ui/displaying-api-data",
order: 2
}, {
path: "master/core-concepts/apis",
order: 3
}, {
path: "master/core-concepts/apis/taking-inputs-from-widgets",
order: 4
}]

function swap(arr, index1, index2) {
let x = arr[index1]
arr[index1] = arr[index2]
arr[index2] = x;
let x = arr[index1];
arr[index1] = arr[index2];
arr[index2] = x;
}

exports.handler = async (event) => {
const response = await new Promise((resolve, reject) => {
const req = https.get("https://api-beta.gitbook.com/v1/spaces/-Lzuzdhj8LjrQPaeyCxr/content", options, (res) => {
exports.handler = async (event, context, callback) => {
const parameters = await loadParametersFromStore("/" + process.env.ENV + "/algolia");
console.log('Received event:', JSON.stringify(event, null, 2));

const client = algoliasearch(parameters.application_id, parameters.api_key);
const algoliaIndex = client.initIndex("test_appsmith");

return await new Promise((resolve, reject) => {
https.get("https://api-beta.gitbook.com/v1/spaces/-Lzuzdhj8LjrQPaeyCxr/content", options, (res) => {
console.log("Setting up response handlers for GitBook API request");
res.setEncoding('utf8');
let rawData = '';
res.on('data', (chunk) => { rawData += chunk; });
res.on('end', () => {
try {
const parsedData = JSON.parse(rawData);
let masterPage = parsedData.variants[0].page;
let requiredIndex = parsedData.variants.findIndex(varaint => varaint.uid === DOCS_VERSION);
let masterPage = parsedData.variants[requiredIndex].page;

pushChildPages(masterPage);
masterPage.pages = undefined;
delete masterPage.pages;

pages.push(masterPage);

let promises = pages.map(page => page.uid).map(getPage);

Promise.all(promises).then(updatedPages => {

updatedPages.forEach((page, index) => {
page.path = pages[index].path;
page.pages = undefined;
delete page.pages;
page.objectID = page.uid;
if (page.path === "master/changelog") {
page.document = undefined
delete page.uid;
if(page.path.endsWith("/changelog")) {
delete page.document;
}
});

orderArr.forEach(order => {
let index = updatedPages.findIndex(i => i.path === order.path)
if (index !== -1) {
swap(updatedPages, index, order.order)
}
})
let index = updatedPages.findIndex(i => i.path === order.path);
if(index !== -1) {
swap(updatedPages, index, order.order);
}
});

updatedPages = updatedPages.map((item, index) => {return {...item, defaultOrder: index}});

updatedPages = updatedPages.map((item, index) => { return { ...item, defaultOrder: index } })
// Truncate large docs.
updatedPages.filter(page => page.document).forEach(page => {
const size = JSON.stringify(page).length;
if (size < 10000) {
return;
}
console.log("Truncating page", page);
page.document = page.document.substr(0, page.document.length - (JSON.stringify(page).length - 9900));
});//*/

console.log("Pages:", updatedPages.map(page => ({objectID: page.objectID, size: JSON.stringify(page).length, title: page.title, path: page.path})));

// resolve({
// statusCode: 200,
// body: JSON.stringify(updatedPages)
// })

index.replaceAllObjects(updatedPages, {
algoliaIndex.replaceAllObjects(updatedPages, {
autoGenerateObjectIDIfNotExist: true
}).then(({ objectIDs }) => {
console.log(objectIDs);
console.log("Algolia upload finished", objectIDs);
callback(null, 'Finished');
resolve({
statusCode: 200,
body: JSON.stringify(updatedPages)
})
});
}).catch(e => {
console.error("Algolia upload failed", e);
reject({
statusCode: 500,
body: 'Algolia upload failed.'
})
})
});
});
});
} catch (e) {
reject({
statusCode: 500,
body: 'Most probably gitbook getPage apis failed'
})
statusCode: 500,
body: 'Most probably gitbook getPage apis failed'
});
}
});
}).on("error", (e) => {
console.error(e);
reject({
statusCode: 500,
body: "Error executing GitBook API request",
error: e
});
});
});

return response;
};

async function loadParametersFromStore(prefix) {
const parametersResponse = await SSM.getParametersByPath({Path: prefix, WithDecryption: true}).promise();
console.log("parametersResponse", parametersResponse);

const parameters = {};
for (const paramObject of parametersResponse.Parameters) {
parameters[paramObject.Name.replace(prefix + "/", "")] = paramObject.Value;
}

console.log("Parameters", parameters);
return parameters;
}