diff --git a/app/client/packages/rts/src/ctl/export_db_pg.ts b/app/client/packages/rts/src/ctl/export_db_pg.ts new file mode 100644 index 000000000000..cfd4aaa24b83 --- /dev/null +++ b/app/client/packages/rts/src/ctl/export_db_pg.ts @@ -0,0 +1,43 @@ +import fsPromises from "fs/promises"; +import * as Constants from "./constants"; +import * as utils from "./utils"; +import { writeDataFromMongoToJsonlFiles } from './move-to-postgres.mjs'; + + +export async function exportDatabase() { + const dbUrl = utils.getDburl(); + try { + await writeDataFromMongoToJsonlFiles(dbUrl); + console.log('MongoDB data exported successfully.'); + } catch (error) { + console.error('Error exporting MongoDB data:', error); + } +} + +export async function run() { + let errorCode = 0; + + await utils.ensureSupervisorIsRunning(); + + try { + console.log("stop backend & rts application before export database"); + await utils.stop(["backend", "rts"]); + await exportDatabase(); + console.log("start backend & rts application after export database"); + console.log(); + console.log("\x1b[0;33m++++++++++++++++++++ NOTE ++++++++++++++++++++"); + console.log(); + console.log( + "Please remember to also copy APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file to the target instance where you intend to import this database dump.", + ); + console.log(); + console.log("++++++++++++++++++++++++++++++++++++++++++++++\x1b[0m"); + console.log(); + } catch (err) { + console.log(err); + errorCode = 1; + } finally { + await utils.start(["backend", "rts"]); + process.exit(errorCode); + } +} diff --git a/app/client/packages/rts/src/ctl/index.ts b/app/client/packages/rts/src/ctl/index.ts index b333d27cdc77..cacb4580d3a4 100755 --- a/app/client/packages/rts/src/ctl/index.ts +++ b/app/client/packages/rts/src/ctl/index.ts @@ -4,6 +4,7 @@ import process from "process"; import { showHelp } from "./utils"; import * as export_db from "./export_db"; import * as import_db from "./import_db"; +import * as export_db_pg from "./export_db_pg"; import * as backup from "./backup"; import * as restore from "./restore"; import * as check_replica_set from "./check_replica_set"; @@ -34,6 +35,10 @@ if (["export-db", "export_db", "ex"].includes(command)) { console.log("Exporting database"); export_db.run(); console.log("Export database done"); +} else if (["export-mongo-to-pg"].includes(command)) { + console.log("Exporting data dump for Postgres migration"); + export_db_pg.run(); + console.log("Exporting data dump for Postgres migration done"); } else if (["import-db", "import_db", "im"].includes(command)) { console.log("Importing database"); // Get Force option flag to run import DB immediately diff --git a/app/client/packages/rts/src/ctl/move-to-postgres.mjs b/app/client/packages/rts/src/ctl/move-to-postgres.mjs index 48018e0a52e4..cb03547fe18c 100644 --- a/app/client/packages/rts/src/ctl/move-to-postgres.mjs +++ b/app/client/packages/rts/src/ctl/move-to-postgres.mjs @@ -1,169 +1,180 @@ +import { spawn } from "child_process"; +import { MongoClient } from "mongodb"; +import * as fs from "node:fs"; + +const EXPORT_ROOT = "/appsmith-stacks/mongo-data"; +const MINIMUM_MONGO_CHANGESET = "add_empty_policyMap_for_null_entries"; +const MONGO_MIGRATION_COLLECTION = "mongockChangeLog"; + /** - * Moves data from MongoDB to Postgres. + * Moves data from MongoDB to JSONL files, with optional baseline mode filtering. + * + * This script connects to a MongoDB instance, optionally restores from a dump file, + * and exports data from all collections to JSONL files. In baseline mode, specific + * filters are applied to exclude certain data. * * @param {string} mongoDbUrl - The URL of the MongoDB. - * @param {string} mongoDumpFile - The path to the MongoDB dump file. - * @param {boolean} isBaselineMode - Flag indicating whether the script is running in baseline mode. * @returns {Promise} - A promise that resolves when the data migration is complete. */ -import { spawn } from "child_process"; -import { MongoClient } from "mongodb"; -import * as fs from "node:fs"; - -let isBaselineMode = false; +export async function writeDataFromMongoToJsonlFiles(mongoDbUrl) { + let isBaselineMode = false; + let mongoDumpFile = null; -// Don't use `localhost` here, it'll try to connect on IPv6, irrespective of whether you have it enabled or not. -let mongoDbUrl; + // Process command line arguments + for (let i = 2; i < process.argv.length; ++i) { + const arg = process.argv[i]; + if (arg.startsWith("--mongodb-url=") && !mongoDbUrl) { + mongoDbUrl = extractValueFromArg(arg); + } else if (arg.startsWith("--mongodb-dump=") && !mongoDumpFile) { + mongoDumpFile = extractValueFromArg(arg); + } else if (arg === "--baseline") { + isBaselineMode = true; + console.warn( + "Running in baseline mode. If you're not an Appsmith team member, we sure hope you know what you're doing.", + ); + } else { + console.error("Unknown/unexpected argument: " + arg); + process.exit(1); + } + } -let mongoDumpFile = null; -const EXPORT_ROOT = "/appsmith-stacks/mongo-data"; + if (!mongoDbUrl && !mongoDumpFile) { + console.error("No source specified"); + process.exit(1); + } -// The minimum version of the MongoDB changeset that must be present in the mongockChangeLog collection to run this script. -// This is to ensure we are migrating the data from the stable version of MongoDB. -const MINIMUM_MONGO_CHANGESET = "add_empty_policyMap_for_null_entries"; -const MONGO_MIGRATION_COLLECTION = "mongockChangeLog"; + let mongoServer; + if (mongoDumpFile) { + fs.mkdirSync("/tmp/db-tmp", { recursive: true }); -for (let i = 2; i < process.argv.length; ++i) { - const arg = process.argv[i]; - if (arg.startsWith("--mongodb-url=") && !mongoDbUrl) { - mongoDbUrl = extractValueFromArg(arg); - } else if (arg.startsWith("--mongodb-dump=") && !mongoDumpFile) { - mongoDumpFile = extractValueFromArg(arg); - } else if (arg === "--baseline") { - isBaselineMode = true; - console.warn( - "Running in baseline mode. If you're not an Appsmith team member, we sure hope you know what you're doing.", + mongoServer = spawn( + "mongod", + ["--bind_ip_all", "--dbpath", "/tmp/db-tmp", "--port", "27500"], + { + stdio: "inherit", + }, ); - } else { - console.error("Unknown/unexpected argument: " + arg); - process.exit(1); + + mongoDbUrl = "mongodb://localhost/tmp"; + + spawn("mongorestore", [ + mongoDbUrl, + "--archive=" + mongoDumpFile, + "--gzip", + "--noIndexRestore", + ]); } -} -if (!mongoDbUrl && !mongoDumpFile) { - console.error("No source specified"); - process.exit(1); -} + const mongoClient = new MongoClient(mongoDbUrl); + mongoClient.on("error", console.error); + await mongoClient.connect(); + const mongoDb = mongoClient.db(); -let mongoServer; -if (mongoDumpFile) { - fs.mkdirSync("/tmp/db-tmp", { recursive: true }); - - mongoServer = spawn( - "mongod", - ["--bind_ip_all", "--dbpath", "/tmp/db-tmp", "--port", "27500"], - { - stdio: "inherit", - }, - ); - - mongoDbUrl = "mongodb://localhost/tmp"; - - // mongorestore 'mongodb://localhost/' --archive=mongodb-data.gz --gzip --nsFrom='appsmith.*' --nsTo='appsmith.*' - spawn("mongorestore", [ - mongoDbUrl, - "--archive=" + mongoDumpFile, - "--gzip", - "--noIndexRestore", - ]); -} + fs.rmSync(EXPORT_ROOT, { recursive: true, force: true }); + fs.mkdirSync(EXPORT_ROOT, { recursive: true }); -const mongoClient = new MongoClient(mongoDbUrl); -mongoClient.on("error", console.error); -await mongoClient.connect(); -const mongoDb = mongoClient.db(); - -// Make sure EXPORT_ROOT directory is empty -fs.rmSync(EXPORT_ROOT, { recursive: true, force: true }); -fs.mkdirSync(EXPORT_ROOT, { recursive: true }); - -const filters = {}; - -if (isBaselineMode) { - filters.config = { - // Remove the "appsmith_registered" value, since this is baseline static data, and we want new instances to do register. - name: { $ne: "appsmith_registered" }, - }; - filters.plugin = { - // Remove saas plugins so they can be fetched from CS again, as usual. - packageName: { $ne: "saas-plugin" }, - }; -} + const filters = {}; -const collectionNames = await mongoDb - .listCollections({}, { nameOnly: true }) - .toArray(); -const sortedCollectionNames = collectionNames - .map((collection) => collection.name) - .sort(); - -// Verify that the MongoDB data has been migrated to a stable version i.e. v1.43 before we start migrating the data to Postgres. -if (!(await isMongoDataMigratedToStableVersion(mongoDb))) { - console.error( - "MongoDB migration check failed: Try upgrading the Appsmith instance to latest before opting for data migration.", - ); - console.error( - `Could not find the valid migration execution entry for "${MINIMUM_MONGO_CHANGESET}" in the "${MONGO_MIGRATION_COLLECTION}" collection.`, - ); - await mongoClient.close(); - mongoServer?.kill(); - process.exit(1); -} + if (isBaselineMode) { + filters.config = { + name: { $ne: "appsmith_registered" }, + }; + filters.plugin = { + packageName: { $ne: "saas-plugin" }, + }; + } -for await (const collectionName of sortedCollectionNames) { - console.log("Collection:", collectionName); - if (isBaselineMode && collectionName.startsWith("mongock")) { - continue; + const collectionNames = await mongoDb + .listCollections({}, { nameOnly: true }) + .toArray(); + const sortedCollectionNames = collectionNames + .map((collection) => collection.name) + .sort(); + + if (!(await isMongoDataMigratedToStableVersion(mongoDb))) { + console.error( + "MongoDB migration check failed: Try upgrading the Appsmith instance to latest before opting for data migration.", + ); + console.error( + `Could not find the valid migration execution entry for "${MINIMUM_MONGO_CHANGESET}" in the "${MONGO_MIGRATION_COLLECTION}" collection.`, + ); + await mongoClient.close(); + mongoServer?.kill(); + process.exit(1); } - let outFile = null; - for await (const doc of mongoDb - .collection(collectionName) - .find(filters[collectionName])) { - // Skip archived objects as they are not migrated during the Mongock migration which may end up failing for the - // constraints in the Postgres DB. - if (isArchivedObject(doc)) { + + for await (const collectionName of sortedCollectionNames) { + console.log("Collection:", collectionName); + if (isBaselineMode && collectionName.startsWith("mongock")) { continue; } - transformFields(doc); // This now handles the _class to type transformation. - if (doc.policyMap == null) { - doc.policyMap = {}; - } + let outFile = null; + for await (const doc of mongoDb + .collection(collectionName) + .find(filters[collectionName])) { + if (isArchivedObject(doc)) { + continue; + } + transformFields(doc); + if (doc.policyMap == null) { + doc.policyMap = {}; + } - if (outFile == null) { - // Don't create the file unless there's data to write. - outFile = fs.openSync(EXPORT_ROOT + "/" + collectionName + ".jsonl", "w"); - } + if (outFile == null) { + outFile = fs.openSync(EXPORT_ROOT + "/" + collectionName + ".jsonl", "w"); + } - fs.writeSync(outFile, toJsonSortedKeys(doc) + "\n"); - } + fs.writeSync(outFile, toJsonSortedKeys(doc) + "\n"); + } - if (outFile != null) { - fs.closeSync(outFile); + if (outFile != null) { + fs.closeSync(outFile); + } } -} -await mongoClient.close(); -mongoServer?.kill(); + await mongoClient.close(); + mongoServer?.kill(); -console.log("done"); + console.log("done"); -// TODO(Shri): We shouldn't need this. -process.exit(0); + process.exit(0); +} +/** + * Extracts value from command line argument + * @param {string} arg - Command line argument in format "--key=value" + * @returns {string} The extracted value + */ function extractValueFromArg(arg) { return arg.replace(/^.*?=/, ""); } +/** + * Checks if a document is marked as archived/deleted + * @param {Object} doc - MongoDB document + * @returns {boolean} True if document is archived/deleted + */ function isArchivedObject(doc) { return doc.deleted === true || doc.deletedAt != null; } +/** + * Converts object to JSON string with sorted keys + * @param {Object} obj - Object to stringify + * @returns {string} JSON string with sorted keys + */ function toJsonSortedKeys(obj) { // We want the keys sorted in the serialized JSON string, so that everytime we run this script, we don't see diffs // that are just keys being reshuffled, which we don't care about, and don't need a diff for. return JSON.stringify(obj, replacer); } +/** + * Replacer function for JSON.stringify that sorts object keys + * @param {string} key - The current key + * @param {any} value - The current value + * @returns {any} The processed value + */ function replacer(key, value) { // Ref: https://gist.github.com/davidfurlong/463a83a33b70a3b6618e97ec9679e490 return value instanceof Object && !Array.isArray(value) @@ -192,9 +203,9 @@ function transformFields(obj) { } else if (key === "_class") { const type = mapClassToType(obj._class); if (type) { - obj.type = type; // Add the type field + obj.type = type; } - delete obj._class; // Remove the _class field + delete obj._class; } else if (typeof obj[key] === "object" && obj[key] !== null) { transformFields(obj[key]); } @@ -229,3 +240,20 @@ async function isMongoDataMigratedToStableVersion(mongoDb) { }); return doc !== null; } + +// Parse command line arguments +const args = process.argv.slice(2); +let mongoUrl; + +for (const arg of args) { + if (arg.startsWith('--mongodb-url=')) { + mongoUrl = arg.split('=')[1]; + } +} + +if (!mongoUrl) { + console.error('Usage: node move-to-postgres.mjs --mongodb-url='); + process.exit(1); +} + +writeDataFromMongoToJsonlFiles(mongoUrl).catch(console.error);