diff --git a/app/client/packages/rts/src/ctl/index.ts b/app/client/packages/rts/src/ctl/index.ts index b333d27cdc77..5cfbc9f5d291 100755 --- a/app/client/packages/rts/src/ctl/index.ts +++ b/app/client/packages/rts/src/ctl/index.ts @@ -9,6 +9,7 @@ import * as restore from "./restore"; import * as check_replica_set from "./check_replica_set"; import * as version from "./version"; import * as mongo_shell_utils from "./mongo_shell_utils"; +import moveToPostgres from "./move-to-postgres"; import { config } from "dotenv"; const APPLICATION_CONFIG_PATH = "/appsmith-stacks/configuration/docker.env"; @@ -60,6 +61,8 @@ if (["export-db", "export_db", "ex"].includes(command)) { version.exec(); } else if (["mongo-eval", "mongo_eval", "mongoEval"].includes(command)) { mongo_shell_utils.exec(); +} else if (command === "move-to-postgres") { + moveToPostgres(); } else { showHelp(); } diff --git a/app/client/packages/rts/src/ctl/move-to-postgres.mjs b/app/client/packages/rts/src/ctl/move-to-postgres.mjs deleted file mode 100644 index 48018e0a52e4..000000000000 --- a/app/client/packages/rts/src/ctl/move-to-postgres.mjs +++ /dev/null @@ -1,231 +0,0 @@ -/** - * Moves data from MongoDB to Postgres. - * - * @param {string} mongoDbUrl - The URL of the MongoDB. - * @param {string} mongoDumpFile - The path to the MongoDB dump file. - * @param {boolean} isBaselineMode - Flag indicating whether the script is running in baseline mode. - * @returns {Promise} - A promise that resolves when the data migration is complete. - */ -import { spawn } from "child_process"; -import { MongoClient } from "mongodb"; -import * as fs from "node:fs"; - -let isBaselineMode = false; - -// Don't use `localhost` here, it'll try to connect on IPv6, irrespective of whether you have it enabled or not. -let mongoDbUrl; - -let mongoDumpFile = null; -const EXPORT_ROOT = "/appsmith-stacks/mongo-data"; - -// The minimum version of the MongoDB changeset that must be present in the mongockChangeLog collection to run this script. -// This is to ensure we are migrating the data from the stable version of MongoDB. -const MINIMUM_MONGO_CHANGESET = "add_empty_policyMap_for_null_entries"; -const MONGO_MIGRATION_COLLECTION = "mongockChangeLog"; - -for (let i = 2; i < process.argv.length; ++i) { - const arg = process.argv[i]; - if (arg.startsWith("--mongodb-url=") && !mongoDbUrl) { - mongoDbUrl = extractValueFromArg(arg); - } else if (arg.startsWith("--mongodb-dump=") && !mongoDumpFile) { - mongoDumpFile = extractValueFromArg(arg); - } else if (arg === "--baseline") { - isBaselineMode = true; - console.warn( - "Running in baseline mode. If you're not an Appsmith team member, we sure hope you know what you're doing.", - ); - } else { - console.error("Unknown/unexpected argument: " + arg); - process.exit(1); - } -} - -if (!mongoDbUrl && !mongoDumpFile) { - console.error("No source specified"); - process.exit(1); -} - -let mongoServer; -if (mongoDumpFile) { - fs.mkdirSync("/tmp/db-tmp", { recursive: true }); - - mongoServer = spawn( - "mongod", - ["--bind_ip_all", "--dbpath", "/tmp/db-tmp", "--port", "27500"], - { - stdio: "inherit", - }, - ); - - mongoDbUrl = "mongodb://localhost/tmp"; - - // mongorestore 'mongodb://localhost/' --archive=mongodb-data.gz --gzip --nsFrom='appsmith.*' --nsTo='appsmith.*' - spawn("mongorestore", [ - mongoDbUrl, - "--archive=" + mongoDumpFile, - "--gzip", - "--noIndexRestore", - ]); -} - -const mongoClient = new MongoClient(mongoDbUrl); -mongoClient.on("error", console.error); -await mongoClient.connect(); -const mongoDb = mongoClient.db(); - -// Make sure EXPORT_ROOT directory is empty -fs.rmSync(EXPORT_ROOT, { recursive: true, force: true }); -fs.mkdirSync(EXPORT_ROOT, { recursive: true }); - -const filters = {}; - -if (isBaselineMode) { - filters.config = { - // Remove the "appsmith_registered" value, since this is baseline static data, and we want new instances to do register. - name: { $ne: "appsmith_registered" }, - }; - filters.plugin = { - // Remove saas plugins so they can be fetched from CS again, as usual. - packageName: { $ne: "saas-plugin" }, - }; -} - -const collectionNames = await mongoDb - .listCollections({}, { nameOnly: true }) - .toArray(); -const sortedCollectionNames = collectionNames - .map((collection) => collection.name) - .sort(); - -// Verify that the MongoDB data has been migrated to a stable version i.e. v1.43 before we start migrating the data to Postgres. -if (!(await isMongoDataMigratedToStableVersion(mongoDb))) { - console.error( - "MongoDB migration check failed: Try upgrading the Appsmith instance to latest before opting for data migration.", - ); - console.error( - `Could not find the valid migration execution entry for "${MINIMUM_MONGO_CHANGESET}" in the "${MONGO_MIGRATION_COLLECTION}" collection.`, - ); - await mongoClient.close(); - mongoServer?.kill(); - process.exit(1); -} - -for await (const collectionName of sortedCollectionNames) { - console.log("Collection:", collectionName); - if (isBaselineMode && collectionName.startsWith("mongock")) { - continue; - } - let outFile = null; - for await (const doc of mongoDb - .collection(collectionName) - .find(filters[collectionName])) { - // Skip archived objects as they are not migrated during the Mongock migration which may end up failing for the - // constraints in the Postgres DB. - if (isArchivedObject(doc)) { - continue; - } - transformFields(doc); // This now handles the _class to type transformation. - if (doc.policyMap == null) { - doc.policyMap = {}; - } - - if (outFile == null) { - // Don't create the file unless there's data to write. - outFile = fs.openSync(EXPORT_ROOT + "/" + collectionName + ".jsonl", "w"); - } - - fs.writeSync(outFile, toJsonSortedKeys(doc) + "\n"); - } - - if (outFile != null) { - fs.closeSync(outFile); - } -} - -await mongoClient.close(); -mongoServer?.kill(); - -console.log("done"); - -// TODO(Shri): We shouldn't need this. -process.exit(0); - -function extractValueFromArg(arg) { - return arg.replace(/^.*?=/, ""); -} - -function isArchivedObject(doc) { - return doc.deleted === true || doc.deletedAt != null; -} - -function toJsonSortedKeys(obj) { - // We want the keys sorted in the serialized JSON string, so that everytime we run this script, we don't see diffs - // that are just keys being reshuffled, which we don't care about, and don't need a diff for. - return JSON.stringify(obj, replacer); -} - -function replacer(key, value) { - // Ref: https://gist.github.com/davidfurlong/463a83a33b70a3b6618e97ec9679e490 - return value instanceof Object && !Array.isArray(value) - ? Object.keys(value) - .sort() - .reduce((sorted, key) => { - sorted[key] = value[key]; - return sorted; - }, {}) - : value; -} - -/** - * Method to transform the data in the object to be compatible with Postgres. - * Updates: - * 1. Changes the _id field to id, and removes the _id field. - * 2. Replaces the _class field with the appropriate type field. - * @param {Document} obj - The object to transform. - * @returns {void} - No return value. - */ -function transformFields(obj) { - for (const key in obj) { - if (key === "_id") { - obj.id = obj._id.toString(); - delete obj._id; - } else if (key === "_class") { - const type = mapClassToType(obj._class); - if (type) { - obj.type = type; // Add the type field - } - delete obj._class; // Remove the _class field - } else if (typeof obj[key] === "object" && obj[key] !== null) { - transformFields(obj[key]); - } - } -} - -/** - * Map the _class field to the appropriate type value. The DatasourceStorage class requires this check - * @param {string} _class - The _class field value. - * @returns {string|null} - The corresponding type value, or null if no match is found. - */ -function mapClassToType(_class) { - switch (_class) { - case "com.appsmith.external.models.DatasourceStructure$PrimaryKey": - return "primary key"; - case "com.appsmith.external.models.DatasourceStructure$ForeignKey": - return "foreign key"; - default: - return null; - } -} - -/** - * Method to check if MongoDB data has migrated to a stable version before we start migrating the data to Postgres. - * @param {*} mongoDb - The MongoDB client. - * @returns {Promise} - A promise that resolves to true if the data has been migrated to a stable version, false otherwise. - */ -async function isMongoDataMigratedToStableVersion(mongoDb) { - const doc = await mongoDb.collection(MONGO_MIGRATION_COLLECTION).findOne({ - changeId: MINIMUM_MONGO_CHANGESET, - state: "EXECUTED", - }); - return doc !== null; -} diff --git a/app/client/packages/rts/src/ctl/move-to-postgres.ts b/app/client/packages/rts/src/ctl/move-to-postgres.ts new file mode 100644 index 000000000000..d960268f7545 --- /dev/null +++ b/app/client/packages/rts/src/ctl/move-to-postgres.ts @@ -0,0 +1,236 @@ +/** + * Moves data from MongoDB to Postgres. + * + * @param {string} mongoDbUrl - The URL of the MongoDB. + * @param {string} mongoDumpFile - The path to the MongoDB dump file. + * @param {boolean} isBaselineMode - Flag indicating whether the script is running in baseline mode. + * @returns {Promise} - A promise that resolves when the data migration is complete. + */ +import { spawn } from "node:child_process"; +import type { ChildProcess } from "node:child_process"; +import { MongoClient } from "mongodb"; +import type { Db, Document } from "mongodb"; +import * as fs from "node:fs"; + +// The minimum version of the MongoDB changeset that must be present in the mongockChangeLog collection to run this script. +// This is to ensure we are migrating the data from the stable version of MongoDB. +const MINIMUM_MONGO_CHANGESET = "add_empty_policyMap_for_null_entries"; +const MONGO_MIGRATION_COLLECTION = "mongockChangeLog"; + +export default async function main() { + let isBaselineMode = false; + + // Don't use `localhost` here, it'll try to connect on IPv6, irrespective of whether you have it enabled or not. + let mongoDbUrl: string; + + let mongoDumpFile = null; + const EXPORT_ROOT = "/appsmith-stacks/mongo-data"; + + for (let i = 2; i < process.argv.length; ++i) { + const arg = process.argv[i]; + if (arg.startsWith("--mongodb-url=") && !mongoDbUrl) { + mongoDbUrl = extractValueFromArg(arg); + } else if (arg.startsWith("--mongodb-dump=") && !mongoDumpFile) { + mongoDumpFile = extractValueFromArg(arg); + } else if (arg === "--baseline") { + isBaselineMode = true; + console.warn( + "Running in baseline mode. If you're not an Appsmith team member, we sure hope you know what you're doing.", + ); + } else { + console.error("Unknown/unexpected argument: " + arg); + process.exit(1); + } + } + + if (!mongoDbUrl && !mongoDumpFile) { + console.error("No source specified"); + process.exit(1); + } + + let mongoServer: ChildProcess; + if (mongoDumpFile) { + fs.mkdirSync("/tmp/db-tmp", { recursive: true }); + + mongoServer = spawn( + "mongod", + ["--bind_ip_all", "--dbpath", "/tmp/db-tmp", "--port", "27500"], + { + stdio: "inherit", + }, + ); + + mongoDbUrl = "mongodb://localhost/tmp"; + + // mongorestore 'mongodb://localhost/' --archive=mongodb-data.gz --gzip --nsFrom='appsmith.*' --nsTo='appsmith.*' + spawn("mongorestore", [ + mongoDbUrl, + "--archive=" + mongoDumpFile, + "--gzip", + "--noIndexRestore", + ]); + } + + const mongoClient = new MongoClient(mongoDbUrl); + mongoClient.on("error", console.error); + await mongoClient.connect(); + const mongoDb: Db = mongoClient.db(); + + // Make sure EXPORT_ROOT directory is empty + fs.rmSync(EXPORT_ROOT, { recursive: true, force: true }); + fs.mkdirSync(EXPORT_ROOT, { recursive: true }); + + const filters: Record> = {}; + + if (isBaselineMode) { + filters.config = { + // Remove the "appsmith_registered" value, since this is baseline static data, and we want new instances to do register. + name: { $ne: "appsmith_registered" }, + }; + filters.plugin = { + // Remove saas plugins so they can be fetched from CS again, as usual. + packageName: { $ne: "saas-plugin" }, + }; + } + + const collectionNames = await mongoDb + .listCollections({}, { nameOnly: true }) + .toArray(); + const sortedCollectionNames = collectionNames + .map((collection) => collection.name) + .sort(); + + // Verify that the MongoDB data has been migrated to a stable version i.e. v1.43 before we start migrating the data to Postgres. + if (!(await isMongoDataMigratedToStableVersion(mongoDb))) { + console.error( + "MongoDB migration check failed: Try upgrading the Appsmith instance to latest before opting for data migration.", + ); + console.error( + `Could not find the valid migration execution entry for "${MINIMUM_MONGO_CHANGESET}" in the "${MONGO_MIGRATION_COLLECTION}" collection.`, + ); + await mongoClient.close(); + mongoServer?.kill(); + process.exit(1); + } + + for await (const collectionName of sortedCollectionNames) { + console.log("Collection:", collectionName); + if (isBaselineMode && collectionName.startsWith("mongock")) { + continue; + } + let outFile = null; + for await (const doc of mongoDb + .collection(collectionName) + .find(filters[collectionName])) { + // Skip archived objects as they are not migrated during the Mongock migration which may end up failing for the + // constraints in the Postgres DB. + if (isArchivedObject(doc)) { + continue; + } + transformFields(doc); // This now handles the _class to type transformation. + if (doc.policyMap == null) { + doc.policyMap = {}; + } + + if (outFile == null) { + // Don't create the file unless there's data to write. + outFile = fs.openSync( + EXPORT_ROOT + "/" + collectionName + ".jsonl", + "w", + ); + } + + fs.writeSync(outFile, toJsonSortedKeys(doc) + "\n"); + } + + if (outFile != null) { + fs.closeSync(outFile); + } + } + + await mongoClient.close(); + mongoServer?.kill(); + + console.log("done"); + + // TODO(Shri): We shouldn't need this. + process.exit(0); +} + +function extractValueFromArg(arg: string): string { + return arg.replace(/^.*?=/, ""); +} + +function isArchivedObject(doc: Document): boolean { + return doc.deleted === true || doc.deletedAt != null; +} + +function toJsonSortedKeys(obj: Document) { + // We want the keys sorted in the serialized JSON string, so that everytime we run this script, we don't see diffs + // that are just keys being reshuffled, which we don't care about, and don't need a diff for. + return JSON.stringify(obj, replacer); +} + +function replacer(_key: string, value: unknown) { + // Ref: https://gist.github.com/davidfurlong/463a83a33b70a3b6618e97ec9679e490 + return value instanceof Object && !Array.isArray(value) + ? Object.keys(value) + .sort() + .reduce((sorted, key) => { + sorted[key] = value[key]; + return sorted; + }, {}) + : value; +} + +/** + * Method to transform the data in the object to be compatible with Postgres. + * Updates: + * 1. Changes the _id field to id, and removes the _id field. + * 2. Replaces the _class field with the appropriate type field. + */ +function transformFields(obj: Document) { + for (const key in obj) { + if (key === "_id") { + obj.id = obj._id.toString(); + delete obj._id; + } else if (key === "_class") { + const type = mapClassToType(obj._class); + if (type) { + obj.type = type; // Add the type field + } + delete obj._class; // Remove the _class field + } else if (typeof obj[key] === "object" && obj[key] !== null) { + transformFields(obj[key]); + } + } +} + +/** + * Map the _class field to the appropriate type value. The DatasourceStorage class requires this check + * @param {string} _class - The _class field value. + * @returns {string|null} - The corresponding type value, or null if no match is found. + */ +function mapClassToType(_class: string): "primary key" | "foreign key" | null { + switch (_class) { + case "com.appsmith.external.models.DatasourceStructure$PrimaryKey": + return "primary key"; + case "com.appsmith.external.models.DatasourceStructure$ForeignKey": + return "foreign key"; + default: + return null; + } +} + +/** + * Method to check if MongoDB data has migrated to a stable version before we start migrating the data to Postgres. + * @param mongoDb - The MongoDB client. + * @returns - A promise that resolves to true if the data has been migrated to a stable version, false otherwise. + */ +async function isMongoDataMigratedToStableVersion(mongoDb: Db): Promise { + const doc = await mongoDb.collection(MONGO_MIGRATION_COLLECTION).findOne({ + changeId: MINIMUM_MONGO_CHANGESET, + state: "EXECUTED", + }); + return doc !== null; +}