From 2e42516b49b63642e3226b96ffa28c387c90cfa6 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 24 Oct 2024 14:47:56 +0530 Subject: [PATCH 1/4] chore: Add mongo data migration check before initializing the postgres dump --- .../appsmith/utils/bin/move-to-postgres.mjs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs index efdae3404f59..f73f84fef153 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs +++ b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs @@ -18,6 +18,11 @@ let mongoDbUrl; let mongoDumpFile = null; const EXPORT_ROOT = "/appsmith-stacks/mongo-data"; +// The minimum version of the MongoDB changeset that must be present in the mongockChangeLog collection to run this script. +// This is to ensure we are migrating the data from the stable version of MongoDB. +const MINIMUM_MONGO_CHANGESET = "add_empty_policyMap_for_null_entries"; +const MONGO_MIGRATION_COLLECTION = "mongockChangeLog"; + for (let i = 2; i < process.argv.length; ++i) { const arg = process.argv[i]; if (arg.startsWith("--mongodb-url=") && !mongoDbUrl) { @@ -77,6 +82,14 @@ if (isBaselineMode) { const collectionNames = await mongoDb.listCollections({}, { nameOnly: true }).toArray(); const sortedCollectionNames = collectionNames.map(collection => collection.name).sort(); +// Verify that the MongoDB data has been migrated to a stable version i.e. v1.43 before we start migrating the data to Postgres. +if (!await isMongoDataMigratedToStableVersion(mongoDb)) { + console.log("Please upgrade the MongoDB data to the latest version before running this script."); + await mongoClient.close(); + mongoServer?.kill(); + process.exit(0); +} + for await (const collectionName of sortedCollectionNames) { console.log("Collection:", collectionName); @@ -183,3 +196,20 @@ function mapClassToType(_class) { return null; } } + +/** + * Method to check if MongoDB data has migrated to a stable version before we start migrating the data to Postgres. + * @param {*} mongoDb - The MongoDB client. + * @returns {Promise} - A promise that resolves to true if the data has been migrated to a stable version, false otherwise. + */ +async function isMongoDataMigratedToStableVersion(mongoDb) { + const migrationFilters = {}; + migrationFilters[MONGO_MIGRATION_COLLECTION] = { + changeId: MINIMUM_MONGO_CHANGESET + }; + let shouldMigrate = false; + for await (const doc of mongoDb.collection(MONGO_MIGRATION_COLLECTION).find(migrationFilters[MONGO_MIGRATION_COLLECTION])) { + shouldMigrate = true; + } + return shouldMigrate; +} From 69355629573b12180fc9085f82ad3065bbea3718 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 24 Oct 2024 15:27:10 +0530 Subject: [PATCH 2/4] chore: Review comments --- .../appsmith/utils/bin/move-to-postgres.mjs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs index f73f84fef153..6ef27d8f59d5 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs +++ b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs @@ -16,7 +16,7 @@ let isBaselineMode = false; let mongoDbUrl; let mongoDumpFile = null; -const EXPORT_ROOT = "/appsmith-stacks/mongo-data"; +const EXPORT_ROOT = "appsmith-stacks/mongo-data"; // The minimum version of the MongoDB changeset that must be present in the mongockChangeLog collection to run this script. // This is to ensure we are migrating the data from the stable version of MongoDB. @@ -84,10 +84,11 @@ const sortedCollectionNames = collectionNames.map(collection => collection.name) // Verify that the MongoDB data has been migrated to a stable version i.e. v1.43 before we start migrating the data to Postgres. if (!await isMongoDataMigratedToStableVersion(mongoDb)) { - console.log("Please upgrade the MongoDB data to the latest version before running this script."); + console.error("MongoDB migration check failed: Try upgrading the Appsmith instance to latest before opting for data migration."); + console.error(`Could not find the valid migration execution entry for "${MINIMUM_MONGO_CHANGESET}" in the "${MONGO_MIGRATION_COLLECTION}" collection.`); await mongoClient.close(); mongoServer?.kill(); - process.exit(0); + process.exit(1); } for await (const collectionName of sortedCollectionNames) { @@ -203,13 +204,12 @@ function mapClassToType(_class) { * @returns {Promise} - A promise that resolves to true if the data has been migrated to a stable version, false otherwise. */ async function isMongoDataMigratedToStableVersion(mongoDb) { - const migrationFilters = {}; - migrationFilters[MONGO_MIGRATION_COLLECTION] = { - changeId: MINIMUM_MONGO_CHANGESET - }; let shouldMigrate = false; - for await (const doc of mongoDb.collection(MONGO_MIGRATION_COLLECTION).find(migrationFilters[MONGO_MIGRATION_COLLECTION])) { - shouldMigrate = true; + for await (const doc of mongoDb.collection(MONGO_MIGRATION_COLLECTION).find({ changeId: MINIMUM_MONGO_CHANGESET })) { + if (doc.state === 'EXECUTED') { + shouldMigrate = true; + break; + } } return shouldMigrate; } From e98a4e189192130953e051b0c95c6c4c56b773e0 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 24 Oct 2024 15:38:37 +0530 Subject: [PATCH 3/4] chore: Refactor --- .../opt/appsmith/utils/bin/move-to-postgres.mjs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs index 6ef27d8f59d5..976b18f9c6c2 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs +++ b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs @@ -16,7 +16,7 @@ let isBaselineMode = false; let mongoDbUrl; let mongoDumpFile = null; -const EXPORT_ROOT = "appsmith-stacks/mongo-data"; +const EXPORT_ROOT = "/appsmith-stacks/mongo-data"; // The minimum version of the MongoDB changeset that must be present in the mongockChangeLog collection to run this script. // This is to ensure we are migrating the data from the stable version of MongoDB. @@ -204,12 +204,10 @@ function mapClassToType(_class) { * @returns {Promise} - A promise that resolves to true if the data has been migrated to a stable version, false otherwise. */ async function isMongoDataMigratedToStableVersion(mongoDb) { - let shouldMigrate = false; - for await (const doc of mongoDb.collection(MONGO_MIGRATION_COLLECTION).find({ changeId: MINIMUM_MONGO_CHANGESET })) { - if (doc.state === 'EXECUTED') { - shouldMigrate = true; - break; - } - } - return shouldMigrate; + const doc = await mongoDb.collection(MONGO_MIGRATION_COLLECTION) + .findOne({ + changeId: MINIMUM_MONGO_CHANGESET, + state: 'EXECUTED' + }); + return doc !== null; } From df02162e5be8e39825ac59bf0ba7ea35f42b5571 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 24 Oct 2024 15:40:07 +0530 Subject: [PATCH 4/4] chore: Refactor --- .../docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs index 976b18f9c6c2..a8cd9c65dcda 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs +++ b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs @@ -204,10 +204,9 @@ function mapClassToType(_class) { * @returns {Promise} - A promise that resolves to true if the data has been migrated to a stable version, false otherwise. */ async function isMongoDataMigratedToStableVersion(mongoDb) { - const doc = await mongoDb.collection(MONGO_MIGRATION_COLLECTION) - .findOne({ - changeId: MINIMUM_MONGO_CHANGESET, - state: 'EXECUTED' + const doc = await mongoDb.collection(MONGO_MIGRATION_COLLECTION).findOne({ + changeId: MINIMUM_MONGO_CHANGESET, + state: "EXECUTED", }); return doc !== null; }