From 09063727dbe70f86c886f49947ef9f589398d423 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 12 Nov 2024 00:03:17 +0530 Subject: [PATCH 01/20] chore: Introduce backup and restore for postgres (cherry picked from commit 588e792fd625d87818a12ac6332e571ae8d05126) --- .../fs/opt/appsmith/utils/bin/backup.js | 14 +++++++-- .../fs/opt/appsmith/utils/bin/backup.test.js | 18 ++++++++--- .../fs/opt/appsmith/utils/bin/export_db.js | 15 +++++++++- .../fs/opt/appsmith/utils/bin/import_db.js | 30 ++++++++++++++++--- .../fs/opt/appsmith/utils/bin/restore.js | 24 +++++++++++++-- .../docker/fs/opt/appsmith/utils/bin/utils.js | 4 +-- 6 files changed, 90 insertions(+), 15 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index a2ef2b85578f..50fccf614348 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -125,7 +125,12 @@ function getEncryptionPasswordFromUser(){ async function exportDatabase(destFolder) { console.log('Exporting database'); - await executeMongoDumpCMD(destFolder, utils.getDburl()) + // Check the DB url + if (utils.getDburl().startsWith('mongodb')) { + await executeMongoDumpCMD(destFolder, utils.getDburl()) + } else if (utils.getDburl().startsWith('postgresql')) { + await executePostgresDumpCMD(destFolder, utils.getDburl()); + } console.log('Exporting database done.'); } @@ -141,7 +146,7 @@ async function createGitStorageArchive(destFolder) { async function createManifestFile(path) { const version = await utils.getCurrentAppsmithVersion() - const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(utils.getDburl()) } + const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromDBURI(utils.getDburl()) } await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data)); } @@ -161,6 +166,10 @@ async function executeMongoDumpCMD(destFolder, appsmithMongoURI) { return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd } +async function executePostgresDumpCMD(destFolder, appsmithDBURI) { + return await utils.execCommand(['pg_dump', appsmithDBURI, '-Fc', '-f', destFolder + '/pg-data.archive']); +} + async function createFinalArchive(destFolder, timestamp) { console.log('Creating final archive'); @@ -252,6 +261,7 @@ module.exports = { generateBackupRootPath, getBackupContentsPath, executeMongoDumpCMD, + executePostgresDumpCMD, getGitRoot, executeCopyCMD, removeSensitiveEnvData, diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js index 335dfd71a427..adb6700de357 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js @@ -56,6 +56,16 @@ test('Test mongodump CMD generaton', async () => { console.log(res) }) +test('Test postgresdump CMD generaton', async () => { + var dest = '/dest' + var appsmithMongoURI = 'postgresql://username:password@host/appsmith' + var cmd = 'pg_dump postgresql://username:password@host/appsmith -Fc -f /dest/pg-data.archive' + utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(' ')); + const res = await backup.executePostgresDumpCMD(dest, appsmithMongoURI) + expect(res).toBe(cmd) + console.log(res) +}) + test('Test get gitRoot path when APPSMITH_GIT_ROOT is \'\' ', () => { expect(backup.getGitRoot('')).toBe('/appsmith-stacks/git-storage') }); @@ -229,28 +239,28 @@ test('Test backup encryption function', async () => { test('Get DB name from Mongo URI 1', async () => { var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" var expectedDBName = 'my_db_name' - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri) + const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri) expect(dbName).toEqual(expectedDBName) }) test('Get DB name from Mongo URI 2', async () => { var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" var expectedDBName = 'test123' - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri) + const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri) expect(dbName).toEqual(expectedDBName) }) test('Get DB name from Mongo URI 3', async () => { var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123" var expectedDBName = 'test123' - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri) + const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri) expect(dbName).toEqual(expectedDBName) }) test('Get DB name from Mongo URI 4', async () => { var mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith" var expectedDBName = 'appsmith' - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri) + const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri) expect(dbName).toEqual(expectedDBName) }) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js index 9072479450da..9d4c17929069 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js @@ -7,9 +7,22 @@ function export_database() { console.log('export_database ....'); dbUrl = utils.getDburl(); shell.mkdir('-p', [Constants.BACKUP_PATH]); + if (utils.getDburl().startsWith('mongodb')) { + executeMongoDumpCMD(destFolder, utils.getDburl()) + } else if (utils.getDburl().startsWith('postgresql')) { + executePostgresDumpCMD(destFolder, utils.getDburl()); + } + console.log('export_database done'); +} + +function executeMongoDumpCMD(dbUrl) { const cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; shell.exec(cmd); - console.log('export_database done'); +} + +function executePostgresDumpCMD(dbUrl) { + const cmd = `pg_dump ${dbUrl} -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`; + shell.exec(cmd); } function stop_application() { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js index 0d575e4ff16e..9feb3e83868e 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js @@ -9,11 +9,24 @@ const utils = require('./utils'); function import_database() { console.log('import_database ....') dbUrl = utils.getDburl(); - const cmd = `mongorestore --uri='${dbUrl}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip` - shell.exec(cmd) + if (utils.getDburl().startsWith('mongodb')) { + restore_mongo_db(); + } else if (utils.getDburl().startsWith('postgresql')) { + restore_postgres_db(); + } console.log('import_database done') } +restore_mongo_db = () => { + const cmd = `mongorestore --uri='${dbUrl}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; + shell.exec(cmd); +} + +restore_postgres_db = () => { + const cmd = `pg_restore -U postgres -d appsmith -c ${Constants.RESTORE_PATH}/${Constants.POSTGRES_DUMP_FILE_NAME}`; + shell.exec(cmd); +} + function stop_application() { shell.exec('/usr/bin/supervisorctl stop backend rts') } @@ -22,6 +35,16 @@ function start_application() { shell.exec('/usr/bin/supervisorctl start backend rts') } +function get_table_or_collection_len() { + let count; + if (utils.getDburl().startsWith('mongodb')) { + count = shell.exec(`mongo ${utils.getDburl()} --quiet --eval "db.getCollectionNames().length"`) + } else if (utils.getDburl().startsWith('postgresql')) { + count = shell.exec(`psql -U postgres -d ${utils.getDburl()} -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`) + } + return parseInt(count.stdout.toString().trimEnd()) +} + // Main application workflow const main = (forceOption) => { let errorCode = 0 @@ -37,8 +60,7 @@ const main = (forceOption) => { shell.echo('stop backend & rts application before import database') stop_application() - const shellCmdResult = shell.exec(`mongo ${process.env.APPSMITH_DB_URL} --quiet --eval "db.getCollectionNames().length"`) - const collectionsLen = parseInt(shellCmdResult.stdout.toString().trimEnd()) + const collectionsLen = get_table_or_collection_len(); if (collectionsLen > 0) { if (forceOption) { import_database() diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js index 780ac3d0050c..71d98a0a2407 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js @@ -59,17 +59,37 @@ async function extractArchive(backupFilePath, restoreRootPath) { async function restoreDatabase(restoreContentsPath, dbUrl) { console.log('Restoring database...'); + if (dbUrl.startsWith('mongodb')) { + await restore_mongo_db(restoreContentsPath, dbUrl); + } else if (dbUrl.includes('postgresql')) { + await restore_postgres_db(restoreContentsPath, dbUrl); + } + console.log('Restoring database completed'); +} + +async function restore_mongo_db(restoreContentsPath, dbUrl) { const cmd = ['mongorestore', `--uri=${dbUrl}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip'] try { const fromDbName = await getBackupDatabaseName(restoreContentsPath); - const toDbName = utils.getDatabaseNameFromMongoURI(dbUrl); + const toDbName = utils.getDatabaseNameFromDBURI(dbUrl); console.log("Restoring database from " + fromDbName + " to " + toDbName) cmd.push('--nsInclude=*', `--nsFrom=${fromDbName}.*`, `--nsTo=${toDbName}.*`) } catch (error) { console.warn('Error reading manifest file. Assuming same database name.', error); } await utils.execCommand(cmd); - console.log('Restoring database completed'); +} + +async function restore_postgres_db(restoreContentsPath, dbUrl) { + const cmd = ['pg_restore', '-U', 'postgres', '-c', `${restoreContentsPath}/pg-data.archive`]; + try { + const toDbName = utils.getDatabaseNameFromDBURI(dbUrl); + console.log("Restoring database to " + toDbName); + cmd.push('-d' , toDbName); + } catch (error) { + console.warn('Error reading manifest file. Assuming same database name.', error); + } + await utils.execCommand(cmd); } async function restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEncryptionKeys) { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/utils.js b/deploy/docker/fs/opt/appsmith/utils/bin/utils.js index 8b70980adc0c..183d0800e65d 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/utils.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/utils.js @@ -179,7 +179,7 @@ function execCommandSilent(cmd, options) { }); } -function getDatabaseNameFromMongoURI(uri) { +function getDatabaseNameFromDBURI(uri) { const uriParts = uri.split("/"); return uriParts[uriParts.length - 1].split("?")[0]; } @@ -195,6 +195,6 @@ module.exports = { getCurrentAppsmithVersion, preprocessMongoDBURI, execCommandSilent, - getDatabaseNameFromMongoURI, + getDatabaseNameFromDBURI, getDburl }; From d52d62f22dc889ebfd440deabf28e5890f6b56c1 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 12 Nov 2024 00:34:35 +0530 Subject: [PATCH 02/20] chore: Dummy server side change --- .../java/com/appsmith/server/configurations/CommonDBConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java index 417cb3de2bdd..3121170140fc 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java @@ -33,7 +33,7 @@ public MongoProperties configureMongoDB() { if (!appsmithDbUrl.startsWith("mongodb")) { return null; } - log.info("Found MongoDB uri configuring now"); + log.info("Found MongoDB uri configuring now."); MongoProperties mongoProperties = new MongoProperties(); mongoProperties.setUri(appsmithDbUrl); return mongoProperties; From f304f6c909e778e503460ebb660cd7414a05119e Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 12 Nov 2024 14:44:10 +0530 Subject: [PATCH 03/20] chore: Fix export DB --- .../fs/opt/appsmith/utils/bin/export_db.js | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js index 9d4c17929069..e1bee4d877b5 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js @@ -7,22 +7,16 @@ function export_database() { console.log('export_database ....'); dbUrl = utils.getDburl(); shell.mkdir('-p', [Constants.BACKUP_PATH]); - if (utils.getDburl().startsWith('mongodb')) { - executeMongoDumpCMD(destFolder, utils.getDburl()) - } else if (utils.getDburl().startsWith('postgresql')) { - executePostgresDumpCMD(destFolder, utils.getDburl()); + let cmd; + if (dbUrl.startsWith('mongodb')) { + cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; + } else if (dbUrl.startsWith('postgresql')) { + cmd = `pg_dump ${dbUrl} -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`; + } else { + throw new Error('Unsupported database URL'); } - console.log('export_database done'); -} - -function executeMongoDumpCMD(dbUrl) { - const cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; - shell.exec(cmd); -} - -function executePostgresDumpCMD(dbUrl) { - const cmd = `pg_dump ${dbUrl} -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`; shell.exec(cmd); + console.log('export_database done'); } function stop_application() { From 5368a683a051ec40b1226e0dd2b2160f4852923b Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 12 Nov 2024 20:14:14 +0530 Subject: [PATCH 04/20] chore: Update import script for postgres DB restore --- deploy/docker/fs/opt/appsmith/utils/bin/import_db.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js index 9feb3e83868e..0d21be089fa1 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js @@ -23,7 +23,7 @@ restore_mongo_db = () => { } restore_postgres_db = () => { - const cmd = `pg_restore -U postgres -d appsmith -c ${Constants.RESTORE_PATH}/${Constants.POSTGRES_DUMP_FILE_NAME}`; + const cmd = `pg_restore -U postgres -d appsmith --verbose --clean ${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}`; shell.exec(cmd); } @@ -40,9 +40,9 @@ function get_table_or_collection_len() { if (utils.getDburl().startsWith('mongodb')) { count = shell.exec(`mongo ${utils.getDburl()} --quiet --eval "db.getCollectionNames().length"`) } else if (utils.getDburl().startsWith('postgresql')) { - count = shell.exec(`psql -U postgres -d ${utils.getDburl()} -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`) + count = shell.exec(`psql -U postgres -d ${utils.getDburl()} -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`) } - return parseInt(count.stdout.toString().trimEnd()) + return parseInt(count.stdout.toString().trimEnd()); } // Main application workflow From 74f96f0fa1302cf3c70200f65ce5172334efe940 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 13 Nov 2024 18:48:09 +0530 Subject: [PATCH 05/20] fix: Remote and local postgres permissions --- .../fs/opt/appsmith/utils/bin/backup.js | 17 +++++++------- .../fs/opt/appsmith/utils/bin/import_db.js | 16 +++++++++----- .../fs/opt/appsmith/utils/bin/restore.js | 22 +++++++++++++------ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index 50fccf614348..09153dd3a9e5 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -125,11 +125,12 @@ function getEncryptionPasswordFromUser(){ async function exportDatabase(destFolder) { console.log('Exporting database'); + const dbUrl = utils.getDburl(); // Check the DB url - if (utils.getDburl().startsWith('mongodb')) { - await executeMongoDumpCMD(destFolder, utils.getDburl()) - } else if (utils.getDburl().startsWith('postgresql')) { - await executePostgresDumpCMD(destFolder, utils.getDburl()); + if (dbUrl.startsWith('mongodb')) { + await executeMongoDumpCMD(destFolder, dbUrl); + } else if (dbUrl.startsWith('postgresql')) { + await executePostgresDumpCMD(destFolder, dbUrl); } console.log('Exporting database done.'); } @@ -162,12 +163,12 @@ async function exportDockerEnvFile(destFolder, encryptArchive) { console.log('Exporting docker environment file done.'); } -async function executeMongoDumpCMD(destFolder, appsmithMongoURI) { - return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd +async function executeMongoDumpCMD(destFolder, dbUrl) { + return await utils.execCommand(['mongodump', `--uri=${dbUrl}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd } -async function executePostgresDumpCMD(destFolder, appsmithDBURI) { - return await utils.execCommand(['pg_dump', appsmithDBURI, '-Fc', '-f', destFolder + '/pg-data.archive']); +async function executePostgresDumpCMD(destFolder, dbUrl) { + return await utils.execCommand(['pg_dump', dbUrl, '-Fc', '-f', destFolder + '/pg-data.archive']); } async function createFinalArchive(destFolder, timestamp) { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js index 0d21be089fa1..33ee198c40d1 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js @@ -10,20 +10,26 @@ function import_database() { console.log('import_database ....') dbUrl = utils.getDburl(); if (utils.getDburl().startsWith('mongodb')) { - restore_mongo_db(); + restore_mongo_db(dbUrl); } else if (utils.getDburl().startsWith('postgresql')) { - restore_postgres_db(); + restore_postgres_db(dbUrl); } console.log('import_database done') } -restore_mongo_db = () => { +restore_mongo_db = (dbUrl) => { const cmd = `mongorestore --uri='${dbUrl}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; shell.exec(cmd); } -restore_postgres_db = () => { - const cmd = `pg_restore -U postgres -d appsmith --verbose --clean ${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}`; +restore_postgres_db = (dbUrl) => { + let cmd; + if (dbUrl.includes('localhost') || dbUrl.includes('127.0.0.1')) { + const toDbName = utils.getDatabaseNameFromDBURI(dbUrl); + cmd = `pg_restore -U postgres -d 'postgresql://localhost:5432/${toDbName}' --verbose --clean ${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}`; + } else { + cmd = `pg_restore -d ${dbUrl} --verbose --clean ${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}`; + } shell.exec(cmd); } diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js index 71d98a0a2407..12f1c2821bd0 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js @@ -81,13 +81,21 @@ async function restore_mongo_db(restoreContentsPath, dbUrl) { } async function restore_postgres_db(restoreContentsPath, dbUrl) { - const cmd = ['pg_restore', '-U', 'postgres', '-c', `${restoreContentsPath}/pg-data.archive`]; - try { - const toDbName = utils.getDatabaseNameFromDBURI(dbUrl); - console.log("Restoring database to " + toDbName); - cmd.push('-d' , toDbName); - } catch (error) { - console.warn('Error reading manifest file. Assuming same database name.', error); + const cmd = ['pg_restore', '--verbose', '--clean', `${restoreContentsPath}/pg-data.archive`]; + if (dbUrl.includes('localhost') || dbUrl.includes('127.0.0.1')) { + let dbName; + try { + dbName = utils.getDatabaseNameFromDBURI(dbUrl); + console.log("Restoring database to " + dbName); + } catch (error) { + console.warn('Error reading manifest file. Assuming same database name as appsmith.', error); + dbName = 'appsmith'; + } + cmd.push('-d' , 'postgresql://localhost:5432/' + dbName); + // Use default user for local postgres + cmd.push('-U', 'postgres'); + } else { + cmd.push('-d', dbUrl); } await utils.execCommand(cmd); } From a6d9be8d02164c1f5663bc918bf16e999aa046b4 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 13 Nov 2024 19:49:49 +0530 Subject: [PATCH 06/20] chore: Only export appsmith schema to avoid backing up un-necessary schemas --- deploy/docker/fs/opt/appsmith/utils/bin/backup.js | 2 +- deploy/docker/fs/opt/appsmith/utils/bin/export_db.js | 5 +++-- deploy/docker/fs/opt/appsmith/utils/bin/import_db.js | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index 09153dd3a9e5..b9738cbf7ed3 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -168,7 +168,7 @@ async function executeMongoDumpCMD(destFolder, dbUrl) { } async function executePostgresDumpCMD(destFolder, dbUrl) { - return await utils.execCommand(['pg_dump', dbUrl, '-Fc', '-f', destFolder + '/pg-data.archive']); + return await utils.execCommand(['pg_dump', dbUrl, '--verbose', '-n', 'appsmith','-Fc', '-f', destFolder + '/pg-data.archive']); } async function createFinalArchive(destFolder, timestamp) { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js index e1bee4d877b5..d561abf2258e 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js @@ -1,4 +1,4 @@ -// Init function export mongodb +// Init function export db const shell = require('shelljs'); const Constants = require('./constants'); const utils = require('./utils'); @@ -11,7 +11,8 @@ function export_database() { if (dbUrl.startsWith('mongodb')) { cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; } else if (dbUrl.startsWith('postgresql')) { - cmd = `pg_dump ${dbUrl} -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`; + // Dump only the appsmith schema with custom format + cmd = `pg_dump ${dbUrl} --verbose -n appsmith -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`; } else { throw new Error('Unsupported database URL'); } diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js index 33ee198c40d1..78a248a80b71 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js @@ -46,7 +46,7 @@ function get_table_or_collection_len() { if (utils.getDburl().startsWith('mongodb')) { count = shell.exec(`mongo ${utils.getDburl()} --quiet --eval "db.getCollectionNames().length"`) } else if (utils.getDburl().startsWith('postgresql')) { - count = shell.exec(`psql -U postgres -d ${utils.getDburl()} -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`) + count = shell.exec(`psql -d ${utils.getDburl()} -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`) } return parseInt(count.stdout.toString().trimEnd()); } From 032356b9a92854647e6ad53d2a7b6e4c8d9b08b5 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 14 Nov 2024 18:35:20 +0530 Subject: [PATCH 07/20] chore: Minor refactors --- .../server/configurations/CommonDBConfig.java | 2 +- .../docker/fs/opt/appsmith/utils/bin/backup.js | 4 +++- .../fs/opt/appsmith/utils/bin/backup.test.js | 14 ++++++++++++++ .../fs/opt/appsmith/utils/bin/export_db.js | 4 ++-- .../fs/opt/appsmith/utils/bin/import_db.js | 17 +++++++++++------ .../docker/fs/opt/appsmith/utils/bin/restore.js | 6 +++++- .../docker/fs/opt/appsmith/utils/bin/utils.js | 6 ++++++ 7 files changed, 42 insertions(+), 11 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java index 3121170140fc..417cb3de2bdd 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java @@ -33,7 +33,7 @@ public MongoProperties configureMongoDB() { if (!appsmithDbUrl.startsWith("mongodb")) { return null; } - log.info("Found MongoDB uri configuring now."); + log.info("Found MongoDB uri configuring now"); MongoProperties mongoProperties = new MongoProperties(); mongoProperties.setUri(appsmithDbUrl); return mongoProperties; diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index b9738cbf7ed3..8689c43d154b 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -131,6 +131,8 @@ async function exportDatabase(destFolder) { await executeMongoDumpCMD(destFolder, dbUrl); } else if (dbUrl.startsWith('postgresql')) { await executePostgresDumpCMD(destFolder, dbUrl); + } else { + throw new Error(`Unsupported database type in URL: ${dbUrl}`); } console.log('Exporting database done.'); } @@ -168,7 +170,7 @@ async function executeMongoDumpCMD(destFolder, dbUrl) { } async function executePostgresDumpCMD(destFolder, dbUrl) { - return await utils.execCommand(['pg_dump', dbUrl, '--verbose', '-n', 'appsmith','-Fc', '-f', destFolder + '/pg-data.archive']); + return await utils.execCommand(['pg_dump', dbUrl, '-n', 'appsmith','-Fc', '-f', destFolder + '/pg-data.archive']); } async function createFinalArchive(destFolder, timestamp) { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js index adb6700de357..a7f72ee83961 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js @@ -264,3 +264,17 @@ test('Get DB name from Mongo URI 4', async () => { expect(dbName).toEqual(expectedDBName) }) +test('Get DB name from PostgreSQL URI', async () => { + var pg_uri = "postgresql://user:password@host:5432/postgres_db" + var expectedDBName = 'postgres_db' + const dbName = utils.getDatabaseNameFromDBURI(pg_uri) + expect(dbName).toEqual(expectedDBName) +}) + +test('Get DB name from PostgreSQL URI with query params', async () => { + var pg_uri = "postgresql://user:password@host:5432/postgres_db?sslmode=disable" + var expectedDBName = 'postgres_db' + const dbName = utils.getDatabaseNameFromDBURI(pg_uri) + expect(dbName).toEqual(expectedDBName) +}) + diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js index d561abf2258e..602f6b958b3d 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js @@ -12,9 +12,9 @@ function export_database() { cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; } else if (dbUrl.startsWith('postgresql')) { // Dump only the appsmith schema with custom format - cmd = `pg_dump ${dbUrl} --verbose -n appsmith -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`; + cmd = `pg_dump ${dbUrl} -n appsmith -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`; } else { - throw new Error('Unsupported database URL'); + throw new Error('Unsupported database type, only MongoDB and PostgreSQL are supported'); } shell.exec(cmd); console.log('export_database done'); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js index 78a248a80b71..f4c5f015707a 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js @@ -9,10 +9,12 @@ const utils = require('./utils'); function import_database() { console.log('import_database ....') dbUrl = utils.getDburl(); - if (utils.getDburl().startsWith('mongodb')) { + if (dbUrl.startsWith('mongodb')) { restore_mongo_db(dbUrl); - } else if (utils.getDburl().startsWith('postgresql')) { + } else if (dbUrl.startsWith('postgresql')) { restore_postgres_db(dbUrl); + } else { + throw new Error('Unsupported database type, only MongoDB and PostgreSQL are supported'); } console.log('import_database done') } @@ -43,10 +45,13 @@ function start_application() { function get_table_or_collection_len() { let count; - if (utils.getDburl().startsWith('mongodb')) { - count = shell.exec(`mongo ${utils.getDburl()} --quiet --eval "db.getCollectionNames().length"`) - } else if (utils.getDburl().startsWith('postgresql')) { - count = shell.exec(`psql -d ${utils.getDburl()} -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`) + const dbUrl = utils.getDburl(); + if (dbUrl.startsWith('mongodb')) { + count = shell.exec(`mongo ${dbUrl} --quiet --eval "db.getCollectionNames().length"`) + } else if (dbUrl.startsWith('postgresql')) { + count = shell.exec(`psql -d ${dbUrl} -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`) + } else { + throw new Error('Unsupported database type, only MongoDB and PostgreSQL are supported'); } return parseInt(count.stdout.toString().trimEnd()); } diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js index 12f1c2821bd0..59ccb234da7d 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js @@ -63,6 +63,8 @@ async function restoreDatabase(restoreContentsPath, dbUrl) { await restore_mongo_db(restoreContentsPath, dbUrl); } else if (dbUrl.includes('postgresql')) { await restore_postgres_db(restoreContentsPath, dbUrl); + } else { + throw new Error('Unsupported database type, only MongoDB and PostgreSQL are supported'); } console.log('Restoring database completed'); } @@ -82,7 +84,9 @@ async function restore_mongo_db(restoreContentsPath, dbUrl) { async function restore_postgres_db(restoreContentsPath, dbUrl) { const cmd = ['pg_restore', '--verbose', '--clean', `${restoreContentsPath}/pg-data.archive`]; - if (dbUrl.includes('localhost') || dbUrl.includes('127.0.0.1')) { + const url = new URL(dbUrl); + const isLocalhost = ['localhost', '127.0.0.1'].includes(url.hostname); + if (isLocalhost) { let dbName; try { dbName = utils.getDatabaseNameFromDBURI(dbUrl); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/utils.js b/deploy/docker/fs/opt/appsmith/utils/bin/utils.js index 183d0800e65d..98522cac9bfa 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/utils.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/utils.js @@ -179,6 +179,12 @@ function execCommandSilent(cmd, options) { }); } +/** + * Extracts database name from MongoDB or PostgreSQL connection URI + * @param {string} uri - Database connection URI + * @returns {string} Database name + * @returns + */ function getDatabaseNameFromDBURI(uri) { const uriParts = uri.split("/"); return uriParts[uriParts.length - 1].split("?")[0]; From c56c283da8b6a6aa4c1d721061137dee1cb5fa98 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Fri, 29 Nov 2024 21:09:14 +0530 Subject: [PATCH 08/20] fix test --- app/client/packages/rts/src/ctl/backup/backup.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index 57533b955056..4c687ffe5a58 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -67,11 +67,11 @@ describe("Backup Tests", () => { console.log(res); }); - test("Test postgresdump CMD generaton", async () => { + test("Test postgres dump CMD generation", async () => { const dest = "/dest"; const appsmithMongoURI = "postgresql://username:password@host/appsmith"; const cmd = - "pg_dump postgresql://username:password@host/appsmith -Fc -f /dest/pg-data.archive"; + "pg_dump postgresql://username:password@host/appsmith ---schema=appsmith --format=custom --file=/dest/pg-data.gz"; const res = await executePostgresDumpCMD(dest, appsmithMongoURI); expect(res).toBe(cmd); From cfba51da016b82da0f92870793eb382e60ba5d1a Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Fri, 29 Nov 2024 21:11:01 +0530 Subject: [PATCH 09/20] Remove extension to pg-data --- .../packages/rts/src/ctl/backup/links/PostgresDumpLink.ts | 2 +- app/client/packages/rts/src/ctl/restore.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts index 70fa15377fc0..17a666860af0 100644 --- a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts @@ -28,6 +28,6 @@ export async function executePostgresDumpCMD( dbUrl, "--schema=appsmith", "--format=custom", - `--file=${destFolder}/pg-data.gz`, + `--file=${destFolder}/pg-data`, ]); } diff --git a/app/client/packages/rts/src/ctl/restore.ts b/app/client/packages/rts/src/ctl/restore.ts index f05a736aaa94..00805df39aa7 100644 --- a/app/client/packages/rts/src/ctl/restore.ts +++ b/app/client/packages/rts/src/ctl/restore.ts @@ -159,7 +159,7 @@ async function restorePostgres(restoreContentsPath: string, dbUrl: string) { "pg_restore", "--verbose", "--clean", - `${restoreContentsPath}/pg-data.gz`, + `${restoreContentsPath}/pg-data`, ]; const url = new URL(dbUrl); const isLocalhost = ["localhost", "127.0.0.1"].includes(url.hostname); From 9bda084b6c92fd99db292f418138139612513b36 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Fri, 29 Nov 2024 21:12:03 +0530 Subject: [PATCH 10/20] cosmetics --- app/client/packages/rts/src/ctl/backup/backup.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index 4c687ffe5a58..3de91a628309 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -69,10 +69,10 @@ describe("Backup Tests", () => { test("Test postgres dump CMD generation", async () => { const dest = "/dest"; - const appsmithMongoURI = "postgresql://username:password@host/appsmith"; + const url = "postgresql://username:password@host/appsmith"; const cmd = "pg_dump postgresql://username:password@host/appsmith ---schema=appsmith --format=custom --file=/dest/pg-data.gz"; - const res = await executePostgresDumpCMD(dest, appsmithMongoURI); + const res = await executePostgresDumpCMD(dest, url); expect(res).toBe(cmd); console.log(res); From b1faadac56c3f6ea470c01af40a5e5a6eba93f9d Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Sat, 30 Nov 2024 05:57:57 +0530 Subject: [PATCH 11/20] fix test --- app/client/packages/rts/src/ctl/backup/backup.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index 3de91a628309..6f829b7f9e6b 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -71,7 +71,7 @@ describe("Backup Tests", () => { const dest = "/dest"; const url = "postgresql://username:password@host/appsmith"; const cmd = - "pg_dump postgresql://username:password@host/appsmith ---schema=appsmith --format=custom --file=/dest/pg-data.gz"; + "pg_dump postgresql://username:password@host/appsmith --schema=appsmith --format=custom --file=/dest/pg-data.gz"; const res = await executePostgresDumpCMD(dest, url); expect(res).toBe(cmd); From 4252c0e4dcbc175e938afa4af77c68e27a26efb8 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Sat, 30 Nov 2024 06:00:57 +0530 Subject: [PATCH 12/20] fix test --- app/client/packages/rts/src/ctl/backup/backup.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index 6f829b7f9e6b..90c21c94e6d2 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -71,7 +71,7 @@ describe("Backup Tests", () => { const dest = "/dest"; const url = "postgresql://username:password@host/appsmith"; const cmd = - "pg_dump postgresql://username:password@host/appsmith --schema=appsmith --format=custom --file=/dest/pg-data.gz"; + "pg_dump postgresql://username:password@host/appsmith --schema=appsmith --format=custom --file=/dest/pg-data"; const res = await executePostgresDumpCMD(dest, url); expect(res).toBe(cmd); From e03d9259f6c5ebade17fab34992e4575565ace0c Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Sat, 30 Nov 2024 06:07:35 +0530 Subject: [PATCH 13/20] fix comment --- .../packages/rts/src/ctl/backup/links/PostgresDumpLink.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts index 17a666860af0..3c369fc0579f 100644 --- a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts @@ -3,7 +3,7 @@ import type { BackupState } from "../BackupState"; import * as utils from "../../utils"; /** - * Exports the Postgres database data using mongodump. + * Exports the Postgres database data using pg_dump. */ export class PostgresDumpLink implements Link { constructor(private readonly state: BackupState) {} From 552044c69eed5f57ad0330b00dc23d9c5d0aa06e Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Mon, 27 Jan 2025 18:05:33 +0530 Subject: [PATCH 14/20] Use Keycloak DB URL if DB URL is that of MongoDB --- .../rts/src/ctl/backup/BackupState.ts | 5 ++++- .../packages/rts/src/ctl/backup/index.ts | 2 +- .../rts/src/ctl/backup/links/MongoDumpLink.ts | 2 +- .../src/ctl/backup/links/PostgresDumpLink.ts | 22 +++++++++++++++---- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/app/client/packages/rts/src/ctl/backup/BackupState.ts b/app/client/packages/rts/src/ctl/backup/BackupState.ts index 9192f654d51e..5f56bd13ef98 100644 --- a/app/client/packages/rts/src/ctl/backup/BackupState.ts +++ b/app/client/packages/rts/src/ctl/backup/BackupState.ts @@ -1,5 +1,7 @@ export class BackupState { readonly args: readonly string[]; + readonly dbUrl: string; + readonly initAt: string = new Date().toISOString().replace(/:/g, "-"); readonly errors: string[] = []; @@ -8,8 +10,9 @@ export class BackupState { isEncryptionEnabled: boolean = false; - constructor(args: string[]) { + constructor(args: string[], url: string) { this.args = Object.freeze([...args]); + this.dbUrl = url; // We seal `this` so that no link in the chain can "add" new properties to the state. This is intentional. If any // link wants to save data in the `BackupState`, which shouldn't even be needed in most cases, it should do so by diff --git a/app/client/packages/rts/src/ctl/backup/index.ts b/app/client/packages/rts/src/ctl/backup/index.ts index 81213ebd5bda..cf133f910bdd 100644 --- a/app/client/packages/rts/src/ctl/backup/index.ts +++ b/app/client/packages/rts/src/ctl/backup/index.ts @@ -19,7 +19,7 @@ export async function run(args: string[]) { await utils.ensureSupervisorIsRunning(); - const state: BackupState = new BackupState(args); + const state: BackupState = new BackupState(args, url); const chain: Link[] = [ new linkClasses.BackupFolderLink(state), diff --git a/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts index 1e7631f5ecbd..b689be8264a7 100644 --- a/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts @@ -9,7 +9,7 @@ export class MongoDumpLink implements Link { constructor(private readonly state: BackupState) {} async doBackup() { - const url = utils.getDburl(); + const url = this.state.dbUrl; if (url.startsWith("mongodb")) { console.log("Exporting database"); diff --git a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts index 3c369fc0579f..5e050e800ba6 100644 --- a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts @@ -6,14 +6,28 @@ import * as utils from "../../utils"; * Exports the Postgres database data using pg_dump. */ export class PostgresDumpLink implements Link { - constructor(private readonly state: BackupState) {} + private postgresUrl: null | string = null; - async doBackup() { - const url = utils.getDburl(); + constructor(private readonly state: BackupState) {} + async preBackup() { + const url = this.state.dbUrl; if (url.startsWith("postgresql")) { + this.postgresUrl = url; + return; + } + + if (process.env.APPSMITH_KEYCLOAK_DB_URL) { + this.postgresUrl = `postgresql://${process.env.APPSMITH_KEYCLOAK_DB_USERNAME}:${process.env.APPSMITH_KEYCLOAK_DB_PASSWORD}@${process.env.APPSMITH_KEYCLOAK_DB_URL}`; + } else { + throw new Error("No Postgres DB URL found"); + } + } + + async doBackup() { + if (this.postgresUrl) { console.log("Exporting database"); - await executePostgresDumpCMD(this.state.backupRootPath, url); + await executePostgresDumpCMD(this.state.backupRootPath, this.postgresUrl); console.log("Exporting database done."); } } From f849878ee1303b6fd1c3985bb970dbd335f569b5 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Tue, 4 Feb 2025 12:15:21 +0530 Subject: [PATCH 15/20] fix lint --- .../packages/rts/src/ctl/backup/links/PostgresDumpLink.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts index 5e050e800ba6..2749ca719eca 100644 --- a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts @@ -12,8 +12,10 @@ export class PostgresDumpLink implements Link { async preBackup() { const url = this.state.dbUrl; + if (url.startsWith("postgresql")) { this.postgresUrl = url; + return; } From 4e54e91add72f18e5837c7b61b30037beb40a2bb Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Thu, 6 Feb 2025 21:45:58 +0530 Subject: [PATCH 16/20] fix: Fix Postgres backup & restore, tested on EE --- .../rts/src/ctl/backup/backup.test.ts | 9 +- .../src/ctl/backup/links/PostgresDumpLink.ts | 120 ++++++++++++++++-- app/client/packages/rts/src/ctl/restore.ts | 36 +++++- 3 files changed, 145 insertions(+), 20 deletions(-) diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index 90c21c94e6d2..fa332f47600a 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -72,7 +72,13 @@ describe("Backup Tests", () => { const url = "postgresql://username:password@host/appsmith"; const cmd = "pg_dump postgresql://username:password@host/appsmith --schema=appsmith --format=custom --file=/dest/pg-data"; - const res = await executePostgresDumpCMD(dest, url); + const res = await executePostgresDumpCMD(dest, { + host: "host", + port: 5432, + username: "username", + password: "password", + database: "appsmith", + }); expect(res).toBe(cmd); console.log(res); @@ -288,6 +294,7 @@ test("Get DB name from Mongo URI 4", async () => { expect(dbName).toEqual(expectedDBName); }); + test("Get DB name from Postgres URL", async () => { const dbName = utils.getDatabaseNameFromUrl( "postgresql://user:password@host:5432/postgres_db", diff --git a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts index 2749ca719eca..0bacc85fb55f 100644 --- a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts @@ -2,11 +2,19 @@ import type { Link } from "."; import type { BackupState } from "../BackupState"; import * as utils from "../../utils"; +interface ConnectionDetails { + host: string; + port: number; + username: string; + password: string; + database: string; +} + /** - * Exports the Postgres database data using pg_dump. + * Backup & restore for Postgres database data using `pg_dump` and `psql`. */ export class PostgresDumpLink implements Link { - private postgresUrl: null | string = null; + private postgresUrl: null | ConnectionDetails = null; constructor(private readonly state: BackupState) {} @@ -14,13 +22,38 @@ export class PostgresDumpLink implements Link { const url = this.state.dbUrl; if (url.startsWith("postgresql")) { - this.postgresUrl = url; + this.postgresUrl = parsePostgresUrl(url); return; } if (process.env.APPSMITH_KEYCLOAK_DB_URL) { - this.postgresUrl = `postgresql://${process.env.APPSMITH_KEYCLOAK_DB_USERNAME}:${process.env.APPSMITH_KEYCLOAK_DB_PASSWORD}@${process.env.APPSMITH_KEYCLOAK_DB_URL}`; + if (process.env.APPSMITH_KEYCLOAK_DB_URL.startsWith("postgresql://")) { + this.postgresUrl = parsePostgresUrl( + process.env.APPSMITH_KEYCLOAK_DB_URL, + ); + } else { + // then it's just the hostname and database in there + const [host, database] = + process.env.APPSMITH_KEYCLOAK_DB_URL.split("/"); + this.postgresUrl = { + host, + port: 5432, + username: process.env.APPSMITH_KEYCLOAK_DB_USERNAME, + password: process.env.APPSMITH_KEYCLOAK_DB_PASSWORD, + database, + }; + } + } else if (process.env.APPSMITH_ENABLE_EMBEDDED_DB !== "0") { + this.postgresUrl = { + // Get unix_socket_directories from postgresql.conf, like in pg-utils.sh/get_unix_socket_directory. + // Unix socket directory + host: "/var/run/postgresql", + port: 5432, + username: "postgres", + password: process.env.APPSMITH_TEMPORAL_PASSWORD, + database: "appsmith", + }; } else { throw new Error("No Postgres DB URL found"); } @@ -28,22 +61,85 @@ export class PostgresDumpLink implements Link { async doBackup() { if (this.postgresUrl) { - console.log("Exporting database"); await executePostgresDumpCMD(this.state.backupRootPath, this.postgresUrl); - console.log("Exporting database done."); } } + + async doRestore(restoreContentsPath: string) { + const env = { + ...process.env, + }; + + const cmd = ["psql", "-v", "ON_ERROR_STOP=1"]; + + const isLocalhost = ["localhost", "127.0.0.1"].includes( + this.postgresUrl.host, + ); + + if (isLocalhost) { + env.PGHOST = "/var/run/postgresql"; + env.PGPORT = "5432"; + env.PGUSER = "postgres"; + env.PGPASSWORD = process.env.APPSMITH_TEMPORAL_PASSWORD; + env.PGDATABASE = this.postgresUrl.database; + } else { + env.PGHOST = this.postgresUrl.host; + env.PGPORT = this.postgresUrl.port.toString(); + env.PGUSER = this.postgresUrl.username; + env.PGPASSWORD = this.postgresUrl.password; + env.PGDATABASE = this.postgresUrl.database; + } + + await utils.execCommand( + [ + ...cmd, + "--command=DROP SCHEMA IF EXISTS public CASCADE; DROP SCHEMA IF EXISTS appsmith CASCADE; DROP SCHEMA IF EXISTS temporal CASCADE;", + ], + { env }, + ); + + await utils.execCommand( + [...cmd, `--file=${restoreContentsPath}/pg-data.sql`], + { env }, + ); + console.log("Restoring Postgres database completed"); + } +} + +function parsePostgresUrl(url: string): ConnectionDetails { + const parsed = new URL(url); + return { + host: parsed.hostname, + port: parseInt(parsed.port || "5432"), + username: parsed.username, + password: decodeURIComponent(parsed.password), + database: parsed.pathname.substring(1), + }; } export async function executePostgresDumpCMD( destFolder: string, - dbUrl: string, + details: ConnectionDetails, ) { - return await utils.execCommand([ + const args = [ "pg_dump", - dbUrl, + `--host=${details.host}`, + `--port=${details.port || "5432"}`, + `--username=${details.username}`, + `--dbname=${details.database}`, "--schema=appsmith", - "--format=custom", - `--file=${destFolder}/pg-data`, - ]); + "--schema=public", // Keycloak + "--schema=temporal", + `--file=${destFolder}/pg-data.sql`, + "--verbose", + "--serializable-deferrable", + ]; + + // Set password in environment since it's not allowed in the CLI + const env = { + ...process.env, + PGPASSWORD: details.password, + }; + + return await utils.execCommand(args, { env }); } diff --git a/app/client/packages/rts/src/ctl/restore.ts b/app/client/packages/rts/src/ctl/restore.ts index 00805df39aa7..6ac0d9912441 100644 --- a/app/client/packages/rts/src/ctl/restore.ts +++ b/app/client/packages/rts/src/ctl/restore.ts @@ -4,6 +4,8 @@ import os from "os"; import readlineSync from "readline-sync"; import * as utils from "./utils"; import * as Constants from "./constants"; +import { PostgresDumpLink } from "./backup/links/PostgresDumpLink"; +import { BackupState } from "./backup/BackupState"; const command_args = process.argv.slice(3); @@ -109,19 +111,22 @@ async function extractArchive(backupFilePath: string, restoreRootPath: string) { console.log("Extracting the backup archive completed"); } -async function restoreDatabase(restoreContentsPath: string, dbUrl: string) { +async function restoreDatabases(restoreContentsPath: string, dbUrl: string) { console.log("Restoring database..."); if (dbUrl.startsWith("mongodb")) { await restoreMongoDB(restoreContentsPath, dbUrl); - } else if (dbUrl.includes("postgresql")) { - await restorePostgres(restoreContentsPath, dbUrl); } else { throw new Error( "Unsupported database type, only MongoDB and Postgres are supported", ); } + // TODO: Get all link classes equipped with `doRestore` and refactor this to be like backup. + const link = new PostgresDumpLink(new BackupState([], "")); + await link.preBackup(); + await link.doRestore(restoreContentsPath); + console.log("Restoring database completed"); } @@ -361,7 +366,24 @@ async function getBackupDatabaseName(restoreContentsPath: string) { return db_name; } -export async function run() { +export async function run() { const processesToPause = ["backend", "rts"]; + if ( + await fsPromises + .access(process.env.TMP + "/supervisor-conf.d/keycloak.conf") + .then(() => true) + .catch(() => false) + ) { + processesToPause.push("keycloak"); + } + if ( + await fsPromises + .access(process.env.TMP + "/supervisor-conf.d/temporal.conf") + .then(() => true) + .catch(() => false) + ) { + processesToPause.push("temporal"); + } + let cleanupArchive = false; let overwriteEncryptionKeys = true; let backupFilePath: string; @@ -415,8 +437,8 @@ export async function run() { console.log( "Restoring Appsmith instance from the backup at " + backupFilePath, ); - await utils.stop(["backend", "rts"]); - await restoreDatabase(restoreContentsPath, utils.getDburl()); + await utils.stop(processesToPause); + await restoreDatabases(restoreContentsPath, utils.getDburl()); await restoreDockerEnvFile( restoreContentsPath, backupName, @@ -434,7 +456,7 @@ export async function run() { await fsPromises.rm(backupFilePath, { force: true }); } - await utils.start(["backend", "rts"]); + await utils.start(processesToPause); process.exit(); } } From 1b43c4ca3aa1678fb81245f8b675125d51f0168f Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Thu, 6 Feb 2025 22:55:26 +0530 Subject: [PATCH 17/20] fmt --- app/client/packages/rts/src/ctl/restore.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/client/packages/rts/src/ctl/restore.ts b/app/client/packages/rts/src/ctl/restore.ts index 6ac0d9912441..5457ad79ae4a 100644 --- a/app/client/packages/rts/src/ctl/restore.ts +++ b/app/client/packages/rts/src/ctl/restore.ts @@ -366,7 +366,8 @@ async function getBackupDatabaseName(restoreContentsPath: string) { return db_name; } -export async function run() { const processesToPause = ["backend", "rts"]; +export async function run() { + const processesToPause = ["backend", "rts"]; if ( await fsPromises .access(process.env.TMP + "/supervisor-conf.d/keycloak.conf") From 9ff1b9c4a4a665aee9f000be7c2dc13e2f5b2824 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Fri, 7 Feb 2025 16:21:04 +0530 Subject: [PATCH 18/20] fix: Fix case where Postgres URL is invalid --- .../rts/src/ctl/backup/backup.test.ts | 15 +++++++++++++-- .../src/ctl/backup/links/PostgresDumpLink.ts | 19 ++++++++++++------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index fa332f47600a..2ecfc5e9b5e2 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -70,8 +70,19 @@ describe("Backup Tests", () => { test("Test postgres dump CMD generation", async () => { const dest = "/dest"; const url = "postgresql://username:password@host/appsmith"; - const cmd = - "pg_dump postgresql://username:password@host/appsmith --schema=appsmith --format=custom --file=/dest/pg-data"; + const cmd = [ + "pg_dump --host=host", + "--port=5432", + "--username=username", + "--dbname=appsmith", + "--schema=appsmith", + "--schema=public", + "--schema=temporal", + "--file=/dest/pg-data.sql", + "--verbose", + "--serializable-deferrable", + ].join(" "); + const res = await executePostgresDumpCMD(dest, { host: "host", port: 5432, diff --git a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts index 0bacc85fb55f..81202e8954a9 100644 --- a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts @@ -28,14 +28,13 @@ export class PostgresDumpLink implements Link { } if (process.env.APPSMITH_KEYCLOAK_DB_URL) { - if (process.env.APPSMITH_KEYCLOAK_DB_URL.startsWith("postgresql://")) { - this.postgresUrl = parsePostgresUrl( - process.env.APPSMITH_KEYCLOAK_DB_URL, - ); - } else { + const dbUrlFromEnv = process.env.APPSMITH_KEYCLOAK_DB_URL; + + if (dbUrlFromEnv.startsWith("postgresql://")) { + this.postgresUrl = parsePostgresUrl(dbUrlFromEnv); + } else if (dbUrlFromEnv.includes("/")) { // then it's just the hostname and database in there - const [host, database] = - process.env.APPSMITH_KEYCLOAK_DB_URL.split("/"); + const [host, database] = dbUrlFromEnv.split("/"); this.postgresUrl = { host, port: 5432, @@ -43,6 +42,12 @@ export class PostgresDumpLink implements Link { password: process.env.APPSMITH_KEYCLOAK_DB_PASSWORD, database, }; + } else { + // Identify this as an invalid value for this env variable. + // But we ignore this fact for now, since Postgres itself is not a critical component yet. + console.warn( + "APPSMITH_KEYCLOAK_DB_URL is set, but it doesn't start with postgresql://. This is not a valid value for this env variable. But we ignore this fact for now, since Postgres itself is not a critical component yet.", + ); } } else if (process.env.APPSMITH_ENABLE_EMBEDDED_DB !== "0") { this.postgresUrl = { From 8e2a2526a41a36d7eb3bc08bfbe7be3283a601e1 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Thu, 20 Feb 2025 12:42:53 +0530 Subject: [PATCH 19/20] fix: check if postgres data exists in backup before trying to restore --- app/client/packages/rts/src/ctl/restore.ts | 55 ++++++---------------- 1 file changed, 15 insertions(+), 40 deletions(-) diff --git a/app/client/packages/rts/src/ctl/restore.ts b/app/client/packages/rts/src/ctl/restore.ts index 5457ad79ae4a..4208cecc18f6 100644 --- a/app/client/packages/rts/src/ctl/restore.ts +++ b/app/client/packages/rts/src/ctl/restore.ts @@ -4,7 +4,7 @@ import os from "os"; import readlineSync from "readline-sync"; import * as utils from "./utils"; import * as Constants from "./constants"; -import { PostgresDumpLink } from "./backup/links/PostgresDumpLink"; +import { PostgresDumpLink } from "./backup/links"; import { BackupState } from "./backup/BackupState"; const command_args = process.argv.slice(3); @@ -123,13 +123,24 @@ async function restoreDatabases(restoreContentsPath: string, dbUrl: string) { } // TODO: Get all link classes equipped with `doRestore` and refactor this to be like backup. - const link = new PostgresDumpLink(new BackupState([], "")); - await link.preBackup(); - await link.doRestore(restoreContentsPath); + if (await isFilePresent(restoreContentsPath + "/pg-data.sql")) { + const link = new PostgresDumpLink(new BackupState([], "")); + await link.preBackup(); + await link.doRestore(restoreContentsPath); + } console.log("Restoring database completed"); } +async function isFilePresent(path: string): Promise { + try { + await fsPromises.access(path); + return true; + } catch (e) { + return false; + } +} + async function restoreMongoDB(restoreContentsPath: string, dbUrl: string) { const cmd = [ "mongorestore", @@ -159,42 +170,6 @@ async function restoreMongoDB(restoreContentsPath: string, dbUrl: string) { console.log("Restoring database completed"); } -async function restorePostgres(restoreContentsPath: string, dbUrl: string) { - const cmd = [ - "pg_restore", - "--verbose", - "--clean", - `${restoreContentsPath}/pg-data`, - ]; - const url = new URL(dbUrl); - const isLocalhost = ["localhost", "127.0.0.1"].includes(url.hostname); - - if (isLocalhost) { - let dbName: string; - - try { - dbName = utils.getDatabaseNameFromUrl(dbUrl); - console.log("Restoring database to", dbName); - } catch (error) { - console.warn( - "Error reading manifest file. Assuming same database name as appsmith.", - error, - ); - dbName = "appsmith"; - } - cmd.push( - "-d", - "postgresql://localhost:5432/" + dbName, - // Use default user for local postgres - "--username=postgres", - ); - } else { - cmd.push("-d", dbUrl); - } - - await utils.execCommand(cmd); -} - async function restoreDockerEnvFile( restoreContentsPath: string, backupName: string, From bbc2ab467a1e86490d5076489dcf90fe4fbcb40a Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Thu, 20 Feb 2025 13:43:05 +0530 Subject: [PATCH 20/20] formatting --- app/client/packages/rts/src/ctl/backup/backup.test.ts | 2 +- .../packages/rts/src/ctl/backup/links/PostgresDumpLink.ts | 2 ++ app/client/packages/rts/src/ctl/restore.ts | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index 2ecfc5e9b5e2..3c988e174bca 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -69,7 +69,7 @@ describe("Backup Tests", () => { test("Test postgres dump CMD generation", async () => { const dest = "/dest"; - const url = "postgresql://username:password@host/appsmith"; + const cmd = [ "pg_dump --host=host", "--port=5432", diff --git a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts index 81202e8954a9..e49bc45d3864 100644 --- a/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts @@ -35,6 +35,7 @@ export class PostgresDumpLink implements Link { } else if (dbUrlFromEnv.includes("/")) { // then it's just the hostname and database in there const [host, database] = dbUrlFromEnv.split("/"); + this.postgresUrl = { host, port: 5432, @@ -113,6 +114,7 @@ export class PostgresDumpLink implements Link { function parsePostgresUrl(url: string): ConnectionDetails { const parsed = new URL(url); + return { host: parsed.hostname, port: parseInt(parsed.port || "5432"), diff --git a/app/client/packages/rts/src/ctl/restore.ts b/app/client/packages/rts/src/ctl/restore.ts index 4208cecc18f6..bc9ee1e14e04 100644 --- a/app/client/packages/rts/src/ctl/restore.ts +++ b/app/client/packages/rts/src/ctl/restore.ts @@ -125,6 +125,7 @@ async function restoreDatabases(restoreContentsPath: string, dbUrl: string) { // TODO: Get all link classes equipped with `doRestore` and refactor this to be like backup. if (await isFilePresent(restoreContentsPath + "/pg-data.sql")) { const link = new PostgresDumpLink(new BackupState([], "")); + await link.preBackup(); await link.doRestore(restoreContentsPath); } @@ -135,6 +136,7 @@ async function restoreDatabases(restoreContentsPath: string, dbUrl: string) { async function isFilePresent(path: string): Promise { try { await fsPromises.access(path); + return true; } catch (e) { return false; @@ -343,6 +345,7 @@ async function getBackupDatabaseName(restoreContentsPath: string) { export async function run() { const processesToPause = ["backend", "rts"]; + if ( await fsPromises .access(process.env.TMP + "/supervisor-conf.d/keycloak.conf") @@ -351,6 +354,7 @@ export async function run() { ) { processesToPause.push("keycloak"); } + if ( await fsPromises .access(process.env.TMP + "/supervisor-conf.d/temporal.conf")