diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index 539d9fedd2d0..a529019aa44b 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -1,12 +1,12 @@ -const fsPromises = require('fs/promises'); -const path = require('path'); -const os = require('os'); -const utils = require('./utils'); -const Constants = require('./constants'); -const logger = require('./logger'); -const mailer = require('./mailer'); -const tty = require('tty'); -const readlineSync = require('readline-sync'); +const fsPromises = require("fs/promises"); +const path = require("path"); +const os = require("os"); +const utils = require("./utils"); +const Constants = require("./constants"); +const logger = require("./logger"); +const mailer = require("./mailer"); +const tty = require("tty"); +const readlineSync = require("readline-sync"); const command_args = process.argv.slice(3); @@ -19,9 +19,10 @@ async function run() { await utils.ensureSupervisorIsRunning(); try { - console.log('Available free space at /appsmith-stacks'); - const availSpaceInBytes = getAvailableBackupSpaceInBytes("/appsmith-stacks"); - console.log('\n'); + console.log("Available free space at /appsmith-stacks"); + const availSpaceInBytes = + getAvailableBackupSpaceInBytes("/appsmith-stacks"); + console.log("\n"); checkAvailableBackupSpace(availSpaceInBytes); @@ -36,10 +37,15 @@ async function run() { await createManifestFile(backupContentsPath); - if (!command_args.includes('--non-interactive') && (tty.isatty(process.stdout.fd))){ + if ( + !command_args.includes("--non-interactive") && + tty.isatty(process.stdout.fd) + ) { encryptionPassword = getEncryptionPasswordFromUser(); - if (encryptionPassword == -1){ - throw new Error('Backup process aborted because a valid enctyption password could not be obtained from the user'); + if (encryptionPassword == -1) { + throw new Error( + "Backup process aborted because a valid enctyption password could not be obtained from the user", + ); } encryptArchive = true; } @@ -47,33 +53,51 @@ async function run() { archivePath = await createFinalArchive(backupRootPath, timestamp); // shell.exec("openssl enc -aes-256-cbc -pbkdf2 -iter 100000 -in " + archivePath + " -out " + archivePath + ".enc"); - if (encryptArchive){ - const encryptedArchivePath = await encryptBackupArchive(archivePath,encryptionPassword); - await logger.backup_info('Finished creating an encrypted a backup archive at ' + encryptedArchivePath); - if (archivePath != null) { - await fsPromises.rm(archivePath, { recursive: true, force: true }); - } - } - else { - await logger.backup_info('Finished creating a backup archive at ' + archivePath); - console.log('********************************************************* IMPORTANT!!! *************************************************************'); - console.log('*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **') - console.log('*** These values are not included in the backup export. **'); - console.log('************************************************************************************************************************************'); + if (encryptArchive) { + const encryptedArchivePath = await encryptBackupArchive( + archivePath, + encryptionPassword, + ); + await logger.backup_info( + "Finished creating an encrypted a backup archive at " + + encryptedArchivePath, + ); + if (archivePath != null) { + await fsPromises.rm(archivePath, { recursive: true, force: true }); + } + } else { + await logger.backup_info( + "Finished creating a backup archive at " + archivePath, + ); + console.log( + "********************************************************* IMPORTANT!!! *************************************************************", + ); + console.log( + "*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **", + ); + console.log( + "*** These values are not included in the backup export. **", + ); + console.log( + "************************************************************************************************************************************", + ); } await fsPromises.rm(backupRootPath, { recursive: true, force: true }); - await logger.backup_info('Finished taking a backup at ' + archivePath); - + await logger.backup_info("Finished taking a backup at " + archivePath); } catch (err) { errorCode = 1; await logger.backup_error(err.stack); - if (command_args.includes('--error-mail')) { + if (command_args.includes("--error-mail")) { const currentTS = new Date().getTime(); const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec(); - if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS) { + if ( + lastMailTS + + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC < + currentTS + ) { await mailer.sendBackupErrorToAdmins(err, timestamp); await utils.updateLastBackupErrorMailSentInMilliSec(currentTS); } @@ -92,137 +116,193 @@ async function run() { } } -async function encryptBackupArchive(archivePath, encryptionPassword){ - const encryptedArchivePath = archivePath + '.enc'; - await utils.execCommand(['openssl', 'enc', '-aes-256-cbc', '-pbkdf2', '-iter', 100000, '-in', archivePath, '-out', encryptedArchivePath, '-k', encryptionPassword ]) +async function encryptBackupArchive(archivePath, encryptionPassword) { + const encryptedArchivePath = archivePath + ".enc"; + await utils.execCommand([ + "openssl", + "enc", + "-aes-256-cbc", + "-pbkdf2", + "-iter", + 100000, + "-in", + archivePath, + "-out", + encryptedArchivePath, + "-k", + encryptionPassword, + ]); return encryptedArchivePath; } -function getEncryptionPasswordFromUser(){ - for (const _ of [1, 2, 3]) - { - const encryptionPwd1 = readlineSync.question('Enter a password to encrypt the backup archive: ', { hideEchoBack: true }); - const encryptionPwd2 = readlineSync.question('Enter the above password again: ', { hideEchoBack: true }); - if (encryptionPwd1 === encryptionPwd2){ - if (encryptionPwd1){ +function getEncryptionPasswordFromUser() { + for (const _ of [1, 2, 3]) { + const encryptionPwd1 = readlineSync.question( + "Enter a password to encrypt the backup archive: ", + { hideEchoBack: true }, + ); + const encryptionPwd2 = readlineSync.question( + "Enter the above password again: ", + { hideEchoBack: true }, + ); + if (encryptionPwd1 === encryptionPwd2) { + if (encryptionPwd1) { return encryptionPwd1; - } - console.error("Invalid input. Empty password is not allowed, please try again.") - } - else { + } + console.error( + "Invalid input. Empty password is not allowed, please try again.", + ); + } else { console.error("The passwords do not match, please try again."); } } - console.error("Aborting backup process, failed to obtain valid encryption password."); - return -1 + console.error( + "Aborting backup process, failed to obtain valid encryption password.", + ); + return -1; } async function exportDatabase(destFolder) { - console.log('Exporting database'); - await executeMongoDumpCMD(destFolder, utils.getDburl()) - console.log('Exporting database done.'); + console.log("Exporting database"); + await executeMongoDumpCMD(destFolder, utils.getDburl()); + console.log("Exporting database done."); } async function createGitStorageArchive(destFolder) { - console.log('Creating git-storage archive'); + console.log("Creating git-storage archive"); const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT); - await executeCopyCMD(gitRoot, destFolder) + await executeCopyCMD(gitRoot, destFolder); - console.log('Created git-storage archive'); + console.log("Created git-storage archive"); } async function createManifestFile(path) { - const version = await utils.getCurrentAppsmithVersion() - const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(utils.getDburl()) } - await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data)); + const version = await utils.getCurrentAppsmithVersion(); + const manifest_data = { + appsmithVersion: version, + dbName: utils.getDatabaseNameFromMongoURI(utils.getDburl()), + }; + await fsPromises.writeFile( + path + "/manifest.json", + JSON.stringify(manifest_data), + ); } async function exportDockerEnvFile(destFolder, encryptArchive) { - console.log('Exporting docker environment file'); - const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', { encoding: 'utf8' }); + console.log("Exporting docker environment file"); + const content = await fsPromises.readFile( + "/appsmith-stacks/configuration/docker.env", + { encoding: "utf8" }, + ); let cleaned_content = removeSensitiveEnvData(content); - if (encryptArchive){ - cleaned_content += '\nAPPSMITH_ENCRYPTION_SALT=' + process.env.APPSMITH_ENCRYPTION_SALT + - '\nAPPSMITH_ENCRYPTION_PASSWORD=' + process.env.APPSMITH_ENCRYPTION_PASSWORD + if (encryptArchive) { + cleaned_content += + "\nAPPSMITH_ENCRYPTION_SALT=" + + process.env.APPSMITH_ENCRYPTION_SALT + + "\nAPPSMITH_ENCRYPTION_PASSWORD=" + + process.env.APPSMITH_ENCRYPTION_PASSWORD; } - await fsPromises.writeFile(destFolder + '/docker.env', cleaned_content); - console.log('Exporting docker environment file done.'); + await fsPromises.writeFile(destFolder + "/docker.env", cleaned_content); + console.log("Exporting docker environment file done."); } async function executeMongoDumpCMD(destFolder, appsmithMongoURI) { - return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd + return await utils.execCommand([ + "mongodump", + `--uri=${appsmithMongoURI}`, + `--archive=${destFolder}/mongodb-data.gz`, + "--gzip", + ]); // generate cmd } async function createFinalArchive(destFolder, timestamp) { - console.log('Creating final archive'); + console.log("Creating final archive"); const archive = `${Constants.BACKUP_PATH}/appsmith-backup-${timestamp}.tar.gz`; - await utils.execCommand(['tar', '-cah', '-C', destFolder, '-f', archive, '.']); - - console.log('Created final archive'); + await utils.execCommand([ + "tar", + "-cah", + "-C", + destFolder, + "-f", + archive, + ".", + ]); + + console.log("Created final archive"); return archive; } async function postBackupCleanup() { - console.log('Starting the cleanup task after taking a backup.'); - let backupArchivesLimit = getBackupArchiveLimit(process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT); + console.log("Starting the cleanup task after taking a backup."); + let backupArchivesLimit = getBackupArchiveLimit( + process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT, + ); const backupFiles = await utils.listLocalBackupFiles(); while (backupFiles.length > backupArchivesLimit) { const fileName = backupFiles.shift(); - await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName); + await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName); } - console.log('Cleanup task completed.'); - + console.log("Cleanup task completed."); } async function executeCopyCMD(srcFolder, destFolder) { - return await utils.execCommand(['ln', '-s', srcFolder, destFolder + '/git-storage']) + return await utils.execCommand([ + "ln", + "-s", + srcFolder, + destFolder + "/git-storage", + ]); } function getGitRoot(gitRoot) { - if (gitRoot == null || gitRoot === '') { - gitRoot = '/appsmith-stacks/git-storage'; + if (gitRoot == null || gitRoot === "") { + gitRoot = "/appsmith-stacks/git-storage"; } - return gitRoot + return gitRoot; } function generateBackupRootPath() { - return fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-')); + return fsPromises.mkdtemp(path.join(os.tmpdir(), "appsmithctl-backup-")); } function getBackupContentsPath(backupRootPath, timestamp) { - return backupRootPath + '/appsmith-backup-' + timestamp; + return backupRootPath + "/appsmith-backup-" + timestamp; } function removeSensitiveEnvData(content) { // Remove encryption and Mongodb data from docker.env - const output_lines = [] - content.split(/\r?\n/).forEach(line => { - if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB") && !line.startsWith("APPSMITH_DB_URL=")) { + const output_lines = []; + content.split(/\r?\n/).forEach((line) => { + if ( + !line.startsWith("APPSMITH_ENCRYPTION") && + !line.startsWith("APPSMITH_MONGODB") && + !line.startsWith("APPSMITH_DB_URL=") + ) { output_lines.push(line); } }); - return output_lines.join('\n') + return output_lines.join("\n"); } function getBackupArchiveLimit(backupArchivesLimit) { if (!backupArchivesLimit) backupArchivesLimit = Constants.APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT; - return backupArchivesLimit + return backupArchivesLimit; } async function removeOldBackups(backupFiles, backupArchivesLimit) { while (backupFiles.length > backupArchivesLimit) { const fileName = backupFiles.shift(); - await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName); + await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName); } - return backupFiles + return backupFiles; } function getTimeStampInISO() { - return new Date().toISOString().replace(/:/g, '-') + return new Date().toISOString().replace(/:/g, "-"); } async function getAvailableBackupSpaceInBytes(path) { @@ -232,12 +312,12 @@ async function getAvailableBackupSpaceInBytes(path) { function checkAvailableBackupSpace(availSpaceInBytes) { if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) { - throw new Error("Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully."); + throw new Error( + "Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.", + ); } } - - module.exports = { run, getTimeStampInISO, diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js index d49e4eff4f59..ac5309d0e109 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js @@ -1,254 +1,274 @@ -const backup = require('./backup'); -const Constants = require('./constants'); -const os = require('os'); -const fsPromises = require('fs/promises'); -const utils = require('./utils'); -const readlineSync = require('readline-sync'); - -describe('Backup Tests', () => { - -test('Timestamp string in ISO format', () => { - console.log(backup.getTimeStampInISO()) - expect(backup.getTimeStampInISO()).toMatch(/(\d{4})-(\d{2})-(\d{2})T(\d{2})\-(\d{2})\-(\d{2})\.(\d{3})Z/) -}); +const backup = require("./backup"); +const Constants = require("./constants"); +const os = require("os"); +const fsPromises = require("fs/promises"); +const utils = require("./utils"); +const readlineSync = require("readline-sync"); + +describe("Backup Tests", () => { + test("Timestamp string in ISO format", () => { + console.log(backup.getTimeStampInISO()); + expect(backup.getTimeStampInISO()).toMatch( + /(\d{4})-(\d{2})-(\d{2})T(\d{2})\-(\d{2})\-(\d{2})\.(\d{3})Z/, + ); + }); -test('Available Space in /appsmith-stacks volume in Bytes', async () => { - const res = expect(await backup.getAvailableBackupSpaceInBytes("/")) - res.toBeGreaterThan(1024 * 1024) -}); + test("Available Space in /appsmith-stacks volume in Bytes", async () => { + const res = expect(await backup.getAvailableBackupSpaceInBytes("/")); + res.toBeGreaterThan(1024 * 1024); + }); -it('Checkx the constant is 2 GB', () => { - let size = 2 * 1024 * 1024 * 1024 - expect(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES).toBe(size) -}); + it("Checkx the constant is 2 GB", () => { + let size = 2 * 1024 * 1024 * 1024; + expect(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES).toBe(size); + }); -it('Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES', () => { - let size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1; - expect(() => backup.checkAvailableBackupSpace(size)).toThrow(); -}); + it("Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { + let size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1; + expect(() => backup.checkAvailableBackupSpace(size)).toThrow(); + }); -it('Should not hould throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES', () => { - expect(() => {backup.checkAvailableBackupSpace(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES)}).not.toThrow('Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.'); -}); + it("Should not hould throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { + expect(() => { + backup.checkAvailableBackupSpace( + Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES, + ); + }).not.toThrow( + "Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.", + ); + }); -it('Generates t', async () => { - os.tmpdir = jest.fn().mockReturnValue('temp/dir'); - fsPromises.mkdtemp = jest.fn().mockImplementation((a) => a); - const res = await backup.generateBackupRootPath() - expect(res).toBe('temp/dir/appsmithctl-backup-') -}); + it("Generates t", async () => { + os.tmpdir = jest.fn().mockReturnValue("temp/dir"); + fsPromises.mkdtemp = jest.fn().mockImplementation((a) => a); + const res = await backup.generateBackupRootPath(); + expect(res).toBe("temp/dir/appsmithctl-backup-"); + }); -test('Test backup contents path generation', () => { - var root = '/rootDir' - var timestamp = '0000-00-0T00-00-00.00Z' - expect(backup.getBackupContentsPath(root, timestamp)).toBe('/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z') -}); + test("Test backup contents path generation", () => { + var root = "/rootDir"; + var timestamp = "0000-00-0T00-00-00.00Z"; + expect(backup.getBackupContentsPath(root, timestamp)).toBe( + "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z", + ); + }); -test('Test mongodump CMD generaton', async () => { - var dest = '/dest' - var appsmithMongoURI = 'mongodb://username:password@host/appsmith' - var cmd = 'mongodump --uri=mongodb://username:password@host/appsmith --archive=/dest/mongodb-data.gz --gzip' - utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(' ')); - const res = await backup.executeMongoDumpCMD(dest, appsmithMongoURI) - expect(res).toBe(cmd) - console.log(res) -}) - -test('Test get gitRoot path when APPSMITH_GIT_ROOT is \'\' ', () => { - expect(backup.getGitRoot('')).toBe('/appsmith-stacks/git-storage') -}); + test("Test mongodump CMD generaton", async () => { + var dest = "/dest"; + var appsmithMongoURI = "mongodb://username:password@host/appsmith"; + var cmd = + "mongodump --uri=mongodb://username:password@host/appsmith --archive=/dest/mongodb-data.gz --gzip"; + utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(" ")); + const res = await backup.executeMongoDumpCMD(dest, appsmithMongoURI); + expect(res).toBe(cmd); + console.log(res); + }); -test('Test get gitRoot path when APPSMITH_GIT_ROOT is null ', () => { - expect(backup.getGitRoot()).toBe('/appsmith-stacks/git-storage') -}); + test("Test get gitRoot path when APPSMITH_GIT_ROOT is '' ", () => { + expect(backup.getGitRoot("")).toBe("/appsmith-stacks/git-storage"); + }); -test('Test get gitRoot path when APPSMITH_GIT_ROOT is defined ', () => { - expect(backup.getGitRoot('/my/git/storage')).toBe('/my/git/storage') -}); + test("Test get gitRoot path when APPSMITH_GIT_ROOT is null ", () => { + expect(backup.getGitRoot()).toBe("/appsmith-stacks/git-storage"); + }); -test('Test ln command generation', async () => { - var gitRoot = '/appsmith-stacks/git-storage' - var dest = '/destdir' - var cmd = 'ln -s /appsmith-stacks/git-storage /destdir/git-storage' - utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(' ')); - const res = await backup.executeCopyCMD(gitRoot, dest) - expect(res).toBe(cmd) - console.log(res) -}) - -it('Checks for the current Appsmith Version.', async () => { - fsPromises.readFile = jest.fn().mockImplementation(async (path) => { - if (path === "/opt/appsmith/info.json") { - return `{"version": "v0.0.0-SNAPSHOT"}` - } else { - throw new Error("Unexpected file to read: " + path) - } - }); - const res = await utils.getCurrentAppsmithVersion() - expect(res).toBe("v0.0.0-SNAPSHOT") -}) - -test('If MONGODB and Encryption env values are being removed', () => { - expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n - `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) -}); + test("Test get gitRoot path when APPSMITH_GIT_ROOT is defined ", () => { + expect(backup.getGitRoot("/my/git/storage")).toBe("/my/git/storage"); + }); -test('If MONGODB and Encryption env values are being removed', () => { - expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n - `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) -}); + test("Test ln command generation", async () => { + var gitRoot = "/appsmith-stacks/git-storage"; + var dest = "/destdir"; + var cmd = "ln -s /appsmith-stacks/git-storage /destdir/git-storage"; + utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(" ")); + const res = await backup.executeCopyCMD(gitRoot, dest); + expect(res).toBe(cmd); + console.log(res); + }); + it("Checks for the current Appsmith Version.", async () => { + fsPromises.readFile = jest.fn().mockImplementation(async (path) => { + if (path === "/opt/appsmith/info.json") { + return `{"version": "v0.0.0-SNAPSHOT"}`; + } else { + throw new Error("Unexpected file to read: " + path); + } + }); + const res = await utils.getCurrentAppsmithVersion(); + expect(res).toBe("v0.0.0-SNAPSHOT"); + }); -test('Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is null', () => { - expect(backup.getBackupArchiveLimit()).toBe(4) -}); + test("If MONGODB and Encryption env values are being removed", () => { + expect( + backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + `), + ).toMatch( + `APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`, + ); + }); + + test("If MONGODB and Encryption env values are being removed", () => { + expect( + backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + `), + ).toMatch( + `APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`, + ); + }); + + test("Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is null", () => { + expect(backup.getBackupArchiveLimit()).toBe(4); + }); + + test("Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is 5", () => { + expect(backup.getBackupArchiveLimit(5)).toBe(5); + }); + + test("Cleanup Backups when limit is 4 and there are 5 files", async () => { + const backupArchivesLimit = 4; + fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); + var backupFiles = ["file1", "file2", "file3", "file4", "file5"]; + var expectedBackupFiles = ["file2", "file3", "file4", "file5"]; + const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); + console.log(res); + + expect(res).toEqual(expectedBackupFiles); + }); + + test("Cleanup Backups when limit is 2 and there are 5 files", async () => { + const backupArchivesLimit = 2; + fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); + var backupFiles = ["file1", "file2", "file3", "file4", "file5"]; + var expectedBackupFiles = ["file4", "file5"]; + const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); + console.log(res); + + expect(res).toEqual(expectedBackupFiles); + }); + + test("Cleanup Backups when limit is 4 and there are 4 files", async () => { + const backupArchivesLimit = 4; + fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); + var backupFiles = ["file1", "file2", "file3", "file4"]; + var expectedBackupFiles = ["file1", "file2", "file3", "file4"]; + const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); + console.log(res); + + expect(res).toEqual(expectedBackupFiles); + }); + + test("Cleanup Backups when limit is 4 and there are 2 files", async () => { + const backupArchivesLimit = 4; + fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); + var backupFiles = ["file1", "file2"]; + var expectedBackupFiles = ["file1", "file2"]; + const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); + console.log(res); + + expect(res).toEqual(expectedBackupFiles); + }); -test('Backup Archive Limit when env APPSMITH_BACKUP_ARCHIVE_LIMIT is 5', () => { - expect(backup.getBackupArchiveLimit(5)).toBe(5) + test("Cleanup Backups when limit is 2 and there is 1 file", async () => { + const backupArchivesLimit = 4; + fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); + var backupFiles = ["file1"]; + var expectedBackupFiles = ["file1"]; + const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); + console.log(res); + expect(res).toEqual(expectedBackupFiles); + }); + + test("Cleanup Backups when limit is 2 and there is no file", async () => { + const backupArchivesLimit = 4; + fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); + var backupFiles = []; + var expectedBackupFiles = []; + const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); + console.log(res); + expect(res).toEqual(expectedBackupFiles); + }); + + test("Test get encryption password from user prompt whene both passords are the same", async () => { + const password = "password#4321"; + readlineSync.question = jest.fn().mockImplementation((a) => { + return password; + }); + const password_res = backup.getEncryptionPasswordFromUser(); + + expect(password_res).toEqual(password); + }); + + test("Test get encryption password from user prompt when both passords are the different", async () => { + const password = "password#4321"; + readlineSync.question = jest.fn().mockImplementation((a) => { + if (a == "Enter the above password again: ") { + return "pass"; + } + return password; + }); + const password_res = backup.getEncryptionPasswordFromUser(); + + expect(password_res).toEqual(-1); + }); + + test("Get encrypted archive path", async () => { + const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; + const encryptionPassword = "password#4321"; + utils.execCommand = jest + .fn() + .mockImplementation(async (a) => console.log(a)); + const encArchivePath = await backup.encryptBackupArchive( + archivePath, + encryptionPassword, + ); + + expect(encArchivePath).toEqual( + "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z" + ".enc", + ); + }); + + test("Test backup encryption function", async () => { + utils.execCommand = jest + .fn() + .mockImplementation(async (a) => console.log(a)); + const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; + const encryptionPassword = "password#123"; + const res = await backup.encryptBackupArchive( + archivePath, + encryptionPassword, + ); + console.log(res); + expect(res).toEqual("/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z.enc"); + }); }); +test("Get DB name from Mongo URI 1", async () => { + var mongodb_uri = + "mongodb+srv://admin:password@test.cluster.mongodb.net/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"; + var expectedDBName = "my_db_name"; + const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); + expect(dbName).toEqual(expectedDBName); +}); -test('Cleanup Backups when limit is 4 and there are 5 files', async () => { - const backupArchivesLimit = 4; - fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ['file1','file2','file3','file4','file5'] - var expectedBackupFiles = ['file2','file3','file4','file5'] - const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit) - console.log(res) - - expect(res).toEqual(expectedBackupFiles) -}) - -test('Cleanup Backups when limit is 2 and there are 5 files', async () => { - const backupArchivesLimit = 2; - fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ['file1','file2','file3','file4','file5'] - var expectedBackupFiles = ['file4','file5'] - const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit) - console.log(res) - - expect(res).toEqual(expectedBackupFiles) -}) - -test('Cleanup Backups when limit is 4 and there are 4 files', async () => { - const backupArchivesLimit = 4; - fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ['file1','file2','file3','file4'] - var expectedBackupFiles = ['file1','file2','file3','file4'] - const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit) - console.log(res) - - expect(res).toEqual(expectedBackupFiles) -}) - -test('Cleanup Backups when limit is 4 and there are 2 files', async () => { - const backupArchivesLimit = 4; - fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ['file1','file2'] - var expectedBackupFiles = ['file1','file2'] - const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit) - console.log(res) - - expect(res).toEqual(expectedBackupFiles) -}) - - -test('Cleanup Backups when limit is 4 and there are 2 files', async () => { - const backupArchivesLimit = 4; - fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ['file1','file2'] - var expectedBackupFiles = ['file1','file2'] - const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit) - console.log(res) - expect(res).toEqual(expectedBackupFiles) -}) - - -test('Cleanup Backups when limit is 2 and there is 1 file', async () => { - const backupArchivesLimit = 4; - fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ['file1'] - var expectedBackupFiles = ['file1'] - const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit) - console.log(res) - expect(res).toEqual(expectedBackupFiles) -}) - -test('Cleanup Backups when limit is 2 and there is no file', async () => { - const backupArchivesLimit = 4; - fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = [] - var expectedBackupFiles = [] - const res = await backup.removeOldBackups(backupFiles,backupArchivesLimit) - console.log(res) - expect(res).toEqual(expectedBackupFiles) -}) - - -test('Test get encryption password from user prompt whene both passords are the same', async () => { - const password = 'password#4321' - readlineSync.question = jest.fn().mockImplementation((a) => {return password}); - const password_res = backup.getEncryptionPasswordFromUser() - - expect(password_res).toEqual(password) -}) - -test('Test get encryption password from user prompt when both passords are the different', async () => { - const password = 'password#4321' - readlineSync.question = jest.fn().mockImplementation((a) => { - if (a=='Enter the above password again: '){ - return 'pass'; - } - return password}); - const password_res = backup.getEncryptionPasswordFromUser() - - expect(password_res).toEqual(-1) -}) - -test('Get encrypted archive path', async () => { - const archivePath = '/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z'; - const encryptionPassword = 'password#4321' - utils.execCommand = jest.fn().mockImplementation( async (a) => console.log(a)); - const encArchivePath = await backup.encryptBackupArchive(archivePath, encryptionPassword) - - expect(encArchivePath).toEqual('/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z' + '.enc') -}) - -test('Test backup encryption function', async () => { - utils.execCommand= jest.fn().mockImplementation(async (a) => console.log(a)); - const archivePath = '/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z' - const encryptionPassword = 'password#123' - const res = await backup.encryptBackupArchive(archivePath,encryptionPassword) - console.log(res) - expect(res).toEqual('/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z.enc') -}) +test("Get DB name from Mongo URI 2", async () => { + var mongodb_uri = + "mongodb+srv://admin:password@test.cluster.mongodb.net/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"; + var expectedDBName = "test123"; + const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); + expect(dbName).toEqual(expectedDBName); }); -test('Get DB name from Mongo URI 1', async () => { - var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" - var expectedDBName = 'my_db_name' - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri) - expect(dbName).toEqual(expectedDBName) -}) - -test('Get DB name from Mongo URI 2', async () => { - var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" - var expectedDBName = 'test123' - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri) - expect(dbName).toEqual(expectedDBName) -}) - -test('Get DB name from Mongo URI 3', async () => { - var mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123" - var expectedDBName = 'test123' - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri) - expect(dbName).toEqual(expectedDBName) -}) - -test('Get DB name from Mongo URI 4', async () => { - var mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith" - var expectedDBName = 'appsmith' - const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri) - expect(dbName).toEqual(expectedDBName) -}) +test("Get DB name from Mongo URI 3", async () => { + var mongodb_uri = + "mongodb+srv://admin:password@test.cluster.mongodb.net/test123"; + var expectedDBName = "test123"; + const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); + expect(dbName).toEqual(expectedDBName); +}); +test("Get DB name from Mongo URI 4", async () => { + var mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith"; + var expectedDBName = "appsmith"; + const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); + expect(dbName).toEqual(expectedDBName); +}); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js b/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js index 3289ebfa479d..0564107094cd 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js @@ -1,11 +1,14 @@ -const { MongoClient, MongoServerError} = require("mongodb"); +const { MongoClient, MongoServerError } = require("mongodb"); const { preprocessMongoDBURI } = require("./utils"); async function exec() { - const client = new MongoClient(preprocessMongoDBURI(process.env.APPSMITH_DB_URL), { - useNewUrlParser: true, - useUnifiedTopology: true, - }); + const client = new MongoClient( + preprocessMongoDBURI(process.env.APPSMITH_DB_URL), + { + useNewUrlParser: true, + useUnifiedTopology: true, + }, + ); let isReplicaSetEnabled = false; @@ -30,7 +33,11 @@ async function checkReplicaSet(client) { .watch() .on("change", (change) => console.log(change)) .on("error", (err) => { - if (err instanceof MongoServerError && err.toString() == "MongoServerError: The $changeStream stage is only supported on replica sets") { + if ( + err instanceof MongoServerError && + err.toString() == + "MongoServerError: The $changeStream stage is only supported on replica sets" + ) { console.log("Replica set not enabled"); } else { console.error("Error even from changeStream", err); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/constants.js b/deploy/docker/fs/opt/appsmith/utils/bin/constants.js index ae44158c82e2..6c0f920cb569 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/constants.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/constants.js @@ -1,30 +1,29 @@ +const BACKUP_PATH = "/appsmith-stacks/data/backup"; -const BACKUP_PATH = "/appsmith-stacks/data/backup" +const RESTORE_PATH = "/appsmith-stacks/data/restore"; -const RESTORE_PATH = "/appsmith-stacks/data/restore" +const DUMP_FILE_NAME = "appsmith-data.archive"; -const DUMP_FILE_NAME = "appsmith-data.archive" +const APPSMITHCTL_LOG_PATH = "/appsmith-stacks/logs/appsmithctl"; -const APPSMITHCTL_LOG_PATH = "/appsmith-stacks/logs/appsmithctl" +const LAST_ERROR_MAIL_TS = "/appsmith-stacks/data/backup/last-error-mail-ts"; -const LAST_ERROR_MAIL_TS = "/appsmith-stacks/data/backup/last-error-mail-ts" +const ENV_PATH = "/appsmith-stacks/configuration/docker.env"; -const ENV_PATH = "/appsmith-stacks/configuration/docker.env" +const MIN_REQUIRED_DISK_SPACE_IN_BYTES = 2_147_483_648; // 2GB -const MIN_REQUIRED_DISK_SPACE_IN_BYTES = 2_147_483_648 // 2GB +const DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC = 21_600_000; // 6 hrs -const DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC = 21_600_000 // 6 hrs - -const APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT = 4 // 4 backup archives +const APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT = 4; // 4 backup archives module.exports = { - BACKUP_PATH, - RESTORE_PATH, - DUMP_FILE_NAME, - LAST_ERROR_MAIL_TS, - APPSMITHCTL_LOG_PATH, - MIN_REQUIRED_DISK_SPACE_IN_BYTES, - DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC, - APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT, - ENV_PATH -} + BACKUP_PATH, + RESTORE_PATH, + DUMP_FILE_NAME, + LAST_ERROR_MAIL_TS, + APPSMITHCTL_LOG_PATH, + MIN_REQUIRED_DISK_SPACE_IN_BYTES, + DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC, + APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT, + ENV_PATH, +}; diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js index 6ef9131c127b..6518f7074c78 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js @@ -1,9 +1,9 @@ const fsPromises = require("fs/promises"); -const Constants = require('./constants'); -const utils = require('./utils'); +const Constants = require("./constants"); +const utils = require("./utils"); async function exportDatabase() { - console.log('export_database ....'); + console.log("export_database ...."); const dbUrl = utils.getDburl(); await fsPromises.mkdir(Constants.BACKUP_PATH, { recursive: true }); await utils.execCommand([ @@ -11,8 +11,8 @@ async function exportDatabase() { "--uri=" + dbUrl, `--archive=${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}`, "--gzip", - ]) - console.log('export_database done'); + ]); + console.log("export_database done"); } async function run() { @@ -21,18 +21,18 @@ async function run() { await utils.ensureSupervisorIsRunning(); try { - console.log('stop backend & rts application before export database'); + console.log("stop backend & rts application before export database"); await utils.stop(["backend", "rts"]); await exportDatabase(); - console.log('start backend & rts application after export database'); + console.log("start backend & rts application after export database"); console.log(); - console.log('\033[0;33m++++++++++++++++++++ NOTE ++++++++++++++++++++'); + console.log("\x1b[0;33m++++++++++++++++++++ NOTE ++++++++++++++++++++"); console.log(); console.log( - 'Please remember to also copy APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file to the target instance where you intend to import this database dump.', + "Please remember to also copy APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file to the target instance where you intend to import this database dump.", ); console.log(); - console.log('++++++++++++++++++++++++++++++++++++++++++++++\033[0m'); + console.log("++++++++++++++++++++++++++++++++++++++++++++++\x1b[0m"); console.log(); } catch (err) { console.log(err); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js index cdb458d6bed0..c7c40f903730 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js @@ -1,7 +1,7 @@ -const readlineSync = require('readline-sync'); -const process = require('process'); -const Constants = require('./constants'); -const utils = require('./utils'); +const readlineSync = require("readline-sync"); +const process = require("process"); +const Constants = require("./constants"); +const utils = require("./utils"); async function importDatabase() { console.log("Importing the database"); @@ -22,12 +22,12 @@ async function importDatabase() { // Main application workflow async function run(forceOption) { - let errorCode = 0 + let errorCode = 0; await utils.ensureSupervisorIsRunning(); try { - console.log('stop backend & rts application before import database') + console.log("stop backend & rts application before import database"); await utils.stop(["backend", "rts"]); try { const shellCmdResult = await utils.execCommandReturningOutput([ @@ -41,37 +41,45 @@ async function run(forceOption) { console.error("Failed to execute mongo command:", error); throw error; } - const collectionsLen = parseInt(shellCmdResult.stdout.toString().trimEnd()) + const collectionsLen = parseInt(shellCmdResult.stdout.toString().trimEnd()); if (collectionsLen > 0) { if (forceOption) { - await importDatabase() - return + await importDatabase(); + return; } - console.log() - console.log('**************************** WARNING ****************************') - console.log(`Your target database is not empty, it has data in ${collectionsLen} collections.`) - const input = readlineSync.question('Importing this DB will erase this data. Are you sure you want to proceed?[Yes/No] ') - const answer = input && input.toLocaleUpperCase() - if (answer === 'Y' || answer === 'YES') { - await importDatabase() - return - } else if (answer === 'N' || answer === 'NO') { - return + console.log(); + console.log( + "**************************** WARNING ****************************", + ); + console.log( + `Your target database is not empty, it has data in ${collectionsLen} collections.`, + ); + const input = readlineSync.question( + "Importing this DB will erase this data. Are you sure you want to proceed?[Yes/No] ", + ); + const answer = input && input.toLocaleUpperCase(); + if (answer === "Y" || answer === "YES") { + await importDatabase(); + return; + } else if (answer === "N" || answer === "NO") { + return; } - console.log(`Your input is invalid. Please try to run import command again.`) + console.log( + `Your input is invalid. Please try to run import command again.`, + ); } else { - await importDatabase() + await importDatabase(); } } catch (err) { - console.log(err) - errorCode = 1 + console.log(err); + errorCode = 1; } finally { - console.log('start backend & rts application after import database') + console.log("start backend & rts application after import database"); await utils.start(["backend", "rts"]); - process.exit(errorCode) + process.exit(errorCode); } } module.exports = { run, -} +}; diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/logger.js b/deploy/docker/fs/opt/appsmith/utils/bin/logger.js index d05b4a77e443..6ce5d89ce0b2 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/logger.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/logger.js @@ -1,5 +1,5 @@ -const fsPromises = require('fs/promises'); -const Constants = require('./constants'); +const fsPromises = require("fs/promises"); +const Constants = require("./constants"); async function backup_error(err) { console.error(err); @@ -8,7 +8,10 @@ async function backup_error(err) { } catch (error) { await fsPromises.mkdir(Constants.APPSMITHCTL_LOG_PATH); } - await fsPromises.appendFile(Constants.APPSMITHCTL_LOG_PATH + '/backup.log', new Date().toISOString() + ' [ ERROR ] ' + err + '\n'); + await fsPromises.appendFile( + Constants.APPSMITHCTL_LOG_PATH + "/backup.log", + new Date().toISOString() + " [ ERROR ] " + err + "\n", + ); } async function backup_info(msg) { @@ -18,7 +21,10 @@ async function backup_info(msg) { } catch (error) { await fsPromises.mkdir(Constants.APPSMITHCTL_LOG_PATH); } - await fsPromises.appendFile(Constants.APPSMITHCTL_LOG_PATH + '/backup.log', new Date().toISOString() + ' [ INFO ] ' + msg + '\n'); + await fsPromises.appendFile( + Constants.APPSMITHCTL_LOG_PATH + "/backup.log", + new Date().toISOString() + " [ INFO ] " + msg + "\n", + ); } module.exports = { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/mailer.js b/deploy/docker/fs/opt/appsmith/utils/bin/mailer.js index 32e11127f5f0..a794cf6a0869 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/mailer.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/mailer.js @@ -1,7 +1,7 @@ -const nodemailer = require('nodemailer'); -const Constants = require('./constants'); -const utils = require('./utils'); -const logger = require('./logger'); +const nodemailer = require("nodemailer"); +const Constants = require("./constants"); +const utils = require("./utils"); +const logger = require("./logger"); const mailEnabled = process.env.APPSMITH_MAIL_ENABLED; const mailFrom = process.env.APPSMITH_MAIL_FROM; @@ -12,53 +12,78 @@ const mailPass = process.env.APPSMITH_MAIL_PASSWORD; const mailTo = process.env.APPSMITH_ADMIN_EMAILS; async function sendBackupErrorToAdmins(err, backupTimestamp) { - console.log('Sending Error mail to admins.'); + console.log("Sending Error mail to admins."); try { - if (!mailEnabled || !mailFrom || !mailHost || !mailPort || !mailUser || !mailPass) { - throw new Error('Failed to send error mail. Email provider is not configured, please refer to https://docs.appsmith.com/setup/instance-configuration/email to configure it.'); - } - else if (!mailTo) { - throw new Error('Failed to send error mail. Admin email(s) not configured, please refer to https://docs.appsmith.com/setup/instance-configuration/disable-user-signup#administrator-emails to configure it.'); - } - else if (!mailEnabled) { - throw new Error('Mail not sent! APPSMITH_MAIL_ENABLED env val is disabled, please refer to https://docs.appsmith.com/setup/instance-configuration/email to enable it.'); - } - else { + if ( + !mailEnabled || + !mailFrom || + !mailHost || + !mailPort || + !mailUser || + !mailPass + ) { + throw new Error( + "Failed to send error mail. Email provider is not configured, please refer to https://docs.appsmith.com/setup/instance-configuration/email to configure it.", + ); + } else if (!mailTo) { + throw new Error( + "Failed to send error mail. Admin email(s) not configured, please refer to https://docs.appsmith.com/setup/instance-configuration/disable-user-signup#administrator-emails to configure it.", + ); + } else if (!mailEnabled) { + throw new Error( + "Mail not sent! APPSMITH_MAIL_ENABLED env val is disabled, please refer to https://docs.appsmith.com/setup/instance-configuration/email to enable it.", + ); + } else { const backupFiles = await utils.listLocalBackupFiles(); const lastBackupfile = backupFiles.pop(); - const lastBackupTimestamp = lastBackupfile.match(/appsmith-backup-(.*)\.tar.gz/)[1]; - const lastBackupPath = Constants.BACKUP_PATH + '/' + lastBackupfile; + const lastBackupTimestamp = lastBackupfile.match( + /appsmith-backup-(.*)\.tar.gz/, + )[1]; + const lastBackupPath = Constants.BACKUP_PATH + "/" + lastBackupfile; const domainName = process.env.APPSMITH_CUSTOM_DOMAIN; const instanceName = process.env.APPSMITH_INSTANCE_NAME; - let text = 'Appsmith backup did not complete successfully.\n\n ' + - 'Backup timestamp: ' + backupTimestamp + '\n\n' + - 'Last Successful Backup timestamp: ' + lastBackupTimestamp + '\n' + - 'Last Successful Backup location: ' + lastBackupPath + '\n\n'; + let text = + "Appsmith backup did not complete successfully.\n\n " + + "Backup timestamp: " + + backupTimestamp + + "\n\n" + + "Last Successful Backup timestamp: " + + lastBackupTimestamp + + "\n" + + "Last Successful Backup location: " + + lastBackupPath + + "\n\n"; if (instanceName) { - text = text + 'Appsmith instance name: ' + instanceName + '\n'; + text = text + "Appsmith instance name: " + instanceName + "\n"; } if (domainName) { - text = text + 'Link to Appsmith admin settings: ' + 'http://' + domainName + '/settings/general' + '\n'; + text = + text + + "Link to Appsmith admin settings: " + + "http://" + + domainName + + "/settings/general" + + "\n"; } - text = text + '\n' + err.stack; + text = text + "\n" + err.stack; const transporter = nodemailer.createTransport({ host: mailHost, port: mailPort, auth: { user: mailUser, - pass: mailPass - } + pass: mailPass, + }, }); await transporter.sendMail({ from: mailFrom, to: mailTo, - subject: '[Appsmith] ERROR: Backup Failed', - text: text + subject: "[Appsmith] ERROR: Backup Failed", + text: text, }); } } catch (err) { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js b/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js index d0ab437b97e7..c2b48a3d9a11 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js @@ -1,5 +1,4 @@ - -const utils = require('./utils'); +const utils = require("./utils"); const command_args = process.argv.slice(3); @@ -9,7 +8,7 @@ async function exec() { await execMongoEval(command_args[0], process.env.APPSMITH_DB_URL); } catch (err) { errorCode = 1; - console.error('Error evaluating the mongo query', err); + console.error("Error evaluating the mongo query", err); } finally { process.exit(errorCode); } @@ -17,12 +16,16 @@ async function exec() { async function execMongoEval(queryExpression, appsmithMongoURI) { queryExpression = queryExpression.trim(); - if (command_args.includes('--pretty')) { - queryExpression += '.pretty()'; + if (command_args.includes("--pretty")) { + queryExpression += ".pretty()"; } - return await utils.execCommand(['mongosh', appsmithMongoURI, `--eval=${queryExpression}`]); + return await utils.execCommand([ + "mongosh", + appsmithMongoURI, + `--eval=${queryExpression}`, + ]); } module.exports = { - exec + exec, }; diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs index a8cd9c65dcda..48018e0a52e4 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs +++ b/deploy/docker/fs/opt/appsmith/utils/bin/move-to-postgres.mjs @@ -6,8 +6,8 @@ * @param {boolean} isBaselineMode - Flag indicating whether the script is running in baseline mode. * @returns {Promise} - A promise that resolves when the data migration is complete. */ -import {spawn} from "child_process"; -import {MongoClient} from "mongodb"; +import { spawn } from "child_process"; +import { MongoClient } from "mongodb"; import * as fs from "node:fs"; let isBaselineMode = false; @@ -31,7 +31,9 @@ for (let i = 2; i < process.argv.length; ++i) { mongoDumpFile = extractValueFromArg(arg); } else if (arg === "--baseline") { isBaselineMode = true; - console.warn("Running in baseline mode. If you're not an Appsmith team member, we sure hope you know what you're doing.") + console.warn( + "Running in baseline mode. If you're not an Appsmith team member, we sure hope you know what you're doing.", + ); } else { console.error("Unknown/unexpected argument: " + arg); process.exit(1); @@ -45,16 +47,25 @@ if (!mongoDbUrl && !mongoDumpFile) { let mongoServer; if (mongoDumpFile) { - fs.mkdirSync("/tmp/db-tmp", {recursive: true}); + fs.mkdirSync("/tmp/db-tmp", { recursive: true }); - mongoServer = spawn("mongod", ["--bind_ip_all", "--dbpath", "/tmp/db-tmp", "--port", "27500"], { - stdio: "inherit", - }); + mongoServer = spawn( + "mongod", + ["--bind_ip_all", "--dbpath", "/tmp/db-tmp", "--port", "27500"], + { + stdio: "inherit", + }, + ); mongoDbUrl = "mongodb://localhost/tmp"; // mongorestore 'mongodb://localhost/' --archive=mongodb-data.gz --gzip --nsFrom='appsmith.*' --nsTo='appsmith.*' - spawn("mongorestore", [mongoDbUrl, "--archive=" + mongoDumpFile, "--gzip", "--noIndexRestore"]); + spawn("mongorestore", [ + mongoDbUrl, + "--archive=" + mongoDumpFile, + "--gzip", + "--noIndexRestore", + ]); } const mongoClient = new MongoClient(mongoDbUrl); @@ -71,41 +82,49 @@ const filters = {}; if (isBaselineMode) { filters.config = { // Remove the "appsmith_registered" value, since this is baseline static data, and we want new instances to do register. - name: {$ne: "appsmith_registered"}, + name: { $ne: "appsmith_registered" }, }; filters.plugin = { // Remove saas plugins so they can be fetched from CS again, as usual. - packageName: {$ne: "saas-plugin"}, + packageName: { $ne: "saas-plugin" }, }; } -const collectionNames = await mongoDb.listCollections({}, { nameOnly: true }).toArray(); -const sortedCollectionNames = collectionNames.map(collection => collection.name).sort(); +const collectionNames = await mongoDb + .listCollections({}, { nameOnly: true }) + .toArray(); +const sortedCollectionNames = collectionNames + .map((collection) => collection.name) + .sort(); // Verify that the MongoDB data has been migrated to a stable version i.e. v1.43 before we start migrating the data to Postgres. -if (!await isMongoDataMigratedToStableVersion(mongoDb)) { - console.error("MongoDB migration check failed: Try upgrading the Appsmith instance to latest before opting for data migration."); - console.error(`Could not find the valid migration execution entry for "${MINIMUM_MONGO_CHANGESET}" in the "${MONGO_MIGRATION_COLLECTION}" collection.`); +if (!(await isMongoDataMigratedToStableVersion(mongoDb))) { + console.error( + "MongoDB migration check failed: Try upgrading the Appsmith instance to latest before opting for data migration.", + ); + console.error( + `Could not find the valid migration execution entry for "${MINIMUM_MONGO_CHANGESET}" in the "${MONGO_MIGRATION_COLLECTION}" collection.`, + ); await mongoClient.close(); mongoServer?.kill(); process.exit(1); } for await (const collectionName of sortedCollectionNames) { - console.log("Collection:", collectionName); if (isBaselineMode && collectionName.startsWith("mongock")) { continue; } let outFile = null; - for await (const doc of mongoDb.collection(collectionName).find(filters[collectionName])) { - + for await (const doc of mongoDb + .collection(collectionName) + .find(filters[collectionName])) { // Skip archived objects as they are not migrated during the Mongock migration which may end up failing for the // constraints in the Postgres DB. if (isArchivedObject(doc)) { continue; } - transformFields(doc); // This now handles the _class to type transformation. + transformFields(doc); // This now handles the _class to type transformation. if (doc.policyMap == null) { doc.policyMap = {}; } @@ -147,14 +166,14 @@ function toJsonSortedKeys(obj) { function replacer(key, value) { // Ref: https://gist.github.com/davidfurlong/463a83a33b70a3b6618e97ec9679e490 - return value instanceof Object && !Array.isArray(value) ? - Object.keys(value) - .sort() - .reduce((sorted, key) => { - sorted[key] = value[key]; - return sorted - }, {}) : - value; + return value instanceof Object && !Array.isArray(value) + ? Object.keys(value) + .sort() + .reduce((sorted, key) => { + sorted[key] = value[key]; + return sorted; + }, {}) + : value; } /** @@ -173,9 +192,9 @@ function transformFields(obj) { } else if (key === "_class") { const type = mapClassToType(obj._class); if (type) { - obj.type = type; // Add the type field + obj.type = type; // Add the type field } - delete obj._class; // Remove the _class field + delete obj._class; // Remove the _class field } else if (typeof obj[key] === "object" && obj[key] !== null) { transformFields(obj[key]); } diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js index a88878ccc93e..550386245fb4 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js @@ -1,170 +1,283 @@ -const fsPromises = require('fs/promises'); -const path = require('path'); -const os = require('os'); -const readlineSync = require('readline-sync'); +const fsPromises = require("fs/promises"); +const path = require("path"); +const os = require("os"); +const readlineSync = require("readline-sync"); -const utils = require('./utils'); -const Constants = require('./constants'); +const utils = require("./utils"); +const Constants = require("./constants"); const command_args = process.argv.slice(3); -const {getCurrentAppsmithVersion} = require("./utils") +const { getCurrentAppsmithVersion } = require("./utils"); async function getBackupFileName() { - const backupFiles = await utils.listLocalBackupFiles(); - console.log("\n" + backupFiles.length + " Appsmith backup file(s) found: [Sorted in ascending/chronological order]"); + console.log( + "\n" + + backupFiles.length + + " Appsmith backup file(s) found: [Sorted in ascending/chronological order]", + ); if (backupFiles.length == 0) { return; } - console.log('----------------------------------------------------------------'); - console.log('Index\t|\tAppsmith Backup Archive File'); - console.log('----------------------------------------------------------------'); + console.log( + "----------------------------------------------------------------", + ); + console.log("Index\t|\tAppsmith Backup Archive File"); + console.log( + "----------------------------------------------------------------", + ); for (let i = 0; i < backupFiles.length; i++) { if (i === backupFiles.length - 1) - console.log(i + '\t|\t' + backupFiles[i] + ' <--Most recent backup'); - else - console.log(i + '\t|\t' + backupFiles[i]); + console.log(i + "\t|\t" + backupFiles[i] + " <--Most recent backup"); + else console.log(i + "\t|\t" + backupFiles[i]); } - console.log('----------------------------------------------------------------'); + console.log( + "----------------------------------------------------------------", + ); - const backupFileIndex = parseInt(readlineSync.question('Please enter the backup file index: '), 10); - if (!isNaN(backupFileIndex) && Number.isInteger(backupFileIndex) && (backupFileIndex >= 0) && (backupFileIndex < backupFiles.length)) { + const backupFileIndex = parseInt( + readlineSync.question("Please enter the backup file index: "), + 10, + ); + if ( + !isNaN(backupFileIndex) && + Number.isInteger(backupFileIndex) && + backupFileIndex >= 0 && + backupFileIndex < backupFiles.length + ) { return backupFiles[parseInt(backupFileIndex, 10)]; - } - else { - console.log('Invalid input, please try the command again with a valid option'); + } else { + console.log( + "Invalid input, please try the command again with a valid option", + ); } } -async function decryptArchive(encryptedFilePath, backupFilePath){ - console.log('Enter the password to decrypt the backup archive:') +async function decryptArchive(encryptedFilePath, backupFilePath) { + console.log("Enter the password to decrypt the backup archive:"); for (const _ of [1, 2, 3]) { - const decryptionPwd = readlineSync.question('', { hideEchoBack: true }); - try{ - await utils.execCommandSilent(['openssl', 'enc', '-d', '-aes-256-cbc', '-pbkdf2', '-iter', 100000, '-in', encryptedFilePath, '-out', backupFilePath, '-k', decryptionPwd]) - return true + const decryptionPwd = readlineSync.question("", { hideEchoBack: true }); + try { + await utils.execCommandSilent([ + "openssl", + "enc", + "-d", + "-aes-256-cbc", + "-pbkdf2", + "-iter", + 100000, + "-in", + encryptedFilePath, + "-out", + backupFilePath, + "-k", + decryptionPwd, + ]); + return true; } catch (error) { - console.log('Invalid password. Please try again:'); + console.log("Invalid password. Please try again:"); } } - return false + return false; } async function extractArchive(backupFilePath, restoreRootPath) { - console.log('Extracting the Appsmith backup archive at ' + backupFilePath); - await utils.execCommand(['tar', '-C', restoreRootPath, '-xf', backupFilePath]); - console.log('Extracting the backup archive completed'); + console.log("Extracting the Appsmith backup archive at " + backupFilePath); + await utils.execCommand([ + "tar", + "-C", + restoreRootPath, + "-xf", + backupFilePath, + ]); + console.log("Extracting the backup archive completed"); } async function restoreDatabase(restoreContentsPath, dbUrl) { - console.log('Restoring database...'); - const cmd = ['mongorestore', `--uri=${dbUrl}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip'] + console.log("Restoring database..."); + const cmd = [ + "mongorestore", + `--uri=${dbUrl}`, + "--drop", + `--archive=${restoreContentsPath}/mongodb-data.gz`, + "--gzip", + ]; try { const fromDbName = await getBackupDatabaseName(restoreContentsPath); const toDbName = utils.getDatabaseNameFromMongoURI(dbUrl); - console.log("Restoring database from " + fromDbName + " to " + toDbName) - cmd.push('--nsInclude=*', `--nsFrom=${fromDbName}.*`, `--nsTo=${toDbName}.*`) + console.log("Restoring database from " + fromDbName + " to " + toDbName); + cmd.push( + "--nsInclude=*", + `--nsFrom=${fromDbName}.*`, + `--nsTo=${toDbName}.*`, + ); } catch (error) { - console.warn('Error reading manifest file. Assuming same database name.', error); + console.warn( + "Error reading manifest file. Assuming same database name.", + error, + ); } await utils.execCommand(cmd); - console.log('Restoring database completed'); + console.log("Restoring database completed"); } -async function restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEncryptionKeys) { - console.log('Restoring docker environment file'); - const dockerEnvFile = '/appsmith-stacks/configuration/docker.env'; +async function restoreDockerEnvFile( + restoreContentsPath, + backupName, + overwriteEncryptionKeys, +) { + console.log("Restoring docker environment file"); + const dockerEnvFile = "/appsmith-stacks/configuration/docker.env"; const updatedbUrl = utils.getDburl(); let encryptionPwd = process.env.APPSMITH_ENCRYPTION_PASSWORD; let encryptionSalt = process.env.APPSMITH_ENCRYPTION_SALT; - await utils.execCommand(['mv', dockerEnvFile, dockerEnvFile + '.' + backupName]); - await utils.execCommand(['cp', restoreContentsPath + '/docker.env', dockerEnvFile]); - if (overwriteEncryptionKeys){ + await utils.execCommand([ + "mv", + dockerEnvFile, + dockerEnvFile + "." + backupName, + ]); + await utils.execCommand([ + "cp", + restoreContentsPath + "/docker.env", + dockerEnvFile, + ]); + if (overwriteEncryptionKeys) { if (encryptionPwd && encryptionSalt) { - const input = readlineSync.question('If you are restoring to the same Appsmith deployment which generated the backup archive, you can use the existing encryption keys on the instance.\n\ + const input = readlineSync.question( + 'If you are restoring to the same Appsmith deployment which generated the backup archive, you can use the existing encryption keys on the instance.\n\ Press Enter to continue with existing encryption keys\n\ - Or Type "n"/"No" to provide encryption key & password corresponding to the original Appsmith instance that is being restored.\n'); + Or Type "n"/"No" to provide encryption key & password corresponding to the original Appsmith instance that is being restored.\n', + ); const answer = input && input.toLocaleUpperCase(); - if (answer === 'N' || answer === 'NO') { - encryptionPwd = readlineSync.question('Enter the APPSMITH_ENCRYPTION_PASSWORD: ', { - hideEchoBack: true - }); - encryptionSalt = readlineSync.question('Enter the APPSMITH_ENCRYPTION_SALT: ', { - hideEchoBack: true - }); - } - else { - console.log('Restoring docker environment file with existing encryption password & salt'); + if (answer === "N" || answer === "NO") { + encryptionPwd = readlineSync.question( + "Enter the APPSMITH_ENCRYPTION_PASSWORD: ", + { + hideEchoBack: true, + }, + ); + encryptionSalt = readlineSync.question( + "Enter the APPSMITH_ENCRYPTION_SALT: ", + { + hideEchoBack: true, + }, + ); + } else { + console.log( + "Restoring docker environment file with existing encryption password & salt", + ); } - } - else { - encryptionPwd = readlineSync.question('Enter the APPSMITH_ENCRYPTION_PASSWORD: ', { - hideEchoBack: true - }); - encryptionSalt = readlineSync.question('Enter the APPSMITH_ENCRYPTION_SALT: ', { - hideEchoBack: true - }); - } - await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd + '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt + '\nAPPSMITH_DB_URL=' + utils.getDburl() + - '\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ; } else { - await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_DB_URL=' + updatedbUrl + - '\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ; + encryptionPwd = readlineSync.question( + "Enter the APPSMITH_ENCRYPTION_PASSWORD: ", + { + hideEchoBack: true, + }, + ); + encryptionSalt = readlineSync.question( + "Enter the APPSMITH_ENCRYPTION_SALT: ", + { + hideEchoBack: true, + }, + ); } - console.log('Restoring docker environment file completed'); + await fsPromises.appendFile( + dockerEnvFile, + "\nAPPSMITH_ENCRYPTION_PASSWORD=" + + encryptionPwd + + "\nAPPSMITH_ENCRYPTION_SALT=" + + encryptionSalt + + "\nAPPSMITH_DB_URL=" + + utils.getDburl() + + "\nAPPSMITH_MONGODB_USER=" + + process.env.APPSMITH_MONGODB_USER + + "\nAPPSMITH_MONGODB_PASSWORD=" + + process.env.APPSMITH_MONGODB_PASSWORD, + ); + } else { + await fsPromises.appendFile( + dockerEnvFile, + "\nAPPSMITH_DB_URL=" + + updatedbUrl + + "\nAPPSMITH_MONGODB_USER=" + + process.env.APPSMITH_MONGODB_USER + + "\nAPPSMITH_MONGODB_PASSWORD=" + + process.env.APPSMITH_MONGODB_PASSWORD, + ); + } + console.log("Restoring docker environment file completed"); } async function restoreGitStorageArchive(restoreContentsPath, backupName) { - console.log('Restoring git-storage archive'); + console.log("Restoring git-storage archive"); // TODO: Consider APPSMITH_GIT_ROOT env for later iterations - const gitRoot = '/appsmith-stacks/git-storage'; - await utils.execCommand(['mv', gitRoot, gitRoot + '-' + backupName]); - await utils.execCommand(['mv', restoreContentsPath + '/git-storage', '/appsmith-stacks']); - console.log('Restoring git-storage archive completed'); - + const gitRoot = "/appsmith-stacks/git-storage"; + await utils.execCommand(["mv", gitRoot, gitRoot + "-" + backupName]); + await utils.execCommand([ + "mv", + restoreContentsPath + "/git-storage", + "/appsmith-stacks", + ]); + console.log("Restoring git-storage archive completed"); } async function checkRestoreVersionCompatability(restoreContentsPath) { const currentVersion = await getCurrentAppsmithVersion(); - const manifest_data = await fsPromises.readFile(restoreContentsPath + '/manifest.json', { encoding: 'utf8' }); + const manifest_data = await fsPromises.readFile( + restoreContentsPath + "/manifest.json", + { encoding: "utf8" }, + ); const manifest_json = JSON.parse(manifest_data); const restoreVersion = manifest_json["appsmithVersion"]; - console.log('Current Appsmith Version: ' + currentVersion); - console.log('Restore Appsmith Version: ' + restoreVersion); + console.log("Current Appsmith Version: " + currentVersion); + console.log("Restore Appsmith Version: " + restoreVersion); if (currentVersion === restoreVersion) { - console.log('The restore instance is compatible with the current appsmith version'); + console.log( + "The restore instance is compatible with the current appsmith version", + ); } else { - console.log('**************************** WARNING ****************************'); - console.log('The Appsmith instance to be restored is not compatible with the current version.'); - console.log('Please update your appsmith image to \"index.docker.io/appsmith/appsmith-ce:' + restoreVersion + - '\" in the \"docker-compose.yml\" file\nand run the cmd: \"docker-compose restart\" ' + - 'after the restore process is completed, to ensure the restored instance runs successfully.'); - const confirm = readlineSync.question('Press Enter to continue \nOr Type "c" to cancel the restore process.\n'); - if (confirm.toLowerCase() === 'c') { + console.log( + "**************************** WARNING ****************************", + ); + console.log( + "The Appsmith instance to be restored is not compatible with the current version.", + ); + console.log( + 'Please update your appsmith image to "index.docker.io/appsmith/appsmith-ce:' + + restoreVersion + + '" in the "docker-compose.yml" file\nand run the cmd: "docker-compose restart" ' + + "after the restore process is completed, to ensure the restored instance runs successfully.", + ); + const confirm = readlineSync.question( + 'Press Enter to continue \nOr Type "c" to cancel the restore process.\n', + ); + if (confirm.toLowerCase() === "c") { process.exit(0); } } } async function getBackupDatabaseName(restoreContentsPath) { - let db_name = "appsmith" - if (command_args.includes('--backup-db-name')) { + let db_name = "appsmith"; + if (command_args.includes("--backup-db-name")) { for (let i = 0; i < command_args.length; i++) { - if (command_args[i].startsWith('--backup-db-name')) { + if (command_args[i].startsWith("--backup-db-name")) { db_name = command_args[i].split("=")[1]; } } - } - else { - const manifest_data = await fsPromises.readFile(restoreContentsPath + '/manifest.json', { encoding: 'utf8' }); + } else { + const manifest_data = await fsPromises.readFile( + restoreContentsPath + "/manifest.json", + { encoding: "utf8" }, + ); const manifest_json = JSON.parse(manifest_data); - if ("dbName" in manifest_json){ + if ("dbName" in manifest_json) { db_name = manifest_json["dbName"]; } } - console.log('Backup Database Name: ' + db_name); - return db_name + console.log("Backup Database Name: " + db_name); + return db_name; } async function run() { @@ -181,17 +294,25 @@ async function run() { process.exit(errorCode); } else { backupFilePath = path.join(Constants.BACKUP_PATH, backupFileName); - if (isArchiveEncrypted(backupFileName)){ - const encryptedBackupFilePath = path.join(Constants.BACKUP_PATH, backupFileName); - backupFileName = backupFileName.replace('.enc', ''); + if (isArchiveEncrypted(backupFileName)) { + const encryptedBackupFilePath = path.join( + Constants.BACKUP_PATH, + backupFileName, + ); + backupFileName = backupFileName.replace(".enc", ""); backupFilePath = path.join(Constants.BACKUP_PATH, backupFileName); cleanupArchive = true; overwriteEncryptionKeys = false; - const decryptSuccess = await decryptArchive(encryptedBackupFilePath, backupFilePath); - if (!decryptSuccess){ - console.log('You have entered the incorrect password multiple times. Aborting the restore process.') + const decryptSuccess = await decryptArchive( + encryptedBackupFilePath, + backupFilePath, + ); + if (!decryptSuccess) { + console.log( + "You have entered the incorrect password multiple times. Aborting the restore process.", + ); await fsPromises.rm(backupFilePath, { force: true }); - process.exit(errorCode) + process.exit(errorCode); } } const backupName = backupFileName.replace(/\.tar\.gz$/, ""); @@ -201,31 +322,37 @@ async function run() { await extractArchive(backupFilePath, restoreRootPath); await checkRestoreVersionCompatability(restoreContentsPath); - console.log('****************************************************************'); - console.log('Restoring Appsmith instance from the backup at ' + backupFilePath); + console.log( + "****************************************************************", + ); + console.log( + "Restoring Appsmith instance from the backup at " + backupFilePath, + ); await utils.stop(["backend", "rts"]); await restoreDatabase(restoreContentsPath, utils.getDburl()); - await restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEncryptionKeys); + await restoreDockerEnvFile( + restoreContentsPath, + backupName, + overwriteEncryptionKeys, + ); await restoreGitStorageArchive(restoreContentsPath, backupName); - console.log('Appsmith instance successfully restored.'); + console.log("Appsmith instance successfully restored."); await fsPromises.rm(restoreRootPath, { recursive: true, force: true }); } } catch (err) { console.log(err); errorCode = 1; - } finally { - if (cleanupArchive){ + if (cleanupArchive) { await fsPromises.rm(backupFilePath, { force: true }); } await utils.start(["backend", "rts"]); process.exit(errorCode); - } } -function isArchiveEncrypted(backupFilePath){ - return backupFilePath.endsWith('.enc'); +function isArchiveEncrypted(backupFilePath) { + return backupFilePath.endsWith(".enc"); } module.exports = { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/utils.js b/deploy/docker/fs/opt/appsmith/utils/bin/utils.js index e19b5092ef6d..337d2472fdb2 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/utils.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/utils.js @@ -1,12 +1,12 @@ const fsPromises = require("fs/promises"); const Constants = require("./constants"); const childProcess = require("child_process"); -const fs = require('node:fs'); +const fs = require("node:fs"); const { ConnectionString } = require("mongodb-connection-string-url"); function showHelp() { console.log( - "\nUsage: appsmith to interact with appsmith utils tool" + "\nUsage: appsmith to interact with appsmith utils tool", ); console.log("\nOptions:\r"); console.log("\tex, export_db\t\tExport internal database.\r"); @@ -21,7 +21,7 @@ async function ensureSupervisorIsRunning() { try { await execCommandSilent(["/usr/bin/supervisorctl"]); } catch (e) { - console.error('Supervisor is not running, exiting.'); + console.error("Supervisor is not running, exiting."); throw e; } } @@ -39,11 +39,17 @@ async function start(apps) { } function getDburl() { - let dbUrl = ''; + let dbUrl = ""; try { - let env_array = fs.readFileSync(Constants.ENV_PATH, 'utf8').toString().split("\n"); + let env_array = fs + .readFileSync(Constants.ENV_PATH, "utf8") + .toString() + .split("\n"); for (let i in env_array) { - if (env_array[i].startsWith("APPSMITH_MONGODB_URI") || env_array[i].startsWith("APPSMITH_DB_URL")) { + if ( + env_array[i].startsWith("APPSMITH_MONGODB_URI") || + env_array[i].startsWith("APPSMITH_DB_URL") + ) { dbUrl = env_array[i].toString().split("=")[1].trim(); break; // Break early when the desired line is found } @@ -51,7 +57,8 @@ function getDburl() { } catch (err) { console.error("Error reading the environment file:", err); } - let dbEnvUrl = process.env.APPSMITH_DB_URL || process.env.APPSMITH_MONGO_DB_URI; + let dbEnvUrl = + process.env.APPSMITH_DB_URL || process.env.APPSMITH_MONGO_DB_URI; // Make sure dbEnvUrl takes precedence over dbUrl if (dbEnvUrl && dbEnvUrl !== "undefined") { dbUrl = dbEnvUrl.trim(); @@ -95,9 +102,10 @@ function execCommandReturningOutput(cmd, options) { return new Promise((resolve, reject) => { const p = childProcess.spawn(cmd[0], cmd.slice(1), options); - p.stdin.end() + p.stdin.end(); - const outChunks = [], errChunks = []; + const outChunks = [], + errChunks = []; p.stdout.setEncoding("utf8"); p.stdout.on("data", (data) => { @@ -107,10 +115,14 @@ function execCommandReturningOutput(cmd, options) { p.stderr.setEncoding("utf8"); p.stderr.on("data", (data) => { errChunks.push(data.toString()); - }) + }); p.on("close", (code) => { - const output = (outChunks.join("").trim() + "\n" + errChunks.join("").trim()).trim(); + const output = ( + outChunks.join("").trim() + + "\n" + + errChunks.join("").trim() + ).trim(); if (code === 0) { resolve(output); } else { @@ -138,7 +150,6 @@ async function listLocalBackupFiles() { return backupFiles; } - async function updateLastBackupErrorMailSentInMilliSec(ts) { await fsPromises.mkdir(Constants.BACKUP_PATH, { recursive: true }); await fsPromises.writeFile(Constants.LAST_ERROR_MAIL_TS, ts.toString()); @@ -154,7 +165,10 @@ async function getLastBackupErrorMailSentInMilliSec() { } async function getCurrentAppsmithVersion() { - return JSON.parse(await fsPromises.readFile("/opt/appsmith/info.json", "utf8")).version ?? ""; + return ( + JSON.parse(await fsPromises.readFile("/opt/appsmith/info.json", "utf8")) + .version ?? "" + ); } function preprocessMongoDBURI(uri /* string */) { @@ -163,22 +177,27 @@ function preprocessMongoDBURI(uri /* string */) { const cs = new ConnectionString(uri); const params = cs.searchParams; - params.set('appName', 'appsmithctl'); + params.set("appName", "appsmithctl"); if ( - !cs.isSRV - && !params.has('replicaSet') - && !params.has('directConnection') - && !params.has('loadBalanced') - && cs.hosts.length === 1 + !cs.isSRV && + !params.has("replicaSet") && + !params.has("directConnection") && + !params.has("loadBalanced") && + cs.hosts.length === 1 ) { - params.set('directConnection', 'true'); + params.set("directConnection", "true"); } // For localhost connections, set a lower timeout to avoid hanging for too long. // Taken from . - if (!params.has('serverSelectionTimeoutMS') && cs.hosts.every(host => ['localhost', '127.0.0.1'].includes(host.split(':')[0]))) { - params.set('serverSelectionTimeoutMS', '2000'); + if ( + !params.has("serverSelectionTimeoutMS") && + cs.hosts.every((host) => + ["localhost", "127.0.0.1"].includes(host.split(":")[0]), + ) + ) { + params.set("serverSelectionTimeoutMS", "2000"); } return cs.toString(); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/utils.test.js b/deploy/docker/fs/opt/appsmith/utils/bin/utils.test.js index 5b96816e111a..d2d67e31f369 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/utils.test.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/utils.test.js @@ -1,38 +1,54 @@ const { describe, test, expect } = require("@jest/globals"); const utils = require("./utils"); - describe("execCommandReturningOutput", () => { - test("Output of echo", async () => { - const result = await utils.execCommandReturningOutput(["echo", "hello", "world"]); + const result = await utils.execCommandReturningOutput([ + "echo", + "hello", + "world", + ]); expect(result).toBe("hello world"); }); test("Node console out", async () => { - const result = await utils.execCommandReturningOutput(["node", "--eval", "console.log('to out')"]); + const result = await utils.execCommandReturningOutput([ + "node", + "--eval", + "console.log('to out')", + ]); expect(result).toBe("to out"); }); test("Node console err", async () => { - const result = await utils.execCommandReturningOutput(["node", "--eval", "console.error('to err')"]); + const result = await utils.execCommandReturningOutput([ + "node", + "--eval", + "console.error('to err')", + ]); expect(result).toBe("to err"); }); test("Node console out and err", async () => { - const result = await utils.execCommandReturningOutput(["node", "--eval", "console.log('to out'); console.error('to err')"]); + const result = await utils.execCommandReturningOutput([ + "node", + "--eval", + "console.log('to out'); console.error('to err')", + ]); expect(result).toBe("to out\nto err"); }); test("Node console err and out", async () => { - const result = await utils.execCommandReturningOutput(["node", "--eval", "console.error('to err'); console.log('to out')"]); + const result = await utils.execCommandReturningOutput([ + "node", + "--eval", + "console.error('to err'); console.log('to out')", + ]); expect(result).toBe("to out\nto err"); }); - }); describe("execCommandSilent", () => { - test("Runs a command", async () => { await utils.execCommandSilent(["echo"]); }); @@ -45,8 +61,8 @@ describe("execCommandSilent", () => { }); test("handles errors silently", async () => { - await expect(utils.execCommandSilent(["nonexistentcommand"])) - .rejects.toThrow(); + await expect( + utils.execCommandSilent(["nonexistentcommand"]), + ).rejects.toThrow(); }); - }); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/version.js b/deploy/docker/fs/opt/appsmith/utils/bin/version.js index b154e1244329..144af3282346 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/version.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/version.js @@ -1,4 +1,4 @@ -const utils = require('./utils'); +const utils = require("./utils"); async function exec() { let version = null; @@ -10,9 +10,8 @@ async function exec() { } if (version) { console.log(version); - } - else { - console.error("Error: could not find the current Appsmith version") + } else { + console.error("Error: could not find the current Appsmith version"); process.exit(1); } }