From 7ef6b5a64bb7f4c1dadbea32264d5d47bebb0375 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Thu, 28 Nov 2024 15:32:38 +0530 Subject: [PATCH 1/3] chore: Move all remaining actions into links --- .../rts/src/ctl/backup/BackupState.ts | 4 +- .../rts/src/ctl/backup/backup.test.ts | 46 ++-- .../packages/rts/src/ctl/backup/index.ts | 212 +----------------- .../src/ctl/backup/links/BackupFolderLink.ts | 20 ++ .../rts/src/ctl/backup/links/DiskSpaceLink.ts | 19 +- .../src/ctl/backup/links/EncryptionLink.ts | 64 +++++- .../rts/src/ctl/backup/links/EnvFileLink.ts | 57 +++++ .../src/ctl/backup/links/GitStorageLink.ts | 39 ++++ .../rts/src/ctl/backup/links/MongoDumpLink.ts | 29 +++ .../rts/src/ctl/backup/links/index.ts | 9 +- 10 files changed, 267 insertions(+), 232 deletions(-) create mode 100644 app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts create mode 100644 app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts create mode 100644 app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts create mode 100644 app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts diff --git a/app/client/packages/rts/src/ctl/backup/BackupState.ts b/app/client/packages/rts/src/ctl/backup/BackupState.ts index 182756ab298f..73d1e0ed926f 100644 --- a/app/client/packages/rts/src/ctl/backup/BackupState.ts +++ b/app/client/packages/rts/src/ctl/backup/BackupState.ts @@ -1,8 +1,6 @@ -import { getTimeStampInISO } from "./index"; - export class BackupState { readonly args: string[]; - readonly initAt: string = getTimeStampInISO(); + readonly initAt: string = new Date().toISOString().replace(/:/g, "-"); readonly errors: string[] = []; backupRootPath: string = ""; diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index fd4d12b73091..e5cd196321c8 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -3,6 +3,17 @@ import * as backup from "."; import * as Constants from "../constants"; import * as utils from "../utils"; import readlineSync from "readline-sync"; +import { + checkAvailableBackupSpace, + encryptBackupArchive, + executeCopyCMD, + executeMongoDumpCMD, + getAvailableBackupSpaceInBytes, + getEncryptionPasswordFromUser, + getGitRoot, + removeSensitiveEnvData +} from "./links"; +import { getTimeStampInISO } from "."; jest.mock("../utils", () => ({ ...jest.requireActual("../utils"), @@ -10,15 +21,8 @@ jest.mock("../utils", () => ({ })); describe("Backup Tests", () => { - test("Timestamp string in ISO format", () => { - console.log(backup.getTimeStampInISO()); - expect(backup.getTimeStampInISO()).toMatch( - /(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})\.(\d{3})Z/, - ); - }); - test("Available Space in /appsmith-stacks volume in Bytes", async () => { - const res = expect(await backup.getAvailableBackupSpaceInBytes("/")); + const res = expect(await getAvailableBackupSpaceInBytes("/")); res.toBeGreaterThan(1024 * 1024); }); @@ -32,12 +36,12 @@ describe("Backup Tests", () => { it("Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { const size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1; - expect(() => backup.checkAvailableBackupSpace(size)).toThrow(); + expect(() => checkAvailableBackupSpace(size)).toThrow(); }); it("Should not should throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { expect(() => { - backup.checkAvailableBackupSpace( + checkAvailableBackupSpace( Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES, ); }).not.toThrow( @@ -59,29 +63,29 @@ describe("Backup Tests", () => { const appsmithMongoURI = "mongodb://username:password@host/appsmith"; const cmd = "mongodump --uri=mongodb://username:password@host/appsmith --archive=/dest/mongodb-data.gz --gzip"; - const res = await backup.executeMongoDumpCMD(dest, appsmithMongoURI); + const res = await executeMongoDumpCMD(dest, appsmithMongoURI); expect(res).toBe(cmd); console.log(res); }); test("Test get gitRoot path when APPSMITH_GIT_ROOT is '' ", () => { - expect(backup.getGitRoot("")).toBe("/appsmith-stacks/git-storage"); + expect(getGitRoot("")).toBe("/appsmith-stacks/git-storage"); }); test("Test get gitRoot path when APPSMITH_GIT_ROOT is null ", () => { - expect(backup.getGitRoot()).toBe("/appsmith-stacks/git-storage"); + expect(getGitRoot()).toBe("/appsmith-stacks/git-storage"); }); test("Test get gitRoot path when APPSMITH_GIT_ROOT is defined ", () => { - expect(backup.getGitRoot("/my/git/storage")).toBe("/my/git/storage"); + expect(getGitRoot("/my/git/storage")).toBe("/my/git/storage"); }); test("Test ln command generation", async () => { const gitRoot = "/appsmith-stacks/git-storage"; const dest = "/destdir"; const cmd = "ln -s /appsmith-stacks/git-storage /destdir/git-storage"; - const res = await backup.executeCopyCMD(gitRoot, dest); + const res = await executeCopyCMD(gitRoot, dest); expect(res).toBe(cmd); console.log(res); @@ -102,7 +106,7 @@ describe("Backup Tests", () => { test("If MONGODB and Encryption env values are being removed", () => { expect( - backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n `), ).toMatch( `APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`, @@ -111,7 +115,7 @@ describe("Backup Tests", () => { test("If MONGODB and Encryption env values are being removed", () => { expect( - backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n `), ).toMatch( `APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`, @@ -199,7 +203,7 @@ describe("Backup Tests", () => { const password = "password#4321"; readlineSync.question = jest.fn().mockImplementation(() => password); - const password_res = backup.getEncryptionPasswordFromUser(); + const password_res = getEncryptionPasswordFromUser(); expect(password_res).toEqual(password); }); @@ -215,13 +219,13 @@ describe("Backup Tests", () => { return password; }); - expect(() => backup.getEncryptionPasswordFromUser()).toThrow(); + expect(() => getEncryptionPasswordFromUser()).toThrow(); }); test("Get encrypted archive path", async () => { const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; const encryptionPassword = "password#4321"; - const encArchivePath = await backup.encryptBackupArchive( + const encArchivePath = await encryptBackupArchive( archivePath, encryptionPassword, ); @@ -234,7 +238,7 @@ describe("Backup Tests", () => { test("Test backup encryption function", async () => { const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; const encryptionPassword = "password#123"; - const res = await backup.encryptBackupArchive( + const res = await encryptBackupArchive( archivePath, encryptionPassword, ); diff --git a/app/client/packages/rts/src/ctl/backup/index.ts b/app/client/packages/rts/src/ctl/backup/index.ts index 8fa2c353583f..9d1782fc7792 100644 --- a/app/client/packages/rts/src/ctl/backup/index.ts +++ b/app/client/packages/rts/src/ctl/backup/index.ts @@ -1,14 +1,10 @@ import fsPromises from "fs/promises"; -import path from "path"; -import os from "os"; import * as utils from "../utils"; import * as Constants from "../constants"; import * as logger from "../logger"; import * as mailer from "../mailer"; -import readlineSync from "readline-sync"; -import { DiskSpaceLink } from "./links/DiskSpaceLink"; import type { Link } from "./links"; -import { EncryptionLink, ManifestLink } from "./links"; +import * as linkClasses from "./links"; import { BackupState } from "./BackupState"; export async function run(args: string[]) { @@ -17,9 +13,13 @@ export async function run(args: string[]) { const state: BackupState = new BackupState(args); const chain: Link[] = [ - new DiskSpaceLink(), - new ManifestLink(state), - new EncryptionLink(state), + new linkClasses.BackupFolderLink(state), + new linkClasses.DiskSpaceLink(), + new linkClasses.ManifestLink(state), + new linkClasses.MongoDumpLink(state), + new linkClasses.GitStorageLink(state), + new linkClasses.EnvFileLink(state), + new linkClasses.EncryptionLink(state), ]; try { @@ -29,19 +29,6 @@ export async function run(args: string[]) { } // BACKUP - state.backupRootPath = await fsPromises.mkdtemp( - path.join(os.tmpdir(), "appsmithctl-backup-"), - ); - - await exportDatabase(state.backupRootPath); - - await createGitStorageArchive(state.backupRootPath); - - await exportDockerEnvFile( - state.backupRootPath, - state.isEncryptionEnabled(), - ); - for (const link of chain) { await link.doBackup?.(); } @@ -58,23 +45,6 @@ export async function run(args: string[]) { console.log("Post-backup done. Final archive at", state.archivePath); - if (!state.isEncryptionEnabled()) { - console.log( - "********************************************************* IMPORTANT!!! *************************************************************", - ); - console.log( - "*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **", - ); - console.log( - "*** These values are not included in the backup export. **", - ); - console.log( - "************************************************************************************************************************************", - ); - } - - await fsPromises.rm(state.backupRootPath, { recursive: true, force: true }); - await logger.backup_info( "Finished taking a backup at " + state.archivePath, ); @@ -116,118 +86,6 @@ export async function run(args: string[]) { } } -export async function encryptBackupArchive( - archivePath: string, - encryptionPassword: string, -) { - const encryptedArchivePath = archivePath + ".enc"; - - await utils.execCommand([ - "openssl", - "enc", - "-aes-256-cbc", - "-pbkdf2", - "-iter", - "100000", - "-in", - archivePath, - "-out", - encryptedArchivePath, - "-k", - encryptionPassword, - ]); - - return encryptedArchivePath; -} - -export function getEncryptionPasswordFromUser(): string { - for (const attempt of [1, 2, 3]) { - if (attempt > 1) { - console.log("Retry attempt", attempt); - } - - const encryptionPwd1: string = readlineSync.question( - "Enter a password to encrypt the backup archive: ", - { hideEchoBack: true }, - ); - const encryptionPwd2: string = readlineSync.question( - "Enter the above password again: ", - { hideEchoBack: true }, - ); - - if (encryptionPwd1 === encryptionPwd2) { - if (encryptionPwd1) { - return encryptionPwd1; - } - - console.error( - "Invalid input. Empty password is not allowed, please try again.", - ); - } else { - console.error("The passwords do not match, please try again."); - } - } - - console.error( - "Aborting backup process, failed to obtain valid encryption password.", - ); - - throw new Error( - "Backup process aborted because a valid encryption password could not be obtained from the user", - ); -} - -async function exportDatabase(destFolder: string) { - console.log("Exporting database"); - await executeMongoDumpCMD(destFolder, utils.getDburl()); - console.log("Exporting database done."); -} - -async function createGitStorageArchive(destFolder: string) { - console.log("Creating git-storage archive"); - - const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT); - - await executeCopyCMD(gitRoot, destFolder); - - console.log("Created git-storage archive"); -} - -async function exportDockerEnvFile( - destFolder: string, - encryptArchive: boolean, -) { - console.log("Exporting docker environment file"); - const content = await fsPromises.readFile( - "/appsmith-stacks/configuration/docker.env", - { encoding: "utf8" }, - ); - let cleaned_content = removeSensitiveEnvData(content); - - if (encryptArchive) { - cleaned_content += - "\nAPPSMITH_ENCRYPTION_SALT=" + - process.env.APPSMITH_ENCRYPTION_SALT + - "\nAPPSMITH_ENCRYPTION_PASSWORD=" + - process.env.APPSMITH_ENCRYPTION_PASSWORD; - } - - await fsPromises.writeFile(destFolder + "/docker.env", cleaned_content); - console.log("Exporting docker environment file done."); -} - -export async function executeMongoDumpCMD( - destFolder: string, - appsmithMongoURI: string, -) { - return await utils.execCommand([ - "mongodump", - `--uri=${appsmithMongoURI}`, - `--archive=${destFolder}/mongodb-data.gz`, - "--gzip", - ]); // generate cmd -} - async function createFinalArchive(destFolder: string, timestamp: string) { console.log("Creating final archive"); @@ -260,23 +118,6 @@ async function postBackupCleanup() { console.log("Cleanup completed."); } -export async function executeCopyCMD(srcFolder: string, destFolder: string) { - return await utils.execCommand([ - "ln", - "-s", - srcFolder, - path.join(destFolder, "git-storage"), - ]); -} - -export function getGitRoot(gitRoot?: string | undefined) { - if (gitRoot == null || gitRoot === "") { - gitRoot = "/appsmith-stacks/git-storage"; - } - - return gitRoot; -} - export function getBackupContentsPath( backupRootPath: string, timestamp: string, @@ -284,23 +125,6 @@ export function getBackupContentsPath( return backupRootPath + "/appsmith-backup-" + timestamp; } -export function removeSensitiveEnvData(content: string): string { - // Remove encryption and Mongodb data from docker.env - const output_lines = []; - - content.split(/\r?\n/).forEach((line) => { - if ( - !line.startsWith("APPSMITH_ENCRYPTION") && - !line.startsWith("APPSMITH_MONGODB") && - !line.startsWith("APPSMITH_DB_URL=") - ) { - output_lines.push(line); - } - }); - - return output_lines.join("\n"); -} - export function getBackupArchiveLimit(backupArchivesLimit?: number): number { return backupArchivesLimit || Constants.APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT; } @@ -318,23 +142,3 @@ export async function removeOldBackups( .map(async (file) => fsPromises.rm(file)), ); } - -export function getTimeStampInISO() { - return new Date().toISOString().replace(/:/g, "-"); -} - -export async function getAvailableBackupSpaceInBytes( - path: string, -): Promise { - const stat = await fsPromises.statfs(path); - - return stat.bsize * stat.bfree; -} - -export function checkAvailableBackupSpace(availSpaceInBytes: number) { - if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) { - throw new Error( - "Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.", - ); - } -} diff --git a/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts b/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts new file mode 100644 index 000000000000..a4a8503622b7 --- /dev/null +++ b/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts @@ -0,0 +1,20 @@ +import type { Link } from "./index"; +import type { BackupState } from "../BackupState"; +import fsPromises from "fs/promises"; +import path from "path"; +import os from "os"; + +export class BackupFolderLink implements Link { + constructor(private readonly state: BackupState) {} + + async preBackup() { + this.state.backupRootPath = await fsPromises.mkdtemp( + path.join(os.tmpdir(), "appsmithctl-backup-"), + ); + } + + async postBackup() { + await fsPromises.rm(this.state.backupRootPath, { recursive: true, force: true }); + } +} + diff --git a/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts b/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts index 8ab16efe7eaf..b4b35a2fd900 100644 --- a/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts @@ -1,5 +1,14 @@ -import { checkAvailableBackupSpace, getAvailableBackupSpaceInBytes } from ".."; import type { Link } from "."; +import * as Constants from "../../constants"; +import fsPromises from "fs/promises"; + +export function checkAvailableBackupSpace(availSpaceInBytes: number) { + if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) { + throw new Error( + "Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.", + ); + } +} export class DiskSpaceLink implements Link { async preBackup() { @@ -9,3 +18,11 @@ export class DiskSpaceLink implements Link { checkAvailableBackupSpace(availSpaceInBytes); } } + +export async function getAvailableBackupSpaceInBytes( + path: string, +): Promise { + const stat = await fsPromises.statfs(path); + + return stat.bsize * stat.bfree; +} diff --git a/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts b/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts index 086c7fc0c8e3..60b827941630 100644 --- a/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts @@ -1,8 +1,9 @@ import type { Link } from "./index"; import tty from "tty"; import fsPromises from "fs/promises"; -import { encryptBackupArchive, getEncryptionPasswordFromUser } from "../index"; import type { BackupState } from "../BackupState"; +import readlineSync from "readline-sync"; +import * as utils from "../../utils"; export class EncryptionLink implements Link { constructor(private readonly state: BackupState) {} @@ -34,3 +35,64 @@ export class EncryptionLink implements Link { }); } } + +export function getEncryptionPasswordFromUser(): string { + for (const attempt of [1, 2, 3]) { + if (attempt > 1) { + console.log("Retry attempt", attempt); + } + + const encryptionPwd1: string = readlineSync.question( + "Enter a password to encrypt the backup archive: ", + {hideEchoBack: true}, + ); + const encryptionPwd2: string = readlineSync.question( + "Enter the above password again: ", + {hideEchoBack: true}, + ); + + if (encryptionPwd1 === encryptionPwd2) { + if (encryptionPwd1) { + return encryptionPwd1; + } + + console.error( + "Invalid input. Empty password is not allowed, please try again.", + ); + } else { + console.error("The passwords do not match, please try again."); + } + } + + console.error( + "Aborting backup process, failed to obtain valid encryption password.", + ); + + throw new Error( + "Backup process aborted because a valid encryption password could not be obtained from the user", + ); +} + +export async function encryptBackupArchive( + archivePath: string, + encryptionPassword: string, +) { + const encryptedArchivePath = archivePath + ".enc"; + + await utils.execCommand([ + "openssl", + "enc", + "-aes-256-cbc", + "-pbkdf2", + "-iter", + "100000", + "-in", + archivePath, + "-out", + encryptedArchivePath, + "-k", + encryptionPassword, + ]); + + return encryptedArchivePath; +} diff --git a/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts b/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts new file mode 100644 index 000000000000..b75811375298 --- /dev/null +++ b/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts @@ -0,0 +1,57 @@ +import type { Link } from "./index"; +import type { BackupState } from "../BackupState"; +import fsPromises from "fs/promises"; + +const SECRETS_WARNING = ` +********************************************************* IMPORTANT!!! ************************************************************* +*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file ** +*** These values are not included in the backup export. ** +************************************************************************************************************************************ +` + +export class EnvFileLink implements Link { + constructor(private readonly state: BackupState) {} + + async doBackup() { + console.log("Exporting docker environment file"); + const content = await fsPromises.readFile( + "/appsmith-stacks/configuration/docker.env", + { encoding: "utf8" }, + ); + let cleanedContent = removeSensitiveEnvData(content); + + if (this.state.isEncryptionEnabled()) { + cleanedContent += + "\nAPPSMITH_ENCRYPTION_SALT=" + + process.env.APPSMITH_ENCRYPTION_SALT + + "\nAPPSMITH_ENCRYPTION_PASSWORD=" + + process.env.APPSMITH_ENCRYPTION_PASSWORD; + } + + await fsPromises.writeFile(this.state.backupRootPath + "/docker.env", cleanedContent); + console.log("Exporting docker environment file done."); + } + + async postBackup() { + if (!this.state.isEncryptionEnabled()) { + console.log(SECRETS_WARNING); + } + } +} + +export function removeSensitiveEnvData(content: string): string { + // Remove encryption and Mongodb data from docker.env + const output_lines = []; + + content.split(/\r?\n/).forEach((line) => { + if ( + !line.startsWith("APPSMITH_ENCRYPTION") && + !line.startsWith("APPSMITH_MONGODB") && + !line.startsWith("APPSMITH_DB_URL=") + ) { + output_lines.push(line); + } + }); + + return output_lines.join("\n"); +} diff --git a/app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts b/app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts new file mode 100644 index 000000000000..81714b408be8 --- /dev/null +++ b/app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts @@ -0,0 +1,39 @@ +import type { Link } from "./index"; +import type { BackupState } from "../BackupState"; +import * as utils from "../../utils"; +import path from "path"; + +export class GitStorageLink implements Link { + constructor(private readonly state: BackupState) {} + + async doBackup() { + await createGitStorageArchive(this.state.backupRootPath); + } +} + +async function createGitStorageArchive(destFolder: string) { + console.log("Creating git-storage archive"); + + const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT); + + await executeCopyCMD(gitRoot, destFolder); + + console.log("Created git-storage archive"); +} + +export function getGitRoot(gitRoot?: string | undefined) { + if (gitRoot == null || gitRoot === "") { + gitRoot = "/appsmith-stacks/git-storage"; + } + + return gitRoot; +} + +export async function executeCopyCMD(srcFolder: string, destFolder: string) { + return await utils.execCommand([ + "ln", + "-s", + srcFolder, + path.join(destFolder, "git-storage"), + ]); +} diff --git a/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts new file mode 100644 index 000000000000..f86d5a7fd211 --- /dev/null +++ b/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts @@ -0,0 +1,29 @@ +import type { Link } from "./index"; +import type { BackupState } from "../BackupState"; +import * as utils from "../../utils"; + +export class MongoDumpLink implements Link { + constructor(private readonly state: BackupState) {} + + async doBackup() { + await exportDatabase(this.state.backupRootPath); + } +} + +export async function executeMongoDumpCMD( + destFolder: string, + appsmithMongoURI: string, +) { + return await utils.execCommand([ + "mongodump", + `--uri=${appsmithMongoURI}`, + `--archive=${destFolder}/mongodb-data.gz`, + "--gzip", + ]); // generate cmd +} + +async function exportDatabase(destFolder: string) { + console.log("Exporting database"); + await executeMongoDumpCMD(destFolder, utils.getDburl()); + console.log("Exporting database done."); +} diff --git a/app/client/packages/rts/src/ctl/backup/links/index.ts b/app/client/packages/rts/src/ctl/backup/links/index.ts index eea6f31bcd55..da48f43c0531 100644 --- a/app/client/packages/rts/src/ctl/backup/links/index.ts +++ b/app/client/packages/rts/src/ctl/backup/links/index.ts @@ -9,5 +9,10 @@ export interface Link { postBackup?(): Promise; } -export { EncryptionLink } from "./EncryptionLink"; -export { ManifestLink } from "./ManifestLink"; +export * from "./BackupFolderLink"; +export * from "./DiskSpaceLink"; +export * from "./EncryptionLink"; +export * from "./EnvFileLink"; +export * from "./GitStorageLink"; +export * from "./ManifestLink"; +export * from "./MongoDumpLink"; From f9c1979868e7107fde28f8b7793934111fdebb0d Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Fri, 29 Nov 2024 11:15:01 +0530 Subject: [PATCH 2/3] fmt --- .../packages/rts/src/ctl/backup/backup.test.ts | 12 +++--------- app/client/packages/rts/src/ctl/backup/index.ts | 3 +++ .../rts/src/ctl/backup/links/BackupFolderLink.ts | 6 ++++-- .../rts/src/ctl/backup/links/DiskSpaceLink.ts | 4 ++-- .../rts/src/ctl/backup/links/EncryptionLink.ts | 4 ++-- .../packages/rts/src/ctl/backup/links/EnvFileLink.ts | 7 +++++-- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/app/client/packages/rts/src/ctl/backup/backup.test.ts b/app/client/packages/rts/src/ctl/backup/backup.test.ts index e5cd196321c8..7fa9ce48d9d3 100644 --- a/app/client/packages/rts/src/ctl/backup/backup.test.ts +++ b/app/client/packages/rts/src/ctl/backup/backup.test.ts @@ -11,9 +11,8 @@ import { getAvailableBackupSpaceInBytes, getEncryptionPasswordFromUser, getGitRoot, - removeSensitiveEnvData + removeSensitiveEnvData, } from "./links"; -import { getTimeStampInISO } from "."; jest.mock("../utils", () => ({ ...jest.requireActual("../utils"), @@ -41,9 +40,7 @@ describe("Backup Tests", () => { it("Should not should throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { expect(() => { - checkAvailableBackupSpace( - Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES, - ); + checkAvailableBackupSpace(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES); }).not.toThrow( "Not enough space available at /appsmith-stacks. Please ensure availability of at least 5GB to backup successfully.", ); @@ -238,10 +235,7 @@ describe("Backup Tests", () => { test("Test backup encryption function", async () => { const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; const encryptionPassword = "password#123"; - const res = await encryptBackupArchive( - archivePath, - encryptionPassword, - ); + const res = await encryptBackupArchive(archivePath, encryptionPassword); console.log(res); expect(res).toEqual("/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z.enc"); diff --git a/app/client/packages/rts/src/ctl/backup/index.ts b/app/client/packages/rts/src/ctl/backup/index.ts index 9d1782fc7792..0e2d70c30868 100644 --- a/app/client/packages/rts/src/ctl/backup/index.ts +++ b/app/client/packages/rts/src/ctl/backup/index.ts @@ -19,6 +19,9 @@ export async function run(args: string[]) { new linkClasses.MongoDumpLink(state), new linkClasses.GitStorageLink(state), new linkClasses.EnvFileLink(state), + + // Encryption link is best placed last so if any of the above links fail, we don't ask the user for a password and + // then do nothing with it. new linkClasses.EncryptionLink(state), ]; diff --git a/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts b/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts index a4a8503622b7..bc6d6fdf2bf0 100644 --- a/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts @@ -14,7 +14,9 @@ export class BackupFolderLink implements Link { } async postBackup() { - await fsPromises.rm(this.state.backupRootPath, { recursive: true, force: true }); + await fsPromises.rm(this.state.backupRootPath, { + recursive: true, + force: true, + }); } } - diff --git a/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts b/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts index b4b35a2fd900..ece5cd74f32b 100644 --- a/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts @@ -5,7 +5,7 @@ import fsPromises from "fs/promises"; export function checkAvailableBackupSpace(availSpaceInBytes: number) { if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) { throw new Error( - "Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.", + "Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.", ); } } @@ -20,7 +20,7 @@ export class DiskSpaceLink implements Link { } export async function getAvailableBackupSpaceInBytes( - path: string, + path: string, ): Promise { const stat = await fsPromises.statfs(path); diff --git a/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts b/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts index 60b827941630..76c683d3eafc 100644 --- a/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts @@ -44,11 +44,11 @@ export function getEncryptionPasswordFromUser(): string { const encryptionPwd1: string = readlineSync.question( "Enter a password to encrypt the backup archive: ", - {hideEchoBack: true}, + { hideEchoBack: true }, ); const encryptionPwd2: string = readlineSync.question( "Enter the above password again: ", - {hideEchoBack: true}, + { hideEchoBack: true }, ); if (encryptionPwd1 === encryptionPwd2) { diff --git a/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts b/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts index b75811375298..bd8518590e5e 100644 --- a/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts @@ -7,7 +7,7 @@ const SECRETS_WARNING = ` *** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file ** *** These values are not included in the backup export. ** ************************************************************************************************************************************ -` +`; export class EnvFileLink implements Link { constructor(private readonly state: BackupState) {} @@ -28,7 +28,10 @@ export class EnvFileLink implements Link { process.env.APPSMITH_ENCRYPTION_PASSWORD; } - await fsPromises.writeFile(this.state.backupRootPath + "/docker.env", cleanedContent); + await fsPromises.writeFile( + this.state.backupRootPath + "/docker.env", + cleanedContent, + ); console.log("Exporting docker environment file done."); } From da90b4ec769ef4997380062bb4541cb22fae6f9f Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Fri, 29 Nov 2024 11:43:25 +0530 Subject: [PATCH 3/3] Add docs and minor refactors --- .../rts/src/ctl/backup/BackupState.ts | 10 +++----- .../src/ctl/backup/links/BackupFolderLink.ts | 6 ++++- .../rts/src/ctl/backup/links/DiskSpaceLink.ts | 19 ++++++++------- .../src/ctl/backup/links/EncryptionLink.ts | 16 +++++++++---- .../rts/src/ctl/backup/links/EnvFileLink.ts | 19 +++++++++------ .../src/ctl/backup/links/GitStorageLink.ts | 20 +++++++--------- .../rts/src/ctl/backup/links/ManifestLink.ts | 5 +++- .../rts/src/ctl/backup/links/MongoDumpLink.ts | 24 ++++++++----------- 8 files changed, 66 insertions(+), 53 deletions(-) diff --git a/app/client/packages/rts/src/ctl/backup/BackupState.ts b/app/client/packages/rts/src/ctl/backup/BackupState.ts index 73d1e0ed926f..9192f654d51e 100644 --- a/app/client/packages/rts/src/ctl/backup/BackupState.ts +++ b/app/client/packages/rts/src/ctl/backup/BackupState.ts @@ -1,23 +1,19 @@ export class BackupState { - readonly args: string[]; + readonly args: readonly string[]; readonly initAt: string = new Date().toISOString().replace(/:/g, "-"); readonly errors: string[] = []; backupRootPath: string = ""; archivePath: string = ""; - encryptionPassword: string = ""; + isEncryptionEnabled: boolean = false; constructor(args: string[]) { - this.args = args; + this.args = Object.freeze([...args]); // We seal `this` so that no link in the chain can "add" new properties to the state. This is intentional. If any // link wants to save data in the `BackupState`, which shouldn't even be needed in most cases, it should do so by // explicitly declaring a property in this class. No surprises. Object.seal(this); } - - isEncryptionEnabled() { - return !!this.encryptionPassword; - } } diff --git a/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts b/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts index bc6d6fdf2bf0..95539fe8a141 100644 --- a/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/BackupFolderLink.ts @@ -1,9 +1,13 @@ -import type { Link } from "./index"; +import type { Link } from "."; import type { BackupState } from "../BackupState"; import fsPromises from "fs/promises"; import path from "path"; import os from "os"; +/** + * Creates the backup folder in pre step, and deletes it in post step. The existence of the backup folder should only + * be assumed in the "doBackup" step, and no other. + */ export class BackupFolderLink implements Link { constructor(private readonly state: BackupState) {} diff --git a/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts b/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts index ece5cd74f32b..950dcde9d7b3 100644 --- a/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/DiskSpaceLink.ts @@ -2,14 +2,9 @@ import type { Link } from "."; import * as Constants from "../../constants"; import fsPromises from "fs/promises"; -export function checkAvailableBackupSpace(availSpaceInBytes: number) { - if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) { - throw new Error( - "Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.", - ); - } -} - +/** + * Checks if there is enough space available at the backup location. + */ export class DiskSpaceLink implements Link { async preBackup() { const availSpaceInBytes: number = @@ -26,3 +21,11 @@ export async function getAvailableBackupSpaceInBytes( return stat.bsize * stat.bfree; } + +export function checkAvailableBackupSpace(availSpaceInBytes: number) { + if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) { + throw new Error( + "Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.", + ); + } +} diff --git a/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts b/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts index 76c683d3eafc..6b6f294544c5 100644 --- a/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/EncryptionLink.ts @@ -1,11 +1,17 @@ -import type { Link } from "./index"; +import type { Link } from "."; import tty from "tty"; import fsPromises from "fs/promises"; import type { BackupState } from "../BackupState"; import readlineSync from "readline-sync"; import * as utils from "../../utils"; +/** + * Asks the user for a password, and then encrypts the backup archive using openssl, with that password. If a TTY is not + * available to ask for a password, then this feature is gracefully disabled, and encryption is not performed. + */ export class EncryptionLink implements Link { + #password: string = ""; + constructor(private readonly state: BackupState) {} async preBackup() { @@ -13,12 +19,14 @@ export class EncryptionLink implements Link { !this.state.args.includes("--non-interactive") && tty.isatty((process.stdout as any).fd) ) { - this.state.encryptionPassword = getEncryptionPasswordFromUser(); + this.#password = getEncryptionPasswordFromUser(); } + + this.state.isEncryptionEnabled = !!this.#password; } async postBackup() { - if (!this.state.isEncryptionEnabled()) { + if (!this.#password) { return; } @@ -26,7 +34,7 @@ export class EncryptionLink implements Link { this.state.archivePath = await encryptBackupArchive( unencryptedArchivePath, - this.state.encryptionPassword, + this.#password, ); await fsPromises.rm(unencryptedArchivePath, { diff --git a/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts b/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts index bd8518590e5e..3c5201aae401 100644 --- a/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/EnvFileLink.ts @@ -1,14 +1,19 @@ -import type { Link } from "./index"; +import type { Link } from "."; import type { BackupState } from "../BackupState"; import fsPromises from "fs/promises"; const SECRETS_WARNING = ` -********************************************************* IMPORTANT!!! ************************************************************* -*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file ** -*** These values are not included in the backup export. ** -************************************************************************************************************************************ +***************************** IMPORTANT!!! ***************************** +*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and *** +*** APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file. *** +*** These values are not included in the backup export. *** +************************************************************************ `; +/** + * Exports the docker environment file to the backup folder. If encryption is not enabled, sensitive information is + * not written to the backup folder. + */ export class EnvFileLink implements Link { constructor(private readonly state: BackupState) {} @@ -20,7 +25,7 @@ export class EnvFileLink implements Link { ); let cleanedContent = removeSensitiveEnvData(content); - if (this.state.isEncryptionEnabled()) { + if (this.state.isEncryptionEnabled) { cleanedContent += "\nAPPSMITH_ENCRYPTION_SALT=" + process.env.APPSMITH_ENCRYPTION_SALT + @@ -36,7 +41,7 @@ export class EnvFileLink implements Link { } async postBackup() { - if (!this.state.isEncryptionEnabled()) { + if (!this.state.isEncryptionEnabled) { console.log(SECRETS_WARNING); } } diff --git a/app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts b/app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts index 81714b408be8..21c2c0934fd4 100644 --- a/app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/GitStorageLink.ts @@ -1,24 +1,22 @@ -import type { Link } from "./index"; +import type { Link } from "."; import type { BackupState } from "../BackupState"; import * as utils from "../../utils"; import path from "path"; +/** + * Copies the `git-storage` folder to the backup folder. + */ export class GitStorageLink implements Link { constructor(private readonly state: BackupState) {} async doBackup() { - await createGitStorageArchive(this.state.backupRootPath); - } -} - -async function createGitStorageArchive(destFolder: string) { - console.log("Creating git-storage archive"); + console.log("Creating git-storage archive"); - const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT); + const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT); - await executeCopyCMD(gitRoot, destFolder); - - console.log("Created git-storage archive"); + await executeCopyCMD(gitRoot, this.state.backupRootPath); + console.log("Created git-storage archive"); + } } export function getGitRoot(gitRoot?: string | undefined) { diff --git a/app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts b/app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts index 16998203ac1b..c9958aa52c9c 100644 --- a/app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts @@ -1,9 +1,12 @@ -import type { Link } from "./index"; +import type { Link } from "."; import type { BackupState } from "../BackupState"; import * as utils from "../../utils"; import fsPromises from "fs/promises"; import path from "path"; +/** + * Creates a manifest file that contains metadata about the backup. + */ export class ManifestLink implements Link { constructor(private readonly state: BackupState) {} diff --git a/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts b/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts index f86d5a7fd211..13400928bb11 100644 --- a/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts +++ b/app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts @@ -1,29 +1,25 @@ -import type { Link } from "./index"; +import type { Link } from "."; import type { BackupState } from "../BackupState"; import * as utils from "../../utils"; +/** + * Exports the MongoDB database data using mongodump. + */ export class MongoDumpLink implements Link { constructor(private readonly state: BackupState) {} async doBackup() { - await exportDatabase(this.state.backupRootPath); + console.log("Exporting database"); + await executeMongoDumpCMD(this.state.backupRootPath, utils.getDburl()); + console.log("Exporting database done."); } } -export async function executeMongoDumpCMD( - destFolder: string, - appsmithMongoURI: string, -) { +export async function executeMongoDumpCMD(destFolder: string, dbUrl: string) { return await utils.execCommand([ "mongodump", - `--uri=${appsmithMongoURI}`, + `--uri=${dbUrl}`, `--archive=${destFolder}/mongodb-data.gz`, "--gzip", - ]); // generate cmd -} - -async function exportDatabase(destFolder: string) { - console.log("Exporting database"); - await executeMongoDumpCMD(destFolder, utils.getDburl()); - console.log("Exporting database done."); + ]); }