diff --git a/app/client/packages/rts/build.js b/app/client/packages/rts/build.js index 414147574c65..7743ae0d076f 100644 --- a/app/client/packages/rts/build.js +++ b/app/client/packages/rts/build.js @@ -44,7 +44,7 @@ const getWorkflowDependencies = () => { const bundle = async () => { return esbuild .build({ - entryPoints: ["src/server.ts", "src/ctl/index.js"], + entryPoints: ["src/server.ts", "src/ctl/index.ts"], bundle: true, sourcemap: true, platform: "node", diff --git a/app/client/packages/rts/package.json b/app/client/packages/rts/package.json index 28c80dda5bbe..8f27c36d6bef 100644 --- a/app/client/packages/rts/package.json +++ b/app/client/packages/rts/package.json @@ -36,6 +36,8 @@ "devDependencies": { "@types/express": "^4.17.14", "@types/jest": "^29.2.3", + "@types/nodemailer": "^6.4.17", + "@types/readline-sync": "^1.4.8", "jest": "^29.3.1", "supertest": "^6.3.3", "ts-jest": "29.1.0", diff --git a/app/client/packages/rts/src/ctl/backup.test.js b/app/client/packages/rts/src/ctl/backup.test.ts similarity index 76% rename from app/client/packages/rts/src/ctl/backup.test.js rename to app/client/packages/rts/src/ctl/backup.test.ts index ac5309d0e109..c6ff06dbee9c 100644 --- a/app/client/packages/rts/src/ctl/backup.test.js +++ b/app/client/packages/rts/src/ctl/backup.test.ts @@ -1,15 +1,20 @@ -const backup = require("./backup"); -const Constants = require("./constants"); -const os = require("os"); -const fsPromises = require("fs/promises"); -const utils = require("./utils"); -const readlineSync = require("readline-sync"); +jest.mock("./utils", () => ({ + ...jest.requireActual("./utils"), + execCommand: jest.fn().mockImplementation(async (a) => a.join(" ")), +})); + +import * as backup from "./backup"; +import * as Constants from "./constants"; +import os from "os"; +import fsPromises from "fs/promises"; +import * as utils from "./utils"; +import readlineSync from "readline-sync"; describe("Backup Tests", () => { test("Timestamp string in ISO format", () => { console.log(backup.getTimeStampInISO()); expect(backup.getTimeStampInISO()).toMatch( - /(\d{4})-(\d{2})-(\d{2})T(\d{2})\-(\d{2})\-(\d{2})\.(\d{3})Z/, + /(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})\.(\d{3})Z/, ); }); @@ -18,23 +23,23 @@ describe("Backup Tests", () => { res.toBeGreaterThan(1024 * 1024); }); - it("Checkx the constant is 2 GB", () => { - let size = 2 * 1024 * 1024 * 1024; + it("Check the constant is 2 GB", () => { + const size = 2 * 1024 * 1024 * 1024; expect(Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES).toBe(size); }); it("Should throw Error when the available size is below MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { - let size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1; + const size = Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES - 1; expect(() => backup.checkAvailableBackupSpace(size)).toThrow(); }); - it("Should not hould throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { + it("Should not should throw Error when the available size is >= MIN_REQUIRED_DISK_SPACE_IN_BYTES", () => { expect(() => { backup.checkAvailableBackupSpace( Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES, ); }).not.toThrow( - "Not enough space avaliable at /appsmith-stacks. Please ensure availability of atleast 5GB to backup successfully.", + "Not enough space available at /appsmith-stacks. Please ensure availability of at least 5GB to backup successfully.", ); }); @@ -46,19 +51,18 @@ describe("Backup Tests", () => { }); test("Test backup contents path generation", () => { - var root = "/rootDir"; - var timestamp = "0000-00-0T00-00-00.00Z"; + const root = "/rootDir"; + const timestamp = "0000-00-0T00-00-00.00Z"; expect(backup.getBackupContentsPath(root, timestamp)).toBe( "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z", ); }); - test("Test mongodump CMD generaton", async () => { - var dest = "/dest"; - var appsmithMongoURI = "mongodb://username:password@host/appsmith"; - var cmd = + test("Test mongodump CMD generation", async () => { + const dest = "/dest"; + const appsmithMongoURI = "mongodb://username:password@host/appsmith"; + const cmd = "mongodump --uri=mongodb://username:password@host/appsmith --archive=/dest/mongodb-data.gz --gzip"; - utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(" ")); const res = await backup.executeMongoDumpCMD(dest, appsmithMongoURI); expect(res).toBe(cmd); console.log(res); @@ -77,10 +81,9 @@ describe("Backup Tests", () => { }); test("Test ln command generation", async () => { - var gitRoot = "/appsmith-stacks/git-storage"; - var dest = "/destdir"; - var cmd = "ln -s /appsmith-stacks/git-storage /destdir/git-storage"; - utils.execCommand = jest.fn().mockImplementation(async (a) => a.join(" ")); + const gitRoot = "/appsmith-stacks/git-storage"; + const dest = "/destdir"; + const cmd = "ln -s /appsmith-stacks/git-storage /destdir/git-storage"; const res = await backup.executeCopyCMD(gitRoot, dest); expect(res).toBe(cmd); console.log(res); @@ -127,8 +130,8 @@ describe("Backup Tests", () => { test("Cleanup Backups when limit is 4 and there are 5 files", async () => { const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ["file1", "file2", "file3", "file4", "file5"]; - var expectedBackupFiles = ["file2", "file3", "file4", "file5"]; + const backupFiles = ["file1", "file2", "file3", "file4", "file5"]; + const expectedBackupFiles = ["file2", "file3", "file4", "file5"]; const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); console.log(res); @@ -138,8 +141,8 @@ describe("Backup Tests", () => { test("Cleanup Backups when limit is 2 and there are 5 files", async () => { const backupArchivesLimit = 2; fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ["file1", "file2", "file3", "file4", "file5"]; - var expectedBackupFiles = ["file4", "file5"]; + const backupFiles = ["file1", "file2", "file3", "file4", "file5"]; + const expectedBackupFiles = ["file4", "file5"]; const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); console.log(res); @@ -149,8 +152,8 @@ describe("Backup Tests", () => { test("Cleanup Backups when limit is 4 and there are 4 files", async () => { const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ["file1", "file2", "file3", "file4"]; - var expectedBackupFiles = ["file1", "file2", "file3", "file4"]; + const backupFiles = ["file1", "file2", "file3", "file4"]; + const expectedBackupFiles = ["file1", "file2", "file3", "file4"]; const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); console.log(res); @@ -160,8 +163,8 @@ describe("Backup Tests", () => { test("Cleanup Backups when limit is 4 and there are 2 files", async () => { const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ["file1", "file2"]; - var expectedBackupFiles = ["file1", "file2"]; + const backupFiles = ["file1", "file2"]; + const expectedBackupFiles = ["file1", "file2"]; const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); console.log(res); @@ -171,8 +174,8 @@ describe("Backup Tests", () => { test("Cleanup Backups when limit is 2 and there is 1 file", async () => { const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = ["file1"]; - var expectedBackupFiles = ["file1"]; + const backupFiles = ["file1"]; + const expectedBackupFiles = ["file1"]; const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); console.log(res); expect(res).toEqual(expectedBackupFiles); @@ -181,24 +184,22 @@ describe("Backup Tests", () => { test("Cleanup Backups when limit is 2 and there is no file", async () => { const backupArchivesLimit = 4; fsPromises.rm = jest.fn().mockImplementation(async (a) => console.log(a)); - var backupFiles = []; - var expectedBackupFiles = []; + const backupFiles = []; + const expectedBackupFiles = []; const res = await backup.removeOldBackups(backupFiles, backupArchivesLimit); console.log(res); expect(res).toEqual(expectedBackupFiles); }); - test("Test get encryption password from user prompt whene both passords are the same", async () => { + test("Test get encryption password from user prompt when both passwords are the same", async () => { const password = "password#4321"; - readlineSync.question = jest.fn().mockImplementation((a) => { - return password; - }); + readlineSync.question = jest.fn().mockImplementation(() => password); const password_res = backup.getEncryptionPasswordFromUser(); expect(password_res).toEqual(password); }); - test("Test get encryption password from user prompt when both passords are the different", async () => { + test("Test get encryption password from user prompt when both passwords are the different", async () => { const password = "password#4321"; readlineSync.question = jest.fn().mockImplementation((a) => { if (a == "Enter the above password again: ") { @@ -214,9 +215,6 @@ describe("Backup Tests", () => { test("Get encrypted archive path", async () => { const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; const encryptionPassword = "password#4321"; - utils.execCommand = jest - .fn() - .mockImplementation(async (a) => console.log(a)); const encArchivePath = await backup.encryptBackupArchive( archivePath, encryptionPassword, @@ -228,9 +226,6 @@ describe("Backup Tests", () => { }); test("Test backup encryption function", async () => { - utils.execCommand = jest - .fn() - .mockImplementation(async (a) => console.log(a)); const archivePath = "/rootDir/appsmith-backup-0000-00-0T00-00-00.00Z"; const encryptionPassword = "password#123"; const res = await backup.encryptBackupArchive( @@ -243,32 +238,32 @@ describe("Backup Tests", () => { }); test("Get DB name from Mongo URI 1", async () => { - var mongodb_uri = + const mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"; - var expectedDBName = "my_db_name"; + const expectedDBName = "my_db_name"; const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); expect(dbName).toEqual(expectedDBName); }); test("Get DB name from Mongo URI 2", async () => { - var mongodb_uri = + const mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"; - var expectedDBName = "test123"; + const expectedDBName = "test123"; const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); expect(dbName).toEqual(expectedDBName); }); test("Get DB name from Mongo URI 3", async () => { - var mongodb_uri = + const mongodb_uri = "mongodb+srv://admin:password@test.cluster.mongodb.net/test123"; - var expectedDBName = "test123"; + const expectedDBName = "test123"; const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); expect(dbName).toEqual(expectedDBName); }); test("Get DB name from Mongo URI 4", async () => { - var mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith"; - var expectedDBName = "appsmith"; + const mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith"; + const expectedDBName = "appsmith"; const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri); expect(dbName).toEqual(expectedDBName); }); diff --git a/app/client/packages/rts/src/ctl/backup.js b/app/client/packages/rts/src/ctl/backup.ts similarity index 85% rename from app/client/packages/rts/src/ctl/backup.js rename to app/client/packages/rts/src/ctl/backup.ts index a529019aa44b..a8eeaebe8e62 100644 --- a/app/client/packages/rts/src/ctl/backup.js +++ b/app/client/packages/rts/src/ctl/backup.ts @@ -1,16 +1,16 @@ -const fsPromises = require("fs/promises"); -const path = require("path"); -const os = require("os"); -const utils = require("./utils"); -const Constants = require("./constants"); -const logger = require("./logger"); -const mailer = require("./mailer"); -const tty = require("tty"); -const readlineSync = require("readline-sync"); +import fsPromises from "fs/promises"; +import path from "path"; +import os from "os"; +import * as utils from "./utils"; +import * as Constants from "./constants"; +import * as logger from "./logger"; +import * as mailer from "./mailer"; +import tty from "tty"; +import readlineSync from "readline-sync"; const command_args = process.argv.slice(3); -async function run() { +export async function run() { const timestamp = getTimeStampInISO(); let errorCode = 0; let backupRootPath, archivePath, encryptionPassword; @@ -116,7 +116,7 @@ async function run() { } } -async function encryptBackupArchive(archivePath, encryptionPassword) { +export async function encryptBackupArchive(archivePath, encryptionPassword) { const encryptedArchivePath = archivePath + ".enc"; await utils.execCommand([ "openssl", @@ -135,7 +135,7 @@ async function encryptBackupArchive(archivePath, encryptionPassword) { return encryptedArchivePath; } -function getEncryptionPasswordFromUser() { +export function getEncryptionPasswordFromUser() { for (const _ of [1, 2, 3]) { const encryptionPwd1 = readlineSync.question( "Enter a password to encrypt the backup archive: ", @@ -208,7 +208,7 @@ async function exportDockerEnvFile(destFolder, encryptArchive) { console.log("Exporting docker environment file done."); } -async function executeMongoDumpCMD(destFolder, appsmithMongoURI) { +export async function executeMongoDumpCMD(destFolder, appsmithMongoURI) { return await utils.execCommand([ "mongodump", `--uri=${appsmithMongoURI}`, @@ -238,7 +238,7 @@ async function createFinalArchive(destFolder, timestamp) { async function postBackupCleanup() { console.log("Starting the cleanup task after taking a backup."); - let backupArchivesLimit = getBackupArchiveLimit( + const backupArchivesLimit = getBackupArchiveLimit( process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT, ); const backupFiles = await utils.listLocalBackupFiles(); @@ -248,7 +248,8 @@ async function postBackupCleanup() { } console.log("Cleanup task completed."); } -async function executeCopyCMD(srcFolder, destFolder) { + +export async function executeCopyCMD(srcFolder, destFolder) { return await utils.execCommand([ "ln", "-s", @@ -257,22 +258,22 @@ async function executeCopyCMD(srcFolder, destFolder) { ]); } -function getGitRoot(gitRoot) { +export function getGitRoot(gitRoot?) { if (gitRoot == null || gitRoot === "") { gitRoot = "/appsmith-stacks/git-storage"; } return gitRoot; } -function generateBackupRootPath() { +export function generateBackupRootPath() { return fsPromises.mkdtemp(path.join(os.tmpdir(), "appsmithctl-backup-")); } -function getBackupContentsPath(backupRootPath, timestamp) { +export function getBackupContentsPath(backupRootPath, timestamp) { return backupRootPath + "/appsmith-backup-" + timestamp; } -function removeSensitiveEnvData(content) { +export function removeSensitiveEnvData(content) { // Remove encryption and Mongodb data from docker.env const output_lines = []; content.split(/\r?\n/).forEach((line) => { @@ -287,13 +288,13 @@ function removeSensitiveEnvData(content) { return output_lines.join("\n"); } -function getBackupArchiveLimit(backupArchivesLimit) { +export function getBackupArchiveLimit(backupArchivesLimit?) { if (!backupArchivesLimit) backupArchivesLimit = Constants.APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT; return backupArchivesLimit; } -async function removeOldBackups(backupFiles, backupArchivesLimit) { +export async function removeOldBackups(backupFiles, backupArchivesLimit) { while (backupFiles.length > backupArchivesLimit) { const fileName = backupFiles.shift(); await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName); @@ -301,36 +302,19 @@ async function removeOldBackups(backupFiles, backupArchivesLimit) { return backupFiles; } -function getTimeStampInISO() { +export function getTimeStampInISO() { return new Date().toISOString().replace(/:/g, "-"); } -async function getAvailableBackupSpaceInBytes(path) { +export async function getAvailableBackupSpaceInBytes(path) { const stat = await fsPromises.statfs(path); return stat.bsize * stat.bfree; } -function checkAvailableBackupSpace(availSpaceInBytes) { +export function checkAvailableBackupSpace(availSpaceInBytes) { if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) { throw new Error( "Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.", ); } } - -module.exports = { - run, - getTimeStampInISO, - getAvailableBackupSpaceInBytes, - checkAvailableBackupSpace, - generateBackupRootPath, - getBackupContentsPath, - executeMongoDumpCMD, - getGitRoot, - executeCopyCMD, - removeSensitiveEnvData, - getBackupArchiveLimit, - removeOldBackups, - getEncryptionPasswordFromUser, - encryptBackupArchive, -}; diff --git a/app/client/packages/rts/src/ctl/check_replica_set.js b/app/client/packages/rts/src/ctl/check_replica_set.ts similarity index 83% rename from app/client/packages/rts/src/ctl/check_replica_set.js rename to app/client/packages/rts/src/ctl/check_replica_set.ts index 0564107094cd..d9b00594357c 100644 --- a/app/client/packages/rts/src/ctl/check_replica_set.js +++ b/app/client/packages/rts/src/ctl/check_replica_set.ts @@ -1,13 +1,14 @@ -const { MongoClient, MongoServerError } = require("mongodb"); -const { preprocessMongoDBURI } = require("./utils"); +import { MongoClient, MongoServerError } from "mongodb"; -async function exec() { +import { preprocessMongoDBURI } from "./utils"; + +export async function exec() { const client = new MongoClient( preprocessMongoDBURI(process.env.APPSMITH_DB_URL), { useNewUrlParser: true, useUnifiedTopology: true, - }, + } as any, ); let isReplicaSetEnabled = false; @@ -23,9 +24,9 @@ async function exec() { process.exit(isReplicaSetEnabled ? 0 : 1); } -async function checkReplicaSet(client) { +async function checkReplicaSet(client: MongoClient) { await client.connect(); - return await new Promise((resolve) => { + return await new Promise((resolve) => { try { const changeStream = client .db() @@ -56,7 +57,3 @@ async function checkReplicaSet(client) { } }); } - -module.exports = { - exec, -}; diff --git a/app/client/packages/rts/src/ctl/constants.js b/app/client/packages/rts/src/ctl/constants.js deleted file mode 100644 index 6c0f920cb569..000000000000 --- a/app/client/packages/rts/src/ctl/constants.js +++ /dev/null @@ -1,29 +0,0 @@ -const BACKUP_PATH = "/appsmith-stacks/data/backup"; - -const RESTORE_PATH = "/appsmith-stacks/data/restore"; - -const DUMP_FILE_NAME = "appsmith-data.archive"; - -const APPSMITHCTL_LOG_PATH = "/appsmith-stacks/logs/appsmithctl"; - -const LAST_ERROR_MAIL_TS = "/appsmith-stacks/data/backup/last-error-mail-ts"; - -const ENV_PATH = "/appsmith-stacks/configuration/docker.env"; - -const MIN_REQUIRED_DISK_SPACE_IN_BYTES = 2_147_483_648; // 2GB - -const DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC = 21_600_000; // 6 hrs - -const APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT = 4; // 4 backup archives - -module.exports = { - BACKUP_PATH, - RESTORE_PATH, - DUMP_FILE_NAME, - LAST_ERROR_MAIL_TS, - APPSMITHCTL_LOG_PATH, - MIN_REQUIRED_DISK_SPACE_IN_BYTES, - DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC, - APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT, - ENV_PATH, -}; diff --git a/app/client/packages/rts/src/ctl/constants.ts b/app/client/packages/rts/src/ctl/constants.ts new file mode 100644 index 000000000000..f47189f5089f --- /dev/null +++ b/app/client/packages/rts/src/ctl/constants.ts @@ -0,0 +1,18 @@ +export const BACKUP_PATH = "/appsmith-stacks/data/backup"; + +export const RESTORE_PATH = "/appsmith-stacks/data/restore"; + +export const DUMP_FILE_NAME = "appsmith-data.archive"; + +export const APPSMITHCTL_LOG_PATH = "/appsmith-stacks/logs/appsmithctl"; + +export const LAST_ERROR_MAIL_TS = + "/appsmith-stacks/data/backup/last-error-mail-ts"; + +export const ENV_PATH = "/appsmith-stacks/configuration/docker.env"; + +export const MIN_REQUIRED_DISK_SPACE_IN_BYTES = 2_147_483_648; // 2GB + +export const DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC = 21_600_000; // 6 hrs + +export const APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT = 4; // 4 backup archives diff --git a/app/client/packages/rts/src/ctl/export_db.js b/app/client/packages/rts/src/ctl/export_db.ts similarity index 84% rename from app/client/packages/rts/src/ctl/export_db.js rename to app/client/packages/rts/src/ctl/export_db.ts index 6518f7074c78..844bd43974e6 100644 --- a/app/client/packages/rts/src/ctl/export_db.js +++ b/app/client/packages/rts/src/ctl/export_db.ts @@ -1,8 +1,8 @@ -const fsPromises = require("fs/promises"); -const Constants = require("./constants"); -const utils = require("./utils"); +import fsPromises from "fs/promises"; +import * as Constants from "./constants"; +import * as utils from "./utils"; -async function exportDatabase() { +export async function exportDatabase() { console.log("export_database ...."); const dbUrl = utils.getDburl(); await fsPromises.mkdir(Constants.BACKUP_PATH, { recursive: true }); @@ -15,7 +15,7 @@ async function exportDatabase() { console.log("export_database done"); } -async function run() { +export async function run() { let errorCode = 0; await utils.ensureSupervisorIsRunning(); @@ -42,8 +42,3 @@ async function run() { process.exit(errorCode); } } - -module.exports = { - run, - exportDatabase, -}; diff --git a/app/client/packages/rts/src/ctl/import_db.js b/app/client/packages/rts/src/ctl/import_db.ts similarity index 84% rename from app/client/packages/rts/src/ctl/import_db.js rename to app/client/packages/rts/src/ctl/import_db.ts index c7c40f903730..d00f16d1885c 100644 --- a/app/client/packages/rts/src/ctl/import_db.js +++ b/app/client/packages/rts/src/ctl/import_db.ts @@ -1,7 +1,7 @@ -const readlineSync = require("readline-sync"); -const process = require("process"); -const Constants = require("./constants"); -const utils = require("./utils"); +import readlineSync from "readline-sync"; +import process from "process"; +import * as Constants from "./constants"; +import * as utils from "./utils"; async function importDatabase() { console.log("Importing the database"); @@ -21,7 +21,7 @@ async function importDatabase() { } // Main application workflow -async function run(forceOption) { +export async function run(forceOption) { let errorCode = 0; await utils.ensureSupervisorIsRunning(); @@ -29,8 +29,9 @@ async function run(forceOption) { try { console.log("stop backend & rts application before import database"); await utils.stop(["backend", "rts"]); + let shellCmdResult: string; try { - const shellCmdResult = await utils.execCommandReturningOutput([ + shellCmdResult = await utils.execCommandReturningOutput([ "mongo", process.env.APPSMITH_DB_URL, "--quiet", @@ -41,7 +42,7 @@ async function run(forceOption) { console.error("Failed to execute mongo command:", error); throw error; } - const collectionsLen = parseInt(shellCmdResult.stdout.toString().trimEnd()); + const collectionsLen = parseInt(shellCmdResult.trimEnd()); if (collectionsLen > 0) { if (forceOption) { await importDatabase(); @@ -79,7 +80,3 @@ async function run(forceOption) { process.exit(errorCode); } } - -module.exports = { - run, -}; diff --git a/app/client/packages/rts/src/ctl/index.js b/app/client/packages/rts/src/ctl/index.ts similarity index 55% rename from app/client/packages/rts/src/ctl/index.js rename to app/client/packages/rts/src/ctl/index.ts index a307792d1cd1..0cec2e5924bb 100755 --- a/app/client/packages/rts/src/ctl/index.js +++ b/app/client/packages/rts/src/ctl/index.ts @@ -1,12 +1,15 @@ #!/usr/bin/env node -const process = require("process"); -const utils = require("./utils"); -const export_db = require("./export_db.js"); -const import_db = require("./import_db.js"); -const check_replica_set = require("./check_replica_set.js"); -const version = require("./version.js"); -const mongo_shell_utils = require("./mongo_shell_utils.js"); +import process from "process"; +import { showHelp } from "./utils"; +import * as export_db from "./export_db"; +import * as import_db from "./import_db"; +import * as backup from "./backup"; +import * as restore from "./restore"; +import * as check_replica_set from "./check_replica_set"; +import * as version from "./version"; +import * as mongo_shell_utils from "./mongo_shell_utils"; +import { config } from "dotenv"; const APPLICATION_CONFIG_PATH = "/appsmith-stacks/configuration/docker.env"; @@ -17,7 +20,7 @@ if (!process.env.APPSMITH_DB_URL) { } // Loading latest application configuration -require("dotenv").config({ path: APPLICATION_CONFIG_PATH }); +config({ path: APPLICATION_CONFIG_PATH }); // AGAIN: Check if APPSMITH_DB_URL is set, if not set, fall back to APPSMITH_MONGODB_URI if (!process.env.APPSMITH_DB_URL) { @@ -31,10 +34,7 @@ if (["export-db", "export_db", "ex"].includes(command)) { console.log("Exporting database"); export_db.run(); console.log("Export database done"); - return; -} - -if (["import-db", "import_db", "im"].includes(command)) { +} else if (["import-db", "import_db", "im"].includes(command)) { console.log("Importing database"); // Get Force option flag to run import DB immediately const forceOption = process.argv[3] === "-f"; @@ -45,26 +45,20 @@ if (["import-db", "import_db", "im"].includes(command)) { console.error("Failed to import database:", error.message); process.exit(1); } - return; -} - -if (["check-replica-set", "check_replica_set", "crs"].includes(command)) { +} else if ( + ["check-replica-set", "check_replica_set", "crs"].includes(command) +) { check_replica_set.exec(); - return; -} - -if (["backup", "restore"].includes(command)) { - require(`./${command}.js`).run(process.argv.slice(3)); - return; -} - -if (["appsmith-version", "appsmith_version", "version"].includes(command)) { +} else if (["backup"].includes(command)) { + backup.run(); +} else if (["restore"].includes(command)) { + restore.run(); +} else if ( + ["appsmith-version", "appsmith_version", "version"].includes(command) +) { version.exec(); - return; +} else if (["mongo-eval", "mongo_eval", "mongoEval"].includes(command)) { + mongo_shell_utils.exec(); +} else { + showHelp(); } -if (["mongo-eval", "mongo_eval", "mongoEval"].includes(command)) { - mongo_shell_utils.exec(process.argv.slice(3)); - return; -} - -utils.showHelp(); diff --git a/app/client/packages/rts/src/ctl/logger.js b/app/client/packages/rts/src/ctl/logger.ts similarity index 75% rename from app/client/packages/rts/src/ctl/logger.js rename to app/client/packages/rts/src/ctl/logger.ts index 6ce5d89ce0b2..742fb6ea6262 100644 --- a/app/client/packages/rts/src/ctl/logger.js +++ b/app/client/packages/rts/src/ctl/logger.ts @@ -1,7 +1,7 @@ -const fsPromises = require("fs/promises"); -const Constants = require("./constants"); +import fsPromises from "fs/promises"; +import * as Constants from "./constants"; -async function backup_error(err) { +export async function backup_error(err) { console.error(err); try { await fsPromises.access(Constants.APPSMITHCTL_LOG_PATH); @@ -14,7 +14,7 @@ async function backup_error(err) { ); } -async function backup_info(msg) { +export async function backup_info(msg) { console.log(msg); try { await fsPromises.access(Constants.APPSMITHCTL_LOG_PATH); @@ -26,8 +26,3 @@ async function backup_info(msg) { new Date().toISOString() + " [ INFO ] " + msg + "\n", ); } - -module.exports = { - backup_error, - backup_info, -}; diff --git a/app/client/packages/rts/src/ctl/mailer.js b/app/client/packages/rts/src/ctl/mailer.ts similarity index 90% rename from app/client/packages/rts/src/ctl/mailer.js rename to app/client/packages/rts/src/ctl/mailer.ts index a794cf6a0869..44fde47d9edb 100644 --- a/app/client/packages/rts/src/ctl/mailer.js +++ b/app/client/packages/rts/src/ctl/mailer.ts @@ -1,7 +1,7 @@ -const nodemailer = require("nodemailer"); -const Constants = require("./constants"); -const utils = require("./utils"); -const logger = require("./logger"); +import nodemailer from "nodemailer"; +import * as Constants from "./constants"; +import * as utils from "./utils"; +import * as logger from "./logger"; const mailEnabled = process.env.APPSMITH_MAIL_ENABLED; const mailFrom = process.env.APPSMITH_MAIL_FROM; @@ -11,7 +11,7 @@ const mailUser = process.env.APPSMITH_MAIL_USERNAME; const mailPass = process.env.APPSMITH_MAIL_PASSWORD; const mailTo = process.env.APPSMITH_ADMIN_EMAILS; -async function sendBackupErrorToAdmins(err, backupTimestamp) { +export async function sendBackupErrorToAdmins(err, backupTimestamp) { console.log("Sending Error mail to admins."); try { if ( @@ -77,7 +77,7 @@ async function sendBackupErrorToAdmins(err, backupTimestamp) { user: mailUser, pass: mailPass, }, - }); + } as any); await transporter.sendMail({ from: mailFrom, @@ -90,7 +90,3 @@ async function sendBackupErrorToAdmins(err, backupTimestamp) { await logger.backup_error(err.stack); } } - -module.exports = { - sendBackupErrorToAdmins, -}; diff --git a/app/client/packages/rts/src/ctl/mongo_shell_utils.js b/app/client/packages/rts/src/ctl/mongo_shell_utils.ts similarity index 87% rename from app/client/packages/rts/src/ctl/mongo_shell_utils.js rename to app/client/packages/rts/src/ctl/mongo_shell_utils.ts index c2b48a3d9a11..b6c44cb2fff5 100644 --- a/app/client/packages/rts/src/ctl/mongo_shell_utils.js +++ b/app/client/packages/rts/src/ctl/mongo_shell_utils.ts @@ -1,8 +1,8 @@ -const utils = require("./utils"); +import * as utils from "./utils"; const command_args = process.argv.slice(3); -async function exec() { +export async function exec() { let errorCode = 0; try { await execMongoEval(command_args[0], process.env.APPSMITH_DB_URL); @@ -25,7 +25,3 @@ async function execMongoEval(queryExpression, appsmithMongoURI) { `--eval=${queryExpression}`, ]); } - -module.exports = { - exec, -}; diff --git a/app/client/packages/rts/src/ctl/restore.js b/app/client/packages/rts/src/ctl/restore.ts similarity index 96% rename from app/client/packages/rts/src/ctl/restore.js rename to app/client/packages/rts/src/ctl/restore.ts index 550386245fb4..fb2f27972c09 100644 --- a/app/client/packages/rts/src/ctl/restore.js +++ b/app/client/packages/rts/src/ctl/restore.ts @@ -1,12 +1,11 @@ -const fsPromises = require("fs/promises"); -const path = require("path"); -const os = require("os"); -const readlineSync = require("readline-sync"); +import fsPromises from "fs/promises"; +import path from "path"; +import os from "os"; +import readlineSync from "readline-sync"; +import * as utils from "./utils"; +import * as Constants from "./constants"; -const utils = require("./utils"); -const Constants = require("./constants"); const command_args = process.argv.slice(3); -const { getCurrentAppsmithVersion } = require("./utils"); async function getBackupFileName() { const backupFiles = await utils.listLocalBackupFiles(); @@ -44,7 +43,7 @@ async function getBackupFileName() { backupFileIndex >= 0 && backupFileIndex < backupFiles.length ) { - return backupFiles[parseInt(backupFileIndex, 10)]; + return backupFiles[backupFileIndex]; } else { console.log( "Invalid input, please try the command again with a valid option", @@ -221,7 +220,7 @@ async function restoreGitStorageArchive(restoreContentsPath, backupName) { } async function checkRestoreVersionCompatability(restoreContentsPath) { - const currentVersion = await getCurrentAppsmithVersion(); + const currentVersion = await utils.getCurrentAppsmithVersion(); const manifest_data = await fsPromises.readFile( restoreContentsPath + "/manifest.json", { encoding: "utf8" }, @@ -280,7 +279,7 @@ async function getBackupDatabaseName(restoreContentsPath) { return db_name; } -async function run() { +export async function run() { let errorCode = 0; let cleanupArchive = false; let overwriteEncryptionKeys = true; @@ -354,7 +353,3 @@ async function run() { function isArchiveEncrypted(backupFilePath) { return backupFilePath.endsWith(".enc"); } - -module.exports = { - run, -}; diff --git a/app/client/packages/rts/src/ctl/utils.test.js b/app/client/packages/rts/src/ctl/utils.test.ts similarity index 94% rename from app/client/packages/rts/src/ctl/utils.test.js rename to app/client/packages/rts/src/ctl/utils.test.ts index d2d67e31f369..fce3dd9cf388 100644 --- a/app/client/packages/rts/src/ctl/utils.test.js +++ b/app/client/packages/rts/src/ctl/utils.test.ts @@ -1,5 +1,6 @@ -const { describe, test, expect } = require("@jest/globals"); -const utils = require("./utils"); +import { describe, expect, test } from "@jest/globals"; + +import * as utils from "./utils"; describe("execCommandReturningOutput", () => { test("Output of echo", async () => { diff --git a/app/client/packages/rts/src/ctl/utils.js b/app/client/packages/rts/src/ctl/utils.ts similarity index 78% rename from app/client/packages/rts/src/ctl/utils.js rename to app/client/packages/rts/src/ctl/utils.ts index 337d2472fdb2..a386eaf1a626 100644 --- a/app/client/packages/rts/src/ctl/utils.js +++ b/app/client/packages/rts/src/ctl/utils.ts @@ -1,10 +1,10 @@ -const fsPromises = require("fs/promises"); -const Constants = require("./constants"); -const childProcess = require("child_process"); -const fs = require("node:fs"); -const { ConnectionString } = require("mongodb-connection-string-url"); +import fsPromises from "fs/promises"; +import * as Constants from "./constants"; +import childProcess from "child_process"; +import fs from "node:fs"; +import { ConnectionString } from "mongodb-connection-string-url"; -function showHelp() { +export function showHelp() { console.log( "\nUsage: appsmith to interact with appsmith utils tool", ); @@ -17,7 +17,7 @@ function showHelp() { console.log("\t--help\t\t\t" + "Show help."); } -async function ensureSupervisorIsRunning() { +export async function ensureSupervisorIsRunning() { try { await execCommandSilent(["/usr/bin/supervisorctl"]); } catch (e) { @@ -26,26 +26,26 @@ async function ensureSupervisorIsRunning() { } } -async function stop(apps) { +export async function stop(apps) { console.log("Stopping", apps); await execCommand(["/usr/bin/supervisorctl", "stop", ...apps]); console.log("Stopped", apps); } -async function start(apps) { +export async function start(apps) { console.log("Starting", apps); await execCommand(["/usr/bin/supervisorctl", "start", ...apps]); console.log("Started", apps); } -function getDburl() { +export function getDburl() { let dbUrl = ""; try { - let env_array = fs + const env_array = fs .readFileSync(Constants.ENV_PATH, "utf8") .toString() .split("\n"); - for (let i in env_array) { + for (const i in env_array) { if ( env_array[i].startsWith("APPSMITH_MONGODB_URI") || env_array[i].startsWith("APPSMITH_DB_URL") @@ -57,7 +57,7 @@ function getDburl() { } catch (err) { console.error("Error reading the environment file:", err); } - let dbEnvUrl = + const dbEnvUrl = process.env.APPSMITH_DB_URL || process.env.APPSMITH_MONGO_DB_URI; // Make sure dbEnvUrl takes precedence over dbUrl if (dbEnvUrl && dbEnvUrl !== "undefined") { @@ -66,8 +66,8 @@ function getDburl() { return dbUrl; } -function execCommand(cmd, options) { - return new Promise((resolve, reject) => { +export function execCommand(cmd: string[], options?) { + return new Promise((resolve, reject) => { let isPromiseDone = false; const p = childProcess.spawn(cmd[0], cmd.slice(1), { @@ -98,8 +98,8 @@ function execCommand(cmd, options) { }); } -function execCommandReturningOutput(cmd, options) { - return new Promise((resolve, reject) => { +export function execCommandReturningOutput(cmd, options?) { + return new Promise((resolve, reject) => { const p = childProcess.spawn(cmd[0], cmd.slice(1), options); p.stdin.end(); @@ -132,13 +132,13 @@ function execCommandReturningOutput(cmd, options) { }); } -async function listLocalBackupFiles() { +export async function listLocalBackupFiles() { // Ascending order const backupFiles = []; await fsPromises .readdir(Constants.BACKUP_PATH) .then((filenames) => { - for (let filename of filenames) { + for (const filename of filenames) { if (filename.match(/^appsmith-backup-.*\.tar\.gz(\.enc)?$/)) { backupFiles.push(filename); } @@ -150,28 +150,28 @@ async function listLocalBackupFiles() { return backupFiles; } -async function updateLastBackupErrorMailSentInMilliSec(ts) { +export async function updateLastBackupErrorMailSentInMilliSec(ts) { await fsPromises.mkdir(Constants.BACKUP_PATH, { recursive: true }); await fsPromises.writeFile(Constants.LAST_ERROR_MAIL_TS, ts.toString()); } -async function getLastBackupErrorMailSentInMilliSec() { +export async function getLastBackupErrorMailSentInMilliSec() { try { - const ts = await fsPromises.readFile(Constants.LAST_ERROR_MAIL_TS); + const ts = await fsPromises.readFile(Constants.LAST_ERROR_MAIL_TS, "utf8"); return parseInt(ts, 10); } catch (error) { return 0; } } -async function getCurrentAppsmithVersion() { +export async function getCurrentAppsmithVersion() { return ( JSON.parse(await fsPromises.readFile("/opt/appsmith/info.json", "utf8")) .version ?? "" ); } -function preprocessMongoDBURI(uri /* string */) { +export function preprocessMongoDBURI(uri /* string */) { // Partially taken from // If we don't add the `directConnection` parameter for non-SRV URIs, we'll see the problem at . const cs = new ConnectionString(uri); @@ -202,8 +202,9 @@ function preprocessMongoDBURI(uri /* string */) { return cs.toString(); } -function execCommandSilent(cmd, options) { - return new Promise((resolve, reject) => { + +export function execCommandSilent(cmd, options?) { + return new Promise((resolve, reject) => { let isPromiseDone = false; const p = childProcess.spawn(cmd[0], cmd.slice(1), { @@ -233,24 +234,7 @@ function execCommandSilent(cmd, options) { }); } -function getDatabaseNameFromMongoURI(uri) { +export function getDatabaseNameFromMongoURI(uri) { const uriParts = uri.split("/"); return uriParts[uriParts.length - 1].split("?")[0]; } - -module.exports = { - showHelp, - ensureSupervisorIsRunning, - start, - stop, - execCommand, - execCommandReturningOutput, - listLocalBackupFiles, - updateLastBackupErrorMailSentInMilliSec, - getLastBackupErrorMailSentInMilliSec, - getCurrentAppsmithVersion, - preprocessMongoDBURI, - execCommandSilent, - getDatabaseNameFromMongoURI, - getDburl, -}; diff --git a/app/client/packages/rts/src/ctl/version.js b/app/client/packages/rts/src/ctl/version.ts similarity index 79% rename from app/client/packages/rts/src/ctl/version.js rename to app/client/packages/rts/src/ctl/version.ts index 144af3282346..79d34be03cbc 100644 --- a/app/client/packages/rts/src/ctl/version.js +++ b/app/client/packages/rts/src/ctl/version.ts @@ -1,6 +1,6 @@ -const utils = require("./utils"); +import * as utils from "./utils"; -async function exec() { +export async function exec() { let version = null; try { version = await utils.getCurrentAppsmithVersion(); @@ -15,7 +15,3 @@ async function exec() { process.exit(1); } } - -module.exports = { - exec, -}; diff --git a/app/client/yarn.lock b/app/client/yarn.lock index 88c6bbf5d0db..ed3d2230d949 100644 --- a/app/client/yarn.lock +++ b/app/client/yarn.lock @@ -10827,6 +10827,15 @@ __metadata: languageName: node linkType: hard +"@types/nodemailer@npm:^6.4.17": + version: 6.4.17 + resolution: "@types/nodemailer@npm:6.4.17" + dependencies: + "@types/node": "*" + checksum: 498b702575111494a42cf9cdd3d399f0308c3b3b0244e7f53008924327955a16b39ba0bf99a5ccf17af9fd2bc50a61a76ae7e8a75498a426ea2f828a05202eab + languageName: node + linkType: hard + "@types/normalize-package-data@npm:^2.4.0": version: 2.4.1 resolution: "@types/normalize-package-data@npm:2.4.1" @@ -11074,6 +11083,13 @@ __metadata: languageName: node linkType: hard +"@types/readline-sync@npm:^1.4.8": + version: 1.4.8 + resolution: "@types/readline-sync@npm:1.4.8" + checksum: 9d69fe944d6a26fb1f6a08d205cb9f6107379570ad0b2081df613ccc7414caa026be9307343afcdd3c93b16939ce2e4a76187c33082301a5c7a2bd5d4a7759ee + languageName: node + linkType: hard + "@types/redux-form@npm:^8.1.9": version: 8.3.0 resolution: "@types/redux-form@npm:8.3.0" @@ -12740,6 +12756,8 @@ __metadata: "@shared/ast": "workspace:^" "@types/express": ^4.17.14 "@types/jest": ^29.2.3 + "@types/nodemailer": ^6.4.17 + "@types/readline-sync": ^1.4.8 axios: ^1.7.4 dotenv: 10.0.0 express: ^4.20.0