diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index ce40d5beb3fd..1f58fa12ffd3 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -2,7 +2,7 @@ name: Quality checks on: pull_request: - branches: [release, master] + branches: [release, master, pg] jobs: path-filter: @@ -45,6 +45,7 @@ jobs: secrets: inherit with: pr: ${{ github.event.pull_request.number }} + is-pg-build: ${{ github.event.pull_request.base.ref == 'pg' }} client-build: name: client-build diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index 6a6d8111b255..eb7ba86f13ab 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -17,6 +17,11 @@ on: description: "This is the branch to be used for the build." required: false type: string + is-pg-build: + description: "This is a boolean value in case the workflow is being called for a PG build" + required: false + type: string + default: "false" # Change the working directory for all the jobs in this workflow defaults: @@ -133,6 +138,12 @@ jobs: distribution: "temurin" java-version: "17" + - name: Conditionally start PostgreSQL + run: | + if [[ inputs.is-pg-build == 'true' ]]; then + docker run --name appsmith-pg -p 5432:5432 -d -e POSTGRES_PASSWORD=password postgres:alpine + fi + # Retrieve maven dependencies from cache. After a successful run, these dependencies are cached again - name: Cache maven dependencies if: steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') @@ -148,16 +159,6 @@ jobs: # Build the code - name: Build if: steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') - env: - ACTIVE_PROFILE: test - APPSMITH_MONGODB_URI: "mongodb://localhost:27017/mobtools" - APPSMITH_CLOUD_SERVICES_BASE_URL: "https://release-cs.appsmith.com" - APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH: ${{ secrets.APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH }} - APPSMITH_REDIS_URL: "redis://127.0.0.1:6379" - APPSMITH_ENCRYPTION_PASSWORD: "password" - APPSMITH_ENCRYPTION_SALT: "salt" - APPSMITH_ENVFILE_PATH: /tmp/dummy.env - APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | ./build.sh -DskipTests @@ -166,7 +167,6 @@ jobs: if: (inputs.skip-tests != 'true' || steps.run_result.outputs.run_result == 'failedtest') && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') env: ACTIVE_PROFILE: test - APPSMITH_MONGODB_URI: "mongodb://localhost:27017/mobtools" APPSMITH_CLOUD_SERVICES_BASE_URL: "https://release-cs.appsmith.com" APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH: ${{ secrets.APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH }} APPSMITH_REDIS_URL: "redis://127.0.0.1:6379" @@ -175,6 +175,11 @@ jobs: APPSMITH_ENVFILE_PATH: /tmp/dummy.env APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | + if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then + export APPSMITH_DB_URL="postgresql://postgres:password@localhost:5432/postgres" + else + export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" + fi args=() if [[ "${{ steps.run_result.outputs.run_result }}" == "failedtest" ]]; then failed_tests="${{ steps.failed_tests.outputs.tests }}" diff --git a/app.json b/app.json index 8cf39b5f7deb..649d48dc3b1c 100644 --- a/app.json +++ b/app.json @@ -18,7 +18,7 @@ "success_url": "/", "stack": "container", "env": { - "APPSMITH_MONGODB_URI": { + "APPSMITH_DB_URL": { "description": "Your Mongo Database URI. Since Heroku doesn't support a managed MongoDB instance, you'll have to create a Mongo DB instance on another service such as https://cloud.mongodb.com", "value": "" }, diff --git a/app/client/src/pages/AdminSettings/config/advanced.ts b/app/client/src/pages/AdminSettings/config/advanced.ts index dfab8254cedf..72d610e26f21 100644 --- a/app/client/src/pages/AdminSettings/config/advanced.ts +++ b/app/client/src/pages/AdminSettings/config/advanced.ts @@ -15,7 +15,7 @@ export const config: AdminConfigType = { canSave: true, settings: [ { - id: "APPSMITH_MONGODB_URI", + id: "APPSMITH_DB_URL", category: SettingCategories.ADVANCED, controlType: SettingTypes.TEXTINPUT, controlSubType: SettingSubtype.TEXT, diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java new file mode 100644 index 000000000000..b7bd7a52f11d --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java @@ -0,0 +1,82 @@ +package com.appsmith.server.configurations; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; +import org.springframework.boot.autoconfigure.mongo.MongoProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Profile; +import org.springframework.util.StringUtils; + +import java.net.URI; +import java.net.URISyntaxException; + +/** + * Class to configure beans based on DB url. This could have been implemented in {@link MongoConfig} or + * {@link DBConfig} but extracted to seperate class as we were facing cyclical dependency issue with the other approach + */ +@Configuration +@Slf4j +public class CommonDBConfig { + + @Value("${appsmith.db.url}") + private String appsmithDbUrl; + + static final String JDBC_PREFIX = "jdbc:"; + + @Bean + @Primary + @Profile("!test") + public MongoProperties configureMongoDB() { + if (!appsmithDbUrl.startsWith("mongodb")) { + return null; + } + log.info("Found MongoDB uri configuring now"); + MongoProperties mongoProperties = new MongoProperties(); + mongoProperties.setUri(appsmithDbUrl); + return mongoProperties; + } + + @Bean + @Primary + public DataSourceProperties configurePostgresDB() { + if (!appsmithDbUrl.contains("postgresql")) { + return null; + } + log.info("Found PostgreSQL uri configuring now"); + return extractJdbcProperties(appsmithDbUrl); + } + + /** + * Method to extract Jdbc props from the given DB URL + * Expected DB URL: postgresql://{username}:{password}@localhost:{port}/{db_name} + */ + public DataSourceProperties extractJdbcProperties(String dbUrl) { + DataSourceProperties ds = new DataSourceProperties(); + try { + URI uri = new URI(dbUrl); + if (!StringUtils.hasLength(uri.getHost())) { + String errorString = String.format( + "Malformed DB URL! Expected format: postgresql://{username}:{password}@localhost:{port}/{db_name}, provided url is %s", + dbUrl); + throw new IllegalArgumentException(errorString); + } + String userInfo = uri.getUserInfo(); + if (StringUtils.hasLength(userInfo)) { + String[] userDetails = userInfo.split(":"); + ds.setUsername(userDetails[0]); + ds.setPassword(userDetails[1]); + } + // If the port is not mentioned default it to standard 5432 + int port = uri.getPort() == -1 ? 5432 : uri.getPort(); + String updatedUrl = + String.format("%s%s://%s:%s%s", JDBC_PREFIX, uri.getScheme(), uri.getHost(), port, uri.getPath()); + ds.setUrl(updatedUrl); + return ds; + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } +} diff --git a/app/server/appsmith-server/src/main/resources/application-production.properties b/app/server/appsmith-server/src/main/resources/application-production.properties deleted file mode 100644 index b4cdd86c2cd9..000000000000 --- a/app/server/appsmith-server/src/main/resources/application-production.properties +++ /dev/null @@ -1,2 +0,0 @@ -# MongoDB Application Database -spring.data.mongodb.uri = ${APPSMITH_MONGODB_URI} \ No newline at end of file diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index 346ee90cefc8..c29d41de90cc 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -8,6 +8,7 @@ spring.lifecycle.timeout-per-shutdown-phase=20s spring.profiles.active=${ACTIVE_PROFILE:production} +appsmith.db.url=${APPSMITH_DB_URL:${APPSMITH_MONGODB_URI}} # This property allows us to override beans during testing. This is useful when we want to set different configurations # and different parameters during test as compared to production. If this property is disabled, some tests will fail. spring.main.allow-bean-definition-overriding=true diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java new file mode 100644 index 000000000000..1332b7241deb --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java @@ -0,0 +1,47 @@ +package com.appsmith.server.configurations; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class CommonDBConfigTest { + + @Test + public void testExtractAndSaveJdbcParams_validDbUrlWithUsernameAndPassword() { + CommonDBConfig commonDBConfig = new CommonDBConfig(); + String dbUrl = "postgresql://postgres:password@localhost/postgres"; + DataSourceProperties ds = commonDBConfig.extractJdbcProperties(dbUrl); + assertEquals("postgres", ds.getUsername()); + assertEquals("password", ds.getPassword()); + assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); + + String dbUrlWithPort = "postgresql://postgres:password@localhost:1234/postgres"; + ds = commonDBConfig.extractJdbcProperties(dbUrlWithPort); + assertEquals("postgres", ds.getUsername()); + assertEquals("password", ds.getPassword()); + assertEquals("jdbc:postgresql://localhost:1234/postgres", ds.getUrl()); + } + + @Test + public void testExtractAndSaveJdbcParams_validDbUrlWithoutUsernameAndPassword() { + CommonDBConfig commonDBConfig = new CommonDBConfig(); + String dbUrl = "postgresql://localhost:5432/postgres"; + DataSourceProperties ds = commonDBConfig.extractJdbcProperties(dbUrl); + assertNull(ds.getUsername()); + assertNull(ds.getPassword()); + assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); + } + + @Test + public void testExtractAndSaveJdbcParams_invalidDbUrl() { + CommonDBConfig commonDBConfig = new CommonDBConfig(); + String dbUrl = "jdbc:postgresql://localhost/postgres"; + String errorString = String.format( + "Malformed DB URL! Expected format: postgresql://{username}:{password}@localhost:{port}/{db_name}, provided url is %s", + dbUrl); + assertThrows(IllegalArgumentException.class, () -> commonDBConfig.extractJdbcProperties(dbUrl), errorString); + } +} diff --git a/app/server/build.sh b/app/server/build.sh index fb6a750697b8..5ff8c860e732 100755 --- a/app/server/build.sh +++ b/app/server/build.sh @@ -34,6 +34,10 @@ if [[ -f .env ]]; then source .env fi +if [[ -f tx/transform.py ]]; then + python3 tx/transform.py +fi + node scripts/check-field-constants.mjs # Build the code. $@ accepts all the parameters from the input command line and uses it in the maven build command diff --git a/app/server/docker-compose.yml b/app/server/docker-compose.yml index e73ba234df72..b18d97d4834d 100644 --- a/app/server/docker-compose.yml +++ b/app/server/docker-compose.yml @@ -8,7 +8,7 @@ services: env_file: envs/docker.env environment: APPSMITH_REDIS_URL: "redis://redis:6379" - APPSMITH_MONGODB_URI: "mongodb://mongo:27017/appsmith" + APPSMITH_DB_URL: "mongodb://mongo:27017/appsmith" ports: - "8080:8080" depends_on: diff --git a/app/server/envs/dev.env.example b/app/server/envs/dev.env.example index 463d4b9b49c2..bcec41238b8b 100644 --- a/app/server/envs/dev.env.example +++ b/app/server/envs/dev.env.example @@ -1,6 +1,6 @@ #!/bin/sh -APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith?replicaSet=rs0" +APPSMITH_DB_URL="mongodb://localhost:27017/appsmith?replicaSet=rs0" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" diff --git a/app/server/envs/test.env.example b/app/server/envs/test.env.example index fdeb33ce7157..1069f4993f2f 100644 --- a/app/server/envs/test.env.example +++ b/app/server/envs/test.env.example @@ -2,7 +2,7 @@ ACTIVE_PROFILE=test -APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" +APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" diff --git a/contributions/ServerSetup.md b/contributions/ServerSetup.md index 1b34e18ec83f..80c62ba5adba 100644 --- a/contributions/ServerSetup.md +++ b/contributions/ServerSetup.md @@ -124,11 +124,11 @@ With the prerequisites met, let's build the code. This command creates a `.env` file in the `app/server` folder. All run scripts pick up environment configuration from this file. -5. Ensure that the environment variables `APPSMITH_MONGODB_URI` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. +5. Ensure that the environment variables `APPSMITH_DB_URL` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. 6. **Update the replica set name with correct value in the mongo connection string in the [.env](#setup-environment-file) file.** The replica name is the same as passed [here](#setting-up-a-local-mongodb-instance) ```bash - APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith?replicaSet=" + APPSMITH_DB_URL="mongodb://localhost:27017/appsmith?replicaSet=" ``` 7. Run the following command to create the final JAR for the Appsmith server: @@ -152,7 +152,7 @@ With the prerequisites met, let's build the code. - On Ubuntu Linux environment docker needs root privilege, hence `./build.sh` script needs to be run with root privilege as well. - On Ubuntu Linux environment, the script may not be able to read `.env` file, so it is advised that you run the cmd like: ```console - sudo APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh + sudo APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh ``` @@ -260,7 +260,7 @@ cp envs/dev.env.example .env This command creates a `.env` file in the `app/server` folder. All run scripts pick up environment configuration from this file. -5. Ensure that the environment variables `APPSMITH_MONGODB_URI` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. +5. Ensure that the environment variables `APPSMITH_DB_URL` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. 6. Run the following command to create the final JAR for the Appsmith server: @@ -274,7 +274,7 @@ Note: - On Ubuntu Linux environment docker needs root privilege, hence ./build.sh script needs to be run with root privilege as well. - On Ubuntu Linux environment, the script may not be able to read .env file, so it is advised that you run the cmd like: ```console -sudo APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh +sudo APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh ``` - If the volume containing docker's data root path (macOS: `~/Library/Containers/com.docker.docker/Data/vms/0/`, Ubuntu: `/var/lib/docker/`) has less than 2 GB of free space, then the script may fail with the following error: ```console diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 8e1c53d5cc1e..2bf6be004ed4 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -168,11 +168,28 @@ unset_unused_variables() { fi } -check_mongodb_uri() { - echo "Checking APPSMITH_MONGODB_URI" +configure_database_connection_url() { + echo "Configuring database connection URL" + isPostgresUrl=0 + isMongoUrl=0 + # Check if APPSMITH_DB_URL is not set + if [[ -z "${APPSMITH_DB_URL}" ]]; then + # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI + export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" + fi + + if [[ "${APPSMITH_DB_URL}" == "postgresql:"* ]]; then + isPostgresUrl=1 + elif [[ "${APPSMITH_DB_URL}" == "mongodb"* ]]; then + isMongoUrl=1 + fi +} + +check_db_uri() { + echo "Checking APPSMITH_DB_URL" isUriLocal=1 - if [[ $APPSMITH_MONGODB_URI == *"localhost"* || $APPSMITH_MONGODB_URI == *"127.0.0.1"* ]]; then - echo "Detected local MongoDB" + if [[ $APPSMITH_DB_URL == *"localhost"* || $APPSMITH_DB_URL == *"127.0.0.1"* ]]; then + echo "Detected local DB" isUriLocal=0 fi } @@ -239,7 +256,7 @@ init_replica_set() { mongod --fork --port 27017 --dbpath "$MONGO_DB_PATH" --logpath "$MONGO_LOG_PATH" --replSet mr1 --keyFile "$MONGODB_TMP_KEY_PATH" --bind_ip localhost echo "Waiting 10s for MongoDB to start with Replica Set" sleep 10 - mongosh "$APPSMITH_MONGODB_URI" --eval 'rs.initiate()' + mongosh "$APPSMITH_DB_URL" --eval 'rs.initiate()' mongod --dbpath "$MONGO_DB_PATH" --shutdown || true fi @@ -477,11 +494,19 @@ print_appsmith_info init_loading_pages unset_unused_variables -check_mongodb_uri +configure_database_connection_url +check_db_uri +# Don't run MongoDB if running in a Heroku dyno. if [[ -z "${DYNO}" ]]; then - # Don't run MongoDB if running in a Heroku dyno. - init_mongodb - init_replica_set + if [[ $isMongoUrl -eq 1 ]]; then + # Setup MongoDB and initialize replica set + echo "Initializing MongoDB" + init_mongodb + init_replica_set + elif [[ $isPostgresUrl -eq 1 ]]; then + echo "Initializing Postgres" + # init_postgres + fi else # These functions are used to limit heap size for Backend process when deployed on Heroku get_maximum_heap diff --git a/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh b/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh index 9d070d4e29d8..3808b2854fa0 100644 --- a/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh +++ b/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh @@ -16,7 +16,7 @@ done echo "MongoDB is RUNNING" for _ in {1..60}; do - if mongosh --quiet "$APPSMITH_MONGODB_URI" --eval ' + if mongosh --quiet "$APPSMITH_DB_URL" --eval ' parseFloat(db.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}).featureCompatibilityVersion.version) < 5 && db.adminCommand({setFeatureCompatibilityVersion: "5.0"}) '; then diff --git a/deploy/docker/fs/opt/appsmith/run-with-env.sh b/deploy/docker/fs/opt/appsmith/run-with-env.sh index 93a7bf65fc9c..a70c8935250c 100755 --- a/deploy/docker/fs/opt/appsmith/run-with-env.sh +++ b/deploy/docker/fs/opt/appsmith/run-with-env.sh @@ -33,4 +33,10 @@ if [[ -z "${APPSMITH_GIT_ROOT:-}" ]]; then fi mkdir -pv "$APPSMITH_GIT_ROOT" +# Check if APPSMITH_DB_URL is set +if [[ -z "${APPSMITH_DB_URL}" ]]; then + # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI + export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" +fi + exec "$@" diff --git a/deploy/docker/fs/opt/appsmith/templates/docker.env.sh b/deploy/docker/fs/opt/appsmith/templates/docker.env.sh index 3aff309e73eb..e80133242808 100644 --- a/deploy/docker/fs/opt/appsmith/templates/docker.env.sh +++ b/deploy/docker/fs/opt/appsmith/templates/docker.env.sh @@ -63,7 +63,7 @@ APPSMITH_RECAPTCHA_SITE_KEY= APPSMITH_RECAPTCHA_SECRET_KEY= APPSMITH_RECAPTCHA_ENABLED= -APPSMITH_MONGODB_URI=mongodb://$MONGO_USER:$MONGO_PASSWORD@localhost:27017/appsmith +APPSMITH_DB_URL=mongodb://$MONGO_USER:$MONGO_PASSWORD@localhost:27017/appsmith APPSMITH_MONGODB_USER=$MONGO_USER APPSMITH_MONGODB_PASSWORD=$MONGO_PASSWORD APPSMITH_API_BASE_URL=http://localhost:8080/api/v1 diff --git a/deploy/docker/fs/opt/appsmith/update-and-restart-supervisor.sh b/deploy/docker/fs/opt/appsmith/update-and-restart-supervisor.sh index 0768b2d13566..36e5d25f35b5 100755 --- a/deploy/docker/fs/opt/appsmith/update-and-restart-supervisor.sh +++ b/deploy/docker/fs/opt/appsmith/update-and-restart-supervisor.sh @@ -13,7 +13,7 @@ set +o allexport check_mongodb_uri() { echo "Check MongoDB uri host" isLocalMongo=1 - if [[ $APPSMITH_MONGODB_URI == *"localhost"* || $APPSMITH_MONGODB_URI == *"127.0.0.1"* ]]; then + if [[ $APPSMITH_DB_URL == *"localhost"* || $APPSMITH_DB_URL == *"127.0.0.1"* ]]; then echo "Use local MongoDB" isLocalMongo=0 fi diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index a6fd65b23ebc..e94c6e6407f0 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -125,7 +125,7 @@ function getEncryptionPasswordFromUser(){ async function exportDatabase(destFolder) { console.log('Exporting database'); - await executeMongoDumpCMD(destFolder, process.env.APPSMITH_MONGODB_URI) + await executeMongoDumpCMD(destFolder, process.env.APPSMITH_DB_URL) console.log('Exporting database done.'); } @@ -141,7 +141,7 @@ async function createGitStorageArchive(destFolder) { async function createManifestFile(path) { const version = await utils.getCurrentAppsmithVersion() - const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_MONGODB_URI) } + const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_DB_URL) } await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data)); } @@ -207,7 +207,7 @@ function removeSensitiveEnvData(content) { // Remove encryption and Mongodb data from docker.env const output_lines = [] content.split(/\r?\n/).forEach(line => { - if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB")) { + if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB") && !line.startsWith("APPSMITH_DB_URL=")) { output_lines.push(line); } }); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js index 2478f7a902e2..335dfd71a427 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js @@ -91,12 +91,12 @@ it('Checks for the current Appsmith Version.', async () => { }) test('If MONGODB and Encryption env values are being removed', () => { - expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_MONGODB_URI=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) }); test('If MONGODB and Encryption env values are being removed', () => { - expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_MONGODB_URI=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) }); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js b/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js index 0a023726da1a..3289ebfa479d 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js @@ -2,7 +2,7 @@ const { MongoClient, MongoServerError} = require("mongodb"); const { preprocessMongoDBURI } = require("./utils"); async function exec() { - const client = new MongoClient(preprocessMongoDBURI(process.env.APPSMITH_MONGODB_URI), { + const client = new MongoClient(preprocessMongoDBURI(process.env.APPSMITH_DB_URL), { useNewUrlParser: true, useUnifiedTopology: true, }); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js index d4554a688180..f5e894b0c434 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js @@ -5,7 +5,7 @@ const Constants = require('./constants'); function export_database() { console.log('export_database ....'); shell.mkdir('-p', [Constants.BACKUP_PATH]); - const cmd = `mongodump --uri='${process.env.APPSMITH_MONGODB_URI}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; + const cmd = `mongodump --uri='${process.env.APPSMITH_DB_URL}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; shell.exec(cmd); console.log('export_database done'); } diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js index f3c2c7254c6f..57c326a0ac9b 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js @@ -6,7 +6,7 @@ const Constants = require('./constants') function import_database() { console.log('import_database ....') - const cmd = `mongorestore --uri='${process.env.APPSMITH_MONGODB_URI}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip` + const cmd = `mongorestore --uri='${process.env.APPSMITH_DB_URL}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip` shell.exec(cmd) console.log('import_database done') } @@ -34,7 +34,7 @@ const main = (forceOption) => { shell.echo('stop backend & rts application before import database') stop_application() - const shellCmdResult = shell.exec(`mongo ${process.env.APPSMITH_MONGODB_URI} --quiet --eval "db.getCollectionNames().length"`) + const shellCmdResult = shell.exec(`mongo ${process.env.APPSMITH_DB_URL} --quiet --eval "db.getCollectionNames().length"`) const collectionsLen = parseInt(shellCmdResult.stdout.toString().trimEnd()) if (collectionsLen > 0) { if (forceOption) { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/index.js b/deploy/docker/fs/opt/appsmith/utils/bin/index.js index 34d4c49492dd..cc9b2f6c2ec2 100755 --- a/deploy/docker/fs/opt/appsmith/utils/bin/index.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/index.js @@ -13,6 +13,11 @@ const APPLICATION_CONFIG_PATH = "/appsmith-stacks/configuration/docker.env"; // Loading latest application configuration require("dotenv").config({ path: APPLICATION_CONFIG_PATH }); +// Check if APPSMITH_DB_URL is set, if not set, fall back to APPSMITH_MONGODB_URI +if (!process.env.APPSMITH_DB_URL) { + process.env.APPSMITH_DB_URL = process.env.APPSMITH_MONGODB_URI; +} + const command = process.argv[2]; if (["export-db", "export_db", "ex"].includes(command)) { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js b/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js index 1ae91de416a0..d0ab437b97e7 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js @@ -6,7 +6,7 @@ const command_args = process.argv.slice(3); async function exec() { let errorCode = 0; try { - await execMongoEval(command_args[0], process.env.APPSMITH_MONGODB_URI); + await execMongoEval(command_args[0], process.env.APPSMITH_DB_URL); } catch (err) { errorCode = 1; console.error('Error evaluating the mongo query', err); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js index b3640fb900e9..f79f84cf8f22 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js @@ -59,10 +59,10 @@ async function extractArchive(backupFilePath, restoreRootPath) { async function restoreDatabase(restoreContentsPath) { console.log('Restoring database...'); - const cmd = ['mongorestore', `--uri=${process.env.APPSMITH_MONGODB_URI}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip'] + const cmd = ['mongorestore', `--uri=${process.env.APPSMITH_DB_URL}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip'] try { const fromDbName = await getBackupDatabaseName(restoreContentsPath); - const toDbName = utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_MONGODB_URI); + const toDbName = utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_DB_URL); console.log("Restoring database from " + fromDbName + " to " + toDbName) cmd.push('--nsInclude=*', `--nsFrom=${fromDbName}.*`, `--nsTo=${toDbName}.*`) } catch (error) { @@ -105,10 +105,10 @@ async function restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEn hideEchoBack: true }); } - await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd + '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt + '\nAPPSMITH_MONGODB_URI=' + process.env.APPSMITH_MONGODB_URI + + await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd + '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt + '\nAPPSMITH_DB_URL=' + process.env.APPSMITH_DB_URL + '\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ; } else { - await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_MONGODB_URI=' + process.env.APPSMITH_MONGODB_URI + + await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_DB_URL=' + process.env.APPSMITH_DB_URL + '\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ; } console.log('Restoring docker environment file completed'); diff --git a/deploy/helm/templates/configMap.yaml b/deploy/helm/templates/configMap.yaml index 96a64b5af9f3..650cf79a9992 100644 --- a/deploy/helm/templates/configMap.yaml +++ b/deploy/helm/templates/configMap.yaml @@ -13,7 +13,7 @@ metadata: {{- include "appsmith.labels" . | nindent 4 }} data: {{- range $key, $value := .Values.applicationConfig }} - {{- if and (eq "APPSMITH_MONGODB_URI" $key) ( not $value) }} + {{- if and (eq "APPSMITH_MONGODB_URI" $key) (not $value) }} {{- if $.Values.mongodb.enabled }} {{ $key }}: mongodb+srv://{{ $mongoUser }}:{{ $mongoPassword }}@{{ $mongoServicename }}.{{ $nameSpace }}.svc.cluster.local/appsmith?retryWrites=true&authSource=admin&ssl=false {{- end }} diff --git a/deploy/heroku/README.MD b/deploy/heroku/README.MD index d189438d8d54..f9290ccae810 100644 --- a/deploy/heroku/README.MD +++ b/deploy/heroku/README.MD @@ -9,7 +9,7 @@ Quickly set up Appsmith to explore product functionality using Heroku. - Fill in the required `Config Variables` including: - `APPSMITH_ENCRYPTION_PASSWORD`: Encryption password to encrypt all credentials in the database - `APPSMITH_ENCRYPTION_SALT`: Encryption salt used to encrypt all credentials in the database - - `APPSMITH_MONGODB_URI`: Your Mongo Database URI + - `APPSMITH_DB_URL`: Your Mongo Database URI - (Optional) Customize the default settings in Heroku - `App Name`: Optionally select a name for your application (this will be used in the app URL)