From 03777b2a8639aed9c81d31f8a39826f20b674902 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 14 May 2024 14:25:00 +0530 Subject: [PATCH 01/41] feat: Enable server tests on PRs raised on PG branch --- .github/workflows/quality-checks.yml | 3 ++- .github/workflows/server-build.yml | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index ce40d5beb3fd..26e7b518f508 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -2,7 +2,7 @@ name: Quality checks on: pull_request: - branches: [release, master] + branches: [release, master, pg] jobs: path-filter: @@ -45,6 +45,7 @@ jobs: secrets: inherit with: pr: ${{ github.event.pull_request.number }} + is-pg: ${{ github.event.pull_request.base.ref == 'pg' }} client-build: name: client-build diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index d19e288cda64..086228a7028b 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -17,6 +17,11 @@ on: description: "This is the branch to be used for the build." required: false type: string + is-pg: + description: "This is a boolean value in case the workflow is being called for a PG build" + required: false + type: string + default: "false" # Change the working directory for all the jobs in this workflow defaults: @@ -133,6 +138,23 @@ jobs: distribution: "temurin" java-version: "17" + # Setup Python for conditional execution + - name: Conditional Python setup and script execution + run: | + if [[ inputs.is-pg == 'true' ]]; then + # Setup Python + echo "Setting up Python" + actions/setup-python@v3 + + # Run transform.py + if [[ -f app/server/tx/transform.py ]]; then + python3 app/server/tx/transform.py + else + echo "Error: transform.py does not exist" + exit 1 + fi + fi + # Retrieve maven dependencies from cache. After a successful run, these dependencies are cached again - name: Cache maven dependencies if: steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') From 7256d6670fab077fbb358270f4c64cf33111cf62 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 14 May 2024 14:54:36 +0530 Subject: [PATCH 02/41] feat: Add postgresql docker image before running the tests --- .github/workflows/quality-checks.yml | 2 +- .github/workflows/server-build.yml | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 26e7b518f508..1f58fa12ffd3 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -45,7 +45,7 @@ jobs: secrets: inherit with: pr: ${{ github.event.pull_request.number }} - is-pg: ${{ github.event.pull_request.base.ref == 'pg' }} + is-pg-build: ${{ github.event.pull_request.base.ref == 'pg' }} client-build: name: client-build diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index 086228a7028b..3d140b359e63 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -17,7 +17,7 @@ on: description: "This is the branch to be used for the build." required: false type: string - is-pg: + is-pg-build: description: "This is a boolean value in case the workflow is being called for a PG build" required: false type: string @@ -138,16 +138,16 @@ jobs: distribution: "temurin" java-version: "17" - # Setup Python for conditional execution + # Setup Python + - name: Set up Python + if: inputs.is-pg-build == 'true' && steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') + uses: actions/setup-python@v3 + + # Setup Python for conditional execution to create cake classes - name: Conditional Python setup and script execution run: | - if [[ inputs.is-pg == 'true' ]]; then - # Setup Python - echo "Setting up Python" - actions/setup-python@v3 - - # Run transform.py - if [[ -f app/server/tx/transform.py ]]; then + if [[ inputs.is-pg-build == 'true' && -f app/server/tx/transform.py ]]; then + # Run transform.py python3 app/server/tx/transform.py else echo "Error: transform.py does not exist" @@ -155,6 +155,12 @@ jobs: fi fi + - name: Conditionally start PostgreSQL + run: | + if [[ inputs.is-pg-build == 'true' ]]; then + docker run -d -p 5432:5432 postgres:alpine + fi + # Retrieve maven dependencies from cache. After a successful run, these dependencies are cached again - name: Cache maven dependencies if: steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') From 72c87182a7f39547aebbb7d1f147b7dc3be4fe65 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 14 May 2024 14:58:51 +0530 Subject: [PATCH 03/41] chore: Minor refactor --- .github/workflows/server-build.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index 3d140b359e63..30024d211ef0 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -143,16 +143,15 @@ jobs: if: inputs.is-pg-build == 'true' && steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') uses: actions/setup-python@v3 - # Setup Python for conditional execution to create cake classes - - name: Conditional Python setup and script execution + # Conditional execution to create cake classes + - name: Generate cake classes conditionally run: | if [[ inputs.is-pg-build == 'true' && -f app/server/tx/transform.py ]]; then - # Run transform.py - python3 app/server/tx/transform.py - else - echo "Error: transform.py does not exist" - exit 1 - fi + # Run transform.py + python3 app/server/tx/transform.py + else + echo "Error: transform.py does not exist, going to exit now" + exit 1 fi - name: Conditionally start PostgreSQL From 15ec4c3e374ae2576cd93ff21c1926028af2e8a1 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 14 May 2024 17:38:22 +0530 Subject: [PATCH 04/41] chore: Move the transform script trigger to build.sh --- .github/workflows/server-build.yml | 21 +++++---------------- app/server/build.sh | 6 ++++++ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index 30024d211ef0..bb811d224a2e 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -138,22 +138,6 @@ jobs: distribution: "temurin" java-version: "17" - # Setup Python - - name: Set up Python - if: inputs.is-pg-build == 'true' && steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') - uses: actions/setup-python@v3 - - # Conditional execution to create cake classes - - name: Generate cake classes conditionally - run: | - if [[ inputs.is-pg-build == 'true' && -f app/server/tx/transform.py ]]; then - # Run transform.py - python3 app/server/tx/transform.py - else - echo "Error: transform.py does not exist, going to exit now" - exit 1 - fi - - name: Conditionally start PostgreSQL run: | if [[ inputs.is-pg-build == 'true' ]]; then @@ -186,6 +170,11 @@ jobs: APPSMITH_ENVFILE_PATH: /tmp/dummy.env APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | + if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then + export APPSMITH_DB_URL="jdbc:postgresql://localhost:5432/postgres" + else + export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" + fi ./build.sh -DskipTests # Test the code diff --git a/app/server/build.sh b/app/server/build.sh index fb6a750697b8..2b9f83d99fe5 100755 --- a/app/server/build.sh +++ b/app/server/build.sh @@ -34,6 +34,12 @@ if [[ -f .env ]]; then source .env fi +# Check the APPSMITH_DB_URL environment variable +if [[ "${APPSMITH_DB_URL}" == *"postgres"* ]]; then + # If it contains "postgres", run the transform script + python3 ./tx/transform.py +fi + node scripts/check-field-constants.mjs # Build the code. $@ accepts all the parameters from the input command line and uses it in the maven build command From b5c0601744ffd4e97b3009016a7253e073855331 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 15 May 2024 11:45:33 +0530 Subject: [PATCH 05/41] chore: Update the DB URI env variable --- .../Migration056UpdateDbUriEnvVariable.java | 50 +++++++++++++++++++ .../application-production.properties | 2 +- app/server/build.sh | 2 +- 3 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java new file mode 100644 index 000000000000..95364a3e8cd3 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java @@ -0,0 +1,50 @@ +package com.appsmith.server.migrations.db.ce; + +import com.appsmith.server.configurations.CommonConfig; +import io.mongock.api.annotations.ChangeUnit; +import io.mongock.api.annotations.Execution; +import io.mongock.api.annotations.RollbackExecution; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Collectors; + +@Slf4j +@RequiredArgsConstructor +@ChangeUnit(order = "056", id = "update-db-uri-env-variable", runAlways = true) +public class Migration056UpdateDbUriEnvVariable { + private final CommonConfig commonConfig; + + @RollbackExecution + public void rollbackExecution() {} + + @Execution + public void executeMigration() throws IOException { + final String currentEnvName = "APPSMITH_MONGODB_URI"; + final String updatedEnvName = "APPSMITH_DB_URL"; + updateEnvInFile(currentEnvName, updatedEnvName, commonConfig.getEnvFilePath()); + } + + private static void updateEnvInFile( + final String currentEnvName, final String updatedEnvName, final String envPathString) throws IOException { + if (StringUtils.isEmpty(envPathString) || StringUtils.isEmpty(updatedEnvName)) { + return; + } + + final Path envPath = Path.of(envPathString); + + final String updatedLines = Files.readAllLines(envPath).stream() + .map(line -> { + if (line.startsWith(currentEnvName + "=")) { + return updatedEnvName + "=" + System.getenv(currentEnvName); + } + return line; + }) + .collect(Collectors.joining("\n")); + Files.writeString(envPath, updatedLines); + } +} diff --git a/app/server/appsmith-server/src/main/resources/application-production.properties b/app/server/appsmith-server/src/main/resources/application-production.properties index b4cdd86c2cd9..7ac803805de9 100644 --- a/app/server/appsmith-server/src/main/resources/application-production.properties +++ b/app/server/appsmith-server/src/main/resources/application-production.properties @@ -1,2 +1,2 @@ # MongoDB Application Database -spring.data.mongodb.uri = ${APPSMITH_MONGODB_URI} \ No newline at end of file +spring.data.mongodb.uri=${APPSMITH_DB_URI:${APPSMITH_MONGODB_URI}} diff --git a/app/server/build.sh b/app/server/build.sh index 2b9f83d99fe5..ef87068fb3ab 100755 --- a/app/server/build.sh +++ b/app/server/build.sh @@ -35,7 +35,7 @@ if [[ -f .env ]]; then fi # Check the APPSMITH_DB_URL environment variable -if [[ "${APPSMITH_DB_URL}" == *"postgres"* ]]; then +if [[ "${APPSMITH_DB_URI}" == *"postgresql://"* ]]; then # If it contains "postgres", run the transform script python3 ./tx/transform.py fi From 4ba50048ba8b9db0e359ff1518286041bc73c9d1 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 15 May 2024 11:51:13 +0530 Subject: [PATCH 06/41] chore: Add verbose comment for postgres URI check --- app/server/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server/build.sh b/app/server/build.sh index ef87068fb3ab..fc47412266e2 100755 --- a/app/server/build.sh +++ b/app/server/build.sh @@ -34,7 +34,7 @@ if [[ -f .env ]]; then source .env fi -# Check the APPSMITH_DB_URL environment variable +# Check APPSMITH_DB_URL environment variable for postgres and run the transform script for postgres databases if [[ "${APPSMITH_DB_URI}" == *"postgresql://"* ]]; then # If it contains "postgres", run the transform script python3 ./tx/transform.py From 0c4f065a5b9e5cc61a892aa22a32719e1f15367e Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 15 May 2024 11:52:44 +0530 Subject: [PATCH 07/41] fix: Updated name for URI in migration --- .../migrations/db/ce/Migration056UpdateDbUriEnvVariable.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java index 95364a3e8cd3..d948f27dc765 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java @@ -15,7 +15,7 @@ @Slf4j @RequiredArgsConstructor -@ChangeUnit(order = "056", id = "update-db-uri-env-variable", runAlways = true) +@ChangeUnit(order = "056", id = "update-db-uri-env-variable") public class Migration056UpdateDbUriEnvVariable { private final CommonConfig commonConfig; @@ -25,7 +25,7 @@ public void rollbackExecution() {} @Execution public void executeMigration() throws IOException { final String currentEnvName = "APPSMITH_MONGODB_URI"; - final String updatedEnvName = "APPSMITH_DB_URL"; + final String updatedEnvName = "APPSMITH_DB_URI"; updateEnvInFile(currentEnvName, updatedEnvName, commonConfig.getEnvFilePath()); } From 35800ceed8e6d6c5874162594d3d4fae9ddcc3fe Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 15 May 2024 13:08:36 +0530 Subject: [PATCH 08/41] refactor: Remove application-production.properties to avoid the confusion --- .../Migration056UpdateDbUriEnvVariable.java | 25 +++++++++++-------- .../application-production.properties | 2 -- .../src/main/resources/application.properties | 1 + 3 files changed, 15 insertions(+), 13 deletions(-) delete mode 100644 app/server/appsmith-server/src/main/resources/application-production.properties diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java index d948f27dc765..492f6e5b52ae 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java @@ -30,21 +30,24 @@ public void executeMigration() throws IOException { } private static void updateEnvInFile( - final String currentEnvName, final String updatedEnvName, final String envPathString) throws IOException { + final String currentEnvName, final String updatedEnvName, final String envPathString) { if (StringUtils.isEmpty(envPathString) || StringUtils.isEmpty(updatedEnvName)) { return; } final Path envPath = Path.of(envPathString); - - final String updatedLines = Files.readAllLines(envPath).stream() - .map(line -> { - if (line.startsWith(currentEnvName + "=")) { - return updatedEnvName + "=" + System.getenv(currentEnvName); - } - return line; - }) - .collect(Collectors.joining("\n")); - Files.writeString(envPath, updatedLines); + try { + final String updatedLines = Files.readAllLines(envPath).stream() + .map(line -> { + if (line.startsWith(currentEnvName + "=")) { + return updatedEnvName + "=" + System.getenv(currentEnvName); + } + return line; + }) + .collect(Collectors.joining("\n")); + Files.writeString(envPath, updatedLines); + } catch (IOException e) { + log.error("Error updating the DB URI environment variable in the file at path {}", envPath, e); + } } } diff --git a/app/server/appsmith-server/src/main/resources/application-production.properties b/app/server/appsmith-server/src/main/resources/application-production.properties deleted file mode 100644 index 7ac803805de9..000000000000 --- a/app/server/appsmith-server/src/main/resources/application-production.properties +++ /dev/null @@ -1,2 +0,0 @@ -# MongoDB Application Database -spring.data.mongodb.uri=${APPSMITH_DB_URI:${APPSMITH_MONGODB_URI}} diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index 521cb4537287..e5c25d605d82 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -7,6 +7,7 @@ server.max-http-request-header-size=16KB spring.lifecycle.timeout-per-shutdown-phase=20s spring.profiles.active=${ACTIVE_PROFILE:production} +spring.data.mongodb.uri=${APPSMITH_DB_URI:${APPSMITH_MONGODB_URI}} # This property allows us to override beans during testing. This is useful when we want to set different configurations # and different parameters during test as compared to production. If this property is disabled, some tests will fail. From 99c6e5c0914ffb2f4fdb2d4ee709ec2ed8cbeb9c Mon Sep 17 00:00:00 2001 From: Abhijeet <41686026+abhvsn@users.noreply.github.com> Date: Thu, 16 May 2024 12:20:36 +0530 Subject: [PATCH 09/41] chore: Run transform script to add the cake classes Co-authored-by: Shrikant Sharat Kandula --- app/server/build.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/server/build.sh b/app/server/build.sh index fc47412266e2..5ff8c860e732 100755 --- a/app/server/build.sh +++ b/app/server/build.sh @@ -34,10 +34,8 @@ if [[ -f .env ]]; then source .env fi -# Check APPSMITH_DB_URL environment variable for postgres and run the transform script for postgres databases -if [[ "${APPSMITH_DB_URI}" == *"postgresql://"* ]]; then - # If it contains "postgres", run the transform script - python3 ./tx/transform.py +if [[ -f tx/transform.py ]]; then + python3 tx/transform.py fi node scripts/check-field-constants.mjs From acd841b783a4dd34c478b0637aed9e18c74ce0ac Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 12:44:57 +0530 Subject: [PATCH 10/41] feat: Add fallback for DB URI in entrypoint.sh --- .github/workflows/server-build.yml | 7 +++++-- deploy/docker/route-tests/entrypoint.sh | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index bb811d224a2e..8eb161e6afdf 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -161,7 +161,6 @@ jobs: if: steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') env: ACTIVE_PROFILE: test - APPSMITH_MONGODB_URI: "mongodb://localhost:27017/mobtools" APPSMITH_CLOUD_SERVICES_BASE_URL: "https://release-cs.appsmith.com" APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH: ${{ secrets.APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH }} APPSMITH_REDIS_URL: "redis://127.0.0.1:6379" @@ -182,7 +181,6 @@ jobs: if: (inputs.skip-tests != 'true' || steps.run_result.outputs.run_result == 'failedtest') && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') env: ACTIVE_PROFILE: test - APPSMITH_MONGODB_URI: "mongodb://localhost:27017/mobtools" APPSMITH_CLOUD_SERVICES_BASE_URL: "https://release-cs.appsmith.com" APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH: ${{ secrets.APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH }} APPSMITH_REDIS_URL: "redis://127.0.0.1:6379" @@ -191,6 +189,11 @@ jobs: APPSMITH_ENVFILE_PATH: /tmp/dummy.env APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | + if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then + export APPSMITH_DB_URL="jdbc:postgresql://localhost:5432/postgres" + else + export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" + fi args=() if [[ "${{ steps.run_result.outputs.run_result }}" == "failedtest" ]]; then failed_tests="${{ steps.failed_tests.outputs.tests }}" diff --git a/deploy/docker/route-tests/entrypoint.sh b/deploy/docker/route-tests/entrypoint.sh index 8e0d98c264fc..6b37d9de3e7c 100644 --- a/deploy/docker/route-tests/entrypoint.sh +++ b/deploy/docker/route-tests/entrypoint.sh @@ -129,3 +129,14 @@ node /caddy-reconfigure.mjs reload-caddy run-hurl --variable frame_ancestors="something.com" \ common/*.hurl + +# Check if APPSMITH_DB_URI is not set +if [[ -z "${APPSMITH_DB_URI}" ]]; then + # If APPSMITH_DB_URI is not set, fall back to APPSMITH_MONGODB_URI + export APPSMITH_DB_URI="${APPSMITH_MONGODB_URI}" +fi + +# Check if APPSMITH_DB_URI is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" +if [[ "${APPSMITH_DB_URI}" == *"postgresql://"* && "${APPSMITH_DB_URI}" != jdbc:* ]]; then + export APPSMITH_DB_URI="jdbc:${APPSMITH_DB_URI}" +fi From 25bbced64810466b97448167d3c6fccfe3e65459 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 12:59:37 +0530 Subject: [PATCH 11/41] chore: Remove envs while building the server --- .github/workflows/server-build.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index 8eb161e6afdf..396d25599dfd 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -159,21 +159,7 @@ jobs: # Build the code - name: Build if: steps.run_result.outputs.run_result != 'success' && (steps.changed-files-specific.outputs.any_changed == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') - env: - ACTIVE_PROFILE: test - APPSMITH_CLOUD_SERVICES_BASE_URL: "https://release-cs.appsmith.com" - APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH: ${{ secrets.APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH }} - APPSMITH_REDIS_URL: "redis://127.0.0.1:6379" - APPSMITH_ENCRYPTION_PASSWORD: "password" - APPSMITH_ENCRYPTION_SALT: "salt" - APPSMITH_ENVFILE_PATH: /tmp/dummy.env - APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | - if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then - export APPSMITH_DB_URL="jdbc:postgresql://localhost:5432/postgres" - else - export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" - fi ./build.sh -DskipTests # Test the code From 17a659f57f1d4cd964aa07752e44f4f14803b8d8 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 13:57:12 +0530 Subject: [PATCH 12/41] fix: DB URI name --- .github/workflows/server-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index 396d25599dfd..2e4556a85d55 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -176,9 +176,9 @@ jobs: APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then - export APPSMITH_DB_URL="jdbc:postgresql://localhost:5432/postgres" + export APPSMITH_DB_URI="jdbc:postgresql://localhost:5432/postgres" else - export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" + export APPSMITH_DB_URI="mongodb://localhost:27017/mobtools" fi args=() if [[ "${{ steps.run_result.outputs.run_result }}" == "failedtest" ]]; then From c84744b8300d8dcd07f01b6e5575bf355ae7535f Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 15:54:30 +0530 Subject: [PATCH 13/41] chore: Update the DB URL variable --- .github/workflows/server-build.yml | 4 ++-- .../db/ce/Migration056UpdateDbUriEnvVariable.java | 2 +- .../src/main/resources/application.properties | 2 +- deploy/docker/route-tests/entrypoint.sh | 14 +++++++------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index 2e4556a85d55..396d25599dfd 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -176,9 +176,9 @@ jobs: APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then - export APPSMITH_DB_URI="jdbc:postgresql://localhost:5432/postgres" + export APPSMITH_DB_URL="jdbc:postgresql://localhost:5432/postgres" else - export APPSMITH_DB_URI="mongodb://localhost:27017/mobtools" + export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" fi args=() if [[ "${{ steps.run_result.outputs.run_result }}" == "failedtest" ]]; then diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java index 492f6e5b52ae..d70c28960226 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java @@ -25,7 +25,7 @@ public void rollbackExecution() {} @Execution public void executeMigration() throws IOException { final String currentEnvName = "APPSMITH_MONGODB_URI"; - final String updatedEnvName = "APPSMITH_DB_URI"; + final String updatedEnvName = "APPSMITH_DB_URL"; updateEnvInFile(currentEnvName, updatedEnvName, commonConfig.getEnvFilePath()); } diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index 2bc3ba36991d..48657b402d37 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -7,7 +7,7 @@ server.max-http-request-header-size=16KB spring.lifecycle.timeout-per-shutdown-phase=20s spring.profiles.active=${ACTIVE_PROFILE:production} -spring.data.mongodb.uri=${APPSMITH_DB_URI:${APPSMITH_MONGODB_URI}} +spring.data.mongodb.uri=${APPSMITH_DB_URL:${APPSMITH_MONGODB_URI}} # This property allows us to override beans during testing. This is useful when we want to set different configurations # and different parameters during test as compared to production. If this property is disabled, some tests will fail. diff --git a/deploy/docker/route-tests/entrypoint.sh b/deploy/docker/route-tests/entrypoint.sh index 6b37d9de3e7c..787d36d9c784 100644 --- a/deploy/docker/route-tests/entrypoint.sh +++ b/deploy/docker/route-tests/entrypoint.sh @@ -130,13 +130,13 @@ reload-caddy run-hurl --variable frame_ancestors="something.com" \ common/*.hurl -# Check if APPSMITH_DB_URI is not set -if [[ -z "${APPSMITH_DB_URI}" ]]; then - # If APPSMITH_DB_URI is not set, fall back to APPSMITH_MONGODB_URI - export APPSMITH_DB_URI="${APPSMITH_MONGODB_URI}" +# Check if APPSMITH_DB_URL is not set +if [[ -z "${APPSMITH_DB_URL}" ]]; then + # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI + export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" fi -# Check if APPSMITH_DB_URI is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" -if [[ "${APPSMITH_DB_URI}" == *"postgresql://"* && "${APPSMITH_DB_URI}" != jdbc:* ]]; then - export APPSMITH_DB_URI="jdbc:${APPSMITH_DB_URI}" +# Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" +if [[ "${APPSMITH_DB_URL}" == *"postgresql://"* && "${APPSMITH_DB_URL}" != jdbc:* ]]; then + export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" fi From ea58c43d6286f6602889be58c40b86a460be59d3 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 19:02:14 +0530 Subject: [PATCH 14/41] refactor: Add DB url config in entrypoint.sh --- deploy/docker/fs/opt/appsmith/entrypoint.sh | 14 ++++++++++++++ deploy/docker/route-tests/entrypoint.sh | 11 ----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 8e1c53d5cc1e..106fb585af77 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -472,6 +472,19 @@ function capture_infra_details(){ bash /opt/appsmith/generate-infra-details.sh || true } +configure_database_connection_url() { + # Check if APPSMITH_DB_URL is not set + if [[ -z "${APPSMITH_DB_URL}" ]]; then + # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI + export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" + fi + + # Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" + if [[ "${APPSMITH_DB_URL}" == *"postgresql://"* && "${APPSMITH_DB_URL}" != jdbc:* ]]; then + export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" + fi +} + # Main Section print_appsmith_info init_loading_pages @@ -507,6 +520,7 @@ export APPSMITH_LOG_DIR="${APPSMITH_LOG_DIR:-/appsmith-stacks/logs}" mkdir -p "$APPSMITH_LOG_DIR"/{supervisor,backend,cron,editor,rts,mongodb,redis,postgres,appsmithctl} setup_auto_heal +configure_database_connection_url capture_infra_details # Handle CMD command diff --git a/deploy/docker/route-tests/entrypoint.sh b/deploy/docker/route-tests/entrypoint.sh index 787d36d9c784..8e0d98c264fc 100644 --- a/deploy/docker/route-tests/entrypoint.sh +++ b/deploy/docker/route-tests/entrypoint.sh @@ -129,14 +129,3 @@ node /caddy-reconfigure.mjs reload-caddy run-hurl --variable frame_ancestors="something.com" \ common/*.hurl - -# Check if APPSMITH_DB_URL is not set -if [[ -z "${APPSMITH_DB_URL}" ]]; then - # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI - export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" -fi - -# Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" -if [[ "${APPSMITH_DB_URL}" == *"postgresql://"* && "${APPSMITH_DB_URL}" != jdbc:* ]]; then - export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" -fi From e34e211a6c4649ec8e354fc811607b23d70fc80b Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 22:01:27 +0530 Subject: [PATCH 15/41] cleanup: Remove un-necessary migration to update db URL name --- .../Migration056UpdateDbUriEnvVariable.java | 53 ------------------- 1 file changed, 53 deletions(-) delete mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java deleted file mode 100644 index d70c28960226..000000000000 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/db/ce/Migration056UpdateDbUriEnvVariable.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.appsmith.server.migrations.db.ce; - -import com.appsmith.server.configurations.CommonConfig; -import io.mongock.api.annotations.ChangeUnit; -import io.mongock.api.annotations.Execution; -import io.mongock.api.annotations.RollbackExecution; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.stream.Collectors; - -@Slf4j -@RequiredArgsConstructor -@ChangeUnit(order = "056", id = "update-db-uri-env-variable") -public class Migration056UpdateDbUriEnvVariable { - private final CommonConfig commonConfig; - - @RollbackExecution - public void rollbackExecution() {} - - @Execution - public void executeMigration() throws IOException { - final String currentEnvName = "APPSMITH_MONGODB_URI"; - final String updatedEnvName = "APPSMITH_DB_URL"; - updateEnvInFile(currentEnvName, updatedEnvName, commonConfig.getEnvFilePath()); - } - - private static void updateEnvInFile( - final String currentEnvName, final String updatedEnvName, final String envPathString) { - if (StringUtils.isEmpty(envPathString) || StringUtils.isEmpty(updatedEnvName)) { - return; - } - - final Path envPath = Path.of(envPathString); - try { - final String updatedLines = Files.readAllLines(envPath).stream() - .map(line -> { - if (line.startsWith(currentEnvName + "=")) { - return updatedEnvName + "=" + System.getenv(currentEnvName); - } - return line; - }) - .collect(Collectors.joining("\n")); - Files.writeString(envPath, updatedLines); - } catch (IOException e) { - log.error("Error updating the DB URI environment variable in the file at path {}", envPath, e); - } - } -} From 285cb3718094be695a015605c873d81f63cf03cd Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 22:02:28 +0530 Subject: [PATCH 16/41] chore: Remove fallback for DB URL --- .../appsmith-server/src/main/resources/application.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index 48657b402d37..f4dd7edcd4e9 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -7,7 +7,7 @@ server.max-http-request-header-size=16KB spring.lifecycle.timeout-per-shutdown-phase=20s spring.profiles.active=${ACTIVE_PROFILE:production} -spring.data.mongodb.uri=${APPSMITH_DB_URL:${APPSMITH_MONGODB_URI}} +spring.data.mongodb.uri=${APPSMITH_DB_URL:} # This property allows us to override beans during testing. This is useful when we want to set different configurations # and different parameters during test as compared to production. If this property is disabled, some tests will fail. From bfc372c614fe4da4e1449ee4159f9f6f4d5aefc7 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 23:21:11 +0530 Subject: [PATCH 17/41] feat: Add more context on DB url to initialise the DB from entrypoint.sh --- deploy/docker/fs/opt/appsmith/entrypoint.sh | 59 +++++++++++++-------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 106fb585af77..254af2a424c9 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -168,11 +168,32 @@ unset_unused_variables() { fi } -check_mongodb_uri() { - echo "Checking APPSMITH_MONGODB_URI" +configure_database_connection_url() { + echo "Configuring database connection URL" + isPostgresUrl=0 + isMongoUrl=0 + # Check if APPSMITH_DB_URL is not set + if [[ -z "${APPSMITH_DB_URL}" ]]; then + # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI + export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" + fi + + if [[ "${APPSMITH_DB_URL}" == *"postgresql"* ]]; then + isPostgresUrl=1 + # Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" + if [[ "${APPSMITH_DB_URL}" != *"jdbc:"* ]]; then + export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" + fi + elif [[ "${APPSMITH_DB_URL}" == *"mongodb"* ]]; then + isMongoUrl=1 + fi +} + +check_db_uri() { + echo "Checking APPSMITH_DB_URL" isUriLocal=1 - if [[ $APPSMITH_MONGODB_URI == *"localhost"* || $APPSMITH_MONGODB_URI == *"127.0.0.1"* ]]; then - echo "Detected local MongoDB" + if [[ $APPSMITH_DB_URL == *"localhost"* || $APPSMITH_DB_URL == *"127.0.0.1"* ]]; then + echo "Detected local DB" isUriLocal=0 fi } @@ -472,29 +493,24 @@ function capture_infra_details(){ bash /opt/appsmith/generate-infra-details.sh || true } -configure_database_connection_url() { - # Check if APPSMITH_DB_URL is not set - if [[ -z "${APPSMITH_DB_URL}" ]]; then - # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI - export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" - fi - - # Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" - if [[ "${APPSMITH_DB_URL}" == *"postgresql://"* && "${APPSMITH_DB_URL}" != jdbc:* ]]; then - export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" - fi -} - # Main Section print_appsmith_info init_loading_pages unset_unused_variables -check_mongodb_uri +configure_database_connection_url +check_db_uri +# Don't run MongoDB if running in a Heroku dyno. if [[ -z "${DYNO}" ]]; then - # Don't run MongoDB if running in a Heroku dyno. - init_mongodb - init_replica_set + # Setup MongoDB and initialize replica set if required + if [[ $isMongoUrl -eq 1 ]]; then + echo "Initializing MongoDB" + init_mongodb + init_replica_set + elif [ $isPostgresUrl -eq 1 ]; then + echo "Initializing Postgres" + # init_postgres + fi else # These functions are used to limit heap size for Backend process when deployed on Heroku get_maximum_heap @@ -520,7 +536,6 @@ export APPSMITH_LOG_DIR="${APPSMITH_LOG_DIR:-/appsmith-stacks/logs}" mkdir -p "$APPSMITH_LOG_DIR"/{supervisor,backend,cron,editor,rts,mongodb,redis,postgres,appsmithctl} setup_auto_heal -configure_database_connection_url capture_infra_details # Handle CMD command From 4ce41e5cdcae5e17451110c29fac932e174dc2e5 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 23:42:39 +0530 Subject: [PATCH 18/41] fix: Postgres container run command for running the tests --- .github/workflows/server-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index 396d25599dfd..ebc9aecb9120 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -141,7 +141,7 @@ jobs: - name: Conditionally start PostgreSQL run: | if [[ inputs.is-pg-build == 'true' ]]; then - docker run -d -p 5432:5432 postgres:alpine + docker run --name appsmith-pg -p 5432:5432 -d -e POSTGRES_PASSWORD=password postgres:alpine fi # Retrieve maven dependencies from cache. After a successful run, these dependencies are cached again From fab1e9b4c3125a5dd41fd34d40ebc1dd3f0785dd Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Thu, 16 May 2024 23:49:42 +0530 Subject: [PATCH 19/41] chore: Update the check to test for jdbc string at the start of the DB url --- deploy/docker/fs/opt/appsmith/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 254af2a424c9..26c2116d3542 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -181,7 +181,7 @@ configure_database_connection_url() { if [[ "${APPSMITH_DB_URL}" == *"postgresql"* ]]; then isPostgresUrl=1 # Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" - if [[ "${APPSMITH_DB_URL}" != *"jdbc:"* ]]; then + if [[ "${APPSMITH_DB_URL}" != "jdbc:"* ]]; then export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" fi elif [[ "${APPSMITH_DB_URL}" == *"mongodb"* ]]; then @@ -502,8 +502,8 @@ configure_database_connection_url check_db_uri # Don't run MongoDB if running in a Heroku dyno. if [[ -z "${DYNO}" ]]; then - # Setup MongoDB and initialize replica set if required if [[ $isMongoUrl -eq 1 ]]; then + # Setup MongoDB and initialize replica set echo "Initializing MongoDB" init_mongodb init_replica_set From 30c7cbae82d4e87f475780152eea9f7b14c25366 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Fri, 17 May 2024 01:34:26 +0530 Subject: [PATCH 20/41] chore: Replace the DB name in few other files --- app.json | 2 +- .../pages/AdminSettings/config/advanced.ts | 2 +- .../server/constants/EnvVariables.java | 2 +- .../server/solutions/EnvManagerTest.java | 63 +++++++++---------- app/server/docker-compose.yml | 2 +- app/server/envs/dev.env.example | 2 +- app/server/envs/test.env.example | 2 +- contributions/ServerSetup.md | 10 +-- deploy/docker/fs/opt/appsmith/entrypoint.sh | 2 +- .../docker/fs/opt/appsmith/mongodb-fixer.sh | 2 +- .../fs/opt/appsmith/templates/docker.env.sh | 2 +- .../appsmith/update-and-restart-supervisor.sh | 2 +- .../fs/opt/appsmith/utils/bin/backup.js | 4 +- .../fs/opt/appsmith/utils/bin/backup.test.js | 4 +- .../appsmith/utils/bin/check_replica_set.js | 2 +- .../fs/opt/appsmith/utils/bin/export_db.js | 2 +- .../fs/opt/appsmith/utils/bin/import_db.js | 4 +- .../appsmith/utils/bin/mongo_shell_utils.js | 2 +- .../fs/opt/appsmith/utils/bin/restore.js | 8 +-- deploy/helm/README.md | 2 +- deploy/helm/templates/configMap.yaml | 2 +- deploy/helm/values.yaml | 2 +- deploy/heroku/README.MD | 2 +- scripts/build_dp_from_branch.sh | 2 +- scripts/deploy_preview.sh | 2 +- 25 files changed, 63 insertions(+), 68 deletions(-) diff --git a/app.json b/app.json index 8cf39b5f7deb..649d48dc3b1c 100644 --- a/app.json +++ b/app.json @@ -18,7 +18,7 @@ "success_url": "/", "stack": "container", "env": { - "APPSMITH_MONGODB_URI": { + "APPSMITH_DB_URL": { "description": "Your Mongo Database URI. Since Heroku doesn't support a managed MongoDB instance, you'll have to create a Mongo DB instance on another service such as https://cloud.mongodb.com", "value": "" }, diff --git a/app/client/src/pages/AdminSettings/config/advanced.ts b/app/client/src/pages/AdminSettings/config/advanced.ts index dfab8254cedf..72d610e26f21 100644 --- a/app/client/src/pages/AdminSettings/config/advanced.ts +++ b/app/client/src/pages/AdminSettings/config/advanced.ts @@ -15,7 +15,7 @@ export const config: AdminConfigType = { canSave: true, settings: [ { - id: "APPSMITH_MONGODB_URI", + id: "APPSMITH_DB_URL", category: SettingCategories.ADVANCED, controlType: SettingTypes.TEXTINPUT, controlSubType: SettingSubtype.TEXT, diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/EnvVariables.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/EnvVariables.java index 09d8212d5e43..0ae80cad9363 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/EnvVariables.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/EnvVariables.java @@ -2,7 +2,7 @@ public enum EnvVariables { APPSMITH_INSTANCE_NAME, - APPSMITH_MONGODB_URI, + APPSMITH_DB_URL, APPSMITH_REDIS_URL, APPSMITH_MAIL_ENABLED, APPSMITH_MAIL_FROM, diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/EnvManagerTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/EnvManagerTest.java index 3458e4610e35..8c798564c2cb 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/EnvManagerTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/EnvManagerTest.java @@ -118,25 +118,25 @@ public void setup() { @Test public void simpleSample() { final String content = - "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL='second value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; + "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL='second value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; - assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_MONGODB_URI", "new first value"))) + assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_DB_URL", "new first value"))) .containsExactly( - "APPSMITH_MONGODB_URI='new first value'", + "APPSMITH_DB_URL='new first value'", "APPSMITH_REDIS_URL='second value'", "", "APPSMITH_INSTANCE_NAME='third value'"); assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_REDIS_URL", "new second value"))) .containsExactly( - "APPSMITH_MONGODB_URI='first value'", + "APPSMITH_DB_URL='first value'", "APPSMITH_REDIS_URL='new second value'", "", "APPSMITH_INSTANCE_NAME='third value'"); assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_INSTANCE_NAME", "new third value"))) .containsExactly( - "APPSMITH_MONGODB_URI='first value'", + "APPSMITH_DB_URL='first value'", "APPSMITH_REDIS_URL='second value'", "", "APPSMITH_INSTANCE_NAME='new third value'"); @@ -144,10 +144,10 @@ public void simpleSample() { assertThat(envManager.transformEnvContent( content, Map.of( - "APPSMITH_MONGODB_URI", "new first value", + "APPSMITH_DB_URL", "new first value", "APPSMITH_INSTANCE_NAME", "new third value"))) .containsExactly( - "APPSMITH_MONGODB_URI='new first value'", + "APPSMITH_DB_URL='new first value'", "APPSMITH_REDIS_URL='second value'", "", "APPSMITH_INSTANCE_NAME='new third value'"); @@ -155,43 +155,39 @@ public void simpleSample() { @Test public void emptyValues() { - final String content = - "APPSMITH_MONGODB_URI=first value\nAPPSMITH_REDIS_URL=\n\nAPPSMITH_INSTANCE_NAME=third value"; + final String content = "APPSMITH_DB_URL=first value\nAPPSMITH_REDIS_URL=\n\nAPPSMITH_INSTANCE_NAME=third value"; assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_REDIS_URL", "new second value"))) .containsExactly( - "APPSMITH_MONGODB_URI=first value", + "APPSMITH_DB_URL=first value", "APPSMITH_REDIS_URL='new second value'", "", "APPSMITH_INSTANCE_NAME=third value"); assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_REDIS_URL", ""))) .containsExactly( - "APPSMITH_MONGODB_URI=first value", - "APPSMITH_REDIS_URL=", - "", - "APPSMITH_INSTANCE_NAME=third value"); + "APPSMITH_DB_URL=first value", "APPSMITH_REDIS_URL=", "", "APPSMITH_INSTANCE_NAME=third value"); } @Test public void quotedValues() { final String content = - "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME='third value'"; + "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME='third value'"; assertThat(envManager.transformEnvContent( content, Map.of( - "APPSMITH_MONGODB_URI", "new first value", + "APPSMITH_DB_URL", "new first value", "APPSMITH_REDIS_URL", "new second value"))) .containsExactly( - "APPSMITH_MONGODB_URI='new first value'", + "APPSMITH_DB_URL='new first value'", "APPSMITH_REDIS_URL='new second value'", "", "APPSMITH_INSTANCE_NAME='third value'"); assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_REDIS_URL", ""))) .containsExactly( - "APPSMITH_MONGODB_URI='first value'", + "APPSMITH_DB_URL='first value'", "APPSMITH_REDIS_URL=", "", "APPSMITH_INSTANCE_NAME='third value'"); @@ -202,7 +198,7 @@ public void quotedValues() { "APPSMITH_INSTANCE_NAME", "Sponge-bob's Instance", "APPSMITH_REDIS_URL", "value with \" char in it"))) .containsExactly( - "APPSMITH_MONGODB_URI='first value'", + "APPSMITH_DB_URL='first value'", "APPSMITH_REDIS_URL='value with \" char in it'", "", "APPSMITH_INSTANCE_NAME='Sponge-bob'\"'\"'s Instance'"); @@ -211,11 +207,10 @@ public void quotedValues() { @Test public void parseEmptyValues() { - assertThat( - envManager.parseToMap( - "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL=\n\nAPPSMITH_INSTANCE_NAME='third value'")) + assertThat(envManager.parseToMap( + "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL=\n\nAPPSMITH_INSTANCE_NAME='third value'")) .containsExactlyInAnyOrderEntriesOf(Map.of( - "APPSMITH_MONGODB_URI", "first value", + "APPSMITH_DB_URL", "first value", "APPSMITH_REDIS_URL", "", "APPSMITH_INSTANCE_NAME", "third value")); } @@ -225,9 +220,9 @@ public void parseQuotedValues() { assertThat( envManager.parseToMap( - "APPSMITH_MONGODB_URI=first\nAPPSMITH_REDIS_URL=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME='third value'")) + "APPSMITH_DB_URL=first\nAPPSMITH_REDIS_URL=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME='third value'")) .containsExactlyInAnyOrderEntriesOf(Map.of( - "APPSMITH_MONGODB_URI", "first", + "APPSMITH_DB_URL", "first", "APPSMITH_REDIS_URL", "quoted value", "APPSMITH_INSTANCE_NAME", "third value")); @@ -247,12 +242,12 @@ public void parseTestWithEscapes() { @Test public void disallowedVariable() { final String content = - "APPSMITH_MONGODB_URI=first value\nDISALLOWED_NASTY_STUFF=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME=third value"; + "APPSMITH_DB_URL=first value\nDISALLOWED_NASTY_STUFF=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME=third value"; assertThatThrownBy(() -> envManager.transformEnvContent( content, Map.of( - "APPSMITH_MONGODB_URI", "new first value", + "APPSMITH_DB_URL", "new first value", "DISALLOWED_NASTY_STUFF", "new second value"))) .matches(value -> value instanceof AppsmithException && AppsmithError.GENERIC_BAD_REQUEST.equals(((AppsmithException) value).getError())); @@ -261,15 +256,15 @@ public void disallowedVariable() { @Test public void addNewVariable() { final String content = - "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL='quoted value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; + "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL='quoted value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; assertThat(envManager.transformEnvContent( content, Map.of( - "APPSMITH_MONGODB_URI", "new first value", + "APPSMITH_DB_URL", "new first value", "APPSMITH_DISABLE_TELEMETRY", "false"))) .containsExactly( - "APPSMITH_MONGODB_URI='new first value'", + "APPSMITH_DB_URL='new first value'", "APPSMITH_REDIS_URL='quoted value'", "", "APPSMITH_INSTANCE_NAME='third value'", @@ -279,15 +274,15 @@ public void addNewVariable() { @Test public void setValueWithQuotes() { final String content = - "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL='quoted value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; + "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL='quoted value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; assertThat(envManager.transformEnvContent( content, Map.of( - "APPSMITH_MONGODB_URI", "'just quotes'", + "APPSMITH_DB_URL", "'just quotes'", "APPSMITH_DISABLE_TELEMETRY", "some quotes 'inside' it"))) .containsExactly( - "APPSMITH_MONGODB_URI=\"'\"'just quotes'\"'\"", + "APPSMITH_DB_URL=\"'\"'just quotes'\"'\"", "APPSMITH_REDIS_URL='quoted value'", "", "APPSMITH_INSTANCE_NAME='third value'", @@ -357,7 +352,7 @@ public void setEnv_AndGetAll() { EnvManager envManagerInner = Mockito.mock(EnvManagerImpl.class); Map envs = new HashMap<>(); - envs.put("APPSMITH_MONGODB_URI", "mongo-url"); + envs.put("APPSMITH_DB_URL", "mongo-url"); envs.put("APPSMITH_DISABLE_TELEMETRY", ""); Mockito.when(envManagerInner.getAll()).thenReturn(Mono.just(envs)); diff --git a/app/server/docker-compose.yml b/app/server/docker-compose.yml index e73ba234df72..b18d97d4834d 100644 --- a/app/server/docker-compose.yml +++ b/app/server/docker-compose.yml @@ -8,7 +8,7 @@ services: env_file: envs/docker.env environment: APPSMITH_REDIS_URL: "redis://redis:6379" - APPSMITH_MONGODB_URI: "mongodb://mongo:27017/appsmith" + APPSMITH_DB_URL: "mongodb://mongo:27017/appsmith" ports: - "8080:8080" depends_on: diff --git a/app/server/envs/dev.env.example b/app/server/envs/dev.env.example index 463d4b9b49c2..bcec41238b8b 100644 --- a/app/server/envs/dev.env.example +++ b/app/server/envs/dev.env.example @@ -1,6 +1,6 @@ #!/bin/sh -APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith?replicaSet=rs0" +APPSMITH_DB_URL="mongodb://localhost:27017/appsmith?replicaSet=rs0" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" diff --git a/app/server/envs/test.env.example b/app/server/envs/test.env.example index fdeb33ce7157..1069f4993f2f 100644 --- a/app/server/envs/test.env.example +++ b/app/server/envs/test.env.example @@ -2,7 +2,7 @@ ACTIVE_PROFILE=test -APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" +APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" diff --git a/contributions/ServerSetup.md b/contributions/ServerSetup.md index 1b34e18ec83f..80c62ba5adba 100644 --- a/contributions/ServerSetup.md +++ b/contributions/ServerSetup.md @@ -124,11 +124,11 @@ With the prerequisites met, let's build the code. This command creates a `.env` file in the `app/server` folder. All run scripts pick up environment configuration from this file. -5. Ensure that the environment variables `APPSMITH_MONGODB_URI` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. +5. Ensure that the environment variables `APPSMITH_DB_URL` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. 6. **Update the replica set name with correct value in the mongo connection string in the [.env](#setup-environment-file) file.** The replica name is the same as passed [here](#setting-up-a-local-mongodb-instance) ```bash - APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith?replicaSet=" + APPSMITH_DB_URL="mongodb://localhost:27017/appsmith?replicaSet=" ``` 7. Run the following command to create the final JAR for the Appsmith server: @@ -152,7 +152,7 @@ With the prerequisites met, let's build the code. - On Ubuntu Linux environment docker needs root privilege, hence `./build.sh` script needs to be run with root privilege as well. - On Ubuntu Linux environment, the script may not be able to read `.env` file, so it is advised that you run the cmd like: ```console - sudo APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh + sudo APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh ``` @@ -260,7 +260,7 @@ cp envs/dev.env.example .env This command creates a `.env` file in the `app/server` folder. All run scripts pick up environment configuration from this file. -5. Ensure that the environment variables `APPSMITH_MONGODB_URI` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. +5. Ensure that the environment variables `APPSMITH_DB_URL` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. 6. Run the following command to create the final JAR for the Appsmith server: @@ -274,7 +274,7 @@ Note: - On Ubuntu Linux environment docker needs root privilege, hence ./build.sh script needs to be run with root privilege as well. - On Ubuntu Linux environment, the script may not be able to read .env file, so it is advised that you run the cmd like: ```console -sudo APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh +sudo APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh ``` - If the volume containing docker's data root path (macOS: `~/Library/Containers/com.docker.docker/Data/vms/0/`, Ubuntu: `/var/lib/docker/`) has less than 2 GB of free space, then the script may fail with the following error: ```console diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 26c2116d3542..00538d9cb993 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -260,7 +260,7 @@ init_replica_set() { mongod --fork --port 27017 --dbpath "$MONGO_DB_PATH" --logpath "$MONGO_LOG_PATH" --replSet mr1 --keyFile "$MONGODB_TMP_KEY_PATH" --bind_ip localhost echo "Waiting 10s for MongoDB to start with Replica Set" sleep 10 - mongosh "$APPSMITH_MONGODB_URI" --eval 'rs.initiate()' + mongosh "$APPSMITH_DB_URL" --eval 'rs.initiate()' mongod --dbpath "$MONGO_DB_PATH" --shutdown || true fi diff --git a/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh b/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh index 9d070d4e29d8..3808b2854fa0 100644 --- a/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh +++ b/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh @@ -16,7 +16,7 @@ done echo "MongoDB is RUNNING" for _ in {1..60}; do - if mongosh --quiet "$APPSMITH_MONGODB_URI" --eval ' + if mongosh --quiet "$APPSMITH_DB_URL" --eval ' parseFloat(db.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}).featureCompatibilityVersion.version) < 5 && db.adminCommand({setFeatureCompatibilityVersion: "5.0"}) '; then diff --git a/deploy/docker/fs/opt/appsmith/templates/docker.env.sh b/deploy/docker/fs/opt/appsmith/templates/docker.env.sh index 3aff309e73eb..e80133242808 100644 --- a/deploy/docker/fs/opt/appsmith/templates/docker.env.sh +++ b/deploy/docker/fs/opt/appsmith/templates/docker.env.sh @@ -63,7 +63,7 @@ APPSMITH_RECAPTCHA_SITE_KEY= APPSMITH_RECAPTCHA_SECRET_KEY= APPSMITH_RECAPTCHA_ENABLED= -APPSMITH_MONGODB_URI=mongodb://$MONGO_USER:$MONGO_PASSWORD@localhost:27017/appsmith +APPSMITH_DB_URL=mongodb://$MONGO_USER:$MONGO_PASSWORD@localhost:27017/appsmith APPSMITH_MONGODB_USER=$MONGO_USER APPSMITH_MONGODB_PASSWORD=$MONGO_PASSWORD APPSMITH_API_BASE_URL=http://localhost:8080/api/v1 diff --git a/deploy/docker/fs/opt/appsmith/update-and-restart-supervisor.sh b/deploy/docker/fs/opt/appsmith/update-and-restart-supervisor.sh index 0768b2d13566..36e5d25f35b5 100755 --- a/deploy/docker/fs/opt/appsmith/update-and-restart-supervisor.sh +++ b/deploy/docker/fs/opt/appsmith/update-and-restart-supervisor.sh @@ -13,7 +13,7 @@ set +o allexport check_mongodb_uri() { echo "Check MongoDB uri host" isLocalMongo=1 - if [[ $APPSMITH_MONGODB_URI == *"localhost"* || $APPSMITH_MONGODB_URI == *"127.0.0.1"* ]]; then + if [[ $APPSMITH_DB_URL == *"localhost"* || $APPSMITH_DB_URL == *"127.0.0.1"* ]]; then echo "Use local MongoDB" isLocalMongo=0 fi diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index a6fd65b23ebc..e39bdbb3caaa 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -125,7 +125,7 @@ function getEncryptionPasswordFromUser(){ async function exportDatabase(destFolder) { console.log('Exporting database'); - await executeMongoDumpCMD(destFolder, process.env.APPSMITH_MONGODB_URI) + await executeMongoDumpCMD(destFolder, process.env.APPSMITH_DB_URL) console.log('Exporting database done.'); } @@ -141,7 +141,7 @@ async function createGitStorageArchive(destFolder) { async function createManifestFile(path) { const version = await utils.getCurrentAppsmithVersion() - const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_MONGODB_URI) } + const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_DB_URL) } await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data)); } diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js index 2478f7a902e2..335dfd71a427 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js @@ -91,12 +91,12 @@ it('Checks for the current Appsmith Version.', async () => { }) test('If MONGODB and Encryption env values are being removed', () => { - expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_MONGODB_URI=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) }); test('If MONGODB and Encryption env values are being removed', () => { - expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_MONGODB_URI=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n + expect(backup.removeSensitiveEnvData(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_ENCRYPTION_PASSWORD=dummy-pass\nAPPSMITH_ENCRYPTION_SALT=dummy-salt\nAPPSMITH_DB_URL=mongodb://appsmith:pass@localhost:27017/appsmith\nAPPSMITH_MONGODB_USER=appsmith\nAPPSMITH_MONGODB_PASSWORD=pass\nAPPSMITH_INSTANCE_NAME=Appsmith\n `)).toMatch(`APPSMITH_REDIS_URL=redis://127.0.0.1:6379\nAPPSMITH_INSTANCE_NAME=Appsmith\n`) }); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js b/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js index 0a023726da1a..3289ebfa479d 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/check_replica_set.js @@ -2,7 +2,7 @@ const { MongoClient, MongoServerError} = require("mongodb"); const { preprocessMongoDBURI } = require("./utils"); async function exec() { - const client = new MongoClient(preprocessMongoDBURI(process.env.APPSMITH_MONGODB_URI), { + const client = new MongoClient(preprocessMongoDBURI(process.env.APPSMITH_DB_URL), { useNewUrlParser: true, useUnifiedTopology: true, }); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js index d4554a688180..f5e894b0c434 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/export_db.js @@ -5,7 +5,7 @@ const Constants = require('./constants'); function export_database() { console.log('export_database ....'); shell.mkdir('-p', [Constants.BACKUP_PATH]); - const cmd = `mongodump --uri='${process.env.APPSMITH_MONGODB_URI}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; + const cmd = `mongodump --uri='${process.env.APPSMITH_DB_URL}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`; shell.exec(cmd); console.log('export_database done'); } diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js index f3c2c7254c6f..57c326a0ac9b 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/import_db.js @@ -6,7 +6,7 @@ const Constants = require('./constants') function import_database() { console.log('import_database ....') - const cmd = `mongorestore --uri='${process.env.APPSMITH_MONGODB_URI}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip` + const cmd = `mongorestore --uri='${process.env.APPSMITH_DB_URL}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip` shell.exec(cmd) console.log('import_database done') } @@ -34,7 +34,7 @@ const main = (forceOption) => { shell.echo('stop backend & rts application before import database') stop_application() - const shellCmdResult = shell.exec(`mongo ${process.env.APPSMITH_MONGODB_URI} --quiet --eval "db.getCollectionNames().length"`) + const shellCmdResult = shell.exec(`mongo ${process.env.APPSMITH_DB_URL} --quiet --eval "db.getCollectionNames().length"`) const collectionsLen = parseInt(shellCmdResult.stdout.toString().trimEnd()) if (collectionsLen > 0) { if (forceOption) { diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js b/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js index 1ae91de416a0..d0ab437b97e7 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/mongo_shell_utils.js @@ -6,7 +6,7 @@ const command_args = process.argv.slice(3); async function exec() { let errorCode = 0; try { - await execMongoEval(command_args[0], process.env.APPSMITH_MONGODB_URI); + await execMongoEval(command_args[0], process.env.APPSMITH_DB_URL); } catch (err) { errorCode = 1; console.error('Error evaluating the mongo query', err); diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js index b3640fb900e9..f79f84cf8f22 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/restore.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/restore.js @@ -59,10 +59,10 @@ async function extractArchive(backupFilePath, restoreRootPath) { async function restoreDatabase(restoreContentsPath) { console.log('Restoring database...'); - const cmd = ['mongorestore', `--uri=${process.env.APPSMITH_MONGODB_URI}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip'] + const cmd = ['mongorestore', `--uri=${process.env.APPSMITH_DB_URL}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip'] try { const fromDbName = await getBackupDatabaseName(restoreContentsPath); - const toDbName = utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_MONGODB_URI); + const toDbName = utils.getDatabaseNameFromMongoURI(process.env.APPSMITH_DB_URL); console.log("Restoring database from " + fromDbName + " to " + toDbName) cmd.push('--nsInclude=*', `--nsFrom=${fromDbName}.*`, `--nsTo=${toDbName}.*`) } catch (error) { @@ -105,10 +105,10 @@ async function restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEn hideEchoBack: true }); } - await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd + '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt + '\nAPPSMITH_MONGODB_URI=' + process.env.APPSMITH_MONGODB_URI + + await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_ENCRYPTION_PASSWORD=' + encryptionPwd + '\nAPPSMITH_ENCRYPTION_SALT=' + encryptionSalt + '\nAPPSMITH_DB_URL=' + process.env.APPSMITH_DB_URL + '\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ; } else { - await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_MONGODB_URI=' + process.env.APPSMITH_MONGODB_URI + + await fsPromises.appendFile(dockerEnvFile, '\nAPPSMITH_DB_URL=' + process.env.APPSMITH_DB_URL + '\nAPPSMITH_MONGODB_USER=' + process.env.APPSMITH_MONGODB_USER + '\nAPPSMITH_MONGODB_PASSWORD=' + process.env.APPSMITH_MONGODB_PASSWORD ) ; } console.log('Restoring docker environment file completed'); diff --git a/deploy/helm/README.md b/deploy/helm/README.md index f0ca5697e9bc..9e18477dd891 100644 --- a/deploy/helm/README.md +++ b/deploy/helm/README.md @@ -188,7 +188,7 @@ To change Appsmith configurations, you can use configuration UI in application o | `applicationConfig.APPSMITH_RECAPTCHA_SITE_KEY` | `""` | | `applicationConfig.APPSMITH_RECAPTCHA_SECRET_KEY` | `""` | | `applicationConfig.APPSMITH_RECAPTCHA_ENABLED` | `""` | -| `applicationConfig.APPSMITH_MONGODB_URI` | `""` | +| `applicationConfig.APPSMITH_DB_URL` | `""` | | `applicationConfig.APPSMITH_REDIS_URL` | `""` | | `applicationConfig.APPSMITH_ENCRYPTION_PASSWORD` | `""` | | `applicationConfig.APPSMITH_ENCRYPTION_SALT` | `""` | diff --git a/deploy/helm/templates/configMap.yaml b/deploy/helm/templates/configMap.yaml index 96a64b5af9f3..367349291cce 100644 --- a/deploy/helm/templates/configMap.yaml +++ b/deploy/helm/templates/configMap.yaml @@ -13,7 +13,7 @@ metadata: {{- include "appsmith.labels" . | nindent 4 }} data: {{- range $key, $value := .Values.applicationConfig }} - {{- if and (eq "APPSMITH_MONGODB_URI" $key) ( not $value) }} + {{- if and (eq "APPSMITH_DB_URL" $key) ( not $value) }} {{- if $.Values.mongodb.enabled }} {{ $key }}: mongodb+srv://{{ $mongoUser }}:{{ $mongoPassword }}@{{ $mongoServicename }}.{{ $nameSpace }}.svc.cluster.local/appsmith?retryWrites=true&authSource=admin&ssl=false {{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 1c54837c0782..030335ccfd33 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -305,7 +305,7 @@ applicationConfig: APPSMITH_RECAPTCHA_SITE_KEY: "" APPSMITH_RECAPTCHA_SECRET_KEY: "" APPSMITH_RECAPTCHA_ENABLED: "" - APPSMITH_MONGODB_URI: "" + APPSMITH_DB_URL: "" APPSMITH_REDIS_URL: "" APPSMITH_ENCRYPTION_PASSWORD: "" APPSMITH_ENCRYPTION_SALT: "" diff --git a/deploy/heroku/README.MD b/deploy/heroku/README.MD index d189438d8d54..f91b5e455eac 100644 --- a/deploy/heroku/README.MD +++ b/deploy/heroku/README.MD @@ -9,7 +9,7 @@ Quickly set up Appsmith to explore product functionality using Heroku. - Fill in the required `Config Variables` including: - `APPSMITH_ENCRYPTION_PASSWORD`: Encryption password to encrypt all credentials in the database - `APPSMITH_ENCRYPTION_SALT`: Encryption salt used to encrypt all credentials in the database - - `APPSMITH_MONGODB_URI`: Your Mongo Database URI + - `APPSMITH_DB_URI`: Your Mongo Database URI - (Optional) Customize the default settings in Heroku - `App Name`: Optionally select a name for your application (this will be used in the app URL) diff --git a/scripts/build_dp_from_branch.sh b/scripts/build_dp_from_branch.sh index 9f182f042436..14cc208eded8 100755 --- a/scripts/build_dp_from_branch.sh +++ b/scripts/build_dp_from_branch.sh @@ -72,5 +72,5 @@ helm upgrade -i $CHARTNAME appsmith/appsmith -n $NAMESPACE \ --set image.pullPolicy="Always" --set autoupdate.enabled="true" --set persistence.size=2Gi \ --set applicationConfig.APPSMITH_SENTRY_DSN="https://abf15a075d1347969df44c746cca7eaa@o296332.ingest.sentry.io/1546547" \ --set applicationConfig.APPSMITH_SENTRY_ENVIRONMENT="$NAMESPACE" \ - --set applicationConfig.APPSMITH_MONGODB_URI="mongodb+srv://$DB_USERNAME:$DB_PASSWORD@$DB_URL/$DBNAME?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" \ + --set applicationConfig.APPSMITH_DB_URL="mongodb+srv://$DB_USERNAME:$DB_PASSWORD@$DB_URL/$DBNAME?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" \ --version $HELMCHART_VERSION diff --git a/scripts/deploy_preview.sh b/scripts/deploy_preview.sh index 7c4000272996..030709331e46 100755 --- a/scripts/deploy_preview.sh +++ b/scripts/deploy_preview.sh @@ -100,7 +100,7 @@ helm upgrade -i $CHARTNAME appsmith-ee/$HELMCHART -n $NAMESPACE --create-namespa --set resources.requests.memory="1Mi" \ --set applicationConfig.APPSMITH_SENTRY_DSN="https://abf15a075d1347969df44c746cca7eaa@o296332.ingest.sentry.io/1546547" \ --set applicationConfig.APPSMITH_SENTRY_ENVIRONMENT="$NAMESPACE" \ - --set applicationConfig.APPSMITH_MONGODB_URI="mongodb+srv://$DB_USERNAME:$DB_PASSWORD@$DB_URL/$DBNAME?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" \ + --set applicationConfig.APPSMITH_DB_URL="mongodb+srv://$DB_USERNAME:$DB_PASSWORD@$DB_URL/$DBNAME?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" \ --set applicationConfig.APPSMITH_DISABLE_EMBEDDED_KEYCLOAK=\"1\" \ --set applicationConfig.APPSMITH_CUSTOMER_PORTAL_URL="https://release-customer.appsmith.com" \ --version $HELMCHART_VERSION From 146a2246a27c785e80bb8fe4ceaffa946b03be57 Mon Sep 17 00:00:00 2001 From: Abhijeet <41686026+abhvsn@users.noreply.github.com> Date: Fri, 17 May 2024 01:38:51 +0530 Subject: [PATCH 21/41] fix: DB URL name in readme --- deploy/heroku/README.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/heroku/README.MD b/deploy/heroku/README.MD index f91b5e455eac..f9290ccae810 100644 --- a/deploy/heroku/README.MD +++ b/deploy/heroku/README.MD @@ -9,7 +9,7 @@ Quickly set up Appsmith to explore product functionality using Heroku. - Fill in the required `Config Variables` including: - `APPSMITH_ENCRYPTION_PASSWORD`: Encryption password to encrypt all credentials in the database - `APPSMITH_ENCRYPTION_SALT`: Encryption salt used to encrypt all credentials in the database - - `APPSMITH_DB_URI`: Your Mongo Database URI + - `APPSMITH_DB_URL`: Your Mongo Database URI - (Optional) Customize the default settings in Heroku - `App Name`: Optionally select a name for your application (this will be used in the app URL) From 1ba2d2e5a342aa3e5ce86480ee484f7a7c2315f2 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Fri, 17 May 2024 12:07:47 +0530 Subject: [PATCH 22/41] chore: Add stricter checks for DB URL and don't provide fallback for DB URL --- .../src/main/resources/application.properties | 2 +- deploy/docker/fs/opt/appsmith/entrypoint.sh | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index f4dd7edcd4e9..ee7830541c16 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -7,7 +7,7 @@ server.max-http-request-header-size=16KB spring.lifecycle.timeout-per-shutdown-phase=20s spring.profiles.active=${ACTIVE_PROFILE:production} -spring.data.mongodb.uri=${APPSMITH_DB_URL:} +spring.data.mongodb.uri=${APPSMITH_DB_URL} # This property allows us to override beans during testing. This is useful when we want to set different configurations # and different parameters during test as compared to production. If this property is disabled, some tests will fail. diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 26c2116d3542..c344d4826c67 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -178,13 +178,12 @@ configure_database_connection_url() { export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" fi - if [[ "${APPSMITH_DB_URL}" == *"postgresql"* ]]; then + if [[ "${APPSMITH_DB_URL}" == "jdbc:postgresql:"* ]]; then + isPostgresUrl=1 + elif [[ "${APPSMITH_DB_URL}" == "postgresql:"* ]]; then isPostgresUrl=1 - # Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" - if [[ "${APPSMITH_DB_URL}" != "jdbc:"* ]]; then - export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" - fi - elif [[ "${APPSMITH_DB_URL}" == *"mongodb"* ]]; then + export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" + elif [[ "${APPSMITH_DB_URL}" == "mongodb"* ]]; then isMongoUrl=1 fi } @@ -507,7 +506,7 @@ if [[ -z "${DYNO}" ]]; then echo "Initializing MongoDB" init_mongodb init_replica_set - elif [ $isPostgresUrl -eq 1 ]; then + elif [[ $isPostgresUrl -eq 1 ]]; then echo "Initializing Postgres" # init_postgres fi From 8ec2ebc0c446867eabac8a93edbd592043a0b149 Mon Sep 17 00:00:00 2001 From: Abhijeet <41686026+abhvsn@users.noreply.github.com> Date: Fri, 17 May 2024 12:38:40 +0530 Subject: [PATCH 23/41] chore: Add Postgres envs for username and password --- .github/workflows/server-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index ebc9aecb9120..f5553cf94312 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -177,6 +177,8 @@ jobs: run: | if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then export APPSMITH_DB_URL="jdbc:postgresql://localhost:5432/postgres" + export APPSMITH_DB_USERNAME="postgres" + export APPSMITH_DB_PASSWORD="password" else export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" fi From 2d5a4ab2fe42f8ca38f8cda874060274dd54b501 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Fri, 17 May 2024 15:22:15 +0530 Subject: [PATCH 24/41] feat: Handle bean initialisation for DB configs --- .../server/configurations/CommonDBConfig.java | 71 +++++++++++++++++++ .../src/main/resources/application.properties | 2 +- .../configurations/CommonDBConfigTest.java | 30 ++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java create mode 100644 app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java new file mode 100644 index 000000000000..3aa8eeeb14a0 --- /dev/null +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java @@ -0,0 +1,71 @@ +package com.appsmith.server.configurations; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; +import org.springframework.boot.autoconfigure.mongo.MongoProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.util.StringUtils; + +import java.net.URI; +import java.net.URISyntaxException; + +/** + * Class to configure beans based on DB url. This could have been implemented in {@link MongoConfig} or + * {@link DBConfig} but extracted to seperate class as we were facing cyclical dependency issue with the other approach + */ +@Configuration +@Slf4j +public class CommonDBConfig { + + @Value("${appsmith.db.url}") + private String appsmithDbUrl; + + @Bean + @Primary + public MongoProperties configureMongoDB() { + if (!appsmithDbUrl.startsWith("mongodb")) { + return null; + } + log.info("Found MongoDB uri configuring now"); + MongoProperties mongoProperties = new MongoProperties(); + mongoProperties.setUri(appsmithDbUrl); + return mongoProperties; + } + + @Bean + @Primary + public DataSourceProperties configurePostgresDB() { + if (!appsmithDbUrl.startsWith("jdbc:postgresql")) { + return null; + } + log.info("Found PostgreSQL uri configuring now"); + return extractJdbcProperties(appsmithDbUrl); + } + + /** + * Method to extract Jdbc props from the given DB URL + * Expected DB URL: jdbc:postgresql://{username}:{password}@localhost:{port}/{db_name} + */ + public DataSourceProperties extractJdbcProperties(String dbUrl) { + DataSourceProperties ds = new DataSourceProperties(); + try { + URI uri = new URI(dbUrl.substring(5)); // remove "jdbc:" prefix + String userInfo = uri.getUserInfo(); + if (StringUtils.hasLength(userInfo)) { + String[] userDetails = userInfo.split(":"); + ds.setUsername(userDetails[0]); + ds.setPassword(userDetails[1]); + } + // If the port is not mentioned default it to standard 5432 + int port = uri.getPort() == -1 ? 5432 : uri.getPort(); + String updatedUrl = String.format("jdbc:%s://%s:%s%s", uri.getScheme(), uri.getHost(), port, uri.getPath()); + ds.setUrl(updatedUrl); + return ds; + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } +} diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index ee7830541c16..ee279b0fe59f 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -7,8 +7,8 @@ server.max-http-request-header-size=16KB spring.lifecycle.timeout-per-shutdown-phase=20s spring.profiles.active=${ACTIVE_PROFILE:production} -spring.data.mongodb.uri=${APPSMITH_DB_URL} +appsmith.db.url=${APPSMITH_DB_URL} # This property allows us to override beans during testing. This is useful when we want to set different configurations # and different parameters during test as compared to production. If this property is disabled, some tests will fail. spring.main.allow-bean-definition-overriding=true diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java new file mode 100644 index 000000000000..f6c2ef63b35c --- /dev/null +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java @@ -0,0 +1,30 @@ +package com.appsmith.server.configurations; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class CommonDBConfigTest { + + @Test + public void testExtractAndSaveJdbcParams_validDbUrlWithUsernameAndPassword() { + CommonDBConfig commonDBConfig = new CommonDBConfig(); + String dbUrl = "jdbc:postgresql://postgres:password@localhost/postgres"; + DataSourceProperties ds = commonDBConfig.extractJdbcProperties(dbUrl); + assertEquals("postgres", ds.getUsername()); + assertEquals("password", ds.getPassword()); + assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); + } + + @Test + public void testExtractAndSaveJdbcParams_validDbUrlWithoutUsernameAndPassword() { + CommonDBConfig commonDBConfig = new CommonDBConfig(); + String dbUrl = "jdbc:postgresql://localhost:5432/postgres"; + DataSourceProperties ds = commonDBConfig.extractJdbcProperties(dbUrl); + assertNull(ds.getUsername()); + assertNull(ds.getPassword()); + assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); + } +} From 382b9ecc990288ad6bd513aa01bd602c5edcf1a7 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Fri, 17 May 2024 16:56:15 +0530 Subject: [PATCH 25/41] chore: Update DB URL name in env and readme files --- app/server/envs/dev.env.example | 2 +- app/server/envs/test.env.example | 2 +- contributions/ServerSetup.md | 10 +++++----- deploy/docker/fs/opt/appsmith/entrypoint.sh | 2 +- deploy/docker/fs/opt/appsmith/mongodb-fixer.sh | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/server/envs/dev.env.example b/app/server/envs/dev.env.example index 463d4b9b49c2..bcec41238b8b 100644 --- a/app/server/envs/dev.env.example +++ b/app/server/envs/dev.env.example @@ -1,6 +1,6 @@ #!/bin/sh -APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith?replicaSet=rs0" +APPSMITH_DB_URL="mongodb://localhost:27017/appsmith?replicaSet=rs0" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" diff --git a/app/server/envs/test.env.example b/app/server/envs/test.env.example index fdeb33ce7157..1069f4993f2f 100644 --- a/app/server/envs/test.env.example +++ b/app/server/envs/test.env.example @@ -2,7 +2,7 @@ ACTIVE_PROFILE=test -APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" +APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" diff --git a/contributions/ServerSetup.md b/contributions/ServerSetup.md index 1b34e18ec83f..80c62ba5adba 100644 --- a/contributions/ServerSetup.md +++ b/contributions/ServerSetup.md @@ -124,11 +124,11 @@ With the prerequisites met, let's build the code. This command creates a `.env` file in the `app/server` folder. All run scripts pick up environment configuration from this file. -5. Ensure that the environment variables `APPSMITH_MONGODB_URI` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. +5. Ensure that the environment variables `APPSMITH_DB_URL` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. 6. **Update the replica set name with correct value in the mongo connection string in the [.env](#setup-environment-file) file.** The replica name is the same as passed [here](#setting-up-a-local-mongodb-instance) ```bash - APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith?replicaSet=" + APPSMITH_DB_URL="mongodb://localhost:27017/appsmith?replicaSet=" ``` 7. Run the following command to create the final JAR for the Appsmith server: @@ -152,7 +152,7 @@ With the prerequisites met, let's build the code. - On Ubuntu Linux environment docker needs root privilege, hence `./build.sh` script needs to be run with root privilege as well. - On Ubuntu Linux environment, the script may not be able to read `.env` file, so it is advised that you run the cmd like: ```console - sudo APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh + sudo APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh ``` @@ -260,7 +260,7 @@ cp envs/dev.env.example .env This command creates a `.env` file in the `app/server` folder. All run scripts pick up environment configuration from this file. -5. Ensure that the environment variables `APPSMITH_MONGODB_URI` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. +5. Ensure that the environment variables `APPSMITH_DB_URL` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis. 6. Run the following command to create the final JAR for the Appsmith server: @@ -274,7 +274,7 @@ Note: - On Ubuntu Linux environment docker needs root privilege, hence ./build.sh script needs to be run with root privilege as well. - On Ubuntu Linux environment, the script may not be able to read .env file, so it is advised that you run the cmd like: ```console -sudo APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh +sudo APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh ``` - If the volume containing docker's data root path (macOS: `~/Library/Containers/com.docker.docker/Data/vms/0/`, Ubuntu: `/var/lib/docker/`) has less than 2 GB of free space, then the script may fail with the following error: ```console diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index c344d4826c67..427995fd7608 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -259,7 +259,7 @@ init_replica_set() { mongod --fork --port 27017 --dbpath "$MONGO_DB_PATH" --logpath "$MONGO_LOG_PATH" --replSet mr1 --keyFile "$MONGODB_TMP_KEY_PATH" --bind_ip localhost echo "Waiting 10s for MongoDB to start with Replica Set" sleep 10 - mongosh "$APPSMITH_MONGODB_URI" --eval 'rs.initiate()' + mongosh "$APPSMITH_DB_URL" --eval 'rs.initiate()' mongod --dbpath "$MONGO_DB_PATH" --shutdown || true fi diff --git a/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh b/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh index 9d070d4e29d8..3808b2854fa0 100644 --- a/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh +++ b/deploy/docker/fs/opt/appsmith/mongodb-fixer.sh @@ -16,7 +16,7 @@ done echo "MongoDB is RUNNING" for _ in {1..60}; do - if mongosh --quiet "$APPSMITH_MONGODB_URI" --eval ' + if mongosh --quiet "$APPSMITH_DB_URL" --eval ' parseFloat(db.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}).featureCompatibilityVersion.version) < 5 && db.adminCommand({setFeatureCompatibilityVersion: "5.0"}) '; then From 3373b5db2061d0bf4ce7ee74b8e469d29c5ecb59 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 10:31:04 +0530 Subject: [PATCH 26/41] revert: Nomenclature change for deploy-preview script --- deploy/helm/values.yaml | 2 +- scripts/build_dp_from_branch.sh | 2 +- scripts/deploy_preview.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 030335ccfd33..1c54837c0782 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -305,7 +305,7 @@ applicationConfig: APPSMITH_RECAPTCHA_SITE_KEY: "" APPSMITH_RECAPTCHA_SECRET_KEY: "" APPSMITH_RECAPTCHA_ENABLED: "" - APPSMITH_DB_URL: "" + APPSMITH_MONGODB_URI: "" APPSMITH_REDIS_URL: "" APPSMITH_ENCRYPTION_PASSWORD: "" APPSMITH_ENCRYPTION_SALT: "" diff --git a/scripts/build_dp_from_branch.sh b/scripts/build_dp_from_branch.sh index 14cc208eded8..9f182f042436 100755 --- a/scripts/build_dp_from_branch.sh +++ b/scripts/build_dp_from_branch.sh @@ -72,5 +72,5 @@ helm upgrade -i $CHARTNAME appsmith/appsmith -n $NAMESPACE \ --set image.pullPolicy="Always" --set autoupdate.enabled="true" --set persistence.size=2Gi \ --set applicationConfig.APPSMITH_SENTRY_DSN="https://abf15a075d1347969df44c746cca7eaa@o296332.ingest.sentry.io/1546547" \ --set applicationConfig.APPSMITH_SENTRY_ENVIRONMENT="$NAMESPACE" \ - --set applicationConfig.APPSMITH_DB_URL="mongodb+srv://$DB_USERNAME:$DB_PASSWORD@$DB_URL/$DBNAME?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" \ + --set applicationConfig.APPSMITH_MONGODB_URI="mongodb+srv://$DB_USERNAME:$DB_PASSWORD@$DB_URL/$DBNAME?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" \ --version $HELMCHART_VERSION diff --git a/scripts/deploy_preview.sh b/scripts/deploy_preview.sh index 030709331e46..7c4000272996 100755 --- a/scripts/deploy_preview.sh +++ b/scripts/deploy_preview.sh @@ -100,7 +100,7 @@ helm upgrade -i $CHARTNAME appsmith-ee/$HELMCHART -n $NAMESPACE --create-namespa --set resources.requests.memory="1Mi" \ --set applicationConfig.APPSMITH_SENTRY_DSN="https://abf15a075d1347969df44c746cca7eaa@o296332.ingest.sentry.io/1546547" \ --set applicationConfig.APPSMITH_SENTRY_ENVIRONMENT="$NAMESPACE" \ - --set applicationConfig.APPSMITH_DB_URL="mongodb+srv://$DB_USERNAME:$DB_PASSWORD@$DB_URL/$DBNAME?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" \ + --set applicationConfig.APPSMITH_MONGODB_URI="mongodb+srv://$DB_USERNAME:$DB_PASSWORD@$DB_URL/$DBNAME?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin" \ --set applicationConfig.APPSMITH_DISABLE_EMBEDDED_KEYCLOAK=\"1\" \ --set applicationConfig.APPSMITH_CUSTOMER_PORTAL_URL="https://release-customer.appsmith.com" \ --version $HELMCHART_VERSION From 0a1199f0767962c7b9e1be274c07104a26c46726 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 10:49:22 +0530 Subject: [PATCH 27/41] chore: Add `ORing` in config map to support backward compatibility --- deploy/helm/templates/configMap.yaml | 2 +- deploy/helm/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/helm/templates/configMap.yaml b/deploy/helm/templates/configMap.yaml index 367349291cce..9b1aecc3643a 100644 --- a/deploy/helm/templates/configMap.yaml +++ b/deploy/helm/templates/configMap.yaml @@ -13,7 +13,7 @@ metadata: {{- include "appsmith.labels" . | nindent 4 }} data: {{- range $key, $value := .Values.applicationConfig }} - {{- if and (eq "APPSMITH_DB_URL" $key) ( not $value) }} + {{- if and (or (eq "APPSMITH_DB_URL" $key) (eq "APPSMITH_MONGODB_URI" $key)) (not $value) }} {{- if $.Values.mongodb.enabled }} {{ $key }}: mongodb+srv://{{ $mongoUser }}:{{ $mongoPassword }}@{{ $mongoServicename }}.{{ $nameSpace }}.svc.cluster.local/appsmith?retryWrites=true&authSource=admin&ssl=false {{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 1c54837c0782..030335ccfd33 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -305,7 +305,7 @@ applicationConfig: APPSMITH_RECAPTCHA_SITE_KEY: "" APPSMITH_RECAPTCHA_SECRET_KEY: "" APPSMITH_RECAPTCHA_ENABLED: "" - APPSMITH_MONGODB_URI: "" + APPSMITH_DB_URL: "" APPSMITH_REDIS_URL: "" APPSMITH_ENCRYPTION_PASSWORD: "" APPSMITH_ENCRYPTION_SALT: "" From f00bdcfcd31d5b3fdd52c4ab4f2e5251cef57b0f Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 11:29:35 +0530 Subject: [PATCH 28/41] feat: Add fallback for APPSMITH_DB_URL in run_with_env script --- deploy/docker/fs/opt/appsmith/run-with-env.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deploy/docker/fs/opt/appsmith/run-with-env.sh b/deploy/docker/fs/opt/appsmith/run-with-env.sh index 93a7bf65fc9c..f100cf9427cc 100755 --- a/deploy/docker/fs/opt/appsmith/run-with-env.sh +++ b/deploy/docker/fs/opt/appsmith/run-with-env.sh @@ -33,4 +33,15 @@ if [[ -z "${APPSMITH_GIT_ROOT:-}" ]]; then fi mkdir -pv "$APPSMITH_GIT_ROOT" +# Check if APPSMITH_DB_URL is set +if [[ -z "${APPSMITH_DB_URL}" ]]; then + # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI + export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" +fi + +if [[ "${APPSMITH_DB_URL}" == "postgresql"* ]]; then + # Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" + export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" +fi + exec "$@" From 87fe4cada37bbef6305a04081be593063c14fb7d Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 11:39:46 +0530 Subject: [PATCH 29/41] feat: Add fallback for APPSMITH_DB_URL in index.js within deploy directory --- deploy/docker/fs/opt/appsmith/utils/bin/index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/index.js b/deploy/docker/fs/opt/appsmith/utils/bin/index.js index 34d4c49492dd..cc9b2f6c2ec2 100755 --- a/deploy/docker/fs/opt/appsmith/utils/bin/index.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/index.js @@ -13,6 +13,11 @@ const APPLICATION_CONFIG_PATH = "/appsmith-stacks/configuration/docker.env"; // Loading latest application configuration require("dotenv").config({ path: APPLICATION_CONFIG_PATH }); +// Check if APPSMITH_DB_URL is set, if not set, fall back to APPSMITH_MONGODB_URI +if (!process.env.APPSMITH_DB_URL) { + process.env.APPSMITH_DB_URL = process.env.APPSMITH_MONGODB_URI; +} + const command = process.argv[2]; if (["export-db", "export_db", "ex"].includes(command)) { From 1562e3e764e7ae377d4f7efd079adec9ff44b0c6 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 12:15:05 +0530 Subject: [PATCH 30/41] revert: EnvManager to use existing APPSMITH_MONGODB_URI, as this will be addressed in a seperate PR --- .../server/constants/EnvVariables.java | 2 +- .../server/solutions/EnvManagerTest.java | 63 ++++++++++--------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/EnvVariables.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/EnvVariables.java index 0ae80cad9363..09d8212d5e43 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/EnvVariables.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/constants/EnvVariables.java @@ -2,7 +2,7 @@ public enum EnvVariables { APPSMITH_INSTANCE_NAME, - APPSMITH_DB_URL, + APPSMITH_MONGODB_URI, APPSMITH_REDIS_URL, APPSMITH_MAIL_ENABLED, APPSMITH_MAIL_FROM, diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/EnvManagerTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/EnvManagerTest.java index 8c798564c2cb..3458e4610e35 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/EnvManagerTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/solutions/EnvManagerTest.java @@ -118,25 +118,25 @@ public void setup() { @Test public void simpleSample() { final String content = - "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL='second value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; + "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL='second value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; - assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_DB_URL", "new first value"))) + assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_MONGODB_URI", "new first value"))) .containsExactly( - "APPSMITH_DB_URL='new first value'", + "APPSMITH_MONGODB_URI='new first value'", "APPSMITH_REDIS_URL='second value'", "", "APPSMITH_INSTANCE_NAME='third value'"); assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_REDIS_URL", "new second value"))) .containsExactly( - "APPSMITH_DB_URL='first value'", + "APPSMITH_MONGODB_URI='first value'", "APPSMITH_REDIS_URL='new second value'", "", "APPSMITH_INSTANCE_NAME='third value'"); assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_INSTANCE_NAME", "new third value"))) .containsExactly( - "APPSMITH_DB_URL='first value'", + "APPSMITH_MONGODB_URI='first value'", "APPSMITH_REDIS_URL='second value'", "", "APPSMITH_INSTANCE_NAME='new third value'"); @@ -144,10 +144,10 @@ public void simpleSample() { assertThat(envManager.transformEnvContent( content, Map.of( - "APPSMITH_DB_URL", "new first value", + "APPSMITH_MONGODB_URI", "new first value", "APPSMITH_INSTANCE_NAME", "new third value"))) .containsExactly( - "APPSMITH_DB_URL='new first value'", + "APPSMITH_MONGODB_URI='new first value'", "APPSMITH_REDIS_URL='second value'", "", "APPSMITH_INSTANCE_NAME='new third value'"); @@ -155,39 +155,43 @@ public void simpleSample() { @Test public void emptyValues() { - final String content = "APPSMITH_DB_URL=first value\nAPPSMITH_REDIS_URL=\n\nAPPSMITH_INSTANCE_NAME=third value"; + final String content = + "APPSMITH_MONGODB_URI=first value\nAPPSMITH_REDIS_URL=\n\nAPPSMITH_INSTANCE_NAME=third value"; assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_REDIS_URL", "new second value"))) .containsExactly( - "APPSMITH_DB_URL=first value", + "APPSMITH_MONGODB_URI=first value", "APPSMITH_REDIS_URL='new second value'", "", "APPSMITH_INSTANCE_NAME=third value"); assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_REDIS_URL", ""))) .containsExactly( - "APPSMITH_DB_URL=first value", "APPSMITH_REDIS_URL=", "", "APPSMITH_INSTANCE_NAME=third value"); + "APPSMITH_MONGODB_URI=first value", + "APPSMITH_REDIS_URL=", + "", + "APPSMITH_INSTANCE_NAME=third value"); } @Test public void quotedValues() { final String content = - "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME='third value'"; + "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME='third value'"; assertThat(envManager.transformEnvContent( content, Map.of( - "APPSMITH_DB_URL", "new first value", + "APPSMITH_MONGODB_URI", "new first value", "APPSMITH_REDIS_URL", "new second value"))) .containsExactly( - "APPSMITH_DB_URL='new first value'", + "APPSMITH_MONGODB_URI='new first value'", "APPSMITH_REDIS_URL='new second value'", "", "APPSMITH_INSTANCE_NAME='third value'"); assertThat(envManager.transformEnvContent(content, Map.of("APPSMITH_REDIS_URL", ""))) .containsExactly( - "APPSMITH_DB_URL='first value'", + "APPSMITH_MONGODB_URI='first value'", "APPSMITH_REDIS_URL=", "", "APPSMITH_INSTANCE_NAME='third value'"); @@ -198,7 +202,7 @@ public void quotedValues() { "APPSMITH_INSTANCE_NAME", "Sponge-bob's Instance", "APPSMITH_REDIS_URL", "value with \" char in it"))) .containsExactly( - "APPSMITH_DB_URL='first value'", + "APPSMITH_MONGODB_URI='first value'", "APPSMITH_REDIS_URL='value with \" char in it'", "", "APPSMITH_INSTANCE_NAME='Sponge-bob'\"'\"'s Instance'"); @@ -207,10 +211,11 @@ public void quotedValues() { @Test public void parseEmptyValues() { - assertThat(envManager.parseToMap( - "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL=\n\nAPPSMITH_INSTANCE_NAME='third value'")) + assertThat( + envManager.parseToMap( + "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL=\n\nAPPSMITH_INSTANCE_NAME='third value'")) .containsExactlyInAnyOrderEntriesOf(Map.of( - "APPSMITH_DB_URL", "first value", + "APPSMITH_MONGODB_URI", "first value", "APPSMITH_REDIS_URL", "", "APPSMITH_INSTANCE_NAME", "third value")); } @@ -220,9 +225,9 @@ public void parseQuotedValues() { assertThat( envManager.parseToMap( - "APPSMITH_DB_URL=first\nAPPSMITH_REDIS_URL=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME='third value'")) + "APPSMITH_MONGODB_URI=first\nAPPSMITH_REDIS_URL=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME='third value'")) .containsExactlyInAnyOrderEntriesOf(Map.of( - "APPSMITH_DB_URL", "first", + "APPSMITH_MONGODB_URI", "first", "APPSMITH_REDIS_URL", "quoted value", "APPSMITH_INSTANCE_NAME", "third value")); @@ -242,12 +247,12 @@ public void parseTestWithEscapes() { @Test public void disallowedVariable() { final String content = - "APPSMITH_DB_URL=first value\nDISALLOWED_NASTY_STUFF=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME=third value"; + "APPSMITH_MONGODB_URI=first value\nDISALLOWED_NASTY_STUFF=\"quoted value\"\n\nAPPSMITH_INSTANCE_NAME=third value"; assertThatThrownBy(() -> envManager.transformEnvContent( content, Map.of( - "APPSMITH_DB_URL", "new first value", + "APPSMITH_MONGODB_URI", "new first value", "DISALLOWED_NASTY_STUFF", "new second value"))) .matches(value -> value instanceof AppsmithException && AppsmithError.GENERIC_BAD_REQUEST.equals(((AppsmithException) value).getError())); @@ -256,15 +261,15 @@ public void disallowedVariable() { @Test public void addNewVariable() { final String content = - "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL='quoted value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; + "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL='quoted value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; assertThat(envManager.transformEnvContent( content, Map.of( - "APPSMITH_DB_URL", "new first value", + "APPSMITH_MONGODB_URI", "new first value", "APPSMITH_DISABLE_TELEMETRY", "false"))) .containsExactly( - "APPSMITH_DB_URL='new first value'", + "APPSMITH_MONGODB_URI='new first value'", "APPSMITH_REDIS_URL='quoted value'", "", "APPSMITH_INSTANCE_NAME='third value'", @@ -274,15 +279,15 @@ public void addNewVariable() { @Test public void setValueWithQuotes() { final String content = - "APPSMITH_DB_URL='first value'\nAPPSMITH_REDIS_URL='quoted value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; + "APPSMITH_MONGODB_URI='first value'\nAPPSMITH_REDIS_URL='quoted value'\n\nAPPSMITH_INSTANCE_NAME='third value'"; assertThat(envManager.transformEnvContent( content, Map.of( - "APPSMITH_DB_URL", "'just quotes'", + "APPSMITH_MONGODB_URI", "'just quotes'", "APPSMITH_DISABLE_TELEMETRY", "some quotes 'inside' it"))) .containsExactly( - "APPSMITH_DB_URL=\"'\"'just quotes'\"'\"", + "APPSMITH_MONGODB_URI=\"'\"'just quotes'\"'\"", "APPSMITH_REDIS_URL='quoted value'", "", "APPSMITH_INSTANCE_NAME='third value'", @@ -352,7 +357,7 @@ public void setEnv_AndGetAll() { EnvManager envManagerInner = Mockito.mock(EnvManagerImpl.class); Map envs = new HashMap<>(); - envs.put("APPSMITH_DB_URL", "mongo-url"); + envs.put("APPSMITH_MONGODB_URI", "mongo-url"); envs.put("APPSMITH_DISABLE_TELEMETRY", ""); Mockito.when(envManagerInner.getAll()).thenReturn(Mono.just(envs)); From 0e32b6ea8f8c8e608521c5b9c525142c4d1c9554 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 13:57:37 +0530 Subject: [PATCH 31/41] chore: Update DB connection url in CI --- .github/workflows/server-build.yml | 4 +--- .../appsmith/server/configurations/CommonDBConfigTest.java | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index f5553cf94312..34fd4270e99f 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -176,9 +176,7 @@ jobs: APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then - export APPSMITH_DB_URL="jdbc:postgresql://localhost:5432/postgres" - export APPSMITH_DB_USERNAME="postgres" - export APPSMITH_DB_PASSWORD="password" + export APPSMITH_DB_URL="jdbc:postgresql://postgres:password@localhost:5432/postgres" else export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" fi diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java index f6c2ef63b35c..f48e9e64f690 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java @@ -16,6 +16,12 @@ public void testExtractAndSaveJdbcParams_validDbUrlWithUsernameAndPassword() { assertEquals("postgres", ds.getUsername()); assertEquals("password", ds.getPassword()); assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); + + String dbUrlWithPort = "jdbc:postgresql://postgres:password@localhost:5432/postgres"; + ds = commonDBConfig.extractJdbcProperties(dbUrlWithPort); + assertEquals("postgres", ds.getUsername()); + assertEquals("password", ds.getPassword()); + assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); } @Test From 22716b4863f2d1cd3e14ab25a06df8a51fdca9dd Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 14:03:11 +0530 Subject: [PATCH 32/41] chore: Keep the fallback DB connection url for local dev till this is completely removed from other places in codebase --- .../appsmith-server/src/main/resources/application.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server/appsmith-server/src/main/resources/application.properties b/app/server/appsmith-server/src/main/resources/application.properties index ee279b0fe59f..c29d41de90cc 100644 --- a/app/server/appsmith-server/src/main/resources/application.properties +++ b/app/server/appsmith-server/src/main/resources/application.properties @@ -8,7 +8,7 @@ spring.lifecycle.timeout-per-shutdown-phase=20s spring.profiles.active=${ACTIVE_PROFILE:production} -appsmith.db.url=${APPSMITH_DB_URL} +appsmith.db.url=${APPSMITH_DB_URL:${APPSMITH_MONGODB_URI}} # This property allows us to override beans during testing. This is useful when we want to set different configurations # and different parameters during test as compared to production. If this property is disabled, some tests will fail. spring.main.allow-bean-definition-overriding=true From 5357654292770ef31d7c08fe326910ad0dd792ca Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 16:21:08 +0530 Subject: [PATCH 33/41] chore: Remove the requirement for jdbc prefix for postgres connection url --- .github/workflows/server-build.yml | 2 +- .../appsmith/server/configurations/CommonDBConfig.java | 10 +++++++--- .../server/configurations/CommonDBConfigTest.java | 6 ++++++ deploy/docker/fs/opt/appsmith/entrypoint.sh | 5 +---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/server-build.yml b/.github/workflows/server-build.yml index a9696bedc30c..7d3f426e1acf 100644 --- a/.github/workflows/server-build.yml +++ b/.github/workflows/server-build.yml @@ -176,7 +176,7 @@ jobs: APPSMITH_VERBOSE_LOGGING_ENABLED: false run: | if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then - export APPSMITH_DB_URL="jdbc:postgresql://postgres:password@localhost:5432/postgres" + export APPSMITH_DB_URL="postgresql://postgres:password@localhost:5432/postgres" else export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools" fi diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java index 3aa8eeeb14a0..736417a9fe03 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java @@ -23,6 +23,8 @@ public class CommonDBConfig { @Value("${appsmith.db.url}") private String appsmithDbUrl; + static final String JDBC_PREFIX = "jdbc:"; + @Bean @Primary public MongoProperties configureMongoDB() { @@ -47,12 +49,13 @@ public DataSourceProperties configurePostgresDB() { /** * Method to extract Jdbc props from the given DB URL - * Expected DB URL: jdbc:postgresql://{username}:{password}@localhost:{port}/{db_name} + * Expected DB URL: postgresql://{username}:{password}@localhost:{port}/{db_name} */ public DataSourceProperties extractJdbcProperties(String dbUrl) { DataSourceProperties ds = new DataSourceProperties(); try { - URI uri = new URI(dbUrl.substring(5)); // remove "jdbc:" prefix + // remove "jdbc:" prefix if exists + URI uri = new URI(dbUrl.replaceFirst("^" + JDBC_PREFIX, "")); String userInfo = uri.getUserInfo(); if (StringUtils.hasLength(userInfo)) { String[] userDetails = userInfo.split(":"); @@ -61,7 +64,8 @@ public DataSourceProperties extractJdbcProperties(String dbUrl) { } // If the port is not mentioned default it to standard 5432 int port = uri.getPort() == -1 ? 5432 : uri.getPort(); - String updatedUrl = String.format("jdbc:%s://%s:%s%s", uri.getScheme(), uri.getHost(), port, uri.getPath()); + String updatedUrl = + String.format("%s%s://%s:%s%s", JDBC_PREFIX, uri.getScheme(), uri.getHost(), port, uri.getPath()); ds.setUrl(updatedUrl); return ds; } catch (URISyntaxException e) { diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java index f48e9e64f690..7ee6a18c8e95 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java @@ -22,6 +22,12 @@ public void testExtractAndSaveJdbcParams_validDbUrlWithUsernameAndPassword() { assertEquals("postgres", ds.getUsername()); assertEquals("password", ds.getPassword()); assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); + + String dbUrlWithoutJdbcPrefix = "postgresql://user:password@localhost:1234/postgres"; + ds = commonDBConfig.extractJdbcProperties(dbUrlWithoutJdbcPrefix); + assertEquals("user", ds.getUsername()); + assertEquals("password", ds.getPassword()); + assertEquals("jdbc:postgresql://localhost:1234/postgres", ds.getUrl()); } @Test diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 427995fd7608..2bf6be004ed4 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -178,11 +178,8 @@ configure_database_connection_url() { export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" fi - if [[ "${APPSMITH_DB_URL}" == "jdbc:postgresql:"* ]]; then - isPostgresUrl=1 - elif [[ "${APPSMITH_DB_URL}" == "postgresql:"* ]]; then + if [[ "${APPSMITH_DB_URL}" == "postgresql:"* ]]; then isPostgresUrl=1 - export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" elif [[ "${APPSMITH_DB_URL}" == "mongodb"* ]]; then isMongoUrl=1 fi From cfe5a1dbe3a0532eedc759b453904533f74c4224 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 16:29:42 +0530 Subject: [PATCH 34/41] test: Update tests --- .../appsmith/server/configurations/CommonDBConfig.java | 2 +- .../server/configurations/CommonDBConfigTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java index 736417a9fe03..1ac891895f6f 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java @@ -40,7 +40,7 @@ public MongoProperties configureMongoDB() { @Bean @Primary public DataSourceProperties configurePostgresDB() { - if (!appsmithDbUrl.startsWith("jdbc:postgresql")) { + if (!appsmithDbUrl.contains("postgresql")) { return null; } log.info("Found PostgreSQL uri configuring now"); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java index 7ee6a18c8e95..9017def11232 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java @@ -17,17 +17,17 @@ public void testExtractAndSaveJdbcParams_validDbUrlWithUsernameAndPassword() { assertEquals("password", ds.getPassword()); assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); - String dbUrlWithPort = "jdbc:postgresql://postgres:password@localhost:5432/postgres"; + String dbUrlWithPort = "jdbc:postgresql://postgres:password@localhost:1234/postgres"; ds = commonDBConfig.extractJdbcProperties(dbUrlWithPort); assertEquals("postgres", ds.getUsername()); assertEquals("password", ds.getPassword()); - assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); + assertEquals("jdbc:postgresql://localhost:1234/postgres", ds.getUrl()); - String dbUrlWithoutJdbcPrefix = "postgresql://user:password@localhost:1234/postgres"; + String dbUrlWithoutJdbcPrefix = "postgresql://user:password@localhost/postgres"; ds = commonDBConfig.extractJdbcProperties(dbUrlWithoutJdbcPrefix); assertEquals("user", ds.getUsername()); assertEquals("password", ds.getPassword()); - assertEquals("jdbc:postgresql://localhost:1234/postgres", ds.getUrl()); + assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); } @Test From 047c56c211655a92c761fd99cf47f8411a1e6691 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 16:32:20 +0530 Subject: [PATCH 35/41] chore: Remove the step to append jdbc --- deploy/docker/fs/opt/appsmith/run-with-env.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/run-with-env.sh b/deploy/docker/fs/opt/appsmith/run-with-env.sh index f100cf9427cc..a70c8935250c 100755 --- a/deploy/docker/fs/opt/appsmith/run-with-env.sh +++ b/deploy/docker/fs/opt/appsmith/run-with-env.sh @@ -39,9 +39,4 @@ if [[ -z "${APPSMITH_DB_URL}" ]]; then export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}" fi -if [[ "${APPSMITH_DB_URL}" == "postgresql"* ]]; then - # Check if APPSMITH_DB_URL is a PostgreSQL URL and doesn't start with "jdbc:", prepend "jdbc:" - export APPSMITH_DB_URL="jdbc:${APPSMITH_DB_URL}" -fi - exec "$@" From 895173d3d203bbd07292161bae88a93ee5c00456 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Mon, 20 May 2024 23:15:18 +0530 Subject: [PATCH 36/41] fix: Add profile check for DB config to configure embedded MongoDB for testcases --- .../java/com/appsmith/server/configurations/CommonDBConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java index 1ac891895f6f..05509cc3d6c0 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java @@ -7,6 +7,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Profile; import org.springframework.util.StringUtils; import java.net.URI; @@ -27,6 +28,7 @@ public class CommonDBConfig { @Bean @Primary + @Profile("!test") public MongoProperties configureMongoDB() { if (!appsmithDbUrl.startsWith("mongodb")) { return null; From f1837b506299f3b7cd5db6d5fe0e9eef9c04169c Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 21 May 2024 12:56:12 +0530 Subject: [PATCH 37/41] fix: Tests --- deploy/docker/fs/opt/appsmith/utils/bin/backup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index a6fd65b23ebc..a1f8731c985c 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -207,7 +207,7 @@ function removeSensitiveEnvData(content) { // Remove encryption and Mongodb data from docker.env const output_lines = [] content.split(/\r?\n/).forEach(line => { - if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB")) { + if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB") && !line.startsWith("APPSMITH_DB")) { output_lines.push(line); } }); From 6b0a9420c909cf96a4ee3f4cd49156b508407d7f Mon Sep 17 00:00:00 2001 From: Abhijeet <41686026+abhvsn@users.noreply.github.com> Date: Tue, 21 May 2024 15:04:00 +0530 Subject: [PATCH 38/41] Update deploy/docker/fs/opt/appsmith/utils/bin/backup.js Co-authored-by: Shrikant Sharat Kandula --- deploy/docker/fs/opt/appsmith/utils/bin/backup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js index df34b7f90019..e94c6e6407f0 100644 --- a/deploy/docker/fs/opt/appsmith/utils/bin/backup.js +++ b/deploy/docker/fs/opt/appsmith/utils/bin/backup.js @@ -207,7 +207,7 @@ function removeSensitiveEnvData(content) { // Remove encryption and Mongodb data from docker.env const output_lines = [] content.split(/\r?\n/).forEach(line => { - if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB") && !line.startsWith("APPSMITH_DB")) { + if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB") && !line.startsWith("APPSMITH_DB_URL=")) { output_lines.push(line); } }); From 1fb63ecf8f976667d119ae94d0dcecef845854f9 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Tue, 21 May 2024 17:05:43 +0530 Subject: [PATCH 39/41] chore: Remove prefix support for Postgres DB URL --- .../server/configurations/CommonDBConfig.java | 9 ++++++-- .../configurations/CommonDBConfigTest.java | 23 +++++++++++-------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java index 05509cc3d6c0..b7bd7a52f11d 100644 --- a/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java +++ b/app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java @@ -56,8 +56,13 @@ public DataSourceProperties configurePostgresDB() { public DataSourceProperties extractJdbcProperties(String dbUrl) { DataSourceProperties ds = new DataSourceProperties(); try { - // remove "jdbc:" prefix if exists - URI uri = new URI(dbUrl.replaceFirst("^" + JDBC_PREFIX, "")); + URI uri = new URI(dbUrl); + if (!StringUtils.hasLength(uri.getHost())) { + String errorString = String.format( + "Malformed DB URL! Expected format: postgresql://{username}:{password}@localhost:{port}/{db_name}, provided url is %s", + dbUrl); + throw new IllegalArgumentException(errorString); + } String userInfo = uri.getUserInfo(); if (StringUtils.hasLength(userInfo)) { String[] userDetails = userInfo.split(":"); diff --git a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java index 9017def11232..1332b7241deb 100644 --- a/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java +++ b/app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java @@ -5,38 +5,43 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; class CommonDBConfigTest { @Test public void testExtractAndSaveJdbcParams_validDbUrlWithUsernameAndPassword() { CommonDBConfig commonDBConfig = new CommonDBConfig(); - String dbUrl = "jdbc:postgresql://postgres:password@localhost/postgres"; + String dbUrl = "postgresql://postgres:password@localhost/postgres"; DataSourceProperties ds = commonDBConfig.extractJdbcProperties(dbUrl); assertEquals("postgres", ds.getUsername()); assertEquals("password", ds.getPassword()); assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); - String dbUrlWithPort = "jdbc:postgresql://postgres:password@localhost:1234/postgres"; + String dbUrlWithPort = "postgresql://postgres:password@localhost:1234/postgres"; ds = commonDBConfig.extractJdbcProperties(dbUrlWithPort); assertEquals("postgres", ds.getUsername()); assertEquals("password", ds.getPassword()); assertEquals("jdbc:postgresql://localhost:1234/postgres", ds.getUrl()); - - String dbUrlWithoutJdbcPrefix = "postgresql://user:password@localhost/postgres"; - ds = commonDBConfig.extractJdbcProperties(dbUrlWithoutJdbcPrefix); - assertEquals("user", ds.getUsername()); - assertEquals("password", ds.getPassword()); - assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); } @Test public void testExtractAndSaveJdbcParams_validDbUrlWithoutUsernameAndPassword() { CommonDBConfig commonDBConfig = new CommonDBConfig(); - String dbUrl = "jdbc:postgresql://localhost:5432/postgres"; + String dbUrl = "postgresql://localhost:5432/postgres"; DataSourceProperties ds = commonDBConfig.extractJdbcProperties(dbUrl); assertNull(ds.getUsername()); assertNull(ds.getPassword()); assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl()); } + + @Test + public void testExtractAndSaveJdbcParams_invalidDbUrl() { + CommonDBConfig commonDBConfig = new CommonDBConfig(); + String dbUrl = "jdbc:postgresql://localhost/postgres"; + String errorString = String.format( + "Malformed DB URL! Expected format: postgresql://{username}:{password}@localhost:{port}/{db_name}, provided url is %s", + dbUrl); + assertThrows(IllegalArgumentException.class, () -> commonDBConfig.extractJdbcProperties(dbUrl), errorString); + } } From fb967159654061322297f2f6b26d10d035f725c8 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 22 May 2024 14:28:32 +0530 Subject: [PATCH 40/41] revert: Helm changes --- deploy/docker/fs/opt/appsmith/entrypoint.sh | 251 +++++++++++++------- deploy/helm/README.md | 2 +- deploy/helm/templates/configMap.yaml | 2 +- deploy/helm/values.yaml | 2 +- 4 files changed, 170 insertions(+), 87 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 2bf6be004ed4..5e9a66604abd 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -194,25 +194,34 @@ check_db_uri() { fi } -init_mongodb() { +init_local_db() { if [[ $isUriLocal -eq 0 ]]; then - echo "Initializing local database" - MONGO_DB_PATH="$stacks_path/data/mongodb" - MONGO_LOG_PATH="$MONGO_DB_PATH/log" - MONGO_DB_KEY="$MONGO_DB_PATH/key" - mkdir -p "$MONGO_DB_PATH" - touch "$MONGO_LOG_PATH" - - if [[ ! -f "$MONGO_DB_KEY" ]]; then - openssl rand -base64 756 > "$MONGO_DB_KEY" + if [[ $isMongoUrl -eq 1 ]]; then + echo "Initializing local MongoDB" + mkdir -p "$MONGO_DB_PATH" + mkdir -p "$MONGO_LOG_PATH" + MONGO_DB_PATH="$stacks_path/data/mongodb" + MONGO_LOG_PATH="$MONGO_DB_PATH/log" + MONGO_DB_KEY="$MONGO_DB_PATH/key" + mkdir -p "$MONGO_DB_PATH" + touch "$MONGO_LOG_PATH" + + if [[ ! -f "$MONGO_DB_KEY" ]]; then + openssl rand -base64 756 > "$MONGO_DB_KEY" + fi + use-mongodb-key "$MONGO_DB_KEY" + ./mongodb-fixer.sh & + elif [[ $isPostgresUrl -eq 1 ]]; then + echo "Initializing local Postgres" + POSTGRES_DB_PATH="$stacks_path/data/postgres/main" + POSTGRES_LOG_PATH="$POSTGRES_DB_PATH/log" + mkdir -p "$POSTGRES_DB_PATH" + touch "$POSTGRES_LOG_PATH" fi - use-mongodb-key "$MONGO_DB_KEY" - - ./mongodb-fixer.sh & fi } -init_replica_set() { +init_mongo_replica_set() { echo "Checking initialized database" shouldPerformInitdb=1 for path in \ @@ -284,6 +293,143 @@ use-mongodb-key() { chmod 600 "$MONGODB_TMP_KEY_PATH" } +init_postgres() { + # Initialize embedded postgres by default; set APPSMITH_ENABLE_EMBEDDED_DB to 0, to use existing cloud postgres mockdb instance + if [[ $isUriLocal -eq 0 ]]; then + echo "" + echo "Checking initialized local postgres" + POSTGRES_DB_PATH="$stacks_path/data/postgres/main" + + mkdir -p "$POSTGRES_DB_PATH" "$TMP/pg-runtime" + + # Postgres does not allow it's server to be run with super user access, we use user postgres and the file system owner also needs to be the same user postgres + chown -R postgres:postgres "$POSTGRES_DB_PATH" "$TMP/pg-runtime" + + if [[ -e "$POSTGRES_DB_PATH/PG_VERSION" ]]; then + echo "Found existing Postgres, Skipping initialization" + cleanup_postgres_mock + else + echo "Initializing local postgresql database" + mkdir -p "$POSTGRES_DB_PATH" + + # Postgres does not allow it's server to be run with super user access, we use user postgres and the file system owner also needs to be the same user postgres + chown postgres:postgres "$POSTGRES_DB_PATH" + + # Initialize the postgres db file system + su postgres -c "/usr/lib/postgresql/13/bin/initdb -D $POSTGRES_DB_PATH" + sed -Ei "s,^#(unix_socket_directories =).*,\\1 '$TMP/pg-runtime'," "$POSTGRES_DB_PATH/postgresql.conf" + + # Start the postgres server in daemon mode + su postgres -c "/usr/lib/postgresql/13/bin/pg_ctl -D $POSTGRES_DB_PATH start" + + # Stop the postgres daemon + su postgres -c "/usr/lib/postgresql/13/bin/pg_ctl stop -D $POSTGRES_DB_PATH" + fi + else + runEmbeddedPostgres=0 + fi + +} + +cleanup_postgres_mock() { + psql -U postgres -c "DROP DATABASE mockdb;" + psql -U postgres -c "DROP ROLE mockdb;" + psql -U postgres -c "DROP DATABASE users;" + psql -U postgres -c "DROP ROLE users;" +} + +ensurePostgresAvailability() { + if pg_isready -h "${TEMPORAL_DB_HOST}" -p "${TEMPORAL_DB_PORT}"; then + echo "PostgreSQL is ready." + else + echo "PostgreSQL is not ready. Error. Exiting Temporal setup" + exit 1 + fi +} + +# Function to extract details from PostgreSQL connection string +# Args: +# conn_string (string): PostgreSQL connection string +# Returns: +# None +# Example: +# extract_postgres_db_params "postgresql://user:password@localhost:5432/appsmith" +# extract_postgres_db_params "postgresql://user:password@localhost/appsmith" +# extract_postgres_db_params "postgresql://user@localhost:5432/appsmith" +# extract_postgres_db_params "postgresql://user@localhost/appsmith" +extract_postgres_db_params() { + local conn_string=$1 + + # Remove the protocol prefix + local temp_string="${conn_string#postgresql://}" + + # Extract username and password (if present) + local credentials="${temp_string%%@*}" + local username="${credentials%%:*}" + local password="" # Default if no password + if [[ "$credentials" == *":"* ]]; then + password="${credentials#*:}" + fi + + # Extract the remaining string after '@' and before '?' + local after_credentials="${temp_string#*@}" + local endpoint_and_db="${after_credentials%%\?*}" + + # Extract endpoint and optional port + local endpoint="${endpoint_and_db%%/*}" + local port="5432" # Default port + if [[ "$endpoint" == *":"* ]]; then + port="${endpoint#*:}" + endpoint="${endpoint%%:*}" + fi + + # Extract database + local db="${endpoint_and_db#*/}" + + dbparams[username]="$username" + dbparams[password]="$password" + dbparams[endpoint]="$endpoint" + dbparams[port]="$port" + dbparams[db]="$db" +} + +setup_temporal_postgres_embedded() { + + # proceed only if postgres is up and running + ensurePostgresAvailability + + # Check if temporal and temporal_visibility schemas exist in postgres. If not, create them. + appsmith_db_exists=$( + psql \ + -U "${postgres_admin_user}" \ + -h "${TEMPORAL_DB_HOST}" \ + -p "${TEMPORAL_DB_PORT}" \ + -tAc "SELECT 1 FROM pg_database WHERE datname = '${TEMPORAL_DB_NAME}'" + ) + if [ "$appsmith_db_exists" != "1" ]; then + echo "Setting up Temporal Postgres embedded for first time ..." + + # Create temporal user + su ${postgres_admin_user} -c "/usr/lib/postgresql/13/bin/createuser -h ${TEMPORAL_DB_HOST} ${TEMPORAL_DB_USER} -s" + + local common_params="--plugin ${temporal_db_plugin} -u ${TEMPORAL_DB_USER} --ep ${TEMPORAL_DB_HOST} -p ${TEMPORAL_DB_PORT} --db ${TEMPORAL_DB_NAME}" + + # Create common appsmith database + "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} create + + # Create temporal namespace and tables + SCHEMA_DIR="${temporal_src}/schema/postgresql/${temporal_postgres_version_dir}/temporal/versioned" + psql -U "${TEMPORAL_DB_USER}" -h "${TEMPORAL_DB_HOST}" -p "${TEMPORAL_DB_PORT}" -d "${TEMPORAL_DB_NAME}" -c "CREATE SCHEMA ${temporal_schemaname};" + "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} --ca "search_path=${temporal_schemaname}" setup-schema -v 0.0 + "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} --ca "search_path=${temporal_schemaname}" update-schema -d "${SCHEMA_DIR}" + + # Create temporal_visibility namespace and tables + VISIBILITY_SCHEMA_DIR="${temporal_src}/schema/postgresql/${temporal_postgres_version_dir}/visibility/versioned" + psql -U "${TEMPORAL_DB_USER}" -h "${TEMPORAL_DB_HOST}" -p "${TEMPORAL_DB_PORT}" -d "${TEMPORAL_DB_NAME}" -c "CREATE SCHEMA ${temporal_visibility_schemaname=};" + "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} --ca "search_path=${temporal_visibility_schemaname}" setup-schema -v 0.0 + "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} --ca "search_path=${temporal_visibility_schemaname}" update-schema -d "${VISIBILITY_SCHEMA_DIR}" + fi +} is_empty_directory() { [[ -d $1 && -z "$(ls -A "$1")" ]] } @@ -361,16 +507,16 @@ configure_supervisord() { # Disable services based on configuration if [[ -z "${DYNO}" ]]; then - if [[ $isUriLocal -eq 0 ]]; then + if [[ $isUriLocal -eq 0 && $isMongoUrl -eq 1 ]]; then cp "$supervisord_conf_source/mongodb.conf" "$SUPERVISORD_CONF_TARGET" fi + if [[ $isUriLocal -eq 0 && $isPostgresUrl -eq 1 ]]; then + cp "$supervisord_conf_source/postgres.conf" "$SUPERVISORD_CONF_TARGET" + fi if [[ $APPSMITH_REDIS_URL == *"localhost"* || $APPSMITH_REDIS_URL == *"127.0.0.1"* ]]; then cp "$supervisord_conf_source/redis.conf" "$SUPERVISORD_CONF_TARGET" mkdir -p "$stacks_path/data/redis" fi - if [[ $runEmbeddedPostgres -eq 1 ]]; then - cp "$supervisord_conf_source/postgres.conf" "$SUPERVISORD_CONF_TARGET" - fi fi } @@ -403,67 +549,6 @@ check_redis_compatible_page_size() { fi } -init_postgres() { - # Initialize embedded postgres by default; set APPSMITH_ENABLE_EMBEDDED_DB to 0, to use existing cloud postgres mockdb instance - if [[ ${APPSMITH_ENABLE_EMBEDDED_DB: -1} != 0 ]]; then - echo "" - echo "Checking initialized local postgres" - POSTGRES_DB_PATH="$stacks_path/data/postgres/main" - - mkdir -p "$POSTGRES_DB_PATH" "$TMP/pg-runtime" - - # Postgres does not allow it's server to be run with super user access, we use user postgres and the file system owner also needs to be the same user postgres - chown -R postgres:postgres "$POSTGRES_DB_PATH" "$TMP/pg-runtime" - - if [[ -e "$POSTGRES_DB_PATH/PG_VERSION" ]]; then - echo "Found existing Postgres, Skipping initialization" - else - echo "Initializing local postgresql database" - mkdir -p "$POSTGRES_DB_PATH" - - # Postgres does not allow it's server to be run with super user access, we use user postgres and the file system owner also needs to be the same user postgres - chown postgres:postgres "$POSTGRES_DB_PATH" - - # Initialize the postgres db file system - su postgres -c "/usr/lib/postgresql/13/bin/initdb -D $POSTGRES_DB_PATH" - sed -Ei "s,^#(unix_socket_directories =).*,\\1 '$TMP/pg-runtime'," "$POSTGRES_DB_PATH/postgresql.conf" - - # Start the postgres server in daemon mode - su postgres -c "/usr/lib/postgresql/13/bin/pg_ctl -D $POSTGRES_DB_PATH start" - - # Create mockdb db and user and populate it with the data - seed_embedded_postgres - # Stop the postgres daemon - su postgres -c "/usr/lib/postgresql/13/bin/pg_ctl stop -D $POSTGRES_DB_PATH" - fi - else - runEmbeddedPostgres=0 - fi - -} - -seed_embedded_postgres(){ - # Create mockdb database - psql -U postgres -c "CREATE DATABASE mockdb;" - # Create mockdb superuser - su postgres -c "/usr/lib/postgresql/13/bin/createuser mockdb -s" - # Dump the sql file containing mockdb data - psql -U postgres -d mockdb --file='/opt/appsmith/templates/mockdb_postgres.sql' - - # Create users database - psql -U postgres -c "CREATE DATABASE users;" - # Create users superuser - su postgres -c "/usr/lib/postgresql/13/bin/createuser users -s" - # Dump the sql file containing mockdb data - psql -U postgres -d users --file='/opt/appsmith/templates/users_postgres.sql' -} - -safe_init_postgres(){ -runEmbeddedPostgres=1 -# fail safe to prevent entrypoint from exiting, and prevent postgres from starting -init_postgres || runEmbeddedPostgres=0 -} - init_loading_pages(){ export XDG_DATA_HOME=/appsmith-stacks/data # so that caddy saves tls certs and other data under stacks/data/caddy export XDG_CONFIG_HOME=/appsmith-stacks/configuration @@ -498,14 +583,14 @@ configure_database_connection_url check_db_uri # Don't run MongoDB if running in a Heroku dyno. if [[ -z "${DYNO}" ]]; then + init_local_db if [[ $isMongoUrl -eq 1 ]]; then # Setup MongoDB and initialize replica set echo "Initializing MongoDB" - init_mongodb - init_replica_set + init_mongo_replica_set elif [[ $isPostgresUrl -eq 1 ]]; then echo "Initializing Postgres" - # init_postgres + init_postgres fi else # These functions are used to limit heap size for Backend process when deployed on Heroku @@ -520,8 +605,6 @@ setup-custom-ca-certificates check_redis_compatible_page_size -safe_init_postgres - configure_supervisord # Ensure the restore path exists in the container, so an archive can be copied to it, if need be. diff --git a/deploy/helm/README.md b/deploy/helm/README.md index 9e18477dd891..f0ca5697e9bc 100644 --- a/deploy/helm/README.md +++ b/deploy/helm/README.md @@ -188,7 +188,7 @@ To change Appsmith configurations, you can use configuration UI in application o | `applicationConfig.APPSMITH_RECAPTCHA_SITE_KEY` | `""` | | `applicationConfig.APPSMITH_RECAPTCHA_SECRET_KEY` | `""` | | `applicationConfig.APPSMITH_RECAPTCHA_ENABLED` | `""` | -| `applicationConfig.APPSMITH_DB_URL` | `""` | +| `applicationConfig.APPSMITH_MONGODB_URI` | `""` | | `applicationConfig.APPSMITH_REDIS_URL` | `""` | | `applicationConfig.APPSMITH_ENCRYPTION_PASSWORD` | `""` | | `applicationConfig.APPSMITH_ENCRYPTION_SALT` | `""` | diff --git a/deploy/helm/templates/configMap.yaml b/deploy/helm/templates/configMap.yaml index 9b1aecc3643a..650cf79a9992 100644 --- a/deploy/helm/templates/configMap.yaml +++ b/deploy/helm/templates/configMap.yaml @@ -13,7 +13,7 @@ metadata: {{- include "appsmith.labels" . | nindent 4 }} data: {{- range $key, $value := .Values.applicationConfig }} - {{- if and (or (eq "APPSMITH_DB_URL" $key) (eq "APPSMITH_MONGODB_URI" $key)) (not $value) }} + {{- if and (eq "APPSMITH_MONGODB_URI" $key) (not $value) }} {{- if $.Values.mongodb.enabled }} {{ $key }}: mongodb+srv://{{ $mongoUser }}:{{ $mongoPassword }}@{{ $mongoServicename }}.{{ $nameSpace }}.svc.cluster.local/appsmith?retryWrites=true&authSource=admin&ssl=false {{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 030335ccfd33..1c54837c0782 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -305,7 +305,7 @@ applicationConfig: APPSMITH_RECAPTCHA_SITE_KEY: "" APPSMITH_RECAPTCHA_SECRET_KEY: "" APPSMITH_RECAPTCHA_ENABLED: "" - APPSMITH_DB_URL: "" + APPSMITH_MONGODB_URI: "" APPSMITH_REDIS_URL: "" APPSMITH_ENCRYPTION_PASSWORD: "" APPSMITH_ENCRYPTION_SALT: "" From 26875950850d073680a096f25c5a2fb67be0d5c8 Mon Sep 17 00:00:00 2001 From: Abhijeet Date: Wed, 22 May 2024 14:41:06 +0530 Subject: [PATCH 41/41] revert: Unwanted changes from entrypoint --- deploy/docker/fs/opt/appsmith/entrypoint.sh | 251 +++++++------------- 1 file changed, 84 insertions(+), 167 deletions(-) diff --git a/deploy/docker/fs/opt/appsmith/entrypoint.sh b/deploy/docker/fs/opt/appsmith/entrypoint.sh index 5e9a66604abd..2bf6be004ed4 100644 --- a/deploy/docker/fs/opt/appsmith/entrypoint.sh +++ b/deploy/docker/fs/opt/appsmith/entrypoint.sh @@ -194,34 +194,25 @@ check_db_uri() { fi } -init_local_db() { +init_mongodb() { if [[ $isUriLocal -eq 0 ]]; then - if [[ $isMongoUrl -eq 1 ]]; then - echo "Initializing local MongoDB" - mkdir -p "$MONGO_DB_PATH" - mkdir -p "$MONGO_LOG_PATH" - MONGO_DB_PATH="$stacks_path/data/mongodb" - MONGO_LOG_PATH="$MONGO_DB_PATH/log" - MONGO_DB_KEY="$MONGO_DB_PATH/key" - mkdir -p "$MONGO_DB_PATH" - touch "$MONGO_LOG_PATH" - - if [[ ! -f "$MONGO_DB_KEY" ]]; then - openssl rand -base64 756 > "$MONGO_DB_KEY" - fi - use-mongodb-key "$MONGO_DB_KEY" - ./mongodb-fixer.sh & - elif [[ $isPostgresUrl -eq 1 ]]; then - echo "Initializing local Postgres" - POSTGRES_DB_PATH="$stacks_path/data/postgres/main" - POSTGRES_LOG_PATH="$POSTGRES_DB_PATH/log" - mkdir -p "$POSTGRES_DB_PATH" - touch "$POSTGRES_LOG_PATH" + echo "Initializing local database" + MONGO_DB_PATH="$stacks_path/data/mongodb" + MONGO_LOG_PATH="$MONGO_DB_PATH/log" + MONGO_DB_KEY="$MONGO_DB_PATH/key" + mkdir -p "$MONGO_DB_PATH" + touch "$MONGO_LOG_PATH" + + if [[ ! -f "$MONGO_DB_KEY" ]]; then + openssl rand -base64 756 > "$MONGO_DB_KEY" fi + use-mongodb-key "$MONGO_DB_KEY" + + ./mongodb-fixer.sh & fi } -init_mongo_replica_set() { +init_replica_set() { echo "Checking initialized database" shouldPerformInitdb=1 for path in \ @@ -293,143 +284,6 @@ use-mongodb-key() { chmod 600 "$MONGODB_TMP_KEY_PATH" } -init_postgres() { - # Initialize embedded postgres by default; set APPSMITH_ENABLE_EMBEDDED_DB to 0, to use existing cloud postgres mockdb instance - if [[ $isUriLocal -eq 0 ]]; then - echo "" - echo "Checking initialized local postgres" - POSTGRES_DB_PATH="$stacks_path/data/postgres/main" - - mkdir -p "$POSTGRES_DB_PATH" "$TMP/pg-runtime" - - # Postgres does not allow it's server to be run with super user access, we use user postgres and the file system owner also needs to be the same user postgres - chown -R postgres:postgres "$POSTGRES_DB_PATH" "$TMP/pg-runtime" - - if [[ -e "$POSTGRES_DB_PATH/PG_VERSION" ]]; then - echo "Found existing Postgres, Skipping initialization" - cleanup_postgres_mock - else - echo "Initializing local postgresql database" - mkdir -p "$POSTGRES_DB_PATH" - - # Postgres does not allow it's server to be run with super user access, we use user postgres and the file system owner also needs to be the same user postgres - chown postgres:postgres "$POSTGRES_DB_PATH" - - # Initialize the postgres db file system - su postgres -c "/usr/lib/postgresql/13/bin/initdb -D $POSTGRES_DB_PATH" - sed -Ei "s,^#(unix_socket_directories =).*,\\1 '$TMP/pg-runtime'," "$POSTGRES_DB_PATH/postgresql.conf" - - # Start the postgres server in daemon mode - su postgres -c "/usr/lib/postgresql/13/bin/pg_ctl -D $POSTGRES_DB_PATH start" - - # Stop the postgres daemon - su postgres -c "/usr/lib/postgresql/13/bin/pg_ctl stop -D $POSTGRES_DB_PATH" - fi - else - runEmbeddedPostgres=0 - fi - -} - -cleanup_postgres_mock() { - psql -U postgres -c "DROP DATABASE mockdb;" - psql -U postgres -c "DROP ROLE mockdb;" - psql -U postgres -c "DROP DATABASE users;" - psql -U postgres -c "DROP ROLE users;" -} - -ensurePostgresAvailability() { - if pg_isready -h "${TEMPORAL_DB_HOST}" -p "${TEMPORAL_DB_PORT}"; then - echo "PostgreSQL is ready." - else - echo "PostgreSQL is not ready. Error. Exiting Temporal setup" - exit 1 - fi -} - -# Function to extract details from PostgreSQL connection string -# Args: -# conn_string (string): PostgreSQL connection string -# Returns: -# None -# Example: -# extract_postgres_db_params "postgresql://user:password@localhost:5432/appsmith" -# extract_postgres_db_params "postgresql://user:password@localhost/appsmith" -# extract_postgres_db_params "postgresql://user@localhost:5432/appsmith" -# extract_postgres_db_params "postgresql://user@localhost/appsmith" -extract_postgres_db_params() { - local conn_string=$1 - - # Remove the protocol prefix - local temp_string="${conn_string#postgresql://}" - - # Extract username and password (if present) - local credentials="${temp_string%%@*}" - local username="${credentials%%:*}" - local password="" # Default if no password - if [[ "$credentials" == *":"* ]]; then - password="${credentials#*:}" - fi - - # Extract the remaining string after '@' and before '?' - local after_credentials="${temp_string#*@}" - local endpoint_and_db="${after_credentials%%\?*}" - - # Extract endpoint and optional port - local endpoint="${endpoint_and_db%%/*}" - local port="5432" # Default port - if [[ "$endpoint" == *":"* ]]; then - port="${endpoint#*:}" - endpoint="${endpoint%%:*}" - fi - - # Extract database - local db="${endpoint_and_db#*/}" - - dbparams[username]="$username" - dbparams[password]="$password" - dbparams[endpoint]="$endpoint" - dbparams[port]="$port" - dbparams[db]="$db" -} - -setup_temporal_postgres_embedded() { - - # proceed only if postgres is up and running - ensurePostgresAvailability - - # Check if temporal and temporal_visibility schemas exist in postgres. If not, create them. - appsmith_db_exists=$( - psql \ - -U "${postgres_admin_user}" \ - -h "${TEMPORAL_DB_HOST}" \ - -p "${TEMPORAL_DB_PORT}" \ - -tAc "SELECT 1 FROM pg_database WHERE datname = '${TEMPORAL_DB_NAME}'" - ) - if [ "$appsmith_db_exists" != "1" ]; then - echo "Setting up Temporal Postgres embedded for first time ..." - - # Create temporal user - su ${postgres_admin_user} -c "/usr/lib/postgresql/13/bin/createuser -h ${TEMPORAL_DB_HOST} ${TEMPORAL_DB_USER} -s" - - local common_params="--plugin ${temporal_db_plugin} -u ${TEMPORAL_DB_USER} --ep ${TEMPORAL_DB_HOST} -p ${TEMPORAL_DB_PORT} --db ${TEMPORAL_DB_NAME}" - - # Create common appsmith database - "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} create - - # Create temporal namespace and tables - SCHEMA_DIR="${temporal_src}/schema/postgresql/${temporal_postgres_version_dir}/temporal/versioned" - psql -U "${TEMPORAL_DB_USER}" -h "${TEMPORAL_DB_HOST}" -p "${TEMPORAL_DB_PORT}" -d "${TEMPORAL_DB_NAME}" -c "CREATE SCHEMA ${temporal_schemaname};" - "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} --ca "search_path=${temporal_schemaname}" setup-schema -v 0.0 - "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} --ca "search_path=${temporal_schemaname}" update-schema -d "${SCHEMA_DIR}" - - # Create temporal_visibility namespace and tables - VISIBILITY_SCHEMA_DIR="${temporal_src}/schema/postgresql/${temporal_postgres_version_dir}/visibility/versioned" - psql -U "${TEMPORAL_DB_USER}" -h "${TEMPORAL_DB_HOST}" -p "${TEMPORAL_DB_PORT}" -d "${TEMPORAL_DB_NAME}" -c "CREATE SCHEMA ${temporal_visibility_schemaname=};" - "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} --ca "search_path=${temporal_visibility_schemaname}" setup-schema -v 0.0 - "$TEMPORAL_HOME"/temporal-sql-tool ${common_params} --ca "search_path=${temporal_visibility_schemaname}" update-schema -d "${VISIBILITY_SCHEMA_DIR}" - fi -} is_empty_directory() { [[ -d $1 && -z "$(ls -A "$1")" ]] } @@ -507,16 +361,16 @@ configure_supervisord() { # Disable services based on configuration if [[ -z "${DYNO}" ]]; then - if [[ $isUriLocal -eq 0 && $isMongoUrl -eq 1 ]]; then + if [[ $isUriLocal -eq 0 ]]; then cp "$supervisord_conf_source/mongodb.conf" "$SUPERVISORD_CONF_TARGET" fi - if [[ $isUriLocal -eq 0 && $isPostgresUrl -eq 1 ]]; then - cp "$supervisord_conf_source/postgres.conf" "$SUPERVISORD_CONF_TARGET" - fi if [[ $APPSMITH_REDIS_URL == *"localhost"* || $APPSMITH_REDIS_URL == *"127.0.0.1"* ]]; then cp "$supervisord_conf_source/redis.conf" "$SUPERVISORD_CONF_TARGET" mkdir -p "$stacks_path/data/redis" fi + if [[ $runEmbeddedPostgres -eq 1 ]]; then + cp "$supervisord_conf_source/postgres.conf" "$SUPERVISORD_CONF_TARGET" + fi fi } @@ -549,6 +403,67 @@ check_redis_compatible_page_size() { fi } +init_postgres() { + # Initialize embedded postgres by default; set APPSMITH_ENABLE_EMBEDDED_DB to 0, to use existing cloud postgres mockdb instance + if [[ ${APPSMITH_ENABLE_EMBEDDED_DB: -1} != 0 ]]; then + echo "" + echo "Checking initialized local postgres" + POSTGRES_DB_PATH="$stacks_path/data/postgres/main" + + mkdir -p "$POSTGRES_DB_PATH" "$TMP/pg-runtime" + + # Postgres does not allow it's server to be run with super user access, we use user postgres and the file system owner also needs to be the same user postgres + chown -R postgres:postgres "$POSTGRES_DB_PATH" "$TMP/pg-runtime" + + if [[ -e "$POSTGRES_DB_PATH/PG_VERSION" ]]; then + echo "Found existing Postgres, Skipping initialization" + else + echo "Initializing local postgresql database" + mkdir -p "$POSTGRES_DB_PATH" + + # Postgres does not allow it's server to be run with super user access, we use user postgres and the file system owner also needs to be the same user postgres + chown postgres:postgres "$POSTGRES_DB_PATH" + + # Initialize the postgres db file system + su postgres -c "/usr/lib/postgresql/13/bin/initdb -D $POSTGRES_DB_PATH" + sed -Ei "s,^#(unix_socket_directories =).*,\\1 '$TMP/pg-runtime'," "$POSTGRES_DB_PATH/postgresql.conf" + + # Start the postgres server in daemon mode + su postgres -c "/usr/lib/postgresql/13/bin/pg_ctl -D $POSTGRES_DB_PATH start" + + # Create mockdb db and user and populate it with the data + seed_embedded_postgres + # Stop the postgres daemon + su postgres -c "/usr/lib/postgresql/13/bin/pg_ctl stop -D $POSTGRES_DB_PATH" + fi + else + runEmbeddedPostgres=0 + fi + +} + +seed_embedded_postgres(){ + # Create mockdb database + psql -U postgres -c "CREATE DATABASE mockdb;" + # Create mockdb superuser + su postgres -c "/usr/lib/postgresql/13/bin/createuser mockdb -s" + # Dump the sql file containing mockdb data + psql -U postgres -d mockdb --file='/opt/appsmith/templates/mockdb_postgres.sql' + + # Create users database + psql -U postgres -c "CREATE DATABASE users;" + # Create users superuser + su postgres -c "/usr/lib/postgresql/13/bin/createuser users -s" + # Dump the sql file containing mockdb data + psql -U postgres -d users --file='/opt/appsmith/templates/users_postgres.sql' +} + +safe_init_postgres(){ +runEmbeddedPostgres=1 +# fail safe to prevent entrypoint from exiting, and prevent postgres from starting +init_postgres || runEmbeddedPostgres=0 +} + init_loading_pages(){ export XDG_DATA_HOME=/appsmith-stacks/data # so that caddy saves tls certs and other data under stacks/data/caddy export XDG_CONFIG_HOME=/appsmith-stacks/configuration @@ -583,14 +498,14 @@ configure_database_connection_url check_db_uri # Don't run MongoDB if running in a Heroku dyno. if [[ -z "${DYNO}" ]]; then - init_local_db if [[ $isMongoUrl -eq 1 ]]; then # Setup MongoDB and initialize replica set echo "Initializing MongoDB" - init_mongo_replica_set + init_mongodb + init_replica_set elif [[ $isPostgresUrl -eq 1 ]]; then echo "Initializing Postgres" - init_postgres + # init_postgres fi else # These functions are used to limit heap size for Backend process when deployed on Heroku @@ -605,6 +520,8 @@ setup-custom-ca-certificates check_redis_compatible_page_size +safe_init_postgres + configure_supervisord # Ensure the restore path exists in the container, so an archive can be copied to it, if need be.