test: Enable server tests for the PRs with base PG branch

.github/workflows/quality-checks.yml

@@ -2,7 +2,7 @@ name: Quality checks
 
 on:
   pull_request:
-    branches: [release, master]
+    branches: [release, master, pg]
 
 jobs:
   path-filter:
@@ -45,6 +45,7 @@ jobs:
     secrets: inherit
     with:
       pr: ${{ github.event.pull_request.number }}
+      is-pg-build: ${{ github.event.pull_request.base.ref == 'pg' }}
 
   client-build:
     name: client-build

.github/workflows/server-build.yml

@@ -17,6 +17,11 @@ on:
         description: "This is the branch to be used for the build."
         required: false
         type: string
+      is-pg-build:
+        description: "This is a boolean value in case the workflow is being called for a PG build"
+        required: false
+        type: string
+        default: "false"
 
 # Change the working directory for all the jobs in this workflow
 defaults:
@@ -133,6 +138,12 @@ jobs:
           distribution: "temurin"
           java-version: "17"
 
+      - name: Conditionally start PostgreSQL
+        run: |
+          if [[ inputs.is-pg-build == 'true' ]]; then
+            docker run --name appsmith-pg -p 5432:5432 -d -e POSTGRES_PASSWORD=password postgres:alpine
+          fi
+
       # Retrieve maven dependencies from cache. After a successful run, these dependencies are cached again
       - name: Cache maven dependencies
         if: steps.run_result.outputs.run_result != 'success'  && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
@@ -148,16 +159,6 @@ jobs:
       # Build the code
       - name: Build
         if: steps.run_result.outputs.run_result != 'success'  && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
-        env:
-          ACTIVE_PROFILE: test
-          APPSMITH_MONGODB_URI: "mongodb://localhost:27017/mobtools"
-          APPSMITH_CLOUD_SERVICES_BASE_URL: "https://release-cs.appsmith.com"
-          APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH: ${{ secrets.APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH }}
-          APPSMITH_REDIS_URL: "redis://127.0.0.1:6379"
-          APPSMITH_ENCRYPTION_PASSWORD: "password"
-          APPSMITH_ENCRYPTION_SALT: "salt"
-          APPSMITH_ENVFILE_PATH: /tmp/dummy.env
-          APPSMITH_VERBOSE_LOGGING_ENABLED: false
         run: |
           ./build.sh -DskipTests
 
@@ -166,7 +167,6 @@ jobs:
         if: (inputs.skip-tests != 'true' || steps.run_result.outputs.run_result == 'failedtest')  && (steps.changed-files-specific.outputs.any_changed == 'true' ||  github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule')
         env:
           ACTIVE_PROFILE: test
-          APPSMITH_MONGODB_URI: "mongodb://localhost:27017/mobtools"
           APPSMITH_CLOUD_SERVICES_BASE_URL: "https://release-cs.appsmith.com"
           APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH: ${{ secrets.APPSMITH_CLOUD_SERVICES_TEMPLATE_UPLOAD_AUTH }}
           APPSMITH_REDIS_URL: "redis://127.0.0.1:6379"
@@ -175,6 +175,11 @@ jobs:
           APPSMITH_ENVFILE_PATH: /tmp/dummy.env
           APPSMITH_VERBOSE_LOGGING_ENABLED: false
         run: |
+          if [[ "${{ inputs.is-pg-build }}" == "true" ]]; then
+            export APPSMITH_DB_URL="postgresql://postgres:password@localhost:5432/postgres"
+          else
+            export APPSMITH_DB_URL="mongodb://localhost:27017/mobtools"
+          fi
           args=()
           if [[ "${{ steps.run_result.outputs.run_result }}" == "failedtest" ]]; then
             failed_tests="${{ steps.failed_tests.outputs.tests }}"

app.json

@@ -18,7 +18,7 @@
     "success_url": "/",
     "stack": "container",
     "env": {
-      "APPSMITH_MONGODB_URI": {
+      "APPSMITH_DB_URL": {
         "description": "Your Mongo Database URI. Since Heroku doesn't support a managed MongoDB instance, you'll have to create a Mongo DB instance on another service such as https://cloud.mongodb.com",
         "value": ""
       },

app/client/src/pages/AdminSettings/config/advanced.ts

@@ -15,7 +15,7 @@ export const config: AdminConfigType = {
   canSave: true,
   settings: [
     {
-      id: "APPSMITH_MONGODB_URI",
+      id: "APPSMITH_DB_URL",
       category: SettingCategories.ADVANCED,
       controlType: SettingTypes.TEXTINPUT,
       controlSubType: SettingSubtype.TEXT,

app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java

@@ -0,0 +1,82 @@
+package com.appsmith.server.configurations;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
+import org.springframework.boot.autoconfigure.mongo.MongoProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.context.annotation.Profile;
+import org.springframework.util.StringUtils;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * Class to configure beans based on DB url. This could have been implemented in {@link MongoConfig} or
+ * {@link DBConfig} but extracted to seperate class as we were facing cyclical dependency issue with the other approach
+ */
+@Configuration
+@Slf4j
+public class CommonDBConfig {
+
+    @Value("${appsmith.db.url}")
+    private String appsmithDbUrl;
+
+    static final String JDBC_PREFIX = "jdbc:";
+
+    @Bean
+    @Primary
+    @Profile("!test")
+    public MongoProperties configureMongoDB() {
+        if (!appsmithDbUrl.startsWith("mongodb")) {
+            return null;
+        }
+        log.info("Found MongoDB uri configuring now");
+        MongoProperties mongoProperties = new MongoProperties();
+        mongoProperties.setUri(appsmithDbUrl);
+        return mongoProperties;
+    }
+
+    @Bean
+    @Primary
+    public DataSourceProperties configurePostgresDB() {
+        if (!appsmithDbUrl.contains("postgresql")) {
+            return null;
+        }
+        log.info("Found PostgreSQL uri configuring now");
+        return extractJdbcProperties(appsmithDbUrl);
+    }
+
+    /**
+     * Method to extract Jdbc props from the given DB URL
+     * Expected DB URL: postgresql://{username}:{password}@localhost:{port}/{db_name}
+     */
+    public DataSourceProperties extractJdbcProperties(String dbUrl) {
+        DataSourceProperties ds = new DataSourceProperties();
+        try {
+            URI uri = new URI(dbUrl);
+            if (!StringUtils.hasLength(uri.getHost())) {
+                String errorString = String.format(
+                        "Malformed DB URL! Expected format: postgresql://{username}:{password}@localhost:{port}/{db_name}, provided url is %s",
+                        dbUrl);
+                throw new IllegalArgumentException(errorString);
+            }
+            String userInfo = uri.getUserInfo();
+            if (StringUtils.hasLength(userInfo)) {
+                String[] userDetails = userInfo.split(":");
+                ds.setUsername(userDetails[0]);
+                ds.setPassword(userDetails[1]);
+            }
+            // If the port is not mentioned default it to standard 5432
+            int port = uri.getPort() == -1 ? 5432 : uri.getPort();
+            String updatedUrl =
+                    String.format("%s%s://%s:%s%s", JDBC_PREFIX, uri.getScheme(), uri.getHost(), port, uri.getPath());
+            ds.setUrl(updatedUrl);
+            return ds;
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

app/server/appsmith-server/src/main/resources/application.properties

@@ -8,6 +8,7 @@ spring.lifecycle.timeout-per-shutdown-phase=20s
 
 spring.profiles.active=${ACTIVE_PROFILE:production}
 
+appsmith.db.url=${APPSMITH_DB_URL:${APPSMITH_MONGODB_URI}}
 # This property allows us to override beans during testing. This is useful when we want to set different configurations
 # and different parameters during test as compared to production. If this property is disabled, some tests will fail.
 spring.main.allow-bean-definition-overriding=true

app/server/appsmith-server/src/test/java/com/appsmith/server/configurations/CommonDBConfigTest.java

@@ -0,0 +1,47 @@
+package com.appsmith.server.configurations;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class CommonDBConfigTest {
+
+    @Test
+    public void testExtractAndSaveJdbcParams_validDbUrlWithUsernameAndPassword() {
+        CommonDBConfig commonDBConfig = new CommonDBConfig();
+        String dbUrl = "postgresql://postgres:password@localhost/postgres";
+        DataSourceProperties ds = commonDBConfig.extractJdbcProperties(dbUrl);
+        assertEquals("postgres", ds.getUsername());
+        assertEquals("password", ds.getPassword());
+        assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl());
+
+        String dbUrlWithPort = "postgresql://postgres:password@localhost:1234/postgres";
+        ds = commonDBConfig.extractJdbcProperties(dbUrlWithPort);
+        assertEquals("postgres", ds.getUsername());
+        assertEquals("password", ds.getPassword());
+        assertEquals("jdbc:postgresql://localhost:1234/postgres", ds.getUrl());
+    }
+
+    @Test
+    public void testExtractAndSaveJdbcParams_validDbUrlWithoutUsernameAndPassword() {
+        CommonDBConfig commonDBConfig = new CommonDBConfig();
+        String dbUrl = "postgresql://localhost:5432/postgres";
+        DataSourceProperties ds = commonDBConfig.extractJdbcProperties(dbUrl);
+        assertNull(ds.getUsername());
+        assertNull(ds.getPassword());
+        assertEquals("jdbc:postgresql://localhost:5432/postgres", ds.getUrl());
+    }
+
+    @Test
+    public void testExtractAndSaveJdbcParams_invalidDbUrl() {
+        CommonDBConfig commonDBConfig = new CommonDBConfig();
+        String dbUrl = "jdbc:postgresql://localhost/postgres";
+        String errorString = String.format(
+                "Malformed DB URL! Expected format: postgresql://{username}:{password}@localhost:{port}/{db_name}, provided url is %s",
+                dbUrl);
+        assertThrows(IllegalArgumentException.class, () -> commonDBConfig.extractJdbcProperties(dbUrl), errorString);
+    }
+}

app/server/build.sh

@@ -34,6 +34,10 @@ if [[ -f .env ]]; then
   source .env
 fi
 
+if [[ -f tx/transform.py ]]; then
+  python3 tx/transform.py
+fi
+
 node scripts/check-field-constants.mjs
 
 # Build the code. $@ accepts all the parameters from the input command line and uses it in the maven build command

app/server/docker-compose.yml

@@ -8,7 +8,7 @@ services:
     env_file: envs/docker.env
     environment:
       APPSMITH_REDIS_URL: "redis://redis:6379"
-      APPSMITH_MONGODB_URI: "mongodb://mongo:27017/appsmith"
+      APPSMITH_DB_URL: "mongodb://mongo:27017/appsmith"
     ports:
       - "8080:8080"
     depends_on:

app/server/envs/dev.env.example

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith?replicaSet=rs0"
+APPSMITH_DB_URL="mongodb://localhost:27017/appsmith?replicaSet=rs0"
 
 APPSMITH_REDIS_URL="redis://127.0.0.1:6379"
 

app/server/envs/test.env.example

@@ -2,7 +2,7 @@
 
 ACTIVE_PROFILE=test
 
-APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith"
+APPSMITH_DB_URL="mongodb://localhost:27017/appsmith"
 
 APPSMITH_REDIS_URL="redis://127.0.0.1:6379"
 

contributions/ServerSetup.md

@@ -124,11 +124,11 @@ With the prerequisites met, let's build the code.
 
         This command creates a `.env` file in the `app/server` folder. All run scripts pick up environment configuration from this file.
 
-5. Ensure that the environment variables `APPSMITH_MONGODB_URI` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis.
+5. Ensure that the environment variables `APPSMITH_DB_URL` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis.
 
 6.  **Update the replica set name with correct value in the mongo connection string in the [.env](#setup-environment-file) file.** The replica name is the same as passed [here](#setting-up-a-local-mongodb-instance)
     ```bash
-    APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith?replicaSet=<replica-set-name>"
+    APPSMITH_DB_URL="mongodb://localhost:27017/appsmith?replicaSet=<replica-set-name>"
     ```
 
 7. Run the following command to create the final JAR for the Appsmith server:
@@ -152,7 +152,7 @@ With the prerequisites met, let's build the code.
         - On Ubuntu Linux environment docker needs root privilege, hence `./build.sh` script needs to be run with root privilege as well.
         - On Ubuntu Linux environment, the script may not be able to read `.env` file, so it is advised that you run the cmd like:
             ```console
-            sudo APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh
+            sudo APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh
             ```
 
 
@@ -260,7 +260,7 @@ cp envs/dev.env.example .env
 
 This command creates a `.env` file in the `app/server` folder. All run scripts pick up environment configuration from this file.
 
-5. Ensure that the environment variables `APPSMITH_MONGODB_URI` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis.
+5. Ensure that the environment variables `APPSMITH_DB_URL` and `APPSMITH_REDIS_URI` in the file `.env` point to your local running instances of MongoDB and Redis.
 
 6. Run the following command to create the final JAR for the Appsmith server:
 
@@ -274,7 +274,7 @@ Note:
 - On Ubuntu Linux environment docker needs root privilege, hence ./build.sh script needs to be run with root privilege as well.
 - On Ubuntu Linux environment, the script may not be able to read .env file, so it is advised that you run the cmd like:
 ```console
-sudo APPSMITH_MONGODB_URI="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh
+sudo APPSMITH_DB_URL="mongodb://localhost:27017/appsmith" APPSMITH_REDIS_URL="redis://127.0.0.1:6379" APPSMITH_MAIL_ENABLED=false APPSMITH_ENCRYPTION_PASSWORD=abcd APPSMITH_ENCRYPTION_SALT=abcd ./build.sh
 ```
 - If the volume containing docker's data root path (macOS: `~/Library/Containers/com.docker.docker/Data/vms/0/`, Ubuntu: `/var/lib/docker/`) has less than 2 GB of free space, then the script may fail with the following error:
 ```console

deploy/docker/fs/opt/appsmith/entrypoint.sh

@@ -168,11 +168,28 @@ unset_unused_variables() {
   fi
 }
 
-check_mongodb_uri() {
-  echo "Checking APPSMITH_MONGODB_URI"
+configure_database_connection_url() {
+  echo "Configuring database connection URL"
+  isPostgresUrl=0
+  isMongoUrl=0
+  # Check if APPSMITH_DB_URL is not set
+  if [[ -z "${APPSMITH_DB_URL}" ]]; then
+    # If APPSMITH_DB_URL is not set, fall back to APPSMITH_MONGODB_URI
+    export APPSMITH_DB_URL="${APPSMITH_MONGODB_URI}"
+  fi
+
+  if [[ "${APPSMITH_DB_URL}" == "postgresql:"* ]]; then
+    isPostgresUrl=1
+  elif [[ "${APPSMITH_DB_URL}" == "mongodb"* ]]; then
+    isMongoUrl=1
+  fi
+}
+
+check_db_uri() {
+  echo "Checking APPSMITH_DB_URL"
   isUriLocal=1
-  if [[ $APPSMITH_MONGODB_URI == *"localhost"* || $APPSMITH_MONGODB_URI == *"127.0.0.1"* ]]; then
-    echo "Detected local MongoDB"
+  if [[ $APPSMITH_DB_URL == *"localhost"* || $APPSMITH_DB_URL == *"127.0.0.1"* ]]; then
+    echo "Detected local DB"
     isUriLocal=0
   fi
 }
@@ -239,7 +256,7 @@ init_replica_set() {
     mongod --fork --port 27017 --dbpath "$MONGO_DB_PATH" --logpath "$MONGO_LOG_PATH" --replSet mr1 --keyFile "$MONGODB_TMP_KEY_PATH" --bind_ip localhost
     echo "Waiting 10s for MongoDB to start with Replica Set"
     sleep 10
-    mongosh "$APPSMITH_MONGODB_URI" --eval 'rs.initiate()'
+    mongosh "$APPSMITH_DB_URL" --eval 'rs.initiate()'
     mongod --dbpath "$MONGO_DB_PATH" --shutdown || true
   fi
 
@@ -477,11 +494,19 @@ print_appsmith_info
 init_loading_pages
 unset_unused_variables
 
-check_mongodb_uri
+configure_database_connection_url
+check_db_uri
+# Don't run MongoDB if running in a Heroku dyno.
 if [[ -z "${DYNO}" ]]; then
-  # Don't run MongoDB if running in a Heroku dyno.
-  init_mongodb
-  init_replica_set
+  if [[ $isMongoUrl -eq 1 ]]; then
+    # Setup MongoDB and initialize replica set
+    echo "Initializing MongoDB"
+    init_mongodb
+    init_replica_set
+  elif [[ $isPostgresUrl -eq 1 ]]; then
+    echo "Initializing Postgres"
+    # init_postgres
+  fi
 else
   # These functions are used to limit heap size for Backend process when deployed on Heroku
   get_maximum_heap

deploy/docker/fs/opt/appsmith/mongodb-fixer.sh

@@ -16,7 +16,7 @@ done
 echo "MongoDB is RUNNING"
 
 for _ in {1..60}; do
-  if mongosh --quiet "$APPSMITH_MONGODB_URI" --eval '
+  if mongosh --quiet "$APPSMITH_DB_URL" --eval '
     parseFloat(db.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}).featureCompatibilityVersion.version) < 5 &&
       db.adminCommand({setFeatureCompatibilityVersion: "5.0"})
   '; then