SDT-157: SDT: Upgrade Postgres database to v15 (#320)

* SDT-149: SDT: Timeout error when manually writing to dead letter queue

* SDT-149: SDT: Timeout error when manually writing to dead letter queue

* SDT-168: SDT: Allow database retention period to be specified on an environment basis (#285)

* Update Terraform azurerm to v3.75.0

* Update dependency commons-io:commons-io to v2.14.0

* SDT-168: SDT: Allow database retention period to be specified on an environment basis

* SDT-168: SDT: Allow database retention period to be specified on an environment basis

---------

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

* SDT-168: SDT: Allow database retention period to be specified on an environment basis

* SDT-157: SDT: Upgrade Postgres database to v15

* SDT-157: SDT: Upgrade Postgres database to v15

* SDT-157: SDT: Upgrade Postgres database to v15

* SDT-157: SDT: Upgrade Postgres database to v15

* Disable git sync on demo branch

* SDT-157: syn branches with master

* suppress CVE-2023-44487

---------

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

build.gradle

@@ -168,7 +168,7 @@ ext {
   jaxwsVersion = "2.3.1"
   activeMQVersion = '5.18.2'
   testcontainers = '1.19.1'
-  sdtCommonVersion = '1.1.0'
+  sdtCommonVersion = '1.1.0-SDT-149.0.1'
   limits = [
     'instruction': 99,
     'branch'     : 99,

config/owasp/suppressions.xml

@@ -29,13 +29,15 @@
       CVE-2023-36479
       CVE-2023-40167
       CVE-2023-41900
+      CVE-2023-44487
     </notes>
     <cve>CVE-2023-4586</cve>
     <cve>CVE-2023-36414</cve>
     <cve>CVE-2023-36415</cve>
     <cve>CVE-2023-36479</cve>
     <cve>CVE-2023-40167</cve>
     <cve>CVE-2023-41900</cve>
+    <cve>CVE-2023-44487</cve>
   </suppress>
   <!--End of temporary suppression section -->
 

docker/database/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgres:11
+FROM postgres:15
 
 COPY init-db.sh /docker-entrypoint-initdb.d
 

infrastructure/aat.tfvars

@@ -1 +1,2 @@
+database_backup_retention_days=7
 sku = "Standard"

infrastructure/database.tf

@@ -7,18 +7,19 @@ module "postgresql" {
     azurerm.postgres_network = azurerm.private_endpoint
   }
 
-  admin_user_object_id = var.jenkins_AAD_objectId
-  business_area        = "cft"
-  common_tags          = local.tags
-  component            = var.component
-  env                  = var.env
+  admin_user_object_id  = var.jenkins_AAD_objectId
+  business_area         = "cft"
+  common_tags           = local.tags
+  component             = var.component
+  env                   = var.env
   pgsql_databases = [
     {
       name = "civil_sdt"
     }
   ]
-  pgsql_version        = "11"
-  product              = var.product
+  pgsql_version         = "11"
+  product               = var.product
+  backup_retention_days = "${var.database_backup_retention_days}"
 }
 
 # Create secret for database user
@@ -41,3 +42,48 @@ resource "azurerm_key_vault_secret" "POSTGRES-HOST" {
   value        = module.postgresql.fqdn
   key_vault_id = module.civil_sdt_key_vault.key_vault_id
 }
+
+
+module "postgresql-v15" {
+  source = "[email protected]:hmcts/terraform-module-postgresql-flexible?ref=master"
+
+  providers = {
+    azurerm.postgres_network = azurerm.private_endpoint
+  }
+
+  admin_user_object_id  = var.jenkins_AAD_objectId
+  business_area         = "cft"
+  common_tags           = local.tags
+  component             = var.component
+  env                   = var.env
+  pgsql_databases = [
+    {
+      name = "civil_sdt"
+    }
+  ]
+  pgsql_version         = "15"
+  product               = var.product
+  name                  = join("-", [var.product, var.component, "v15"])
+  backup_retention_days = "${var.database_backup_retention_days}"
+}
+
+# Create secret for database user
+resource "azurerm_key_vault_secret" "POSTGRES-USER-V15" {
+  name         = "civil-sdt-POSTGRES-USER-V15"
+  value        = module.postgresql-v15.username
+  key_vault_id = module.civil_sdt_key_vault.key_vault_id
+}
+
+# Create secret for database password
+resource "azurerm_key_vault_secret" "POSTGRES-PASS-V15" {
+  name         = "civil-sdt-POSTGRES-PASS-V15"
+  value        = module.postgresql-v15.password
+  key_vault_id = module.civil_sdt_key_vault.key_vault_id
+}
+
+# Create secret for database host
+resource "azurerm_key_vault_secret" "POSTGRES-HOST-V15" {
+  name         = "civil-sdt-POSTGRES-HOST-V15"
+  value        = module.postgresql-v15.fqdn
+  key_vault_id = module.civil_sdt_key_vault.key_vault_id
+}

infrastructure/demo.tfvars

@@ -1 +1,2 @@
+database_backup_retention_days=7
 sku = "Standard"

infrastructure/ithc.tfvars

@@ -1 +1,2 @@
+database_backup_retention_days=7
 sku = "Standard"

infrastructure/perftest.tfvars

@@ -1 +1,2 @@
+database_backup_retention_days=7
 sku = "Standard"

infrastructure/variables.tf

@@ -48,3 +48,9 @@ variable "jenkins_AAD_objectId" {
 }
 
 variable "aks_subscription_id" {}
+
+variable "database_backup_retention_days" {
+  default     = 35
+  description = "Backup retention period in days for the PGSql instance. Valid values are between 7 & 35 days"
+}
+

services/src/integ-test/java/uk/gov/moj/sdt/services/messaging/IndividualRequestMdbIntTest.java

@@ -68,7 +68,7 @@ public void setUp() {
         sdtMessage.setMessageSentTimestamp(System.currentTimeMillis());
         sdtMessage.setEnqueueLoggingId(1);
         final IMessageWriter messageWriter = (IMessageWriter) this.applicationContext.getBean("MessageWriter");
-        messageWriter.queueMessage(sdtMessage, "MCOLS", false);
+        messageWriter.queueMessage(sdtMessage, "MCOLS");
     }
 
     /**

services/src/integ-test/java/uk/gov/moj/sdt/services/messaging/MessageWriterIntAsbTest.java

@@ -84,14 +84,14 @@ public void testQueueMessage() throws JMSException, InterruptedException {
         final String strMessage1 =
                 "TestMessage1" + dateFormat.format(new java.util.Date(System.currentTimeMillis()));
         message1.setSdtRequestReference(strMessage1);
-        messageWriter.queueMessage(message1, "TEST1", false);
+        messageWriter.queueMessage(message1, "TEST1");
 
         // Send the second message.
         final ISdtMessage message2 = new SdtMessage();
         final String strMessage2 =
                 "TestMessage2" + dateFormat.format(new java.util.Date(System.currentTimeMillis()));
         message2.setSdtRequestReference(strMessage2);
-        messageWriter.queueMessage(message2, "TEST1", false);
+        messageWriter.queueMessage(message2, "TEST1");
 
         readMessageFromQueue(Lists.newArrayList(strMessage1, strMessage2));
 
@@ -113,7 +113,7 @@ public void testAzureServiceBusDown() throws JMSException {
         String requestReference =  "Test message" + dateFormat.format(new java.util.Date(System.currentTimeMillis()));
         message.setSdtRequestReference(requestReference);
 
-        messageWriter.queueMessage(message, "TEST1", false);
+        messageWriter.queueMessage(message, "TEST1");
         Assert.assertTrue("Test completed", true);
 
         readMessageFromQueue(Lists.newArrayList(requestReference));

services/src/integ-test/java/uk/gov/moj/sdt/services/messaging/MessageWriterIntTest.java

@@ -83,14 +83,14 @@ public void testQueueMessage() throws JMSException, InterruptedException {
         final String strMessage1 =
                 "TestMessage1" + dateFormat.format(new java.util.Date(System.currentTimeMillis()));
         message1.setSdtRequestReference(strMessage1);
-        messageWriter.queueMessage(message1, "TEST1", false);
+        messageWriter.queueMessage(message1, "TEST1");
 
         // Send the second message.
         final ISdtMessage message2 = new SdtMessage();
         final String strMessage2 =
                 "TestMessage2" + dateFormat.format(new java.util.Date(System.currentTimeMillis()));
         message2.setSdtRequestReference(strMessage2);
-        messageWriter.queueMessage(message2, "TEST1", false);
+        messageWriter.queueMessage(message2, "TEST1");
 
         readMessageFromQueue(3, Lists.newArrayList(strMessage1, strMessage2));
 
@@ -110,7 +110,7 @@ public void testAzureServiceBusDown() throws JMSException {
         final ISdtMessage message = new SdtMessage();
         message.setSdtRequestReference("Test message");
 
-        messageWriter.queueMessage(message, "TEST1", false);
+        messageWriter.queueMessage(message, "TEST1");
         Assert.assertTrue("Test completed", true);
 
         readMessageFromQueue(1);

services/src/main/java/uk/gov/moj/sdt/services/TargetApplicationSubmissionService.java

@@ -350,16 +350,6 @@ private void handleSoapFaultAndWebServiceException(final IIndividualRequest indi
 
         // now persist the request.
         this.getIndividualRequestDao().persist(individualRequest);
-
-        // Create a new message for DLQ.
-        final ISdtMessage messageObj = new SdtMessage();
-        messageObj.setSdtRequestReference(individualRequest.getSdtRequestReference());
-
-        final String targetAppCode =
-                individualRequest.getBulkSubmission().getTargetApplication().getTargetApplicationCode();
-
-        // Write to dead letter queue.
-        this.getMessageWriter().queueMessage(messageObj, targetAppCode, true);
     }
 
     /**
@@ -477,7 +467,7 @@ private void reQueueRequest(final IIndividualRequest individualRequest, Boolean
             final String targetAppCode =
                 individualRequest.getBulkSubmission().getTargetApplication().getTargetApplicationCode();
 
-            this.getMessageWriter().queueMessage(messageObj, targetAppCode, false);
+            this.getMessageWriter().queueMessage(messageObj, targetAppCode);
         } else {
             LOGGER.error("Maximum forwarding attempts exceeded for request {}",
                     individualRequest.getSdtRequestReference());

services/src/main/java/uk/gov/moj/sdt/services/messaging/MessageWriter.java

@@ -59,11 +59,6 @@ public class MessageWriter implements IMessageWriter {
      */
     private static final Logger LOGGER = LoggerFactory.getLogger(MessageWriter.class);
 
-    /**
-     * Default suffix for DLQ name.
-     */
-    private static final String DLQ_SUFFIX = "/$deadletterqueue";
-
     /**
      * The last id used for a queued message.
      */
@@ -89,10 +84,10 @@ public MessageWriter(final JmsTemplate jmsTemplate,
     }
 
     @Override
-    public void queueMessage(final ISdtMessage sdtMessage, final String targetAppCode, final boolean deadLetter) {
+    public void queueMessage(final ISdtMessage sdtMessage, final String targetAppCode) {
 
         // Check the target application code is valid and return queue name.
-        final String queueName = getQueueName(targetAppCode, deadLetter);
+        final String queueName = getQueueName(targetAppCode);
 
         LOGGER.debug("Sending message with SDT request reference [{}] to queue [{}]",
                      sdtMessage.getSdtRequestReference(),
@@ -186,12 +181,9 @@ public void setQueueNameMap(final Map<String, String> queueNameMap) {
      * that is mapped to the target application code.
      *
      * @param targetApplicationCode the target application code.
-     * @param deadLetter            flags whether dead letter queue name should be returned.
      * @return the queue name matching to the target application code.
      */
-    private String getQueueName(final String targetApplicationCode, final boolean deadLetter) {
-        String queueName;
-
+    private String getQueueName(final String targetApplicationCode) {
         // The target application code should be supplied.
         if (targetApplicationCode == null || targetApplicationCode.trim().length() == 0) {
             throw new IllegalArgumentException("Target application code must be supplied.");
@@ -202,13 +194,7 @@ private String getQueueName(final String targetApplicationCode, final boolean de
                 throw new IllegalArgumentException("Target application code [" + targetApplicationCode +
                         "] does not have a JMS queue mapped.");
             } else {
-                queueName = this.getQueueNameMap().get(targetApplicationCode);
-                if (deadLetter) {
-                    // Append DLQ suffix for dead letter.
-                    queueName += DLQ_SUFFIX;
-
-                }
-                return queueName;
+                return this.getQueueNameMap().get(targetApplicationCode);
             }
         }
     }

services/src/main/java/uk/gov/moj/sdt/services/utils/MessagingUtility.java

@@ -91,7 +91,7 @@ private void queueRequest(IIndividualRequest individualRequest) {
 
         messageObj.setSdtRequestReference(individualRequest.getSdtRequestReference());
 
-        getMessageWriter().queueMessage(messageObj, targetAppCode, false);
+        getMessageWriter().queueMessage(messageObj, targetAppCode);
     }
 
     /**

services/src/unit-test/java/uk/gov/moj/sdt/services/TargetApplicationSubmissionServiceTest.java

@@ -452,7 +452,7 @@ void processRequestToSubmitTimeOut() {
         verify(mockErrorMsgCacheable).getValue(IErrorMessage.class, REQ_NOT_ACK);
         verify(mockCacheable).getValue(IGlobalParameter.class, TARGET_APP_TIMEOUT);
         verify(mockCacheable).getValue(IGlobalParameter.class, TARGET_APP_RESP_TIMEOUT);
-        verify(mockMessageWriter).queueMessage(any(ISdtMessage.class),any(String.class), eq(false));
+        verify(mockMessageWriter).queueMessage(any(ISdtMessage.class),any(String.class));
         verify(mockIndividualRequestDao).getRequestBySdtReference(TEST_1);
 
     }
@@ -518,7 +518,6 @@ void processRequestToSubmitForWebServiceException() {
         verify(mockCacheable).getValue(IGlobalParameter.class, TARGET_APP_RESP_TIMEOUT);
         verify(mockCacheable).getValue(IGlobalParameter.class, MCOL_INDV_REQ_DELAY);
         verify(mockIndividualRequestDao).getRequestBySdtReference(TEST_1);
-        verify(mockMessageWriter).queueMessage(any(ISdtMessage.class), any(String.class), eq(true));
     }
 
     /**
@@ -571,7 +570,6 @@ void processRequestToSubmitSoapFault() {
         verify(mockCacheable).getValue(IGlobalParameter.class, TARGET_APP_RESP_TIMEOUT);
         verify(mockCacheable).getValue(IGlobalParameter.class, MCOL_INDV_REQ_DELAY);
         verify(mockIndividualRequestDao).getRequestBySdtReference(TEST_1);
-        verify(mockMessageWriter).queueMessage(any(ISdtMessage.class), any(String.class), eq(true));
     }
 
     /**