Merge branch 'main' into auto_upgrade_mappings

.github/workflows/backport.yml

@@ -13,6 +13,15 @@ on:
 jobs:
   backport:
     runs-on: ubuntu-latest
+    if: >
+      github.event.pull_request.merged
+      && (
+        github.event.action == 'closed'
+        || (
+          github.event.action == 'labeled'
+          && contains(github.event.label.name, 'backport')
+        )
+      )
     permissions:
       contents: write
       pull-requests: write
@@ -27,7 +36,9 @@ jobs:
           installation_id: 22958780
 
       - name: Backport
-        uses: VachaShah/backport@v1.1.4
+        uses: VachaShah/backport@v2.2.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
-          branch_name: backport/backport-${{ github.event.number }}
+          head_template: backport/backport-<%= number %>-to-<%= base %>
+          labels_template: "<%= JSON.stringify([...labels, 'autocut']) %>"
+          failure_labels: "failed backport" 

.github/workflows/maven-publish.yml

@@ -3,11 +3,10 @@ name: Publish snapshots to maven
 on:
   workflow_dispatch:
   push:
-      branches: [
-        main
-        1.*
-        2.*
-      ]
+      branches: 
+        - main
+        - '[0-9]+.[0-9]+'
+        - '[0-9]+.x'
 
 jobs:
   build-and-publish-snapshots:

notifications/build-tools/merged-coverage.gradle

@@ -15,6 +15,9 @@ allprojects {
 
 task jacocoReport(type: JacocoReport, group: 'verification') {
     description = 'Generates an aggregate report from all subprojects'
+    onlyIf {
+        gradle.taskGraph.hasTask("jacocoTestReport")
+    }
     gradle.projectsEvaluated {
         subprojects.each {
             jacocoReport.dependsOn it.tasks.withType(JacocoReport)

notifications/build.gradle

@@ -21,7 +21,7 @@ buildscript {
         }
         opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
-        kotlin_version = System.getProperty("kotlin.version", "1.6.10")
+        kotlin_version = System.getProperty("kotlin.version", "1.8.21")
         junit_version = System.getProperty("junit.version", "5.7.2")
         aws_version = System.getProperty("aws.version", "1.12.48")
     }
@@ -40,7 +40,7 @@ buildscript {
         classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
 // TODO: enable detekt only when snakeyaml vulnerability is fixed
 //        classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.17.1"
-        classpath "org.jacoco:org.jacoco.agent:0.8.5"
+        classpath "org.jacoco:org.jacoco.agent:0.8.7"
     }
 }
 
@@ -83,7 +83,7 @@ dependencies {
 
 task ktlint(type: JavaExec, group: "verification") {
     description = "Check Kotlin code style."
-    main = "com.pinterest.ktlint.Main"
+    mainClass = "com.pinterest.ktlint.Main"
     classpath = configurations.ktlint
     args "notifications/**/*.kt", "core/**/*.kt", "core-spi/**/*.kt"
     // to generate report in checkstyle format prepend following args:
@@ -93,7 +93,7 @@ task ktlint(type: JavaExec, group: "verification") {
 
 task ktlintFormat(type: JavaExec, group: "formatting") {
     description = "Fix Kotlin code style deviations."
-    main = "com.pinterest.ktlint.Main"
+    mainClass = "com.pinterest.ktlint.Main"
     classpath = configurations.ktlint
     args "-F", "notifications/**/*.kt", "core/**/*.kt", "core-spi/**/*.kt"
     // https://github.com/pinterest/ktlint/issues/1391

notifications/core-spi/build.gradle

@@ -20,8 +20,6 @@ repositories {
 }
 
 apply plugin: 'opensearch.java'
-apply plugin: 'opensearch.testclusters'
-apply plugin: 'opensearch.java-rest-test'
 apply plugin: 'org.jetbrains.kotlin.jvm'
 apply plugin: 'org.jetbrains.kotlin.plugin.allopen'
 

notifications/core-spi/src/main/kotlin/org/opensearch/notifications/spi/model/SecureDestinationSettings.kt

@@ -5,6 +5,6 @@
 
 package org.opensearch.notifications.spi.model
 
-import org.opensearch.common.settings.SecureString
+import org.opensearch.core.common.settings.SecureString
 
 data class SecureDestinationSettings(val emailUsername: SecureString, val emailPassword: SecureString)

notifications/core/build.gradle

@@ -74,14 +74,6 @@ publishing {
     }
 }
 
-task integTest(type: RestIntegTestTask) {
-    description = "Run tests against a cluster that has security enabled"
-    testClassesDirs = sourceSets.test.output.classesDirs
-    classpath = sourceSets.test.runtimeClasspath
-    systemProperty 'tests.security.manager', 'false'
-}
-
-tasks.named("check").configure { dependsOn(integTest) }
 check.dependsOn jacocoTestReport
 
 // turn off javadoc as it barfs on Kotlin code
@@ -107,6 +99,11 @@ test {
     systemProperty 'tests.security.manager', 'false'
 }
 
+// disable for core, we don't need to run test cluster
+run {
+    onlyIf { false }
+}
+
 //TODO: see if this can be only set at top level and apply to subprojects, or we remove from top project if possible
 configurations.all {
     if (it.state != Configuration.State.UNRESOLVED) return
@@ -137,6 +134,7 @@ dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
     implementation "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}"
     implementation "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}"
+    implementation "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}"
     implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
     implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
     implementation "com.amazonaws:aws-java-sdk-core:${aws_version}"

notifications/core/src/main/kotlin/org/opensearch/notifications/core/NotificationCorePlugin.kt

@@ -8,9 +8,9 @@ package org.opensearch.notifications.core
 import org.opensearch.client.Client
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver
 import org.opensearch.cluster.service.ClusterService
-import org.opensearch.common.io.stream.NamedWriteableRegistry
 import org.opensearch.common.settings.Setting
 import org.opensearch.common.settings.Settings
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry
 import org.opensearch.core.xcontent.NamedXContentRegistry
 import org.opensearch.env.Environment
 import org.opensearch.env.NodeEnvironment

notifications/core/src/main/kotlin/org/opensearch/notifications/core/client/DestinationHttpClient.kt

@@ -20,8 +20,9 @@ import org.apache.hc.core5.http.HttpResponse
 import org.apache.hc.core5.http.io.entity.EntityUtils
 import org.apache.hc.core5.http.io.entity.StringEntity
 import org.apache.hc.core5.util.Timeout
-import org.opensearch.common.xcontent.XContentFactory
 import org.opensearch.common.xcontent.XContentType
+import org.opensearch.core.rest.RestStatus
+import org.opensearch.core.xcontent.MediaTypeRegistry
 import org.opensearch.notifications.core.setting.PluginSettings
 import org.opensearch.notifications.core.utils.OpenForTesting
 import org.opensearch.notifications.core.utils.logger
@@ -32,7 +33,6 @@ import org.opensearch.notifications.spi.model.destination.ChimeDestination
 import org.opensearch.notifications.spi.model.destination.CustomWebhookDestination
 import org.opensearch.notifications.spi.model.destination.SlackDestination
 import org.opensearch.notifications.spi.model.destination.WebhookDestination
-import org.opensearch.rest.RestStatus
 import java.io.IOException
 import java.nio.charset.StandardCharsets
 import java.util.Collections
@@ -155,7 +155,7 @@ class DestinationHttpClient {
     }
 
     fun buildRequestBody(destination: WebhookDestination, message: MessageContent): String {
-        val builder = XContentFactory.contentBuilder(XContentType.JSON)
+        val builder = MediaTypeRegistry.contentBuilder(XContentType.JSON)
         val keyName = when (destination) {
             // Slack webhook request body has required "text" as key name https://api.slack.com/messaging/webhooks
             // Chime webhook request body has required "Content" as key name

notifications/core/src/main/kotlin/org/opensearch/notifications/core/client/DestinationSesClient.kt

@@ -15,6 +15,7 @@ import com.amazonaws.services.simpleemail.model.MailFromDomainNotVerifiedExcepti
 import com.amazonaws.services.simpleemail.model.MessageRejectedException
 import com.amazonaws.services.simpleemail.model.RawMessage
 import com.amazonaws.services.simpleemail.model.SendRawEmailRequest
+import org.opensearch.core.rest.RestStatus
 import org.opensearch.notifications.core.NotificationCorePlugin.Companion.LOG_PREFIX
 import org.opensearch.notifications.core.credentials.SesClientFactory
 import org.opensearch.notifications.core.setting.PluginSettings
@@ -23,7 +24,6 @@ import org.opensearch.notifications.core.utils.logger
 import org.opensearch.notifications.spi.model.DestinationMessageResponse
 import org.opensearch.notifications.spi.model.MessageContent
 import org.opensearch.notifications.spi.model.destination.SesDestination
-import org.opensearch.rest.RestStatus
 import java.io.ByteArrayOutputStream
 import java.nio.ByteBuffer
 import java.util.Properties

notifications/core/src/main/kotlin/org/opensearch/notifications/core/client/DestinationSmtpClient.kt

@@ -6,15 +6,15 @@
 package org.opensearch.notifications.core.client
 
 import com.sun.mail.util.MailConnectException
-import org.opensearch.common.settings.SecureString
+import org.opensearch.core.common.settings.SecureString
+import org.opensearch.core.rest.RestStatus
 import org.opensearch.notifications.core.setting.PluginSettings
 import org.opensearch.notifications.core.utils.SecurityAccess
 import org.opensearch.notifications.core.utils.logger
 import org.opensearch.notifications.spi.model.DestinationMessageResponse
 import org.opensearch.notifications.spi.model.MessageContent
 import org.opensearch.notifications.spi.model.SecureDestinationSettings
 import org.opensearch.notifications.spi.model.destination.SmtpDestination
-import org.opensearch.rest.RestStatus
 import java.util.Properties
 import javax.mail.Authenticator
 import javax.mail.Message

notifications/core/src/main/kotlin/org/opensearch/notifications/core/client/DestinationSnsClient.kt

@@ -23,14 +23,17 @@ import com.amazonaws.services.sns.model.KMSOptInRequiredException
 import com.amazonaws.services.sns.model.KMSThrottlingException
 import com.amazonaws.services.sns.model.NotFoundException
 import com.amazonaws.services.sns.model.PlatformApplicationDisabledException
+import com.amazonaws.services.sns.model.PublishRequest
 import com.amazonaws.services.sns.model.PublishResult
+import org.opensearch.core.rest.RestStatus
 import org.opensearch.notifications.core.NotificationCorePlugin.Companion.LOG_PREFIX
 import org.opensearch.notifications.core.credentials.SnsClientFactory
+import org.opensearch.notifications.core.setting.PluginSettings
 import org.opensearch.notifications.core.utils.logger
 import org.opensearch.notifications.spi.model.DestinationMessageResponse
 import org.opensearch.notifications.spi.model.MessageContent
 import org.opensearch.notifications.spi.model.destination.SnsDestination
-import org.opensearch.rest.RestStatus
+import java.util.UUID
 
 /**
  * This class handles the SNS connections to the given Destination.
@@ -118,6 +121,14 @@ class DestinationSnsClient(private val snsClientFactory: SnsClientFactory) {
      */
     @Throws(Exception::class)
     fun sendMessage(amazonSNS: AmazonSNS, destination: SnsDestination, message: MessageContent): PublishResult {
-        return amazonSNS.publish(destination.topicArn, message.textDescription, message.title)
+        val request: PublishRequest = PublishRequest()
+            .withTopicArn(destination.topicArn)
+            .withMessage(message.textDescription)
+            .withSubject(message.title)
+        if (destination.topicArn.endsWith(".fifo")) {
+            request.withMessageDeduplicationId(UUID.randomUUID().toString())
+                .withMessageGroupId(String.format("opensearch-%s", PluginSettings.clusterName))
+        }
+        return amazonSNS.publish(request)
     }
 }

notifications/core/src/main/kotlin/org/opensearch/notifications/core/setting/PluginSettings.kt

@@ -6,15 +6,16 @@
 package org.opensearch.notifications.core.setting
 
 import org.opensearch.bootstrap.BootstrapInfo
+import org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING
 import org.opensearch.cluster.service.ClusterService
 import org.opensearch.common.settings.SecureSetting
-import org.opensearch.common.settings.SecureString
 import org.opensearch.common.settings.Setting
 import org.opensearch.common.settings.Setting.Property.Deprecated
 import org.opensearch.common.settings.Setting.Property.Dynamic
 import org.opensearch.common.settings.Setting.Property.Final
 import org.opensearch.common.settings.Setting.Property.NodeScope
 import org.opensearch.common.settings.Settings
+import org.opensearch.core.common.settings.SecureString
 import org.opensearch.notifications.core.NotificationCorePlugin.Companion.LOG_PREFIX
 import org.opensearch.notifications.core.NotificationCorePlugin.Companion.PLUGIN_NAME
 import org.opensearch.notifications.core.utils.OpenForTesting
@@ -105,6 +106,16 @@ internal object PluginSettings {
      */
     private const val TOOLTIP_SUPPORT_KEY = "$KEY_PREFIX.tooltip_support"
 
+    /**
+     * Setting to provide cluster name, which is <AWS-account-number:AWS-domain-name> on the managed service
+     */
+    private const val CLUSTER_NAME = "cluster.name"
+
+    /**
+     * Default cluster name if it cannot be retrieved.
+     */
+    private const val DEFAULT_CLUSTER_NAME = "OpenSearch:DefaultClusterName"
+
     /**
      * Default email size limit as 10MB.
      */
@@ -223,6 +234,12 @@ internal object PluginSettings {
     @Volatile
     var hostDenyList: List<String>
 
+    /**
+     * cluster name
+     */
+    @Volatile
+    var clusterName: String
+
     /**
      * Destination Settings
      */
@@ -258,6 +275,7 @@ internal object PluginSettings {
         allowedConfigTypes = settings?.getAsList(ALLOWED_CONFIG_TYPE_KEY, null) ?: DEFAULT_ALLOWED_CONFIG_TYPES
         tooltipSupport = settings?.getAsBoolean(TOOLTIP_SUPPORT_KEY, true) ?: DEFAULT_TOOLTIP_SUPPORT
         hostDenyList = settings?.getAsList(HOST_DENY_LIST_KEY, null) ?: DEFAULT_HOST_DENY_LIST
+        clusterName = settings?.get(CLUSTER_NAME, DEFAULT_CLUSTER_NAME) ?: DEFAULT_CLUSTER_NAME
         destinationSettings = if (settings != null) loadDestinationSettings(settings) else DEFAULT_DESTINATION_SETTINGS
 
         defaultSettings = mapOf(
@@ -411,6 +429,7 @@ internal object PluginSettings {
         tooltipSupport = TOOLTIP_SUPPORT.get(clusterService.settings)
         hostDenyList = HOST_DENY_LIST.get(clusterService.settings)
         destinationSettings = loadDestinationSettings(clusterService.settings)
+        clusterName = clusterService.clusterName.value()
     }
 
     /**
@@ -464,6 +483,11 @@ internal object PluginSettings {
             log.debug("$LOG_PREFIX:$HOST_DENY_LIST_KEY -autoUpdatedTo-> $clusterHostDenyList")
             hostDenyList = clusterHostDenyList
         }
+        val clusterClusterName = clusterService.clusterName
+        if (clusterClusterName != null) {
+            log.debug("$LOG_PREFIX:$CLUSTER_NAME_SETTING -autoUpdatedTo-> $clusterClusterName")
+            clusterName = clusterClusterName.value()
+        }
     }
 
     /**

notifications/core/src/main/kotlin/org/opensearch/notifications/core/transport/SesDestinationTransport.kt

@@ -5,14 +5,14 @@
 
 package org.opensearch.notifications.core.transport
 
+import org.opensearch.core.rest.RestStatus
 import org.opensearch.notifications.core.client.DestinationClientPool
 import org.opensearch.notifications.core.client.DestinationSesClient
 import org.opensearch.notifications.core.utils.OpenForTesting
 import org.opensearch.notifications.core.utils.logger
 import org.opensearch.notifications.spi.model.DestinationMessageResponse
 import org.opensearch.notifications.spi.model.MessageContent
 import org.opensearch.notifications.spi.model.destination.SesDestination
-import org.opensearch.rest.RestStatus
 import java.io.IOException
 import javax.mail.MessagingException
 import javax.mail.internet.AddressException

notifications/core/src/main/kotlin/org/opensearch/notifications/core/transport/SmtpDestinationTransport.kt

@@ -5,14 +5,14 @@
 
 package org.opensearch.notifications.core.transport
 
+import org.opensearch.core.rest.RestStatus
 import org.opensearch.notifications.core.client.DestinationClientPool
 import org.opensearch.notifications.core.client.DestinationSmtpClient
 import org.opensearch.notifications.core.utils.OpenForTesting
 import org.opensearch.notifications.core.utils.logger
 import org.opensearch.notifications.spi.model.DestinationMessageResponse
 import org.opensearch.notifications.spi.model.MessageContent
 import org.opensearch.notifications.spi.model.destination.SmtpDestination
-import org.opensearch.rest.RestStatus
 import java.io.IOException
 import javax.mail.MessagingException
 import javax.mail.internet.AddressException

notifications/core/src/main/kotlin/org/opensearch/notifications/core/transport/SnsDestinationTransport.kt

@@ -5,14 +5,14 @@
 
 package org.opensearch.notifications.core.transport
 
+import org.opensearch.core.rest.RestStatus
 import org.opensearch.notifications.core.client.DestinationClientPool
 import org.opensearch.notifications.core.client.DestinationSnsClient
 import org.opensearch.notifications.core.utils.OpenForTesting
 import org.opensearch.notifications.core.utils.logger
 import org.opensearch.notifications.spi.model.DestinationMessageResponse
 import org.opensearch.notifications.spi.model.MessageContent
 import org.opensearch.notifications.spi.model.destination.SnsDestination
-import org.opensearch.rest.RestStatus
 import java.io.IOException
 
 /**

notifications/core/src/main/kotlin/org/opensearch/notifications/core/transport/WebhookDestinationTransport.kt

@@ -5,14 +5,14 @@
 
 package org.opensearch.notifications.core.transport
 
+import org.opensearch.core.rest.RestStatus
 import org.opensearch.notifications.core.client.DestinationClientPool
 import org.opensearch.notifications.core.client.DestinationHttpClient
 import org.opensearch.notifications.core.utils.OpenForTesting
 import org.opensearch.notifications.core.utils.logger
 import org.opensearch.notifications.spi.model.DestinationMessageResponse
 import org.opensearch.notifications.spi.model.MessageContent
 import org.opensearch.notifications.spi.model.destination.WebhookDestination
-import org.opensearch.rest.RestStatus
 import java.io.IOException
 
 /**

notifications/core/src/main/kotlin/org/opensearch/notifications/core/utils/Helpers.kt

@@ -7,7 +7,7 @@ package org.opensearch.notifications.core.utils
 
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
-import org.opensearch.common.bytes.BytesReference
+import org.opensearch.core.common.bytes.BytesReference
 import org.opensearch.core.xcontent.XContentBuilder
 
 fun <T : Any> logger(forClass: Class<T>): Lazy<Logger> {