First Nginx integration bundle

.github/workflows/opensearch-observability-test-and-build-workflow.yml

@@ -4,19 +4,10 @@ on: [pull_request, push]
 
 jobs:
   build:
-    env:
-      BUILD_ARGS: ${{ matrix.os_build_args }}
     strategy:
-      # Run all jobs
-      fail-fast: false
       matrix:
         java: [11, 17]
         os: [ubuntu-latest, windows-latest, macos-latest]
-        include:
-          - os: windows-latest
-            os_build_args: -x integTest -x jacocoTestReport
-          - os: macos-latest
-            os_build_args: -x integTest -x jacocoTestReport
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -36,7 +27,7 @@ jobs:
 
       - name: Build with Gradle
         run: |
-          ./gradlew build ${{ env.BUILD_ARGS }}
+          ./gradlew build
 
       - name: Upload coverage
         if: ${{ matrix.os == 'ubuntu-latest' }}

build-tools/opensearchplugin-coverage.gradle

@@ -17,6 +17,9 @@
  *  break if there are multiple nodes in the integTestCluster. But for now... it sorta works.
  */
 
+import org.apache.tools.ant.taskdefs.condition.Os
+apply plugin: 'jacoco'
+
 // Get gradle to generate the required jvm agent arg for us using a dummy tasks of type Test. Unfortunately Elastic's
 // testing tasks don't derive from Test so the jacoco plugin can't do this automatically.
 def jacocoDir = "${buildDir}/jacoco"
@@ -61,7 +64,12 @@ allprojects{
         jacocoTestReport.dependsOn integTest
 
         testClusters.integTest {
-            jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:','javaagent:/')
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                // Replacing build with absolute path to fix the error "error opening zip file or JAR manifest missing : /build/tmp/expandedArchives/..../jacocoagent.jar"
+                jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('build',"${buildDir}")
+            } else {
+                jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:','javaagent:/')
+            }
             systemProperty 'com.sun.management.jmxremote', "true"
             systemProperty 'com.sun.management.jmxremote.authenticate', "false"
             systemProperty 'com.sun.management.jmxremote.port', "7777"

build.gradle

@@ -10,7 +10,7 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 buildscript {
     ext {
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.6.0-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
         opensearch_build = version_tokens[0] + '.0'
@@ -104,6 +104,30 @@ publishing {
     }
 }
 
+publishing {
+    publications {
+        pluginZip(MavenPublication) { publication ->
+            pom {
+                name = 'opensearch-observability'
+                description = 'OpenSearch Observability plugin'
+                groupId = "org.opensearch.plugin"
+                licenses {
+                    license {
+                        name = 'The Apache License, Version 2.0'
+                        url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+                    }
+                }
+                developers {
+                    developer {
+                        name = 'OpenSearch'
+                        url = 'https://github.com/opensearch-project/observability'
+                    }
+                }
+            }
+        }
+    }
+}
+
 allOpen {
     annotation("org.opensearch.observability.util.OpenForTesting")
 }
@@ -128,6 +152,7 @@ configurations.all {
         force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
         force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}"
         force "org.mockito:mockito-core:4.6.1"
+        force "org.yaml:snakeyaml:1.32"
     }
 }
 
@@ -167,6 +192,7 @@ dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9"
     implementation "${group}:common-utils:${common_utils_version}"
+    implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
     implementation 'org.json:json:20220924'
     implementation group: 'com.github.wnameless.json', name: 'json-flattener', version: '0.15.1'
     // json-base, jackson-databind, jackson-annotations are only used by json-flattener.
@@ -326,59 +352,70 @@ testClusters.integTest {
     setting 'path.repo', repo.absolutePath
 }
 
-String baseVersion = "2.6.0"
-String bwcVersion = baseVersion + ".0"
+String bwcMinVersion = "1.2.0-SNAPSHOT"
+String bwcBundleVersion = "1.3.2.0"
+Boolean bwcBundleTest = (project.findProperty('customDistributionDownloadType') != null &&
+        project.properties['customDistributionDownloadType'] == "bundle");
+String bwcVersion = bwcBundleTest ? bwcBundleVersion : bwcMinVersion
+String bwcCurrentVersion = opensearch_version.replace("-SNAPSHOT", "")
 String baseName = "obsBwcCluster"
 String bwcFilePath = "src/test/resources/bwc/"
-String bwcObservabilityPlugin = "opensearch-observability-" + bwcVersion + ".zip"
-String bwcJobSchedulerPlugin = "opensearch-job-scheduler-" + bwcVersion + ".zip" 
-String remoteFileURL = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${baseVersion}/latest/linux/x64/tar/builds/opensearch/plugins/" + bwcObservabilityPlugin
-String bwcJobSchedulerURL = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${baseVersion}/latest/linux/x64/tar/builds/opensearch/plugins/" + bwcJobSchedulerPlugin
+String remoteFileURL = "https://github.com/opensearch-project/observability/releases/download/1.2.0.0/opensearch-observability-1.2.0.0.zip"
 
 2.times {i ->
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = [baseVersion,opensearch_version]
             numberOfNodes = 3
-            plugin(provider(new Callable<RegularFile>(){
-                @Override
-                RegularFile call() throws Exception {
-                    return new RegularFile() {
-                        @Override
-                        File getAsFile() {
-                            File dir = new File(bwcFilePath + "/job-scheduler/" + bwcVersion)
-                            if (!dir.exists()) {
-                                dir.mkdirs()
-                            }
-                            File file = new File(dir, bwcJobSchedulerPlugin)
-                            if (!file.exists()) {
-                                new URL(bwcJobSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
-                            }
-                            return fileTree(bwcFilePath + "/job-scheduler/" + bwcVersion).getSingleFile()
-                        }
-                    }
+            if (bwcBundleTest) {
+                versions = ["1.3.2", bwcCurrentVersion]
+                nodes.each { node ->
+                    node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem"))
+                    node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem"))
+                    node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem"))
+                    node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem"))
+                    node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem"))
+                    node.setting("plugins.security.disabled", "true")
+                    node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+                    node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+                    node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+                    node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+                    node.setting("plugins.security.ssl.http.enabled", "true")
+                    node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+                    node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+                    node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+                    node.setting("plugins.security.allow_unsafe_democertificates", "true")
+                    node.setting("plugins.security.allow_default_init_securityindex", "true")
+                    node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
+                    node.setting("plugins.security.audit.type", "internal_elasticsearch")
+                    node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
+                    node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
+                    node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
+                    node.setting("plugins.security.system_indices.enabled", "true")
                 }
-            }))
-            plugin(provider(new Callable<RegularFile>(){
-                @Override
-                RegularFile call() throws Exception {
-                    return new RegularFile() {
-                        @Override
-                        File getAsFile() {
-                            File dir = new File(bwcFilePath + "observability/" + bwcVersion)
-                            if (!dir.exists()) {
-                                dir.mkdirs()
-                            }
-                            File file = new File(dir, bwcObservabilityPlugin)
-                            if (!file.exists()) {
-                                new URL(remoteFileURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+            } else {
+                versions = ["1.2.0",opensearch_version]
+                plugin(provider(new Callable<RegularFile>(){
+                    @Override
+                    RegularFile call() throws Exception {
+                        return new RegularFile() {
+                            @Override
+                            File getAsFile() {
+                                File dir = new File(bwcFilePath + "observability/" + bwcVersion)
+                                if (!dir.exists()) {
+                                    dir.mkdirs()
+                                }
+                                File file = new File(dir, "opensearch-observability-1.2.0.0-SNAPSHOT.zip")
+                                if (!file.exists()) {
+                                    new URL(remoteFileURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                                }
+                                return fileTree(bwcFilePath + "observability/" + bwcVersion).getSingleFile()
                             }
-                            return fileTree(bwcFilePath + "observability/" + bwcVersion).getSingleFile()
                         }
                     }
-                }
-            }))
+                }))
+            }
+
             setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
             setting 'http.content_type.required', 'true'
         }
@@ -420,7 +457,9 @@ task prepareBwcTests {
 // Creates 2 test clusters with 3 nodes of the old version.
 2.times {i ->
     task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
-        dependsOn 'prepareBwcTests'
+        if (!bwcBundleTest){
+            dependsOn 'prepareBwcTests'
+        }
         useCluster testClusters."${baseName}$i"
         filter {
             includeTestsMatching "org.opensearch.observability.bwc.*IT"
@@ -439,8 +478,14 @@ task prepareBwcTests {
 task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
     useCluster testClusters."${baseName}0"
     dependsOn "${baseName}#oldVersionClusterTask0"
-    doFirst {
-        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    if (bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}0".nextNodeToNextVersion()
+        }
+    } else {
+        doFirst {
+            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        }
     }
     filter {
         includeTestsMatching "org.opensearch.observability.bwc.*IT"
@@ -458,8 +503,14 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
 task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#mixedClusterTask"
     useCluster testClusters."${baseName}0"
-    doFirst {
-        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    if (bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}0".nextNodeToNextVersion()
+        }
+    } else {
+        doFirst {
+            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        }
     }
     filter {
         includeTestsMatching "org.opensearch.observability.bwc.*IT"
@@ -477,8 +528,14 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
 task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
     useCluster testClusters."${baseName}0"
-    doFirst {
-        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    if (bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}0".nextNodeToNextVersion()
+        }
+    } else {
+        doFirst {
+            testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+        }
     }
     filter {
         includeTestsMatching "org.opensearch.observability.bwc.*IT"
@@ -496,8 +553,14 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
 task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
     dependsOn "${baseName}#oldVersionClusterTask1"
     useCluster testClusters."${baseName}1"
-    doFirst {
-        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+    if (bwcBundleTest){
+        doFirst {
+            testClusters."${baseName}1".goToNextVersion()
+        }
+    } else {
+        doFirst {
+            testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+        }
     }
     filter {
         includeTestsMatching "org.opensearch.observability.bwc.*IT"
@@ -582,11 +645,5 @@ task updateVersion {
         println "Setting version to ${newVersion}."
          // String tokenization to support -SNAPSHOT
         ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
-        ant.replaceregexp(file:'../.github/workflows/dashboards-observability-test-and-build-workflow.yml', match:'OPENSEARCH_PLUGIN_VERSION: \\d+.\\d+.\\d+.\\d+', replace:'OPENSEARCH_PLUGIN_VERSION: ' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
-        // Match key version in JSON files.
-        ant.replaceregexp(file:'../dashboards-observability/opensearch_dashboards.json', match:'"version": "\\d+.\\d+.\\d+.\\d+', replace:'"version": ' + '"' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
-        ant.replaceregexp(file:'../dashboards-observability/package.json', match:'"version": "\\d+.\\d+.\\d+.\\d+', replace:'"version": ' + '"' + newVersion.tokenize('-')[0] + '.0', flags:'g', byline:true)
-        // Match key opensearchDashboardsVersion in JSON files.
-        ant.replaceregexp(file:'../dashboards-observability/opensearch_dashboards.json', match:'"opensearchDashboardsVersion": "\\d+.\\d+.\\d+', replace:'"opensearchDashboardsVersion": ' + '"' + newVersion.tokenize('-')[0], flags:'g', byline:true)
     }
 }