build.gradle

/*
 * Copyright OpenSearch Contributors
 * SPDX-License-Identifier: Apache-2.0
 */

import org.opensearch.gradle.testclusters.TestClusterConfiguration
import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask

import java.util.concurrent.Callable
import java.util.concurrent.TimeUnit
import java.util.function.Predicate


buildscript {
    ext {
        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
        opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT")
        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
        // 2.2.0-SNAPSHOT -> 2.2.0.0-SNAPSHOT
        version_tokens = opensearch_version.tokenize('-')
        opensearch_build = version_tokens[0] + '.0'
        job_scheduler_no_snapshot = opensearch_build
        notifications_no_snapshot = opensearch_build
        if (buildVersionQualifier) {
            opensearch_build += "-${buildVersionQualifier}"
            job_scheduler_no_snapshot += "-${buildVersionQualifier}"
            notifications_no_snapshot += "-${buildVersionQualifier}"
        }
        if (isSnapshot) {
            opensearch_build += "-SNAPSHOT"
        }
        opensearch_no_snapshot = opensearch_version.replace("-SNAPSHOT","")
        job_scheduler_resource_folder = "src/test/resources/job-scheduler"

        notifications_resource_folder = "src/test/resources/notifications"
        notifications_core_resource_folder = "src/test/resources/notifications-core"
        // notification_version = System.getProperty("notification.version", opensearch_build)
        common_utils_version = System.getProperty("common_utils.version", opensearch_build)
        job_scheduler_version = System.getProperty("job_scheduler_version.version", opensearch_build)
        job_scheduler_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
                '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-job-scheduler-' + job_scheduler_no_snapshot + '.zip'
        notifications_version = System.getProperty("notifications.version", opensearch_build)
        notifications_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
                '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-notifications-' + notifications_no_snapshot + '.zip'
        notifications_core_build_download = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot +
                '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-notifications-core-' + notifications_no_snapshot + '.zip'

        kotlin_version = System.getProperty("kotlin.version", "1.6.10")
    }

    repositories {
        mavenLocal()
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
    }

    dependencies {
        classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
        classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
        classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.21.0"
        classpath "org.jacoco:org.jacoco.agent:0.8.7"
    }
}

plugins {
    id 'nebula.ospackage' version "8.3.0"
    id "com.dorongold.task-tree" version "1.5"
}

apply plugin: 'java'
apply plugin: 'jacoco'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.testclusters'
apply plugin: 'opensearch.rest-test'
apply plugin: 'io.gitlab.arturbosch.detekt'
apply plugin: 'org.jetbrains.kotlin.jvm'
apply plugin: 'org.jetbrains.kotlin.plugin.allopen'
apply plugin: 'opensearch.pluginzip'

configurations.all {
    if (it.state != Configuration.State.UNRESOLVED) return
    resolutionStrategy {
        force "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
        force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
        force 'junit:junit:4.13.1'
        force 'commons-beanutils:commons-beanutils:1.9.4'
        force 'com.google.guava:guava:30.0-jre'
        force 'com.puppycrawl.tools:checkstyle:8.29'
        force 'commons-codec:commons-codec:1.13'
        force 'org.apache.httpcomponents:httpclient:4.5.13'
        force 'org.apache.httpcomponents:httpclient-osgi:4.5.13'
        force 'org.apache.httpcomponents.client5:httpclient5:5.0.3'
        force 'org.apache.httpcomponents.client5:httpclient5-osgi:5.0.3'
        force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
        force 'org.yaml:snakeyaml:1.32'
        force 'org.codehaus.plexus:plexus-utils:3.0.24'
    }
}

idea.module {
    excludeDirs -= file("$buildDir")
}

def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
def usingMultiNode = project.properties.containsKey('numNodes')
// Only apply jacoco test coverage if we are running a local single node cluster
if (!usingRemoteCluster && !usingMultiNode) {
    apply from: 'build-tools/coverage.gradle'
}
check.dependsOn jacocoTestReport

opensearchplugin {
    name 'opensearch-index-management'
    description 'OpenSearch Index Management Plugin'
    classname 'org.opensearch.indexmanagement.IndexManagementPlugin'
    extendedPlugins = ['opensearch-job-scheduler']
}

tasks.named("integTest").configure {
    it.dependsOn(project.tasks.named("bundlePlugin"))
}

allOpen {
    annotation("org.opensearch.indexmanagement.util.OpenForTesting")
}

configurations {
    ktlint
}

task ktlint(type: JavaExec, group: "verification") {
    description = "Check Kotlin code style."
    main = "com.pinterest.ktlint.Main"
    classpath = configurations.ktlint
    args "src/**/*.kt", "spi/src/main/**/*.kt"
}

check.dependsOn ktlint

task ktlintFormat(type: JavaExec, group: "formatting") {
    description = "Fix Kotlin code style deviations."
    main = "com.pinterest.ktlint.Main"
    classpath = configurations.ktlint
    args "-F", "src/**/*.kt", "spi/src/main/**/*.kt"
}

detekt {
    config = files("detekt.yml")
    buildUponDefaultConfig = true
}

configurations.testImplementation {
    exclude module: "securemock"
}

ext {
    projectSubstitutions = [:]
    licenseFile = rootProject.file('LICENSE')
    noticeFile = rootProject.file('NOTICE')
}

allprojects {
    group = "org.opensearch"
    version = "${opensearch_build}"
}

dependencies {
    compileOnly "org.opensearch:opensearch:${opensearch_version}"
    compileOnly "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}"
    implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
    implementation "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${kotlin_version}"
    implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9'
    implementation "org.jetbrains:annotations:13.0"
    implementation project(path: ":${rootProject.name}-spi", configuration: 'shadow')
    implementation "org.opensearch:common-utils:${common_utils_version}"
    implementation "com.github.seancfoley:ipaddress:5.3.3"
    implementation "commons-codec:commons-codec:${versions.commonscodec}"
    implementation "org.apache.httpcomponents:httpclient:4.5.13"
    implementation "org.apache.httpcomponents:httpcore:4.4.15"

    testImplementation "org.opensearch.test:framework:${opensearch_version}"
    testImplementation "org.jetbrains.kotlin:kotlin-test:${kotlin_version}"
    testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
    testImplementation "org.mockito:mockito-core:4.7.0"

    add("ktlint", "com.pinterest:ktlint:0.45.1") {
        attributes {
            attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL))
        }
    }
}

repositories {
    mavenLocal()
    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
}

publishing {
    publications {
        pluginZip(MavenPublication) { publication ->
            pom {
                name = opensearchplugin.name
                description = opensearchplugin.description
                groupId = "org.opensearch.plugin"
                licenses {
                    license {
                        name = "The Apache License, Version 2.0"
                        url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
                    }
                }
                developers {
                    developer {
                        name = "OpenSearch"
                        url = "https://github.com/opensearch-project/index-management"
                    }
                }
            }
        }
    }
}

plugins.withId('java') {
    sourceCompatibility = targetCompatibility = JavaVersion.VERSION_11
}

plugins.withId('org.jetbrains.kotlin.jvm') {
    compileKotlin.kotlinOptions.jvmTarget = compileTestKotlin.kotlinOptions.jvmTarget = JavaVersion.VERSION_11
    compileKotlin.dependsOn ktlint
}

javadoc.enabled = false // turn off javadoc as it barfs on Kotlin code
licenseHeaders.enabled = true
dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
loggerUsageCheck.enabled = false
validateNebulaPom.enabled = false

def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()

def securityEnabled = System.getProperty("security", "false") == "true"
afterEvaluate {
    testClusters.integTest.nodes.each { node ->
        def plugins = node.plugins
        def firstPlugin = plugins.get(0)
        plugins.remove(0)
        plugins.add(firstPlugin)

        if (securityEnabled) {
            node.extraConfigFile("kirk.pem", file("src/test/resources/security/kirk.pem"))
            node.extraConfigFile("kirk-key.pem", file("src/test/resources/security/kirk-key.pem"))
            node.extraConfigFile("esnode.pem", file("src/test/resources/security/esnode.pem"))
            node.extraConfigFile("esnode-key.pem", file("src/test/resources/security/esnode-key.pem"))
            node.extraConfigFile("root-ca.pem", file("src/test/resources/security/root-ca.pem"))
            node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
            node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
            node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
            node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
            node.setting("plugins.security.ssl.http.enabled", "true")
            node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
            node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
            node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
            node.setting("plugins.security.allow_unsafe_democertificates", "true")
            node.setting("plugins.security.allow_default_init_securityindex", "true")
            node.setting("plugins.security.authcz.admin_dn", "CN=kirk,OU=client,O=client,L=test,C=de")
            node.setting("plugins.security.audit.type", "internal_elasticsearch")
            node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
            node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
            node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
            node.setting("plugins.security.system_indices.enabled", "true")
            // node.setting("plugins.security.system_indices.indices", "[\".opendistro-ism-config\"]")
        }
    }
}

test {
    systemProperty 'tests.security.manager', 'false'
}

ext.getPluginResource = { download_to_folder, download_from_src ->
    project.mkdir download_to_folder
    ant.get(src: download_from_src,
            dest: download_to_folder,
            httpusecaches: false)
    return fileTree(download_to_folder).getSingleFile()
}


File repo = file("$buildDir/testclusters/repo")
def _numNodes = findProperty('numNodes') as Integer ?: 1
testClusters.integTest {
    plugin(project.tasks.bundlePlugin.archiveFile)
    testDistribution = "ARCHIVE"
    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("cluster.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
            debugPort += 1
        }
    }

    plugin(provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    if (new File("$project.rootDir/$job_scheduler_resource_folder").exists()) {
                        project.delete(files("$project.rootDir/$job_scheduler_resource_folder"))
                    }
                    project.mkdir job_scheduler_resource_folder
                    ant.get(src: job_scheduler_build_download,
                            dest: job_scheduler_resource_folder,
                            httpusecaches: false)
                    return fileTree(job_scheduler_resource_folder).getSingleFile()
                }
            }
        }
    }))

    plugin(provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    if (new File("$project.rootDir/$notifications_core_resource_folder").exists()) {
                        project.delete(files("$project.rootDir/$notifications_core_resource_folder"))
                    }
                    project.mkdir notifications_core_resource_folder
                    ant.get(src: notifications_core_build_download,
                            dest: notifications_core_resource_folder,
                            httpusecaches: false)
                    return fileTree(notifications_core_resource_folder).getSingleFile()
                }
            }
        }
    }))

    plugin(provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    if (new File("$project.rootDir/$notifications_resource_folder").exists()) {
                        project.delete(files("$project.rootDir/$notifications_resource_folder"))
                    }
                    project.mkdir notifications_resource_folder
                    ant.get(src: notifications_build_download,
                            dest: notifications_resource_folder,
                            httpusecaches: false)
                    return fileTree(notifications_resource_folder).getSingleFile()
                }
            }
        }
    }))

    if (securityEnabled) {
        plugin(provider({
            new RegularFile() {
                @Override
                File getAsFile() { fileTree("src/test/resources/security") { include "opensearch-security*" }.getSingleFile() }
            }
        }))
    }
    setting 'path.repo', repo.absolutePath
}

integTest {
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
    systemProperty 'buildDir', buildDir.path
    systemProperty "https", System.getProperty("https")
    systemProperty "security", System.getProperty("security")
    systemProperty "user", System.getProperty("user", "admin")
    systemProperty "password", System.getProperty("password", "admin")
    // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
    // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
    doFirst {
        systemProperty 'cluster.debug', getDebug()
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000'
    }

    if (System.getProperty("tests.rest.bwcsuite") == null) {
        filter {
            excludeTestsMatching "org.opensearch.indexmanagement.bwc.*IT"
        }
    }
    // Exclude security test
    if (System.getProperty("https") == null || System.getProperty("https") == "false") {
        filter {
            excludeTestsMatching "org.opensearch.*Security*IT"
        }
    }

    // TODO: Fix running notification test against remote cluster with security plugin installed
    if (System.getProperty("https") != null) {
        filter {
            excludeTestsMatching "org.opensearch.indexmanagement.indexstatemanagement.action.NotificationActionIT"
        }
    }

    // TODO: raise issue in Core, this is because of the test framework
    if (System.getProperty("tests.clustername") != null) {
        exclude 'org/opensearch/indexmanagement/indexstatemanagement/MetadataRegressionIT.class'
    }
}

task integTestRemote(type: RestIntegTestTask)  {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    if (System.getProperty("tests.rest.bwcsuite") == null) {
        filter {
            excludeTestsMatching "org.opensearch.indexmanagement.bwc.*IT"
        }
    }

    // Only rest case can run with remote cluster
    if (System.getProperty("tests.rest.cluster") != null) {
        exclude 'org/opensearch/indexmanagement/indexstatemanagement/MetadataRegressionIT.class'
    }
    // TODO: Fix running notification test against remote cluster with security plugin installed
    if (System.getProperty("https") != null) {
        filter {
            excludeTestsMatching "org.opensearch.indexmanagement.indexstatemanagement.action.NotificationActionIT"
        }
    }

    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000'
    }

    // Snapshot action integration tests rely on node level setting path.repo which we can't set remotely
    exclude 'org/opensearch/indexmanagement/indexstatemanagement/action/SnapshotActionIT.class'
}

String bwcVersion = "1.13.2.0"
String bwcJobSchedulerVersion = "1.13.0.0"
String baseName = "indexmanagementBwcCluster"
String bwcFilePath = "src/test/resources/bwc/"
String bwc_js_resource_location = bwcFilePath + "job-scheduler/" + bwcJobSchedulerVersion
String bwc_im_resource_location = bwcFilePath + "indexmanagement/" + bwcVersion

// Downloads the bwc job scheduler version
String bwc_js_download_url = "https://github.com/opendistro-for-elasticsearch/job-scheduler/releases/download/v" +
        bwcJobSchedulerVersion + "/job-scheduler-artifacts.zip"
getPluginResource(bwc_js_resource_location, bwc_js_download_url)

// Downloads the bwc index management version
String bwc_im_download_url = "https://github.com/opendistro-for-elasticsearch/index-management/releases/download/v" +
        bwcVersion + "/index-management-artifacts.zip"
getPluginResource(bwc_im_resource_location, bwc_im_download_url)

2.times {i ->
    testClusters {
        "${baseName}$i" {
            testDistribution = "ARCHIVE"
            versions = ["7.10.2", opensearch_version]
            numberOfNodes = 3
            plugin(provider(new Callable<RegularFile>(){
                @Override
                RegularFile call() throws Exception {
                    return new RegularFile() {
                        @Override
                        File getAsFile() {
                            return fileTree(bwc_js_resource_location).getSingleFile()
                        }
                    }
                }
            }))

            plugin(provider(new Callable<RegularFile>(){
                @Override
                RegularFile call() throws Exception {
                    return new RegularFile() {
                        @Override
                        File getAsFile() {
                            return fileTree(bwc_im_resource_location).getSingleFile()
                        }
                    }
                }
            }))
            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
            setting 'http.content_type.required', 'true'
        }
    }
}

List<Provider<RegularFile>> plugins = []

// Ensure the artifact for the current project version is available to be used for the bwc tests

task prepareBwcTests {
    dependsOn bundlePlugin
    doLast {
        // Download the job scheduler test dependency
        getPluginResource(job_scheduler_resource_folder, job_scheduler_build_download)
        plugins = [
                provider(new Callable<RegularFile>(){
                    @Override
                    RegularFile call() throws Exception {
                        return new RegularFile() {
                            @Override
                            File getAsFile() {
                                return fileTree(job_scheduler_resource_folder).getSingleFile()
                            }
                        }
                    }
                }),
                project.getObjects().fileProperty().value(project.tasks.bundlePlugin.archiveFile)
        ]
    }
}

// Create two test clusters with 3 nodes of the old version
2.times {i ->
    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
        dependsOn 'prepareBwcTests'
        useCluster testClusters."${baseName}$i"
        filter {
            includeTestsMatching "org.opensearch.indexmanagement.bwc.*IT"
        }
        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
        systemProperty 'tests.rest.bwcsuite_round', 'old'
        systemProperty 'tests.plugin_bwc_version', bwcVersion
        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
    }
}

// Upgrade one node of the old cluster to new OpenSearch version with upgraded plugin version.
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
    useCluster testClusters."${baseName}0"
    dependsOn "${baseName}#oldVersionClusterTask0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.indexmanagement.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'first'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrade the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#mixedClusterTask"
    useCluster testClusters."${baseName}0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.indexmanagement.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'second'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrade the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
    useCluster testClusters."${baseName}0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.indexmanagement.bwc.*IT"
    }
    mustRunAfter "${baseName}#mixedClusterTask"
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'third'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrade all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#oldVersionClusterTask1"
    useCluster testClusters."${baseName}1"
    doFirst {
        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.indexmanagement.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
}

// A bwc test suite which runs all the bwc tasks combined
task bwcTestSuite(type: StandaloneRestIntegTestTask) {
    exclude '**/*Test*'
    exclude '**/*IT*'
    dependsOn tasks.named("${baseName}#mixedClusterTask")
    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
    dependsOn tasks.named("${baseName}#fullRestartClusterTask")
}

run {
    useCluster project.testClusters.integTest
    doFirst {
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            if (securityEnabled) {
                // TODO: This is a bit of a hack
                LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions = new LinkedHashMap<>()
                cluster.waitForConditions(waitConditions, System.currentTimeMillis(), 40, TimeUnit.SECONDS, cluster)
            } else {
                cluster.waitForAllConditions()
            }
        }
    }
}

compileKotlin { kotlinOptions.freeCompilerArgs = ['-Xjsr305=strict'] }

apply from: 'build-tools/pkgbuild.gradle'

// This IT is to simulate the situation
// when there are old version (without metadata change)
// and new version mixed in one cluster
import org.opensearch.gradle.test.RestIntegTestTask

def mixedClusterTest = project.tasks.create('mixedCluster', RestIntegTestTask.class)
def mixedClusterFlag = findProperty('mixed') as Boolean ?: false
println("mixed cluster flag: $mixedClusterFlag")
mixedClusterTest.dependsOn(bundlePlugin)

testClusters.mixedCluster {
    testDistribution = "ARCHIVE"
    if (_numNodes > 1) numberOfNodes = _numNodes
    getNodes().each { node ->
        node.plugin(provider({
            new RegularFile() {
                @Override
                File getAsFile() { fileTree(job_scheduler_resource_folder).getSingleFile() }
            }
        }))

        node.plugin(provider({
            new RegularFile() {
                @Override
                File getAsFile() { fileTree(notifications_core_resource_folder).getSingleFile() }
            }
        }))

        node.plugin(provider({
            new RegularFile() {
                @Override
                File getAsFile() { fileTree(notifications_resource_folder).getSingleFile() }
            }
        }))

        if (mixedClusterFlag && node.name == "mixedCluster-1") {
            node.plugin(provider({
                new RegularFile() {
                    @Override
                    File getAsFile() { fileTree("src/test/resources/index-management").getSingleFile() }
                }
            }))
        } else {
            node.plugin(project.tasks.bundlePlugin.archiveFile)
        }
    }
    setting 'path.repo', repo.absolutePath
}

mixedCluster {
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'tests.path.repo', repo.absolutePath
    systemProperty 'cluster.mixed', "$mixedClusterFlag"
    systemProperty 'cluster.number_of_nodes', "${_numNodes}"
}

// updateVersion: Task to auto increment to the next development iteration
task updateVersion {
    onlyIf { System.getProperty('newVersion') }
    doLast {
        ext.newVersion = System.getProperty('newVersion')
        println "Setting version to ${newVersion}."
        // String tokenization to support -SNAPSHOT
        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
    }
}