Merge branch 'main' into DynamicSignInOptions

.github/workflows/ci.yml

@@ -81,7 +81,7 @@ jobs:
         working-directory: downloaded-artifacts
 
       - name: Upload Coverage with retry
-        uses: Wandalen/wretry.action@v1.3.0
+        uses: Wandalen/wretry.action@v1.4.8
         with:
           attempt_limit: 5
           attempt_delay: 2000

.github/workflows/plugin_install.yml

@@ -39,27 +39,12 @@ jobs:
         run: mv ./build/distributions/${{ env.PLUGIN_NAME }}-*.zip ${{ env.PLUGIN_NAME }}.zip
         shell: bash
 
-      - name: Create Setup Script
-        if: ${{ runner.os == 'Linux' }}
-        run: |
-          cat > setup.sh <<'EOF'
-          chmod +x  ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh
-          /bin/bash -c "yes | ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/plugins/${{ env.PLUGIN_NAME }}/tools/install_demo_configuration.sh -t"
-          EOF
-
-      - name: Create Setup Script
-        if: ${{ runner.os == 'Windows' }}
-        run: |
-          New-Item .\setup.bat -type file
-          Set-Content .\setup.bat -Value "powershell.exe .\opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT\plugins\${{ env.PLUGIN_NAME }}\tools\install_demo_configuration.bat -i -c -y -t"
-          Get-Content .\setup.bat
-
       - name: Run Opensearch with A Single Plugin
-        uses: ./.github/actions/start-opensearch-with-one-plugin
+        uses: derek-ho/start-opensearch@v2
         with:
           opensearch-version: ${{ env.OPENSEARCH_VERSION }}
-          plugin-name: ${{ env.PLUGIN_NAME }}
-          setup-script-name: setup
+          plugins: "file:$(pwd)/${{ env.PLUGIN_NAME }}.zip"
+          security-enabled: true
           admin-password: ${{ steps.random-password.outputs.generated_name }}
 
       - name: Run sanity tests

DEVELOPER_GUIDE.md

@@ -86,21 +86,23 @@ rm -rf config/
 ## ROOT
 
 openssl genrsa -out root-ca-key.pem 2048
-openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "/DC=com/DC=example/O=Example Com Inc./OU=Example Com Inc. Root CA/CN=Example Com Inc. Root CA" -addext "basicConstraints = critical,CA:TRUE" -addext "keyUsage = critical, digitalSignature, keyCertSign, cRLSign" -addext "subjectKeyIdentifier = hash" -addext "authorityKeyIdentifier = keyid:always,issuer:always" -out root-ca.pem
+openssl req -new -x509 -sha256 -days 3650 -key root-ca-key.pem -subj "/DC=com/DC=example/O=Example Com Inc./OU=Example Com Inc. Root CA/CN=Example Com Inc. Root CA" -addext "basicConstraints = critical,CA:TRUE" -addext "keyUsage = critical, digitalSignature, keyCertSign, cRLSign" -addext "subjectKeyIdentifier = hash" -addext "authorityKeyIdentifier = keyid:always,issuer:always" -out root-ca.pem
 
 
 ## NODE
 
 openssl genrsa -out esnode-key-temp.pem 2048
 openssl pkcs8 -inform PEM -outform PEM -in esnode-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out esnode-key.pem
 openssl req -new -key esnode-key.pem -subj "/C=de/L=test/O=node/OU=node/CN=node-0.example.com" -out esnode.csr
-openssl x509 -req -in esnode.csr -out esnode.pem -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -days 3650 -extfile <(printf "subjectAltName = RID:1.2.3.4.5.5, DNS:node-0.example.com, DNS:localhost, IP:::1, IP:127.0.0.1\nkeyUsage = digitalSignature, nonRepudiation, keyEncipherment\nextendedKeyUsage = serverAuth, clientAuth\nbasicConstraints = critical,CA:FALSE")
+printf "subjectAltName = RID:1.2.3.4.5.5, DNS:node-0.example.com, DNS:localhost, IP:::1, IP:127.0.0.1\nkeyUsage = digitalSignature, nonRepudiation, keyEncipherment\nextendedKeyUsage = serverAuth, clientAuth\nbasicConstraints = critical,CA:FALSE" > esnode_ext.conf
+openssl x509 -req -in esnode.csr -out esnode.pem -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -days 3650 -extfile esnode_ext.conf
 
 
 ## ADMIN
 
 openssl req -new -newkey rsa:2048 -keyout kirk-key.pem -out kirk.csr -nodes -subj "/C=de/L=test/O=client/OU=client/CN=kirk"
-openssl x509 -req -in kirk.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -out kirk.pem -days 3650 -extfile <(printf "basicConstraints = critical,CA:FALSE\nkeyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment\nextendedKeyUsage = critical,clientAuth\nauthorityKeyIdentifier = keyid,issuer:always\nsubjectKeyIdentifier = hash")
+printf "basicConstraints = critical,CA:FALSE\nkeyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment\nextendedKeyUsage = critical,clientAuth\nauthorityKeyIdentifier=keyid,issuer:always\nsubjectKeyIdentifier = hash" > kirk_ext.conf
+openssl x509 -req -in kirk.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -out kirk.pem -days 3650 -extfile kirk_ext.conf
 
 ## Remove root-ca-key.pem and other temp keys
 

TRIAGING.md

@@ -12,7 +12,7 @@ Each meeting we seek to  address all new issues. However, should we run out of t
 
 ### How do I join the Backlog & Triage meeting?
 
-Meetings are hosted regularly at 3 PM Eastern Time (Noon Pacific Time) and can be joined via the links posted on the [Upcoming Events](https://opensearch.org/events) webpage.
+Meetings are hosted regularly at 11 AM Eastern Time (8AM Pacific Time) and can be joined via the links posted on the [OpenSearch Meetup Group](https://www.meetup.com/opensearch/events/) list of events. The event will be titled `Development Backlog & Triage Meeting - Security`.
 
 After joining the Zoom meeting, you can enable your video / voice to join the discussion.  If you do not have a webcam or microphone available, you can still join in via the text chat.
 

build.gradle

@@ -25,14 +25,14 @@ buildscript {
         opensearch_build = version_tokens[0] + '.0'
 
         common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT')
-        kafka_version  = '3.6.1'
+        kafka_version  = '3.7.0'
         apache_cxf_version = '4.0.3'
         open_saml_version = '4.3.0'
         one_login_java_saml = '2.9.0'
         jjwt_version = '0.12.5'
         guava_version = '32.1.3-jre'
         jaxb_version = '2.3.9'
-        spring_version = '5.3.31'
+        spring_version = '5.3.32'
 
         if (buildVersionQualifier) {
             opensearch_build += "-${buildVersionQualifier}"
@@ -64,7 +64,7 @@ plugins {
     id 'maven-publish'
     id 'com.diffplug.spotless' version '6.25.0'
     id 'checkstyle'
-    id 'com.netflix.nebula.ospackage' version "11.7.0"
+    id 'com.netflix.nebula.ospackage' version "11.8.1"
     id "org.gradle.test-retry" version "1.5.8"
     id 'eclipse'
     id "com.github.spotbugs" version "5.2.5"
@@ -474,9 +474,9 @@ bundlePlugin {
 configurations {
     all {
         resolutionStrategy {
-            force 'commons-codec:commons-codec:1.16.0'
+            force 'commons-codec:commons-codec:1.16.1'
             force 'org.slf4j:slf4j-api:1.7.36'
-            force 'org.scala-lang:scala-library:2.13.12'
+            force 'org.scala-lang:scala-library:2.13.13'
             force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
             force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
             force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
@@ -499,9 +499,9 @@ configurations {
             // For integrationTest
             force "org.apache.httpcomponents:httpclient:4.5.14"
             force "org.apache.httpcomponents:httpcore:4.4.16"
-            force "com.google.errorprone:error_prone_annotations:2.24.1"
+            force "com.google.errorprone:error_prone_annotations:2.25.0"
             force "org.checkerframework:checker-qual:3.42.0"
-            force "ch.qos.logback:logback-classic:1.2.13"
+            force "ch.qos.logback:logback-classic:1.5.3"
         }
     }
 
@@ -582,7 +582,7 @@ dependencies {
     implementation "com.google.guava:guava:${guava_version}"
     implementation 'org.greenrobot:eventbus-java:3.3.1'
     implementation 'commons-cli:commons-cli:1.6.0'
-    implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
+    implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
     implementation 'org.ldaptive:ldaptive:1.2.3'
     implementation 'com.nimbusds:nimbus-jose-jwt:9.37.3'
 
@@ -607,11 +607,11 @@ dependencies {
 
     runtimeOnly 'com.sun.activation:jakarta.activation:1.2.2'
     runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5'
-    runtimeOnly 'commons-codec:commons-codec:1.16.0'
+    runtimeOnly 'commons-codec:commons-codec:1.16.1'
     runtimeOnly 'org.cryptacular:cryptacular:1.2.6'
-    compileOnly 'com.google.errorprone:error_prone_annotations:2.24.1'
+    compileOnly 'com.google.errorprone:error_prone_annotations:2.25.0'
     runtimeOnly 'com.sun.istack:istack-commons-runtime:4.2.0'
-    runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1'
+    runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.2'
     runtimeOnly 'org.ow2.asm:asm:9.6'
 
     testImplementation 'org.apache.camel:camel-xmlsecurity:3.22.1'
@@ -649,12 +649,12 @@ dependencies {
     runtimeOnly 'org.xerial.snappy:snappy-java:1.1.10.5'
     runtimeOnly 'org.codehaus.woodstox:stax2-api:4.2.2'
     runtimeOnly "org.glassfish.jaxb:txw2:${jaxb_version}"
-    runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.0'
+    runtimeOnly 'com.fasterxml.woodstox:woodstox-core:6.6.1'
     runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1'
     runtimeOnly 'org.apache.santuario:xmlsec:2.3.4'
     runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}"
     runtimeOnly 'org.checkerframework:checker-qual:3.42.0'
-    runtimeOnly "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}"
+    runtimeOnly "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
     runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2'
 
 
@@ -677,6 +677,7 @@ dependencies {
     testImplementation 'org.apache.httpcomponents:fluent-hc:4.5.14'
     testImplementation "org.apache.httpcomponents.client5:httpclient5-fluent:${versions.httpclient5}"
     testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}"
+    testImplementation "org.apache.kafka:kafka-server:${kafka_version}"
     testImplementation "org.apache.kafka:kafka-server-common:${kafka_version}"
     testImplementation "org.apache.kafka:kafka-server-common:${kafka_version}:test"
     testImplementation "org.apache.kafka:kafka-group-coordinator:${kafka_version}"
@@ -702,11 +703,12 @@ dependencies {
     testRuntimeOnly ("org.springframework:spring-core:${spring_version}") {
         exclude(group:'org.springframework', module: 'spring-jcl' )
     }
-    testRuntimeOnly 'org.scala-lang:scala-library:2.13.12'
+    testRuntimeOnly 'org.scala-lang:scala-library:2.13.13'
     testRuntimeOnly 'com.yammer.metrics:metrics-core:2.2.0'
     testRuntimeOnly 'com.typesafe.scala-logging:scala-logging_3:3.9.5'
     testRuntimeOnly('org.apache.zookeeper:zookeeper:3.9.1') {
         exclude(group:'ch.qos.logback', module: 'logback-classic' )
+        exclude(group:'ch.qos.logback', module: 'logback-core' )
     }
     testRuntimeOnly "org.apache.kafka:kafka-metadata:${kafka_version}"
     testRuntimeOnly "org.apache.kafka:kafka-storage:${kafka_version}"
@@ -727,8 +729,8 @@ dependencies {
     integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
     integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
-    integrationTestImplementation "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}"
-    integrationTestImplementation "org.bouncycastle:bcutil-jdk15to18:${versions.bouncycastle}"
+    integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
+    integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}"
     integrationTestImplementation('org.awaitility:awaitility:4.2.0') {
         exclude(group: 'org.hamcrest', module: 'hamcrest')
     }
@@ -740,7 +742,7 @@ dependencies {
     integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
 
     //spotless
-    implementation('com.google.googlejavaformat:google-java-format:1.19.2') {
+    implementation('com.google.googlejavaformat:google-java-format:1.21.0') {
         exclude group: 'com.google.guava'
     }
 }

bwc-test/src/test/resources/security/esnode.pem

@@ -1,9 +1,9 @@
 -----BEGIN CERTIFICATE-----
-MIIEPDCCAySgAwIBAgIUZjrlDPP8azRDPZchA/XEsx0X2iMwDQYJKoZIhvcNAQEL
+MIIEPDCCAySgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLYwDQYJKoZIhvcNAQEL
 BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt
 cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl
 IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v
-dCBDQTAeFw0yMzA4MjkxOTQ0NDJaFw0zMzA4MjYxOTQ0NDJaMFcxCzAJBgNVBAYT
+dCBDQTAeFw0yNDAyMjAxNzAzMjVaFw0zNDAyMTcxNzAzMjVaMFcxCzAJBgNVBAYT
 AmRlMQ0wCwYDVQQHDAR0ZXN0MQ0wCwYDVQQKDARub2RlMQ0wCwYDVQQLDARub2Rl
 MRswGQYDVQQDDBJub2RlLTAuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
 A4IBDwAwggEKAoIBAQCm93kXteDQHMAvbUPNPW5pyRHKDD42XGWSgq0k1D29C/Ud
@@ -16,10 +16,10 @@ BEAwPogFKgMEBQWCEm5vZGUtMC5leGFtcGxlLmNvbYIJbG9jYWxob3N0hxAAAAAA
 AAAAAAAAAAAAAAABhwR/AAABMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAUBggrBgEF
 BQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQU0/qDQaY10jIo
 wCjLUpz/HfQXyt8wHwYDVR0jBBgwFoAUF4ffoFrrZhKn1dD4uhJFPLcrAJwwDQYJ
-KoZIhvcNAQELBQADggEBAHfKc6NUXJfsT5nL1CtaGy3dutV13UQkdUwEkB0BikkX
-1PNaz2NeHyO2yQvp4G6WlovZB78tVqn5hbEZL7v8kUlAOTkjEJjsOu1Ib746eBdT
-gmUBKpIeBrm3a+tsLR9OBOuDb8aQO6fnFehFs/70y0sbyRbVqSmxLaYgRkPhhqwl
-3U7Ha1TpdJrckETk/iRcma0igvym1SvlUahgFXN4ZCLG3SycH+YRFtM749GVZBo5
-5E5gSfkWCj9jao3LjJn3ThtMsiL405uIPbFNm+5iXtflMk2aW666j1jlpeaZySuy
-DWBtA+T5Y6HhDECSjHOV131UekvHLF+SbWrv0S+ptjA=
+KoZIhvcNAQELBQADggEBAGbij5WyF0dKhQodQfTiFDb73ygU6IyeJkFSnxF67gDz
+pQJZKFvXuVBa3cGP5e7Qp3TK50N+blXGH0xXeIV9lXeYUk4hVfBlp9LclZGX8tGi
+7Xa2enMvIt5q/Yg3Hh755ZxnDYxCoGkNOXUmnMusKstE0YzvZ5Gv6fcRKFBUgZLh
+hUBqIEAYly1EqH/y45APiRt3Nor1yF6zEI4TnL0yNrHw6LyQkUNCHIGMJLfnJQ9L
+camMGIXOx60kXNMTigF9oXXwixWAnDM9y3QT8QXA7hej/4zkbO+vIeV/7lGUdkyg
+PAi92EvyxmsliEMyMR0VINl8emyobvfwa7oMeWMR+hg=
 -----END CERTIFICATE-----