Backport FIPS 140-3

.buildkite/pipelines/periodic.template.yml

@@ -119,6 +119,73 @@ steps:
             allowed: true
             permit_on_passed: false
             reason: "Retry with smart test selection if desired"
+  - group: java-fips-140-3-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-140-3-matrix"
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 $$GRADLE_TASK
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - adoptopenjdk17
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+            - exit_status: "1"
+              limit: 1
+          manual:
+            allowed: true
+            permit_on_passed: false
+            reason: "Retry with smart test selection if desired"
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-fips-140-3-matrix-bwc"
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - adoptopenjdk17
+            BWC_VERSION: $BWC_LIST
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+            - exit_status: "1"
+              limit: 1
+          manual:
+            allowed: true
+            permit_on_passed: false
+            reason: "Retry with smart test selection if desired"
   - group: java-matrix
     steps:
       - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"

.buildkite/pipelines/periodic.yml

@@ -918,6 +918,73 @@ steps:
             allowed: true
             permit_on_passed: false
             reason: "Retry with smart test selection if desired"
+  - group: java-fips-140-3-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-140-3-matrix"
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 $$GRADLE_TASK
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - adoptopenjdk17
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkPart6
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+            - exit_status: "1"
+              limit: 1
+          manual:
+            allowed: true
+            permit_on_passed: false
+            reason: "Retry with smart test selection if desired"
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.BWC_VERSION}} / java-fips-140-3-matrix-bwc"
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 v$$BWC_VERSION#bwcTest
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - adoptopenjdk17
+            BWC_VERSION: ["7.17.30", "8.19.11"]
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          BWC_VERSION: "{{matrix.BWC_VERSION}}"
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+            - exit_status: "1"
+              limit: 1
+          manual:
+            allowed: true
+            permit_on_passed: false
+            reason: "Retry with smart test selection if desired"
   - group: java-matrix
     steps:
       - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"

.buildkite/pipelines/pull-request/part-1-fips-140-3.yml

@@ -0,0 +1,12 @@
+config:
+  allow-labels:
+    - test-fips
+steps:
+  - label: part-1-fips-140-3
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 checkPart1
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-2-fips-140-3.yml

@@ -0,0 +1,12 @@
+config:
+  allow-labels:
+    - test-fips
+steps:
+  - label: part-2-fips-140-3
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 checkPart2
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-3-fips-140-3.yml

@@ -0,0 +1,12 @@
+config:
+  allow-labels:
+    - test-fips
+steps:
+  - label: part-3-fips-140-3
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 checkPart3
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-4-fips-140-3.yml

@@ -0,0 +1,12 @@
+config:
+  allow-labels:
+    - test-fips
+steps:
+  - label: part-4-fips-140-3
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: n1-standard-32
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-5-fips-140-3.yml

@@ -0,0 +1,12 @@
+config:
+  allow-labels:
+    - test-fips
+steps:
+  - label: part-5-fips-140-3
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 checkPart5
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/part-6-fips-140-3.yml

@@ -0,0 +1,12 @@
+config:
+  allow-labels:
+    - test-fips
+steps:
+  - label: part-6-fips-140-3
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true -Dtests.fips.mode=140-3 checkPart6
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

build-tools-internal/src/main/groovy/elasticsearch.fips.gradle

@@ -23,30 +23,58 @@ if (buildParams.inFipsJvm) {
     String javaSecurityFilename = buildParams.runtimeJavaDetails.get().toLowerCase().contains('oracle') ? 'fips_java_oracle.security' : 'fips_java.security'
     File fipsResourcesDir = new File(project.buildDir, 'fips-resources')
     File fipsSecurity = new File(fipsResourcesDir, javaSecurityFilename)
-    File fipsPolicy = new File(fipsResourcesDir, 'fips_java.policy')
     File fipsTrustStore = new File(fipsResourcesDir, 'cacerts.bcfks')
-    def bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.6')
-    def bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19')
+
+    def bcFips
+    def bcTlsFips
+    def bcUtilFips
     def manualDebug = false; //change this to manually debug bouncy castle in an IDE
-    if(manualDebug) {
-      bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.6')
-      bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19'){
-        exclude group: 'org.bouncycastle', module: 'bc-fips'  // to avoid jar hell
+    def isFips140_3 = buildParams.fipsMode == '140-3'
+    String javaPolicyFilename = isFips140_3 ? 'fips_java_bc2.policy' : 'fips_java.policy'
+    File fipsPolicy = new File(fipsResourcesDir, javaPolicyFilename)
+
+    if (isFips140_3) {
+      // FIPS 140-3 certified BouncyCastle libraries
+      bcFips = dependencies.create('org.bouncycastle:bc-fips:2.0.1')
+      bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:2.0.22')
+      bcUtilFips = dependencies.create('org.bouncycastle:bcutil-fips:2.0.5')
+      if (manualDebug) {
+        bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:2.0.1')
+        bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:2.0.22'){
+          exclude group: 'org.bouncycastle', module: 'bcutil-fips'  // to avoid jar hell
+        }
+        bcUtilFips = dependencies.create('org.bouncycastle:bcutil-fips:2.0.5'){
+          exclude group: 'org.bouncycastle', module: 'bc-fips'  // to avoid jar hell
+        }
+      }
+    } else {
+      // FIPS 140-2 certified BouncyCastle libraries (default)
+      bcFips = dependencies.create('org.bouncycastle:bc-fips:1.0.2.6')
+      bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19')
+      bcUtilFips = null
+      if (manualDebug) {
+        bcFips = dependencies.create('org.bouncycastle:bc-fips-debug:1.0.2.6')
+        bcTlsFips = dependencies.create('org.bouncycastle:bctls-fips:1.0.19'){
+          exclude group: 'org.bouncycastle', module: 'bc-fips'  // to avoid jar hell
+        }
       }
     }
     pluginManager.withPlugin('java-base') {
       TaskProvider<ExportElasticsearchBuildResourcesTask> fipsResourcesTask = project.tasks.register('fipsResources', ExportElasticsearchBuildResourcesTask)
       fipsResourcesTask.configure {
         outputDir = fipsResourcesDir
         copy javaSecurityFilename
-        copy 'fips_java.policy'
+        copy javaPolicyFilename
         copy 'cacerts.bcfks'
       }
 
       def extraFipsJarsConfiguration = configurations.create("fipsImplementation") {
         withDependencies {
           add(bcFips)
           add(bcTlsFips)
+          if (bcUtilFips != null) {
+            add(bcUtilFips)
+          }
         }
       }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ExportElasticsearchBuildResourcesTask.java

@@ -27,8 +27,8 @@
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.HashMap;
+import java.util.Map;
 
 import javax.inject.Inject;
 
@@ -43,7 +43,8 @@ public class ExportElasticsearchBuildResourcesTask extends DefaultTask {
 
     private static final Logger logger = Logging.getLogger(ExportElasticsearchBuildResourcesTask.class);
 
-    private final Set<String> resources = new HashSet<>();
+    // Maps resource path -> destination filename
+    private final Map<String, String> resources = new HashMap<>();
 
     private DirectoryProperty outputDir;
 
@@ -58,8 +59,8 @@ public DirectoryProperty getOutputDir() {
     }
 
     @Input
-    public Set<String> getResources() {
-        return Collections.unmodifiableSet(resources);
+    public Map<String, String> getResources() {
+        return Collections.unmodifiableMap(resources);
     }
 
     @Classpath
@@ -73,13 +74,23 @@ public void setOutputDir(File outputDir) {
         this.outputDir.set(outputDir);
     }
 
+    /**
+     * Copy a resource to the output directory, keeping the original filename.
+     */
     public void copy(String resource) {
+        copy(resource, resource);
+    }
+
+    /**
+     * Copy a resource to the output directory with a different filename.
+     */
+    public void copy(String resource, String destName) {
         if (getState().getExecuted() || getState().getExecuting()) {
             throw new GradleException(
                 "buildResources can't be configured after the task ran. " + "Make sure task is not used after configuration time"
             );
         }
-        resources.add(resource);
+        resources.put(resource, destName);
     }
 
     @TaskAction
@@ -88,8 +99,10 @@ public void doExport() {
             setDidWork(false);
             throw new StopExecutionException();
         }
-        resources.stream().parallel().forEach(resourcePath -> {
-            Path destination = outputDir.get().file(resourcePath).getAsFile().toPath();
+        resources.entrySet().stream().parallel().forEach(entry -> {
+            String resourcePath = entry.getKey();
+            String destName = entry.getValue();
+            Path destination = outputDir.get().file(destName).getAsFile().toPath();
             try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath)) {
                 Files.createDirectories(destination.getParent());
                 if (is == null) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java

@@ -71,4 +71,6 @@ public interface BuildParameterExtension {
     Provider<Random> getRandom();
 
     Boolean getGraalVmRuntime();
+
+    String getFipsMode();
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java

@@ -27,6 +27,7 @@
 
 public abstract class DefaultBuildParameterExtension implements BuildParameterExtension {
     private final Provider<Boolean> inFipsJvm;
+    private final Provider<String> fipsMode;
     private final Provider<File> runtimeJavaHome;
     private final RuntimeJava runtimeJava;
     private final List<JavaHome> javaVersions;
@@ -65,6 +66,7 @@ public DefaultBuildParameterExtension(
         Provider<BwcVersions> bwcVersions
     ) {
         this.inFipsJvm = providers.systemProperty("tests.fips.enabled").map(DefaultBuildParameterExtension::parseBoolean);
+        this.fipsMode = providers.systemProperty("tests.fips.mode");
         this.runtimeJava = runtimeJava;
         this.runtimeJavaHome = cache(providers, runtimeJava.getJavahome());
         this.javaToolChainSpec = cache(providers, javaToolChainSpec);
@@ -101,6 +103,11 @@ public boolean getInFipsJvm() {
         return inFipsJvm.getOrElse(false);
     }
 
+    @Override
+    public String getFipsMode() {
+        return fipsMode.getOrNull();
+    }
+
     @Override
     public Provider<File> getRuntimeJavaHome() {
         return runtimeJavaHome;