Consistently use slf4j/log4j bridge (#139226)

gradle/verification-metadata.xml

@@ -3110,6 +3110,11 @@
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.logging.log4j" name="log4j-slf4j2-impl" version="2.19.0">
+         <artifact name="log4j-slf4j2-impl-2.19.0.jar">
+            <sha256 value="825605eacb2d5605b105c53d4108c18125e0f82f62960d0be583278b9c524f3c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-common" version="10.2.2">
          <artifact name="lucene-analysis-common-10.2.2.jar">
             <sha256 value="4dc1acc6b5a43be0049057bc99772dd24717cf0084c7ff79c68c984f120c140b" origin="Generated by Gradle"/>
@@ -4928,6 +4933,11 @@
             <sha256 value="2f2a92d410b268139d7d63b75ed25e21995cfe4100c19bf23577cfdbc8077bda" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.slf4j" name="slf4j-ext" version="2.0.6">
+         <artifact name="slf4j-ext-2.0.6.jar">
+            <sha256 value="0f6ef03bc0291899f3fb324baba0dee02fa8c6c1adc7b465f5b923ac70379efd" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.slf4j" name="slf4j-log4j12" version="1.7.10">
          <artifact name="slf4j-log4j12-1.7.10.jar">
             <sha256 value="2e4eebc6e346c92c417aa4e662738802645ef21c5eb4435132dc78d631f2eebb" origin="Generated by Gradle"/>

modules/repository-azure/build.gradle

@@ -83,8 +83,7 @@ dependencies {
 
   implementation project(":modules:transport-netty4")
   implementation("org.slf4j:slf4j-api:${versions.slf4j}")
-  runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
-//  runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
+  runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}")
 
   testImplementation project(':test:fixtures:azure-fixture')
   yamlRestTestImplementation project(':test:fixtures:azure-fixture')
@@ -104,6 +103,7 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /stax-.*/, to: 'stax'
   mapping from: /reactor-netty-.*/, to: 'reactor-netty'
   mapping from: /reactive-streams.*/, to: 'reactive-streams'
+  mapping from: /log4j.*/, to: 'log4j'
 }
 
 tasks.named("thirdPartyAudit").configure {
@@ -173,8 +173,6 @@ tasks.named("thirdPartyAudit").configure {
 
     // from com.ctc.wstx.shaded.msv_core.driver.textui.Driver (woodstox-core)
     'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
-    // [missing classes] SLF4j includes an optional class that depends on an extension class. see Log4jLogger#createConverter
-    //    'org.slf4j.ext.EventData' - bring back when https://github.com/elastic/elasticsearch/issues/93714 is done
 
     // Optional dependency of tink
     'com.google.crypto.tink.subtle.Ed25519Sign',

modules/repository-s3/build.gradle

@@ -46,6 +46,7 @@ dependencies {
   runtimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   runtimeOnly "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
   runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}"
   runtimeOnly "software.amazon.awssdk:arns:${versions.awsv2sdk}"
   runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}"
   runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}"
@@ -119,6 +120,7 @@ tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: 'sts',                      to: 'aws-sdk-2'
   mapping from: 'third-party-jackson-core', to: 'aws-sdk-2'
   mapping from: 'utils',                    to: 'aws-sdk-2'
+  mapping from: /log4j.*/,                  to: 'log4j'
 }
 
 esplugin.bundleSpec.from('config/repository-s3') {

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java

@@ -50,6 +50,8 @@ public class RepositoryS3BasicCredentialsRestIT extends AbstractRepositoryS3Rest
         .keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY)
         .keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY)
         .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress)
+        .systemProperty("es.insecure_network_trace_enabled", "true")
+        .setting("logger.org.apache.http.headers", "TRACE")
         .build();
 
     @ClassRule

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3WireLoggingRestIT.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import fixture.aws.DynamicRegionSupplier;
+import fixture.s3.S3HttpFixture;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.LogType;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
+import static org.hamcrest.Matchers.hasSize;
+
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
+public class RepositoryS3WireLoggingRestIT extends AbstractRepositoryS3RestTestCase {
+
+    private static final String PREFIX = getIdentifierPrefix("RepositoryS3WireLoggingRestIT");
+    private static final String BUCKET = PREFIX + "bucket";
+    private static final String BASE_PATH = PREFIX + "base_path";
+    private static final String ACCESS_KEY = PREFIX + "access-key";
+    private static final String SECRET_KEY = PREFIX + "secret-key";
+    private static final String CLIENT = "wire_logging_client";
+
+    private static final Supplier<String> regionSupplier = new DynamicRegionSupplier();
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        BUCKET,
+        BASE_PATH,
+        fixedAccessKey(ACCESS_KEY, regionSupplier, "s3")
+    );
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .systemProperty("aws.region", regionSupplier)
+        .systemProperty("es.insecure_network_trace_enabled", "true")
+        .setting("logger.org.apache.http.headers", "DEBUG")
+        .setting("logger.org.apache.http.wire", "DEBUG")
+        .setting("logger.software.amazon.awssdk.request", "DEBUG")
+        .keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY)
+        .keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY)
+        .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected String getBucketName() {
+        return BUCKET;
+    }
+
+    @Override
+    protected String getBasePath() {
+        return BASE_PATH;
+    }
+
+    @Override
+    protected String getClientName() {
+        return CLIENT;
+    }
+
+    @Override
+    public void testSnapshotAndRestore() throws Exception {
+        super.testSnapshotAndRestore();
+        try (
+            var logReader = new BufferedReader(
+                new InputStreamReader(cluster.getNodeLog(0, LogType.SERVER_JSON), StandardCharsets.ISO_8859_1)
+            )
+        ) {
+            final var neededLoggers = new HashSet<>(
+                List.of("org.apache.http.wire", "org.apache.http.headers", "software.amazon.awssdk.request")
+            );
+            String currentLine;
+
+            while ((currentLine = logReader.readLine()) != null && neededLoggers.isEmpty() == false) {
+                if (XContentHelper.convertToMap(new BytesArray(currentLine), false, XContentType.JSON)
+                    .v2()
+                    .get("log.logger") instanceof String loggerName) {
+                    neededLoggers.remove(loggerName);
+                }
+            }
+            assertThat(neededLoggers, hasSize(0));
+        }
+    }
+}

server/src/main/java/org/elasticsearch/common/logging/Loggers.java

@@ -46,7 +46,7 @@ public class Loggers {
      */
     static final List<String> RESTRICTED_LOGGERS = NetworkTraceFlag.TRACE_ENABLED
         ? Collections.emptyList()
-        : List.of("org.apache.http", "com.amazonaws.request");
+        : List.of("org.apache.http", "com.amazonaws.request", "software.amazon.awssdk");
 
     public static final Setting<Level> LOG_DEFAULT_LEVEL_SETTING = new Setting<>(
         "logger.level",

x-pack/plugin/core/build.gradle

@@ -33,6 +33,8 @@ esplugin {
 tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
   mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
+  mapping from: /slf4j-.*/, to: 'slf4j'
+  mapping from: /log4j-.*/, to: 'log4j'
 }
 
 configurations {
@@ -48,6 +50,16 @@ dependencies {
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
   api "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
+
+  // Ideally this would be `runtimeOnly` so that we don't accidentally write code against the SLF4j API
+  // However, some child plugins (like security) need to use slf4j directly in order to manage logging for their dependencies :(
+  // And due to the way our plugin loading handles java modules between dependent plugins, this plugin (x-pack-core) needs to declare
+  //  a module dependency on slf4j (`requires org.slf4j`) so that security can also be dependent on it
+  // And having a module dependency counts as using the jar, so we can't make it `runtimeOnly` :(
+
+  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}"
+
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
@@ -61,9 +73,6 @@ dependencies {
   implementation project(":x-pack:plugin:core:template-resources")
 
   testImplementation "org.elasticsearch:mocksocket:${versions.mocksocket}"
-  testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
-  // this might suffer from https://github.com/elastic/elasticsearch/issues/93714
-  testImplementation "org.slf4j:slf4j-api:${versions.slf4j}"
   testImplementation project(path: ':modules:reindex')
   testImplementation project(path: ':modules:parent-join')
   testImplementation project(path: ':modules:lang-mustache')

x-pack/plugin/core/src/main/java/module-info.java

@@ -23,6 +23,7 @@
     requires unboundid.ldapsdk;
     requires org.elasticsearch.tdigest;
     requires org.elasticsearch.xcore.templates;
+    requires org.slf4j;
 
     exports org.elasticsearch.index.engine.frozen;
     exports org.elasticsearch.license;

x-pack/plugin/ent-search/build.gradle

@@ -25,8 +25,6 @@ dependencies {
   implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
   implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
-  implementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
   implementation "com.networknt:json-schema-validator:${versions.networknt_json_schema_validator}"
 
   testImplementation(testArtifact(project(xpackModule('core'))))
@@ -63,13 +61,6 @@ tasks.named("dependencyLicenses") {
   mapping from: /jackson.*/, to: 'jackson'
 }
 
-tasks.named("thirdPartyAudit") {
-  ignoreMissingClasses(
-    // [missing classes] SLF4j includes an optional class that depends on an extension class (!)
-    'org.slf4j.ext.EventData'
-  )
-}
-
 tasks.named("yamlRestTest") {
   usesDefaultDistribution("uses the xpack/usage api")
 }

x-pack/plugin/ent-search/src/main/java/module-info.java

@@ -11,7 +11,6 @@
     requires com.fasterxml.jackson.databind;
     requires json.schema.validator;
     requires org.apache.lucene.core;
-    requires org.slf4j;
 
     requires org.elasticsearch.base;
     requires org.elasticsearch.logging;

x-pack/plugin/identity-provider/build.gradle

@@ -47,9 +47,8 @@ dependencies {
       exclude group: 'org.bouncycastle'
   }
 
-  api "org.slf4j:slf4j-api:${versions.slf4j}"
-  runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
-  //  runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
+  // provided by x-pack-core
+  compileOnly "org.slf4j:slf4j-api:${versions.slf4j}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"

x-pack/plugin/security/build.gradle

@@ -69,9 +69,9 @@ dependencies {
   api ( "org.cryptacular:cryptacular:1.2.5") {
     exclude group: 'org.bouncycastle'
   }
-  api "org.slf4j:slf4j-api:${versions.slf4j}"
-  runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}" // workaround for https://github.com/elastic/elasticsearch/issues/93714
-  // api "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"  see above
+
+  // provided by x-pack-core
+  compileOnly "org.slf4j:slf4j-api:${versions.slf4j}"
 
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"