Consistently use slf4j/log4j bridge (#139226)

gradle/verification-metadata.xml

@@ -3215,6 +3215,11 @@
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.logging.log4j" name="log4j-slf4j2-impl" version="2.19.0">
+         <artifact name="log4j-slf4j2-impl-2.19.0.jar">
+            <sha256 value="825605eacb2d5605b105c53d4108c18125e0f82f62960d0be583278b9c524f3c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.lucene" name="lucene-analysis-common" version="10.3.2">
          <artifact name="lucene-analysis-common-10.3.2.jar">
             <sha256 value="1403e0df203a9037628015bcd9ca46cfa899430bdfb939374ccecc34cdcdf46b" origin="Generated by Gradle"/>

modules/repository-azure/build.gradle

@@ -83,8 +83,7 @@ dependencies {
 
   implementation project(":modules:transport-netty4")
   implementation("org.slf4j:slf4j-api:${versions.slf4j}")
-  runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
-//  runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
+  runtimeOnly("org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}")
 
   testImplementation project(':test:fixtures:azure-fixture')
   yamlRestTestImplementation project(':test:fixtures:azure-fixture')
@@ -104,6 +103,7 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /stax-.*/, to: 'stax'
   mapping from: /reactor-netty-.*/, to: 'reactor-netty'
   mapping from: /reactive-streams.*/, to: 'reactive-streams'
+  mapping from: /log4j.*/, to: 'log4j'
 }
 
 tasks.named("thirdPartyAudit").configure {
@@ -173,8 +173,6 @@ tasks.named("thirdPartyAudit").configure {
 
     // from com.ctc.wstx.shaded.msv_core.driver.textui.Driver (woodstox-core)
     'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
-    // [missing classes] SLF4j includes an optional class that depends on an extension class. see Log4jLogger#createConverter
-    //    'org.slf4j.ext.EventData' - bring back when https://github.com/elastic/elasticsearch/issues/93714 is done
 
     // Optional dependency of tink
     'com.google.crypto.tink.subtle.Ed25519Sign',

modules/repository-s3/build.gradle

@@ -46,6 +46,7 @@ dependencies {
   runtimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   runtimeOnly "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
   runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}"
   runtimeOnly "software.amazon.awssdk:arns:${versions.awsv2sdk}"
   runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}"
   runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}"
@@ -119,6 +120,7 @@ tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: 'sts',                      to: 'aws-sdk-2'
   mapping from: 'third-party-jackson-core', to: 'aws-sdk-2'
   mapping from: 'utils',                    to: 'aws-sdk-2'
+  mapping from: /log4j.*/,                  to: 'log4j'
 }
 
 esplugin.bundleSpec.from('config/repository-s3') {

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3BasicCredentialsRestIT.java

@@ -51,6 +51,8 @@ public class RepositoryS3BasicCredentialsRestIT extends AbstractRepositoryS3Rest
         .keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY)
         .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress)
         .setting("s3.client." + CLIENT + ".disable_chunked_encoding", () -> randomFrom("true", "false"), ignored -> randomBoolean())
+        .systemProperty("es.insecure_network_trace_enabled", "true")
+        .setting("logger.org.apache.http.headers", "TRACE")
         .build();
 
     @ClassRule

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3WireLoggingRestIT.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import fixture.aws.DynamicRegionSupplier;
+import fixture.s3.S3HttpFixture;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.LogType;
+import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
+import org.elasticsearch.xcontent.XContentType;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.List;
+import java.util.function.Supplier;
+
+import static fixture.aws.AwsCredentialsUtils.fixedAccessKey;
+import static org.hamcrest.Matchers.hasSize;
+
+@ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
+public class RepositoryS3WireLoggingRestIT extends AbstractRepositoryS3RestTestCase {
+
+    private static final String PREFIX = getIdentifierPrefix("RepositoryS3WireLoggingRestIT");
+    private static final String BUCKET = PREFIX + "bucket";
+    private static final String BASE_PATH = PREFIX + "base_path";
+    private static final String ACCESS_KEY = PREFIX + "access-key";
+    private static final String SECRET_KEY = PREFIX + "secret-key";
+    private static final String CLIENT = "wire_logging_client";
+
+    private static final Supplier<String> regionSupplier = new DynamicRegionSupplier();
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        BUCKET,
+        BASE_PATH,
+        fixedAccessKey(ACCESS_KEY, regionSupplier, "s3")
+    );
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .systemProperty("aws.region", regionSupplier)
+        .systemProperty("es.insecure_network_trace_enabled", "true")
+        .setting("logger.org.apache.http.headers", "DEBUG")
+        .setting("logger.org.apache.http.wire", "DEBUG")
+        .setting("logger.software.amazon.awssdk.request", "DEBUG")
+        .keystore("s3.client." + CLIENT + ".access_key", ACCESS_KEY)
+        .keystore("s3.client." + CLIENT + ".secret_key", SECRET_KEY)
+        .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected String getBucketName() {
+        return BUCKET;
+    }
+
+    @Override
+    protected String getBasePath() {
+        return BASE_PATH;
+    }
+
+    @Override
+    protected String getClientName() {
+        return CLIENT;
+    }
+
+    @Override
+    public void testSnapshotAndRestore() throws Exception {
+        super.testSnapshotAndRestore();
+        try (
+            var logReader = new BufferedReader(
+                new InputStreamReader(cluster.getNodeLog(0, LogType.SERVER_JSON), StandardCharsets.ISO_8859_1)
+            )
+        ) {
+            final var neededLoggers = new HashSet<>(
+                List.of("org.apache.http.wire", "org.apache.http.headers", "software.amazon.awssdk.request")
+            );
+            String currentLine;
+
+            while ((currentLine = logReader.readLine()) != null && neededLoggers.isEmpty() == false) {
+                if (XContentHelper.convertToMap(new BytesArray(currentLine), false, XContentType.JSON)
+                    .v2()
+                    .get("log.logger") instanceof String loggerName) {
+                    neededLoggers.remove(loggerName);
+                }
+            }
+            assertThat(neededLoggers, hasSize(0));
+        }
+    }
+}

server/src/main/java/org/elasticsearch/common/logging/Loggers.java

@@ -46,7 +46,7 @@ public class Loggers {
      */
     static final List<String> RESTRICTED_LOGGERS = NetworkTraceFlag.TRACE_ENABLED
         ? Collections.emptyList()
-        : List.of("org.apache.http", "com.amazonaws.request");
+        : List.of("org.apache.http", "com.amazonaws.request", "software.amazon.awssdk");
 
     public static final Setting<Level> LOG_DEFAULT_LEVEL_SETTING = new Setting<>(
         "logger.level",

x-pack/plugin/core/build.gradle

@@ -34,6 +34,7 @@ tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
   mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
   mapping from: /slf4j-.*/, to: 'slf4j'
+  mapping from: /log4j-.*/, to: 'log4j'
 }
 
 configurations {
@@ -62,7 +63,7 @@ dependencies {
   // And having a module dependency counts as using the jar, so we can't make it `runtimeOnly` :(
 
   implementation "org.slf4j:slf4j-api:${versions.slf4j}"
-  runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+  runtimeOnly "org.apache.logging.log4j:log4j-slf4j2-impl:${versions.log4j}"
 
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
@@ -158,8 +159,6 @@ tasks.named("thirdPartyAudit").configure {
     'javax.jms.Message',
     // HttpClient5 can use Conscrypt (TLS using BoringSSL), but we don't want that
     'org.conscrypt.Conscrypt',
-    // SLF4j via HttpClient5
-    'org.slf4j.ext.EventData'
   )
 }
 

x-pack/plugin/ent-search/build.gradle

@@ -25,8 +25,6 @@ dependencies {
   implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
   implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
-  implementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
   implementation "com.networknt:json-schema-validator:${versions.networknt_json_schema_validator}"
 
   testImplementation(testArtifact(project(xpackModule('core'))))
@@ -63,13 +61,6 @@ tasks.named("dependencyLicenses") {
   mapping from: /jackson.*/, to: 'jackson'
 }
 
-tasks.named("thirdPartyAudit") {
-  ignoreMissingClasses(
-    // [missing classes] SLF4j includes an optional class that depends on an extension class (!)
-    'org.slf4j.ext.EventData'
-  )
-}
-
 tasks.named("yamlRestTest") {
   usesDefaultDistribution("uses the xpack/usage api")
 }

x-pack/plugin/ent-search/src/main/java/module-info.java

@@ -11,7 +11,6 @@
     requires com.fasterxml.jackson.databind;
     requires json.schema.validator;
     requires org.apache.lucene.core;
-    requires org.slf4j;
 
     requires org.elasticsearch.base;
     requires org.elasticsearch.logging;

x-pack/plugin/identity-provider/build.gradle

@@ -47,8 +47,8 @@ dependencies {
       exclude group: 'org.bouncycastle'
   }
 
-  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
-  runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}")
+  // provided by x-pack-core
+  compileOnly "org.slf4j:slf4j-api:${versions.slf4j}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
@@ -251,8 +251,6 @@ tasks.named("thirdPartyAudit").configure {
         'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
         'org.bouncycastle.util.Arrays',
         'org.bouncycastle.util.io.Streams',
-        // SLF4j
-        'org.slf4j.ext.EventData'
     )
 
   ignoreViolations(

x-pack/plugin/security/build.gradle

@@ -70,8 +70,8 @@ dependencies {
     exclude group: 'org.bouncycastle'
   }
 
-  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
-  runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+  // provided by x-pack-core
+  compileOnly "org.slf4j:slf4j-api:${versions.slf4j}"
 
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@@ -431,8 +431,6 @@ tasks.named("thirdPartyAudit").configure {
     'javax.activation.DataSource',
     'javax.activation.FileDataSource',
     'javax.activation.FileTypeMap',
-    // SLF4j
-    'org.slf4j.ext.EventData'
   )
 }
 

x-pack/plugin/vector-tile/build.gradle

@@ -30,8 +30,6 @@ dependencies {
   api "com.wdtinc:mapbox-vector-tile:3.1.0"
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
   runtimeOnly("org.slf4j:slf4j-api:${versions.slf4j}")
-  runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
-  //  runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
   javaRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0")
   javaRestTestImplementation("com.google.protobuf:protobuf-java:${versions.protobuf}")