From 9eff0dfeb5a0c2899ca782516226d888f5332187 Mon Sep 17 00:00:00 2001 From: Artem Prigoda Date: Tue, 18 Jan 2022 16:16:37 +0100 Subject: [PATCH 1/2] Add support for HTTP Proxies for the GCS repository The change adds 3 new client properties for the GCS repository: * gcs.client.default.proxy.type * gcs.client.default.proxy.host * gcs.client.default.proxy.port They allow to configure a [java.net.Proxy](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/net/Proxy.html) for the GCS SDK to use when communicating with the GCS API. Resolves #82444 --- .../snapshot-restore/repository-gcs.asciidoc | 13 +++++ .../gcs/GoogleCloudStorageClientSettings.java | 55 ++++++++++++++++++- .../gcs/GoogleCloudStorageService.java | 9 +++ ...GoogleCloudStorageClientSettingsTests.java | 37 ++++++++++++- .../gcs/GoogleCloudStorageServiceTests.java | 14 ++++- 5 files changed, 123 insertions(+), 5 deletions(-) diff --git a/docs/reference/snapshot-restore/repository-gcs.asciidoc b/docs/reference/snapshot-restore/repository-gcs.asciidoc index 215dc23bbcb59..b1ddf221952e4 100644 --- a/docs/reference/snapshot-restore/repository-gcs.asciidoc +++ b/docs/reference/snapshot-restore/repository-gcs.asciidoc @@ -191,6 +191,19 @@ are marked as `Secure`. can be specified explicitly. For example, it can be used to switch between projects when the same credentials are usable for both the production and the development projects. +`proxy.host`:: + The host name of a proxy to connect to the Google Cloud Storage through. + For example: `gcs.client.default.proxy.host: proxy.host`. + +`proxy.port`:: + The port of a proxy to connect to the Google Cloud Storage through. + For example, `gcs.client.default.proxy.port: 8888`. + +`proxy.type`:: + The proxy type for the client. Supported values are `direct`, `http`, and `socks`. + The default value is `direct` (no proxy). + For example: `gcs.client.default.proxy.type: http`. + [[repository-gcs-repository]] ==== Repository Settings diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java index cd8076386777a..7a510e8215bbb 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java @@ -14,15 +14,22 @@ import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsException; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Proxy; import java.net.URI; +import java.net.UnknownHostException; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.function.Function; @@ -90,6 +97,29 @@ public class GoogleCloudStorageClientSettings { key -> new Setting<>(key, "repository-gcs", Function.identity(), Setting.Property.NodeScope, Setting.Property.DeprecatedWarning) ); + /** The type of the proxy to connect to the GCS through. Can be DIRECT (aka no proxy), HTTP or SOCKS */ + public static final Setting.AffixSetting PROXY_TYPE_SETTING = Setting.affixKeySetting( + PREFIX, + "proxy.type", + (key) -> new Setting<>(key, "DIRECT", s -> Proxy.Type.valueOf(s.toUpperCase(Locale.ROOT)), Setting.Property.NodeScope) + ); + + /** The host name of a proxy to connect to the GCS through. */ + static final Setting.AffixSetting PROXY_HOST_SETTING = Setting.affixKeySetting( + PREFIX, + "proxy.host", + (key) -> Setting.simpleString(key, Setting.Property.NodeScope), + () -> PROXY_TYPE_SETTING + ); + + /** The port of a proxy to connect to the GCS through. */ + static final Setting.AffixSetting PROXY_PORT_SETTING = Setting.affixKeySetting( + PREFIX, + "proxy.port", + (key) -> Setting.intSetting(key, 0, 0, 65535, Setting.Property.NodeScope), + () -> PROXY_HOST_SETTING + ); + /** The credentials used by the client to connect to the Storage endpoint. */ private final ServiceAccountCredentials credential; @@ -111,6 +141,9 @@ public class GoogleCloudStorageClientSettings { /** The token server URI. This leases access tokens in the oauth flow. */ private final URI tokenUri; + @Nullable + private final Proxy proxy; + GoogleCloudStorageClientSettings( final ServiceAccountCredentials credential, final String endpoint, @@ -118,7 +151,10 @@ public class GoogleCloudStorageClientSettings { final TimeValue connectTimeout, final TimeValue readTimeout, final String applicationName, - final URI tokenUri + final URI tokenUri, + final Proxy.Type proxyType, + final String proxyHost, + final Integer proxyPort ) { this.credential = credential; this.endpoint = endpoint; @@ -127,6 +163,13 @@ public class GoogleCloudStorageClientSettings { this.readTimeout = readTimeout; this.applicationName = applicationName; this.tokenUri = tokenUri; + try { + proxy = proxyType.equals(Proxy.Type.DIRECT) + ? null + : new Proxy(proxyType, new InetSocketAddress(InetAddress.getByName(proxyHost), proxyPort)); + } catch (UnknownHostException e) { + throw new SettingsException("GCS proxy host is unknown.", e); + } } public ServiceAccountCredentials getCredential() { @@ -157,6 +200,11 @@ public URI getTokenUri() { return tokenUri; } + @Nullable + public Proxy getProxy() { + return proxy; + } + public static Map load(final Settings settings) { final Map clients = new HashMap<>(); for (final String clientName : settings.getGroups(PREFIX).keySet()) { @@ -178,7 +226,10 @@ static GoogleCloudStorageClientSettings getClientSettings(final Settings setting getConfigValue(settings, clientName, CONNECT_TIMEOUT_SETTING), getConfigValue(settings, clientName, READ_TIMEOUT_SETTING), getConfigValue(settings, clientName, APPLICATION_NAME_SETTING), - getConfigValue(settings, clientName, TOKEN_URI_SETTING) + getConfigValue(settings, clientName, TOKEN_URI_SETTING), + getConfigValue(settings, clientName, PROXY_TYPE_SETTING), + getConfigValue(settings, clientName, PROXY_HOST_SETTING), + getConfigValue(settings, clientName, PROXY_PORT_SETTING) ); } diff --git a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java index b900eaa4541e1..42de547f504ae 100644 --- a/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java +++ b/modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java @@ -34,6 +34,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.HttpURLConnection; +import java.net.Proxy; import java.net.URI; import java.net.URL; import java.security.KeyStore; @@ -140,6 +141,11 @@ private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings, SecurityUtils.loadKeyStore(certTrustStore, keyStoreStream, "notasecret"); } builder.trustCertificates(certTrustStore); + Proxy proxy = gcsClientSettings.getProxy(); + if (proxy != null) { + builder.setProxy(proxy); + notifyProxyIsSet(proxy); + } return builder.build(); }); @@ -272,4 +278,7 @@ static Integer toTimeout(final TimeValue timeout) { } return Math.toIntExact(timeout.getMillis()); } + + // used for unit testing + void notifyProxyIsSet(Proxy proxy) {} } diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java index e761e77ec7224..0ce4932ad4edd 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java @@ -17,6 +17,9 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.test.ESTestCase; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Proxy; import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.KeyPair; @@ -34,6 +37,9 @@ import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROJECT_ID_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_HOST_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_PORT_SETTING; +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_TYPE_SETTING; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.getClientSettings; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.loadCredential; @@ -94,11 +100,35 @@ public void testProjectIdDefaultsToCredentials() throws Exception { CONNECT_TIMEOUT_SETTING.getDefault(Settings.EMPTY), READ_TIMEOUT_SETTING.getDefault(Settings.EMPTY), APPLICATION_NAME_SETTING.getDefault(Settings.EMPTY), - new URI("") + new URI(""), + PROXY_TYPE_SETTING.getDefault(Settings.EMPTY), + PROXY_HOST_SETTING.getDefault(Settings.EMPTY), + PROXY_PORT_SETTING.getDefault(Settings.EMPTY) ); assertEquals(credential.getProjectId(), googleCloudStorageClientSettings.getProjectId()); } + public void testLoadsProxySettings() throws Exception { + final String clientName = randomAlphaOfLength(5); + final ServiceAccountCredentials credential = randomCredential(clientName).v1(); + final GoogleCloudStorageClientSettings googleCloudStorageClientSettings = new GoogleCloudStorageClientSettings( + credential, + ENDPOINT_SETTING.getDefault(Settings.EMPTY), + PROJECT_ID_SETTING.getDefault(Settings.EMPTY), + CONNECT_TIMEOUT_SETTING.getDefault(Settings.EMPTY), + READ_TIMEOUT_SETTING.getDefault(Settings.EMPTY), + APPLICATION_NAME_SETTING.getDefault(Settings.EMPTY), + new URI(""), + Proxy.Type.HTTP, + "192.168.15.1", + 8080 + ); + assertEquals( + new Proxy(Proxy.Type.HTTP, new InetSocketAddress(InetAddress.getByName("192.168.15.1"), 8080)), + googleCloudStorageClientSettings.getProxy() + ); + } + /** Generates a given number of GoogleCloudStorageClientSettings along with the Settings to build them from **/ private Tuple, Settings> randomClients( final int nbClients, @@ -192,7 +222,10 @@ private static GoogleCloudStorageClientSettings randomClient( connectTimeout, readTimeout, applicationName, - new URI("") + new URI(""), + PROXY_TYPE_SETTING.getDefault(Settings.EMPTY), + PROXY_HOST_SETTING.getDefault(Settings.EMPTY), + PROXY_PORT_SETTING.getDefault(Settings.EMPTY) ); } diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java index 5734bd35046d5..2d858dad2849b 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java @@ -12,6 +12,7 @@ import com.google.cloud.http.HttpTransportOptions; import com.google.cloud.storage.Storage; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Setting; @@ -21,6 +22,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.hamcrest.Matchers; +import java.net.Proxy; import java.security.KeyPair; import java.security.KeyPairGenerator; import java.util.Base64; @@ -58,8 +60,17 @@ public void testClientInitializer() throws Exception { ) .put(GoogleCloudStorageClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint) .put(GoogleCloudStorageClientSettings.PROJECT_ID_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectIdName) + .put(GoogleCloudStorageClientSettings.PROXY_TYPE_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "HTTP") + .put(GoogleCloudStorageClientSettings.PROXY_HOST_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "192.168.52.15") + .put(GoogleCloudStorageClientSettings.PROXY_PORT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), 8080) .build(); - final GoogleCloudStorageService service = new GoogleCloudStorageService(); + SetOnce proxy = new SetOnce<>(); + final GoogleCloudStorageService service = new GoogleCloudStorageService() { + @Override + void notifyProxyIsSet(Proxy p) { + proxy.set(p); + } + }; service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(settings)); GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket"); final IllegalArgumentException e = expectThrows( @@ -84,6 +95,7 @@ public void testClientInitializer() throws Exception { Matchers.is((int) readTimeValue.millis()) ); assertThat(storage.getOptions().getCredentials(), Matchers.nullValue(Credentials.class)); + assertThat(proxy.get().toString(), equalTo("HTTP @ /192.168.52.15:8080")); } public void testReinitClientSettings() throws Exception { From b70c99ba5f472993b6fe2c038a4375b031911ca9 Mon Sep 17 00:00:00 2001 From: Artem Prigoda Date: Thu, 20 Jan 2022 10:31:48 +0100 Subject: [PATCH 2/2] Remove examples of proxy parameters --- docs/reference/snapshot-restore/repository-gcs.asciidoc | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/reference/snapshot-restore/repository-gcs.asciidoc b/docs/reference/snapshot-restore/repository-gcs.asciidoc index b1ddf221952e4..a3cd7e43cb207 100644 --- a/docs/reference/snapshot-restore/repository-gcs.asciidoc +++ b/docs/reference/snapshot-restore/repository-gcs.asciidoc @@ -193,16 +193,13 @@ are marked as `Secure`. `proxy.host`:: The host name of a proxy to connect to the Google Cloud Storage through. - For example: `gcs.client.default.proxy.host: proxy.host`. `proxy.port`:: The port of a proxy to connect to the Google Cloud Storage through. - For example, `gcs.client.default.proxy.port: 8888`. `proxy.type`:: The proxy type for the client. Supported values are `direct`, `http`, and `socks`. The default value is `direct` (no proxy). - For example: `gcs.client.default.proxy.type: http`. [[repository-gcs-repository]] ==== Repository Settings