Skip to content

Commit 1ddaf25

Browse files
arteamarteam
authored
Add support for HTTP Proxies for the GCS repository (#82737)
* Add support for HTTP Proxies for the GCS repository The change adds 3 new client properties for the GCS repository: * gcs.client.default.proxy.type * gcs.client.default.proxy.host * gcs.client.default.proxy.port They allow to configure a [java.net.Proxy](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/net/Proxy.html) for the GCS SDK to use when communicating with the GCS API. Resolves #82444
1 parent febf22c commit 1ddaf25

File tree

5 files changed

+120
-5
lines changed

5 files changed

+120
-5
lines changed

‎docs/reference/snapshot-restore/repository-gcs.asciidoc‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,16 @@ are marked as `Secure`.
191191
can be specified explicitly. For example, it can be used to switch between projects when the
192192
same credentials are usable for both the production and the development projects.
193193

194+
`proxy.host`::
195+
The host name of a proxy to connect to the Google Cloud Storage through.
196+
197+
`proxy.port`::
198+
The port of a proxy to connect to the Google Cloud Storage through.
199+
200+
`proxy.type`::
201+
The proxy type for the client. Supported values are `direct`, `http`, and `socks`.
202+
The default value is `direct` (no proxy).
203+
194204
[[repository-gcs-repository]]
195205
==== Repository Settings
196206

‎modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,22 @@
1414
import org.elasticsearch.common.settings.SecureSetting;
1515
import org.elasticsearch.common.settings.Setting;
1616
import org.elasticsearch.common.settings.Settings;
17+
import org.elasticsearch.common.settings.SettingsException;
18+
import org.elasticsearch.core.Nullable;
1719
import org.elasticsearch.core.TimeValue;
1820

1921
import java.io.IOException;
2022
import java.io.InputStream;
2123
import java.io.UncheckedIOException;
24+
import java.net.InetAddress;
25+
import java.net.InetSocketAddress;
26+
import java.net.Proxy;
2227
import java.net.URI;
28+
import java.net.UnknownHostException;
2329
import java.util.Collection;
2430
import java.util.Collections;
2531
import java.util.HashMap;
32+
import java.util.Locale;
2633
import java.util.Map;
2734
import java.util.function.Function;
2835

@@ -90,6 +97,29 @@ public class GoogleCloudStorageClientSettings {
9097
key -> new Setting<>(key, "repository-gcs", Function.identity(), Setting.Property.NodeScope, Setting.Property.DeprecatedWarning)
9198
);
9299

100+
/** The type of the proxy to connect to the GCS through. Can be DIRECT (aka no proxy), HTTP or SOCKS */
101+
public static final Setting.AffixSetting<Proxy.Type> PROXY_TYPE_SETTING = Setting.affixKeySetting(
102+
PREFIX,
103+
"proxy.type",
104+
(key) -> new Setting<>(key, "DIRECT", s -> Proxy.Type.valueOf(s.toUpperCase(Locale.ROOT)), Setting.Property.NodeScope)
105+
);
106+
107+
/** The host name of a proxy to connect to the GCS through. */
108+
static final Setting.AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(
109+
PREFIX,
110+
"proxy.host",
111+
(key) -> Setting.simpleString(key, Setting.Property.NodeScope),
112+
() -> PROXY_TYPE_SETTING
113+
);
114+
115+
/** The port of a proxy to connect to the GCS through. */
116+
static final Setting.AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(
117+
PREFIX,
118+
"proxy.port",
119+
(key) -> Setting.intSetting(key, 0, 0, 65535, Setting.Property.NodeScope),
120+
() -> PROXY_HOST_SETTING
121+
);
122+
93123
/** The credentials used by the client to connect to the Storage endpoint. */
94124
private final ServiceAccountCredentials credential;
95125

@@ -111,14 +141,20 @@ public class GoogleCloudStorageClientSettings {
111141
/** The token server URI. This leases access tokens in the oauth flow. */
112142
private final URI tokenUri;
113143

144+
@Nullable
145+
private final Proxy proxy;
146+
114147
GoogleCloudStorageClientSettings(
115148
final ServiceAccountCredentials credential,
116149
final String endpoint,
117150
final String projectId,
118151
final TimeValue connectTimeout,
119152
final TimeValue readTimeout,
120153
final String applicationName,
121-
final URI tokenUri
154+
final URI tokenUri,
155+
final Proxy.Type proxyType,
156+
final String proxyHost,
157+
final Integer proxyPort
122158
) {
123159
this.credential = credential;
124160
this.endpoint = endpoint;
@@ -127,6 +163,13 @@ public class GoogleCloudStorageClientSettings {
127163
this.readTimeout = readTimeout;
128164
this.applicationName = applicationName;
129165
this.tokenUri = tokenUri;
166+
try {
167+
proxy = proxyType.equals(Proxy.Type.DIRECT)
168+
? null
169+
: new Proxy(proxyType, new InetSocketAddress(InetAddress.getByName(proxyHost), proxyPort));
170+
} catch (UnknownHostException e) {
171+
throw new SettingsException("GCS proxy host is unknown.", e);
172+
}
130173
}
131174

132175
public ServiceAccountCredentials getCredential() {
@@ -157,6 +200,11 @@ public URI getTokenUri() {
157200
return tokenUri;
158201
}
159202

203+
@Nullable
204+
public Proxy getProxy() {
205+
return proxy;
206+
}
207+
160208
public static Map<String, GoogleCloudStorageClientSettings> load(final Settings settings) {
161209
final Map<String, GoogleCloudStorageClientSettings> clients = new HashMap<>();
162210
for (final String clientName : settings.getGroups(PREFIX).keySet()) {
@@ -178,7 +226,10 @@ static GoogleCloudStorageClientSettings getClientSettings(final Settings setting
178226
getConfigValue(settings, clientName, CONNECT_TIMEOUT_SETTING),
179227
getConfigValue(settings, clientName, READ_TIMEOUT_SETTING),
180228
getConfigValue(settings, clientName, APPLICATION_NAME_SETTING),
181-
getConfigValue(settings, clientName, TOKEN_URI_SETTING)
229+
getConfigValue(settings, clientName, TOKEN_URI_SETTING),
230+
getConfigValue(settings, clientName, PROXY_TYPE_SETTING),
231+
getConfigValue(settings, clientName, PROXY_HOST_SETTING),
232+
getConfigValue(settings, clientName, PROXY_PORT_SETTING)
182233
);
183234
}
184235

‎modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.io.InputStream;
3535
import java.io.InputStreamReader;
3636
import java.net.HttpURLConnection;
37+
import java.net.Proxy;
3738
import java.net.URI;
3839
import java.net.URL;
3940
import java.security.KeyStore;
@@ -140,6 +141,11 @@ private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings,
140141
SecurityUtils.loadKeyStore(certTrustStore, keyStoreStream, "notasecret");
141142
}
142143
builder.trustCertificates(certTrustStore);
144+
Proxy proxy = gcsClientSettings.getProxy();
145+
if (proxy != null) {
146+
builder.setProxy(proxy);
147+
notifyProxyIsSet(proxy);
148+
}
143149
return builder.build();
144150
});
145151

@@ -272,4 +278,7 @@ static Integer toTimeout(final TimeValue timeout) {
272278
}
273279
return Math.toIntExact(timeout.getMillis());
274280
}
281+
282+
// used for unit testing
283+
void notifyProxyIsSet(Proxy proxy) {}
275284
}

‎modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
import org.elasticsearch.core.Tuple;
1818
import org.elasticsearch.test.ESTestCase;
1919

20+
import java.net.InetAddress;
21+
import java.net.InetSocketAddress;
22+
import java.net.Proxy;
2023
import java.net.URI;
2124
import java.nio.charset.StandardCharsets;
2225
import java.security.KeyPair;
@@ -34,6 +37,9 @@
3437
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING;
3538
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING;
3639
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROJECT_ID_SETTING;
40+
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_HOST_SETTING;
41+
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_PORT_SETTING;
42+
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_TYPE_SETTING;
3743
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING;
3844
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.getClientSettings;
3945
import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.loadCredential;
@@ -94,11 +100,35 @@ public void testProjectIdDefaultsToCredentials() throws Exception {
94100
CONNECT_TIMEOUT_SETTING.getDefault(Settings.EMPTY),
95101
READ_TIMEOUT_SETTING.getDefault(Settings.EMPTY),
96102
APPLICATION_NAME_SETTING.getDefault(Settings.EMPTY),
97-
new URI("")
103+
new URI(""),
104+
PROXY_TYPE_SETTING.getDefault(Settings.EMPTY),
105+
PROXY_HOST_SETTING.getDefault(Settings.EMPTY),
106+
PROXY_PORT_SETTING.getDefault(Settings.EMPTY)
98107
);
99108
assertEquals(credential.getProjectId(), googleCloudStorageClientSettings.getProjectId());
100109
}
101110

111+
public void testLoadsProxySettings() throws Exception {
112+
final String clientName = randomAlphaOfLength(5);
113+
final ServiceAccountCredentials credential = randomCredential(clientName).v1();
114+
final GoogleCloudStorageClientSettings googleCloudStorageClientSettings = new GoogleCloudStorageClientSettings(
115+
credential,
116+
ENDPOINT_SETTING.getDefault(Settings.EMPTY),
117+
PROJECT_ID_SETTING.getDefault(Settings.EMPTY),
118+
CONNECT_TIMEOUT_SETTING.getDefault(Settings.EMPTY),
119+
READ_TIMEOUT_SETTING.getDefault(Settings.EMPTY),
120+
APPLICATION_NAME_SETTING.getDefault(Settings.EMPTY),
121+
new URI(""),
122+
Proxy.Type.HTTP,
123+
"192.168.15.1",
124+
8080
125+
);
126+
assertEquals(
127+
new Proxy(Proxy.Type.HTTP, new InetSocketAddress(InetAddress.getByName("192.168.15.1"), 8080)),
128+
googleCloudStorageClientSettings.getProxy()
129+
);
130+
}
131+
102132
/** Generates a given number of GoogleCloudStorageClientSettings along with the Settings to build them from **/
103133
private Tuple<Map<String, GoogleCloudStorageClientSettings>, Settings> randomClients(
104134
final int nbClients,
@@ -192,7 +222,10 @@ private static GoogleCloudStorageClientSettings randomClient(
192222
connectTimeout,
193223
readTimeout,
194224
applicationName,
195-
new URI("")
225+
new URI(""),
226+
PROXY_TYPE_SETTING.getDefault(Settings.EMPTY),
227+
PROXY_HOST_SETTING.getDefault(Settings.EMPTY),
228+
PROXY_PORT_SETTING.getDefault(Settings.EMPTY)
196229
);
197230
}
198231

‎modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.google.cloud.http.HttpTransportOptions;
1313
import com.google.cloud.storage.Storage;
1414

15+
import org.apache.lucene.util.SetOnce;
1516
import org.elasticsearch.common.bytes.BytesReference;
1617
import org.elasticsearch.common.settings.MockSecureSettings;
1718
import org.elasticsearch.common.settings.Setting;
@@ -21,6 +22,7 @@
2122
import org.elasticsearch.xcontent.XContentBuilder;
2223
import org.hamcrest.Matchers;
2324

25+
import java.net.Proxy;
2426
import java.security.KeyPair;
2527
import java.security.KeyPairGenerator;
2628
import java.util.Base64;
@@ -58,8 +60,17 @@ public void testClientInitializer() throws Exception {
5860
)
5961
.put(GoogleCloudStorageClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint)
6062
.put(GoogleCloudStorageClientSettings.PROJECT_ID_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectIdName)
63+
.put(GoogleCloudStorageClientSettings.PROXY_TYPE_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "HTTP")
64+
.put(GoogleCloudStorageClientSettings.PROXY_HOST_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "192.168.52.15")
65+
.put(GoogleCloudStorageClientSettings.PROXY_PORT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), 8080)
6166
.build();
62-
final GoogleCloudStorageService service = new GoogleCloudStorageService();
67+
SetOnce<Proxy> proxy = new SetOnce<>();
68+
final GoogleCloudStorageService service = new GoogleCloudStorageService() {
69+
@Override
70+
void notifyProxyIsSet(Proxy p) {
71+
proxy.set(p);
72+
}
73+
};
6374
service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(settings));
6475
GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
6576
final IllegalArgumentException e = expectThrows(
@@ -84,6 +95,7 @@ public void testClientInitializer() throws Exception {
8495
Matchers.is((int) readTimeValue.millis())
8596
);
8697
assertThat(storage.getOptions().getCredentials(), Matchers.nullValue(Credentials.class));
98+
assertThat(proxy.get().toString(), equalTo("HTTP @ /192.168.52.15:8080"));
8799
}
88100

89101
public void testReinitClientSettings() throws Exception {

0 commit comments

Comments
 (0)