Skip to content

Commit be8ae87

Browse files
committed
Use fixture to test the repository-gcs plugin (#28788)
This commit adds a GoogleCloudStorageFixture that uses the logic of a GoogleCloudStorageTestServer (added in #28576) to emulate a remote Google Cloud Storage service. By adding this fixture and a more complete integration test, we should be able to catch more bugs when upgrading the client library. The fixture is started by the googleCloudStorageFixture task and a custom Service Account file is created and added to the Elasticsearch keystore for each test.
1 parent 240aa34 commit be8ae87

File tree

6 files changed

+536
-61
lines changed

6 files changed

+536
-61
lines changed

plugins/repository-gcs/build.gradle

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
import org.elasticsearch.gradle.test.AntFixture
2+
3+
import java.security.KeyPair
4+
import java.security.KeyPairGenerator
5+
16
/*
27
* Licensed to Elasticsearch under one or more contributor
38
* license agreements. See the NOTICE file distributed with
@@ -52,3 +57,48 @@ thirdPartyAudit.excludes = [
5257
'org.apache.log.Hierarchy',
5358
'org.apache.log.Logger',
5459
]
60+
61+
/** A task to start the GoogleCloudStorageFixture which emulates a Google Cloud Storage service **/
62+
task googleCloudStorageFixture(type: AntFixture) {
63+
dependsOn compileTestJava
64+
executable = new File(project.runtimeJavaHome, 'bin/java')
65+
args '-cp', "${ -> project.sourceSets.test.runtimeClasspath.asPath }",
66+
'org.elasticsearch.repositories.gcs.GoogleCloudStorageFixture',
67+
baseDir, 'bucket_test'
68+
}
69+
70+
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
71+
File serviceAccountFile = new File(project.buildDir, "generated-resources/service_account_test.json")
72+
task createServiceAccountFile() {
73+
dependsOn googleCloudStorageFixture
74+
doLast {
75+
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
76+
keyPairGenerator.initialize(1024)
77+
KeyPair keyPair = keyPairGenerator.generateKeyPair()
78+
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
79+
80+
serviceAccountFile.parentFile.mkdirs()
81+
serviceAccountFile.setText("{\n" +
82+
' "type": "service_account",\n' +
83+
' "project_id": "integration_test",\n' +
84+
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
85+
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
86+
' "client_email": "[email protected]",\n' +
87+
' "client_id": "123456789101112130594",\n' +
88+
" \"auth_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/auth\",\n" +
89+
" \"token_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token\",\n" +
90+
' "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",\n' +
91+
' "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/integration_test%40appspot.gserviceaccount.com"\n' +
92+
'}', 'UTF-8')
93+
}
94+
}
95+
96+
integTestCluster {
97+
dependsOn createServiceAccountFile, googleCloudStorageFixture
98+
setupCommand 'create-elasticsearch-keystore', 'bin/elasticsearch-keystore', 'create'
99+
setupCommand 'add-credentials-to-elasticsearch-keystore',
100+
'bin/elasticsearch-keystore', 'add-file', 'gcs.client.integration_test.credentials_file', "${serviceAccountFile.absolutePath}"
101+
102+
/* Use a closure on the string to delay evaluation until tests are executed */
103+
setting 'gcs.client.integration_test.endpoint', "http://${ -> googleCloudStorageFixture.addressAndPort }"
104+
}
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.repositories.gcs;
20+
21+
import com.sun.net.httpserver.HttpExchange;
22+
import com.sun.net.httpserver.HttpHandler;
23+
import com.sun.net.httpserver.HttpServer;
24+
import org.elasticsearch.common.SuppressForbidden;
25+
import org.elasticsearch.common.io.Streams;
26+
import org.elasticsearch.mocksocket.MockHttpServer;
27+
import org.elasticsearch.repositories.gcs.GoogleCloudStorageTestServer.Response;
28+
29+
import java.io.ByteArrayOutputStream;
30+
import java.io.IOException;
31+
import java.lang.management.ManagementFactory;
32+
import java.net.Inet6Address;
33+
import java.net.InetAddress;
34+
import java.net.InetSocketAddress;
35+
import java.net.SocketAddress;
36+
import java.nio.file.Files;
37+
import java.nio.file.Path;
38+
import java.nio.file.Paths;
39+
import java.nio.file.StandardCopyOption;
40+
import java.util.List;
41+
import java.util.Map;
42+
43+
import static java.util.Collections.singleton;
44+
import static java.util.Collections.singletonList;
45+
46+
/**
47+
* {@link GoogleCloudStorageFixture} is a fixture that emulates a Google Cloud Storage service.
48+
* <p>
49+
* It starts an asynchronous socket server that binds to a random local port. The server parses
50+
* HTTP requests and uses a {@link GoogleCloudStorageTestServer} to handle them before returning
51+
* them to the client as HTTP responses.
52+
*/
53+
public class GoogleCloudStorageFixture {
54+
55+
@SuppressForbidden(reason = "PathUtils#get is fine - we don't have environment here")
56+
public static void main(String[] args) throws Exception {
57+
if (args == null || args.length != 2) {
58+
throw new IllegalArgumentException("GoogleCloudStorageFixture <working directory> <bucket>");
59+
}
60+
61+
final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 43635);
62+
final HttpServer httpServer = MockHttpServer.createHttp(socketAddress, 0);
63+
64+
try {
65+
final Path workingDirectory = Paths.get(args[0]);
66+
/// Writes the PID of the current Java process in a `pid` file located in the working directory
67+
writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
68+
69+
final String addressAndPort = addressToString(httpServer.getAddress());
70+
// Writes the address and port of the http server in a `ports` file located in the working directory
71+
writeFile(workingDirectory, "ports", addressAndPort);
72+
73+
// Emulates a Google Cloud Storage server
74+
final String storageUrl = "http://" + addressAndPort;
75+
final GoogleCloudStorageTestServer storageTestServer = new GoogleCloudStorageTestServer(storageUrl);
76+
storageTestServer.createBucket(args[1]);
77+
78+
httpServer.createContext("/", new ResponseHandler(storageTestServer));
79+
httpServer.start();
80+
81+
// Wait to be killed
82+
Thread.sleep(Long.MAX_VALUE);
83+
84+
} finally {
85+
httpServer.stop(0);
86+
}
87+
}
88+
89+
private static void writeFile(final Path dir, final String fileName, final String content) throws IOException {
90+
final Path tempPidFile = Files.createTempFile(dir, null, null);
91+
Files.write(tempPidFile, singleton(content));
92+
Files.move(tempPidFile, dir.resolve(fileName), StandardCopyOption.ATOMIC_MOVE);
93+
}
94+
95+
private static String addressToString(final SocketAddress address) {
96+
final InetSocketAddress inetSocketAddress = (InetSocketAddress) address;
97+
if (inetSocketAddress.getAddress() instanceof Inet6Address) {
98+
return "[" + inetSocketAddress.getHostString() + "]:" + inetSocketAddress.getPort();
99+
} else {
100+
return inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort();
101+
}
102+
}
103+
104+
@SuppressForbidden(reason = "Use a http server")
105+
static class ResponseHandler implements HttpHandler {
106+
107+
private final GoogleCloudStorageTestServer storageServer;
108+
109+
private ResponseHandler(final GoogleCloudStorageTestServer storageServer) {
110+
this.storageServer = storageServer;
111+
}
112+
113+
@Override
114+
public void handle(HttpExchange exchange) throws IOException {
115+
String method = exchange.getRequestMethod();
116+
String path = storageServer.getEndpoint() + exchange.getRequestURI().getRawPath();
117+
String query = exchange.getRequestURI().getRawQuery();
118+
Map<String, List<String>> headers = exchange.getRequestHeaders();
119+
120+
ByteArrayOutputStream out = new ByteArrayOutputStream();
121+
Streams.copy(exchange.getRequestBody(), out);
122+
123+
final Response storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray());
124+
125+
Map<String, List<String>> responseHeaders = exchange.getResponseHeaders();
126+
responseHeaders.put("Content-Type", singletonList(storageResponse.contentType));
127+
storageResponse.headers.forEach((k, v) -> responseHeaders.put(k, singletonList(v)));
128+
exchange.sendResponseHeaders(storageResponse.status.getStatus(), storageResponse.body.length);
129+
if (storageResponse.body.length > 0) {
130+
exchange.getResponseBody().write(storageResponse.body);
131+
}
132+
exchange.close();
133+
}
134+
}
135+
}

0 commit comments

Comments
 (0)