Skip to content

Commit ef3d72e

Browse files
ywelschywelsch
committed
Add support for reading older codecs
1 parent 098c29f commit ef3d72e

File tree

33 files changed

+2226
-7
lines changed

33 files changed

+2226
-7
lines changed

‎modules/lucene-5-codecs/build.gradle‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
import org.apache.tools.ant.taskdefs.condition.Os
10+
import org.elasticsearch.gradle.Architecture
11+
import org.elasticsearch.gradle.OS
12+
import org.elasticsearch.gradle.internal.info.BuildParams
13+
import org.elasticsearch.gradle.internal.test.AntFixture
14+
15+
apply plugin: 'elasticsearch.test-with-dependencies'
16+
apply plugin: 'elasticsearch.jdk-download'
17+
apply plugin: 'elasticsearch.internal-yaml-rest-test'
18+
apply plugin: 'elasticsearch.java-rest-test'
19+
apply plugin: 'elasticsearch.yaml-rest-compat-test'
20+
apply plugin: 'elasticsearch.internal-cluster-test'
21+
22+
esplugin {
23+
description 'Provides codecs for reading Lucene 5 formats'
24+
classname 'org.elasticsearch.oldcodecs.Lucene5Plugin'
25+
}
26+
27+
project.afterEvaluate(p -> {
28+
project.getTasks().withType(JavaCompile.class).configureEach(compileTask -> {
29+
CompileOptions compileOptions = compileTask.getOptions()
30+
List<String> compilerArgs = compileOptions.getCompilerArgs()
31+
compilerArgs.remove("-Werror")
32+
compilerArgs.remove("-Xlint:all,-path,-serial,-options,-deprecation,-try")
33+
compilerArgs.remove("-Xdoclint:all")
34+
compilerArgs.remove("-Xdoclint:-missing")
35+
})
36+
})
37+
38+
39+
40+
configurations {
41+
oldesFixture
42+
es2
43+
es1
44+
}
45+
46+
dependencies {
47+
oldesFixture project(':test:fixtures:old-elasticsearch')
48+
es2 'org.elasticsearch.distribution.zip:elasticsearch:2.4.5@zip'
49+
es1 'org.elasticsearch:elasticsearch:1.7.6@zip'
50+
}
51+
52+
jdks {
53+
legacy {
54+
vendor = 'adoptium'
55+
version = '8u302+b08'
56+
platform = OS.current().name().toLowerCase()
57+
architecture = Architecture.current().name().toLowerCase()
58+
}
59+
}
60+
61+
String repoLocation = "${buildDir}/cluster/shared/repo"
62+
testClusters.matching { it.name == "javaRestTest" }.configureEach {
63+
setting 'path.repo', repoLocation
64+
}
65+
66+
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
67+
logger.warn("Disabling reindex-from-old tests because we can't get the pid file on windows")
68+
tasks.named("javaRestTest").configure {
69+
systemProperty "tests.fromOld", "false"
70+
}
71+
} else if (rootProject.rootDir.toString().contains(" ")) {
72+
logger.warn("Disabling reindex-from-old tests because Elasticsearch 1.7 won't start with spaces in the path")
73+
tasks.named("javaRestTest").configure {
74+
systemProperty "tests.fromOld", "false"
75+
}
76+
} else {
77+
/* Set up tasks to unzip and run the old versions of ES before running the
78+
* integration tests. */
79+
def versions = ['2', '1']
80+
versions.each { version ->
81+
// TODO Rene: we should be able to replace these unzip tasks with gradle artifact transforms
82+
TaskProvider<Sync> unzip = tasks.register("unzipEs${version}", Sync) {
83+
Configuration oldEsDependency = configurations['es' + version]
84+
dependsOn oldEsDependency
85+
/* Use a closure here to delay resolution of the dependency until we need
86+
* it */
87+
from {
88+
oldEsDependency.collect { zipTree(it) }
89+
}
90+
into temporaryDir
91+
}
92+
93+
TaskProvider<AntFixture> fixture = tasks.register("oldEs${version}Fixture", AntFixture) {
94+
dependsOn project.configurations.oldesFixture, jdks.legacy
95+
dependsOn unzip
96+
executable = "${BuildParams.runtimeJavaHome}/bin/java"
97+
env 'CLASSPATH', "${-> project.configurations.oldesFixture.asPath}"
98+
// old versions of Elasticsearch need JAVA_HOME
99+
env 'JAVA_HOME', jdks.legacy.javaHomePath
100+
// If we are running on certain arm systems we need to explicitly set the stack size to overcome JDK page size bug
101+
if (Architecture.current() == Architecture.AARCH64) {
102+
env 'ES_JAVA_OPTS', '-Xss512k'
103+
}
104+
args 'oldes.OldElasticsearch',
105+
baseDir,
106+
unzip.get().temporaryDir,
107+
version == '090',
108+
"path.repo: ${repoLocation}"
109+
waitCondition = { fixture, ant ->
110+
// the fixture writes the ports file when Elasticsearch's HTTP service
111+
// is ready, so we can just wait for the file to exist
112+
return fixture.portsFile.exists()
113+
}
114+
}
115+
116+
tasks.named("javaRestTest").configure {
117+
dependsOn fixture
118+
systemProperty "tests.fromOld", "true"
119+
systemProperty "tests.repoLocation", repoLocation
120+
/* Use a closure on the string to delay evaluation until right before we
121+
* run the integration tests so that we can be sure that the file is
122+
* ready. */
123+
nonInputProperties.systemProperty "es${version}.port", "${-> fixture.get().addressAndPort}"
124+
}
125+
}
126+
}
127+

‎modules/lucene-5-codecs/src/javaRestTest/java/org/elasticsearch/oldcodecs/BWCCodecIT.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.oldcodecs;
10+
11+
import org.apache.http.HttpHost;
12+
import org.apache.http.util.EntityUtils;
13+
import org.elasticsearch.client.Request;
14+
import org.elasticsearch.client.Response;
15+
import org.elasticsearch.client.RestClient;
16+
import org.elasticsearch.common.io.Streams;
17+
import org.elasticsearch.core.Booleans;
18+
import org.elasticsearch.test.rest.ESRestTestCase;
19+
20+
import java.io.IOException;
21+
22+
import static org.hamcrest.Matchers.containsString;
23+
24+
public class BWCCodecIT extends ESRestTestCase {
25+
private static final int DOCS = 5;
26+
27+
private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException {
28+
boolean enabled = Booleans.parseBoolean(System.getProperty("tests.fromOld"));
29+
assumeTrue("test is disabled, probably because this is windows", enabled);
30+
String repoLocation = System.getProperty("tests.repoLocation");
31+
repoLocation = repoLocation + "/" + randomAlphaOfLength(10);
32+
33+
int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName));
34+
try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) {
35+
try {
36+
Request createIndex = new Request("PUT", "/test");
37+
createIndex.setJsonEntity("{\"settings\":{\"number_of_shards\": 1}}");
38+
oldEs.performRequest(createIndex);
39+
40+
for (int i = 0; i < DOCS; i++) {
41+
Request doc = new Request("PUT", "/test/doc/testdoc" + i);
42+
doc.addParameter("refresh", "true");
43+
doc.setJsonEntity("{\"test\":\"test" + i + "\"}");
44+
oldEs.performRequest(doc);
45+
}
46+
47+
// register repo on old ES and take snapshot
48+
Request createRepoRequest = new Request("PUT", "/_snapshot/testrepo");
49+
createRepoRequest.setJsonEntity("{\"type\":\"fs\",\"settings\":{\"location\":\"" + repoLocation + "\"}}");
50+
oldEs.performRequest(createRepoRequest);
51+
52+
Request createSnapshotRequest = new Request("PUT", "/_snapshot/testrepo/snap1");
53+
createSnapshotRequest.addParameter("wait_for_completion", "true");
54+
oldEs.performRequest(createSnapshotRequest);
55+
56+
// register repo on new ES
57+
Request createReadRepoRequest = new Request("PUT", "/_snapshot/testrepo");
58+
createReadRepoRequest.setJsonEntity(
59+
"{\"type\":\"lucene5\",\"settings\":{\"delegate_type\":\"fs\",\"location\":\"" + repoLocation +
60+
"\",\"readonly\":true}}");
61+
client().performRequest(createReadRepoRequest);
62+
63+
// list snapshots on new ES
64+
Request listSnapshotsRequest = new Request("GET", "/_snapshot/testrepo/snap1");
65+
listSnapshotsRequest.addParameter("error_trace", "true");
66+
Response listSnapshotsResponse = client().performRequest(listSnapshotsRequest);
67+
logger.info(Streams.readFully(listSnapshotsResponse.getEntity().getContent()).utf8ToString());
68+
assertEquals(200, listSnapshotsResponse.getStatusLine().getStatusCode());
69+
70+
// list advanced snapshot info on new ES
71+
Request listSnapshotStatusRequest = new Request("GET", "/_snapshot/testrepo/snap1/_status");
72+
listSnapshotStatusRequest.addParameter("error_trace", "true");
73+
Response listSnapshotStatusResponse = client().performRequest(listSnapshotStatusRequest);
74+
logger.info(Streams.readFully(listSnapshotStatusResponse.getEntity().getContent()).utf8ToString());
75+
assertEquals(200, listSnapshotStatusResponse.getStatusLine().getStatusCode());
76+
77+
// restore snapshot on new ES
78+
Request restoreSnapshotRequest = new Request("POST", "/_snapshot/testrepo/snap1/_restore");
79+
restoreSnapshotRequest.addParameter("error_trace", "true");
80+
restoreSnapshotRequest.addParameter("wait_for_completion", "true");
81+
restoreSnapshotRequest.setJsonEntity("{\"indices\":\"test\"}");
82+
Response restoreSnapshotResponse = client().performRequest(restoreSnapshotRequest);
83+
logger.info(Streams.readFully(restoreSnapshotResponse.getEntity().getContent()).utf8ToString());
84+
assertEquals(200, restoreSnapshotResponse.getStatusLine().getStatusCode());
85+
86+
// run a search against the restored index
87+
Request search = new Request("POST", "/test/_search");
88+
search.addParameter("pretty", "true");
89+
//search.setJsonEntity("{\"stored_fields\": [\"_uid\", \"_source\"]}");
90+
Response response = client().performRequest(search);
91+
String result = EntityUtils.toString(response.getEntity());
92+
for (int i = 0; i < DOCS; i++) {
93+
// check that source_ is present
94+
assertThat(result, containsString("\"test\" : \"test" + i + "\""));
95+
// check that _id is present
96+
// TODO: _id is stored as _uid in older ES versions, not _id (can extract _type and _id from that)
97+
// assertThat(result, containsString("\"_id\" : \"testdoc" + i + "\""));
98+
}
99+
100+
} finally {
101+
oldEs.performRequest(new Request("DELETE", "/test"));
102+
}
103+
}
104+
}
105+
106+
public void testEs2() throws IOException {
107+
oldEsTestCase("es2.port", null);
108+
}
109+
110+
public void testEs1() throws IOException {
111+
oldEsTestCase("es1.port", null);
112+
}
113+
114+
}

0 commit comments

Comments
 (0)