Skip to content

Commit 750f676

Browse files
author
Christoph Büscher
committed
WIP
1 parent c95a95b commit 750f676

File tree

8 files changed

+456
-62
lines changed

8 files changed

+456
-62
lines changed

‎modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/SynonymTokenFilterFactory.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.elasticsearch.index.IndexSettings;
3131
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
3232
import org.elasticsearch.index.analysis.Analysis;
33+
import org.elasticsearch.index.analysis.AnalysisMode;
3334
import org.elasticsearch.index.analysis.CharFilterFactory;
3435
import org.elasticsearch.index.analysis.CustomAnalyzer;
3536
import org.elasticsearch.index.analysis.TokenFilterFactory;
@@ -50,6 +51,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
5051
private final boolean lenient;
5152
protected final Settings settings;
5253
protected final Environment environment;
54+
private final boolean updateable;
5355

5456
SynonymTokenFilterFactory(IndexSettings indexSettings, Environment env,
5557
String name, Settings settings) {
@@ -65,9 +67,15 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory {
6567
this.expand = settings.getAsBoolean("expand", true);
6668
this.lenient = settings.getAsBoolean("lenient", false);
6769
this.format = settings.get("format", "");
70+
this.updateable = settings.getAsBoolean("updateable", false);
6871
this.environment = env;
6972
}
7073

74+
@Override
75+
public AnalysisMode getAnalysisMode() {
76+
return this.updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL;
77+
}
78+
7179
@Override
7280
public TokenStream create(TokenStream tokenStream) {
7381
throw new IllegalStateException("Call createPerAnalyzerSynonymFactory to specialize this factory for an analysis chain first");
@@ -98,6 +106,11 @@ public TokenFilterFactory getSynonymFilter() {
98106
// which doesn't support stacked input tokens
99107
return IDENTITY_FILTER;
100108
}
109+
110+
@Override
111+
public AnalysisMode getAnalysisMode() {
112+
return updateable ? AnalysisMode.SEARCH_TIME : AnalysisMode.ALL;
113+
}
101114
};
102115
}
103116

‎modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerIT.java‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.analysis.common;
21+
22+
import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
23+
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
24+
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
25+
import org.elasticsearch.action.search.SearchResponse;
26+
import org.elasticsearch.common.settings.Settings;
27+
import org.elasticsearch.index.query.QueryBuilders;
28+
import org.elasticsearch.plugins.Plugin;
29+
import org.elasticsearch.test.ESIntegTestCase;
30+
import org.junit.BeforeClass;
31+
32+
import java.io.FileNotFoundException;
33+
import java.io.IOException;
34+
import java.io.OutputStreamWriter;
35+
import java.io.PrintWriter;
36+
import java.nio.charset.StandardCharsets;
37+
import java.nio.file.Files;
38+
import java.nio.file.Path;
39+
import java.nio.file.StandardOpenOption;
40+
import java.util.Arrays;
41+
import java.util.Collection;
42+
import java.util.HashSet;
43+
import java.util.Set;
44+
45+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
46+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
47+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
48+
49+
@AwaitsFix(bugUrl="Cannot be run outside IDE yet")
50+
public class SynonymAnalyzerIT extends ESIntegTestCase {
51+
52+
private static Path config;
53+
private static Path synonymsFile;
54+
private static final String synonymsFileName = "synonyms.txt";
55+
56+
@BeforeClass
57+
public static void initConfigDir() throws IOException {
58+
config = createTempDir().resolve("config");
59+
if (Files.exists(config) == false) {
60+
Files.createDirectory(config);
61+
}
62+
synonymsFile = config.resolve(synonymsFileName);
63+
Files.createFile(synonymsFile);
64+
assertTrue(Files.exists(synonymsFile));
65+
}
66+
67+
68+
@Override
69+
protected Collection<Class<? extends Plugin>> nodePlugins() {
70+
return Arrays.asList(CommonAnalysisPlugin.class);
71+
}
72+
73+
@Override
74+
protected Path nodeConfigPath(int nodeOrdinal) {
75+
return config;
76+
}
77+
78+
public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
79+
try (PrintWriter out = new PrintWriter(
80+
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.CREATE), StandardCharsets.UTF_8))) {
81+
out.println("foo, baz");
82+
}
83+
assertTrue(Files.exists(synonymsFile));
84+
assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder()
85+
.put("index.number_of_shards", 1)
86+
.put("index.number_of_replicas", 0)
87+
.put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard")
88+
.put("analysis.analyzer.my_synonym_analyzer.filter", "my_synonym_filter")
89+
.put("analysis.filter.my_synonym_filter.type", "synonym")
90+
.put("analysis.filter.my_synonym_filter.updateable", "true")
91+
.put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName))
92+
.addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer"));
93+
94+
client().prepareIndex("test", "_doc", "1").setSource("field", "foo").get();
95+
assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet());
96+
97+
SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get();
98+
assertHitCount(response, 1L);
99+
response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
100+
assertHitCount(response, 0L);
101+
AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get();
102+
assertEquals(2, analyzeResponse.getTokens().size());
103+
assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm());
104+
assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm());
105+
106+
// now update synonyms file and trigger reloading
107+
try (PrintWriter out = new PrintWriter(
108+
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
109+
out.println("foo, baz, buzz");
110+
}
111+
// TODO don't use refresh here but something more specific
112+
assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet());
113+
114+
analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get();
115+
assertEquals(3, analyzeResponse.getTokens().size());
116+
Set<String> tokens = new HashSet<>();
117+
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
118+
assertTrue(tokens.contains("foo"));
119+
assertTrue(tokens.contains("baz"));
120+
assertTrue(tokens.contains("buzz"));
121+
122+
response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get();
123+
assertHitCount(response, 1L);
124+
response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
125+
assertHitCount(response, 1L);
126+
}
127+
}

‎modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/SynonymAnalyzerTests.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.analysis.common;
21+
22+
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
23+
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
24+
import org.elasticsearch.action.search.SearchResponse;
25+
import org.elasticsearch.common.settings.Settings;
26+
import org.elasticsearch.index.query.QueryBuilders;
27+
import org.elasticsearch.plugins.Plugin;
28+
import org.elasticsearch.test.ESSingleNodeTestCase;
29+
30+
import java.io.FileNotFoundException;
31+
import java.io.IOException;
32+
import java.io.OutputStreamWriter;
33+
import java.io.PrintWriter;
34+
import java.nio.charset.StandardCharsets;
35+
import java.nio.file.Files;
36+
import java.nio.file.Path;
37+
import java.nio.file.StandardOpenOption;
38+
import java.util.Arrays;
39+
import java.util.Collection;
40+
import java.util.HashSet;
41+
import java.util.Set;
42+
43+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
44+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
45+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
46+
47+
public class SynonymAnalyzerTests extends ESSingleNodeTestCase {
48+
49+
@Override
50+
protected Collection<Class<? extends Plugin>> getPlugins() {
51+
return Arrays.asList(CommonAnalysisPlugin.class);
52+
}
53+
54+
public void testSynonymsUpdateable() throws FileNotFoundException, IOException {
55+
String synonymsFileName = "synonyms.txt";
56+
Path configDir = node().getEnvironment().configFile();
57+
if (Files.exists(configDir) == false) {
58+
Files.createDirectory(configDir);
59+
}
60+
Path synonymsFile = configDir.resolve(synonymsFileName);
61+
if (Files.exists(synonymsFile) == false) {
62+
Files.createFile(synonymsFile);
63+
}
64+
try (PrintWriter out = new PrintWriter(
65+
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
66+
out.println("foo, baz");
67+
}
68+
69+
assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder()
70+
.put("index.number_of_shards", 1)
71+
.put("index.number_of_replicas", 0)
72+
.put("analysis.analyzer.my_synonym_analyzer.tokenizer", "standard")
73+
.putList("analysis.analyzer.my_synonym_analyzer.filter", "lowercase", "my_synonym_filter")
74+
.put("analysis.filter.my_synonym_filter.type", "synonym")
75+
.put("analysis.filter.my_synonym_filter.updateable", "true")
76+
.put("analysis.filter.my_synonym_filter.synonyms_path", synonymsFileName))
77+
.addMapping("_doc", "field", "type=text,analyzer=standard,search_analyzer=my_synonym_analyzer"));
78+
79+
client().prepareIndex("test", "_doc", "1").setSource("field", "Foo").get();
80+
assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet());
81+
82+
SearchResponse response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get();
83+
assertHitCount(response, 1L);
84+
response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
85+
assertHitCount(response, 0L);
86+
AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("test", "foo").setAnalyzer("my_synonym_analyzer").get();
87+
assertEquals(2, analyzeResponse.getTokens().size());
88+
assertEquals("foo", analyzeResponse.getTokens().get(0).getTerm());
89+
assertEquals("baz", analyzeResponse.getTokens().get(1).getTerm());
90+
91+
// now update synonyms file and trigger reloading
92+
try (PrintWriter out = new PrintWriter(
93+
new OutputStreamWriter(Files.newOutputStream(synonymsFile, StandardOpenOption.WRITE), StandardCharsets.UTF_8))) {
94+
out.println("foo, baz, buzz");
95+
}
96+
// TODO don't use refresh here but something more specific
97+
assertNoFailures(client().admin().indices().prepareRefresh("test").execute().actionGet());
98+
99+
analyzeResponse = client().admin().indices().prepareAnalyze("test", "Foo").setAnalyzer("my_synonym_analyzer").get();
100+
assertEquals(3, analyzeResponse.getTokens().size());
101+
Set<String> tokens = new HashSet<>();
102+
analyzeResponse.getTokens().stream().map(AnalyzeToken::getTerm).forEach(t -> tokens.add(t));
103+
assertTrue(tokens.contains("foo"));
104+
assertTrue(tokens.contains("baz"));
105+
assertTrue(tokens.contains("buzz"));
106+
107+
response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "baz")).get();
108+
assertHitCount(response, 1L);
109+
response = client().prepareSearch("test").setQuery(QueryBuilders.matchQuery("field", "buzz")).get();
110+
assertHitCount(response, 1L);
111+
}
112+
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,12 @@ protected void shardOperationOnPrimary(BasicReplicationRequest shardRequest, Ind
5858
ActionListener<PrimaryResult<BasicReplicationRequest, ReplicationResponse>> listener) {
5959
ActionListener.completeWith(listener, () -> {
6060
primary.refresh("api");
61+
try {
62+
primary.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis());
63+
} catch (Exception ex) {
64+
logger.error(ex.getLocalizedMessage(), ex);
65+
return new PrimaryResult(null, null, ex);
66+
}
6167
logger.trace("{} refresh request executed on primary", primary.shardId());
6268
return new PrimaryResult<>(shardRequest, new ReplicationResponse());
6369
});
@@ -66,6 +72,12 @@ protected void shardOperationOnPrimary(BasicReplicationRequest shardRequest, Ind
6672
@Override
6773
protected ReplicaResult shardOperationOnReplica(BasicReplicationRequest request, IndexShard replica) {
6874
replica.refresh("api");
75+
try {
76+
replica.mapperService().reloadSearchAnalyzers(indicesService.getAnalysis());
77+
} catch (Exception ex) {
78+
logger.error(ex.getLocalizedMessage(), ex);
79+
return new ReplicaResult(ex);
80+
}
6981
logger.trace("{} refresh request executed on replica", replica.shardId());
7082
return new ReplicaResult();
7183
}

0 commit comments

Comments
 (0)