Skip to content

Use settings from LogsdbIndexModeSettingsProvider in SyntheticSourceIndexSettingsProvider #115437

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kkrik-es merged 3 commits into from
Oct 24, 2024
Merged

Use settings from LogsdbIndexModeSettingsProvider in SyntheticSourceIndexSettingsProvider #115437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e22b1ad
Use settings from LogsdbIndexModeSettingsProvider in SyntheticSourceI…
kkrik-es Oct 23, 2024
74d9f95
Merge branch 'main' into logsdb/index-mode-in-synthetic-source-provider
kkrik-es Oct 24, 2024
44cf421
update
kkrik-es Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions ...ith-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
package org.elasticsearch.xpack.logsdb;

import org.elasticsearch.client.Request;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
Expand Down Expand Up @@ -171,4 +172,35 @@ public void testLogsdbOverrideNullInTemplate() throws IOException {
assertEquals("logsdb", settings.get("index.mode"));
assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
}

public void testLogsdbOverrideDefaultModeForLogsIndex() throws IOException {
Request request = new Request("PUT", "/_cluster/settings");
request.setJsonEntity("{ \"transient\": { \"cluster.logsdb.enabled\": true } }");
assertOK(client().performRequest(request));

request = new Request("POST", "/_index_template/1");
request.setJsonEntity("""
{
"index_patterns": ["logs-test-*"],
"data_stream": {
}
}
""");
assertOK(client().performRequest(request));

request = new Request("POST", "/logs-test-foo/_doc");
request.setJsonEntity("""
{
"@timestamp": "2020-01-01T00:00:00.000Z",
"host.name": "foo",
"message": "bar"
}
""");
assertOK(client().performRequest(request));

String index = DataStream.getDefaultBackingIndexName("logs-test-foo", 1);
var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
assertEquals("logsdb", settings.get("index.mode"));
assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
}
}
39 changes: 33 additions & 6 deletions x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

package org.elasticsearch.xpack.logsdb;

import org.elasticsearch.client.Request;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
Expand Down Expand Up @@ -35,12 +37,6 @@ protected String getTestRestCluster() {
}

public void testFeatureUsageWithLogsdbIndex() throws IOException {
{
var response = getAsMap("/_license/feature_usage");
@SuppressWarnings("unchecked")
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
assertThat(features, Matchers.empty());
}
{
if (randomBoolean()) {
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
Expand Down Expand Up @@ -81,4 +77,35 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
}
}

public void testLogsdbSourceModeForLogsIndex() throws IOException {
Request request = new Request("PUT", "/_cluster/settings");
request.setJsonEntity("{ \"transient\": { \"cluster.logsdb.enabled\": true } }");
assertOK(client().performRequest(request));

request = new Request("POST", "/_index_template/1");
request.setJsonEntity("""
{
"index_patterns": ["logs-test-*"],
"data_stream": {
}
}
""");
assertOK(client().performRequest(request));

request = new Request("POST", "/logs-test-foo/_doc");
request.setJsonEntity("""
{
"@timestamp": "2020-01-01T00:00:00.000Z",
"host.name": "foo",
"message": "bar"
}
""");
assertOK(client().performRequest(request));

String index = DataStream.getDefaultBackingIndexName("logs-test-foo", 1);
var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
assertEquals("logsdb", settings.get("index.mode"));
assertNull(settings.get("index.mapping.source.mode"));
}

}
2 changes: 1 addition & 1 deletion x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
return List.of(logsdbIndexModeSettingsProvider);
}
return List.of(
new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory()),
new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory(), logsdbIndexModeSettingsProvider),
logsdbIndexModeSettingsProvider
);
}
Expand Down
18 changes: 7 additions & 11 deletions .../logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,19 +48,16 @@ public Settings getAdditionalIndexSettings(
final Settings settings,
final List<CompressedXContent> combinedTemplateMappings
) {
if (isLogsdbEnabled == false || dataStreamName == null) {
return Settings.EMPTY;
}

final IndexMode indexMode = resolveIndexMode(settings.get(IndexSettings.MODE.getKey()));
if (indexMode != null) {
return Settings.EMPTY;
}
return getLogsdbModeSetting(dataStreamName, settings);
}

if (matchesLogsPattern(dataStreamName)) {
Settings getLogsdbModeSetting(final String dataStreamName, final Settings settings) {
if (isLogsdbEnabled
&& dataStreamName != null
&& resolveIndexMode(settings.get(IndexSettings.MODE.getKey())) == null
&& matchesLogsPattern(dataStreamName)) {
return Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build();
}

return Settings.EMPTY;
}

Expand All @@ -71,5 +68,4 @@ private static boolean matchesLogsPattern(final String name) {
private IndexMode resolveIndexMode(final String mode) {
return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null;
}

}
13 changes: 12 additions & 1 deletion ...db/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,16 @@ final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider

private final SyntheticSourceLicenseService syntheticSourceLicenseService;
private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider;

SyntheticSourceIndexSettingsProvider(
SyntheticSourceLicenseService syntheticSourceLicenseService,
CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory
CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory,
LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider
) {
this.syntheticSourceLicenseService = syntheticSourceLicenseService;
this.mapperServiceFactory = mapperServiceFactory;
this.logsdbIndexModeSettingsProvider = logsdbIndexModeSettingsProvider;
}

@Override
Expand All @@ -63,6 +66,14 @@ public Settings getAdditionalIndexSettings(
Settings indexTemplateAndCreateRequestSettings,
List<CompressedXContent> combinedTemplateMappings
) {
var logsdbSettings = logsdbIndexModeSettingsProvider.getLogsdbModeSetting(dataStreamName, indexTemplateAndCreateRequestSettings);
if (logsdbSettings != Settings.EMPTY) {
indexTemplateAndCreateRequestSettings = Settings.builder()
.put(logsdbSettings)
.put(indexTemplateAndCreateRequestSettings)
.build();
}

// This index name is used when validating component and index templates, we should skip this check in that case.
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
boolean isTemplateValidation = "validate-index-name".equals(indexName);
Expand Down
75 changes: 74 additions & 1 deletion ...c/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MapperTestUtils;
import org.elasticsearch.index.mapper.SourceFieldMapper;
import org.elasticsearch.license.MockLicenseState;
Expand All @@ -35,6 +36,10 @@ public class SyntheticSourceIndexSettingsProviderTests extends ESTestCase {
private SyntheticSourceLicenseService syntheticSourceLicenseService;
private SyntheticSourceIndexSettingsProvider provider;

private static LogsdbIndexModeSettingsProvider getLogsdbIndexModeSettingsProvider(boolean enabled) {
return new LogsdbIndexModeSettingsProvider(Settings.builder().put("cluster.logsdb.enabled", enabled).build());
}

@Before
public void setup() {
MockLicenseState licenseState = mock(MockLicenseState.class);
Expand All @@ -46,7 +51,8 @@ public void setup() {

provider = new SyntheticSourceIndexSettingsProvider(
syntheticSourceLicenseService,
im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName())
im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
getLogsdbIndexModeSettingsProvider(false)
);
}

Expand Down Expand Up @@ -310,4 +316,71 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws
assertThat(result.size(), equalTo(1));
assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));
}

public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException {
syntheticSourceLicenseService.setSyntheticSourceFallback(true);
provider = new SyntheticSourceIndexSettingsProvider(
syntheticSourceLicenseService,
im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
getLogsdbIndexModeSettingsProvider(true)
);
final Settings settings = Settings.EMPTY;

String dataStreamName = "logs-app1";
Metadata.Builder mb = Metadata.builder(
DataStreamTestHelper.getClusterStateWithDataStreams(
List.of(Tuple.tuple(dataStreamName, 1)),
List.of(),
Instant.now().toEpochMilli(),
builder().build(),
1
).getMetadata()
);
Metadata metadata = mb.build();
Settings result = provider.getAdditionalIndexSettings(
DataStream.getDefaultBackingIndexName(dataStreamName, 2),
dataStreamName,
null,
metadata,
Instant.ofEpochMilli(1L),
settings,
List.of()
);
assertThat(result.size(), equalTo(0));

dataStreamName = "logs-app1-0";
mb = Metadata.builder(
DataStreamTestHelper.getClusterStateWithDataStreams(
List.of(Tuple.tuple(dataStreamName, 1)),
List.of(),
Instant.now().toEpochMilli(),
builder().build(),
1
).getMetadata()
);
metadata = mb.build();

result = provider.getAdditionalIndexSettings(
DataStream.getDefaultBackingIndexName(dataStreamName, 2),
dataStreamName,
null,
metadata,
Instant.ofEpochMilli(1L),
settings,
List.of()
);
assertThat(result.size(), equalTo(1));
assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result));

result = provider.getAdditionalIndexSettings(
DataStream.getDefaultBackingIndexName(dataStreamName, 2),
dataStreamName,
null,
metadata,
Instant.ofEpochMilli(1L),
builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.toString()).build(),
List.of()
);
assertThat(result.size(), equalTo(0));
}
}