Skip to content

Commit d57a858

Browse files
mtoffl01paullegranddcdatadog-datadog-prod-us1[bot]mcculls
authored
Stable Configuration implementation (#8338)
* Initial commit * Add Stable config source * Add snakeyaml to parse file * Initial tests for StableConfigSource * Add @OverRide for stableconfig get and configorigin methods * apply github code qual suggestions * passing initial tests * support config_id * Additional tests for invalid files * Introduce StableConfigSource to the list of ConfigProvider sources * add comments to current tests * Update internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java Co-authored-by: datadog-datadog-prod-us1[bot] <88084959+datadog-datadog-prod-us1[bot]@users.noreply.github.com> * fix typo * Refactor: Introduce StableConfigSource sub-class, StableConfig * nits: docs and cleanup * Update internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java Co-authored-by: datadog-datadog-prod-us1[bot] <88084959+datadog-datadog-prod-us1[bot]@users.noreply.github.com> * Update internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java Co-authored-by: datadog-datadog-prod-us1[bot] <88084959+datadog-datadog-prod-us1[bot]@users.noreply.github.com> * add doc for getStableConfig * Optimize: lazily load configuration information from file, cache results * nits: variable renaming, comments to describe variables * nit: move comment above variable * fix typo in managed file path * Initial StableConfigParser implementation and testing * Finalize StableConfigParser and tests * User parser in StableconfigSource; no more lazy loading * Add new logs for testing * remove snakeyaml from dependency tree in internal-api * Fix class initialization order * update parser test to use raw file input as opposed to map * make ParserTest handle unexpected input format as well * Revert fileretries stuff * Change managed path to reflect system tests * remove superfluous comments/logs used for testing * snakeyaml is only used for testing now * Register StableConfigSource with native-image since we need it at build time * Update internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java Co-authored-by: Stuart McCulloch <[email protected]> * Address PR comments * Change javadoc on getStableConfig function * Update dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java Co-authored-by: Stuart McCulloch <[email protected]> * Update internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java Co-authored-by: Stuart McCulloch <[email protected]> * Update internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java Co-authored-by: Stuart McCulloch <[email protected]> * Apply PR suggestions round 2 * Apply PR suggestions round 3 --------- Co-authored-by: paullegranddc <[email protected]> Co-authored-by: datadog-datadog-prod-us1[bot] <88084959+datadog-datadog-prod-us1[bot]@users.noreply.github.com> Co-authored-by: Stuart McCulloch <[email protected]>
1 parent 502adf7 commit d57a858

File tree

9 files changed

+490
-0
lines changed

9 files changed

+490
-0
lines changed

‎dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import datadog.trace.api.profiling.ProfilingEnablement;
3434
import datadog.trace.api.scopemanager.ScopeListener;
3535
import datadog.trace.bootstrap.benchmark.StaticEventLogger;
36+
import datadog.trace.bootstrap.config.provider.StableConfigSource;
3637
import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
3738
import datadog.trace.bootstrap.instrumentation.api.AgentTracer.TracerAPI;
3839
import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration;
@@ -1214,9 +1215,15 @@ private static boolean isFeatureEnabled(AgentFeature feature) {
12141215
// must be kept in sync with logic from Config!
12151216
final String featureEnabledSysprop = feature.getSystemProp();
12161217
String featureEnabled = System.getProperty(featureEnabledSysprop);
1218+
if (featureEnabled == null) {
1219+
featureEnabled = getStableConfig(StableConfigSource.MANAGED, featureEnabledSysprop);
1220+
}
12171221
if (featureEnabled == null) {
12181222
featureEnabled = ddGetEnv(featureEnabledSysprop);
12191223
}
1224+
if (featureEnabled == null) {
1225+
featureEnabled = getStableConfig(StableConfigSource.USER, featureEnabledSysprop);
1226+
}
12201227

12211228
if (feature.isEnabledByDefault()) {
12221229
// true unless it's explicitly set to "false"
@@ -1353,6 +1360,11 @@ private static String ddGetProperty(final String sysProp) {
13531360
return value;
13541361
}
13551362

1363+
/** Looks for sysProp in the Stable Configuration input */
1364+
private static String getStableConfig(StableConfigSource source, final String sysProp) {
1365+
return source.get(sysProp);
1366+
}
1367+
13561368
/** Looks for the "DD_" environment variable equivalent of the given "dd." system property. */
13571369
private static String ddGetEnv(final String sysProp) {
13581370
return System.getenv(toEnvVar(sysProp));

‎dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/NativeImageGeneratorRunnerInstrumentation.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ public static void onEnter(@Advice.Argument(value = 0, readOnly = false) String[
7878
+ "datadog.trace.api.env.CapturedEnvironment:build_time,"
7979
+ "datadog.trace.api.ConfigCollector:rerun,"
8080
+ "datadog.trace.api.ConfigDefaults:build_time,"
81+
+ "datadog.trace.api.ConfigOrigin:build_time,"
8182
+ "datadog.trace.api.ConfigSetting:build_time,"
8283
+ "datadog.trace.api.EventTracker:build_time,"
8384
+ "datadog.trace.api.InstrumenterConfig:build_time,"
@@ -106,6 +107,8 @@ public static void onEnter(@Advice.Argument(value = 0, readOnly = false) String[
106107
+ "datadog.trace.bootstrap.config.provider.EnvironmentConfigSource:build_time,"
107108
+ "datadog.trace.bootstrap.config.provider.OtelEnvironmentConfigSource:build_time,"
108109
+ "datadog.trace.bootstrap.config.provider.SystemPropertiesConfigSource:build_time,"
110+
+ "datadog.trace.bootstrap.config.provider.StableConfigSource:build_time,"
111+
+ "datadog.trace.bootstrap.config.provider.StableConfigSource$StableConfig:build_time,"
109112
+ "datadog.trace.bootstrap.Agent:build_time,"
110113
+ "datadog.trace.bootstrap.BootstrapProxy:build_time,"
111114
+ "datadog.trace.bootstrap.CallDepthThreadLocalMap:build_time,"

‎internal-api/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,7 @@ dependencies {
235235
testImplementation libs.commons.math
236236
testImplementation libs.bundles.mockito
237237
testImplementation libs.truth
238+
testImplementation 'org.yaml:snakeyaml:2.0'
238239
}
239240

240241
jmh {

‎internal-api/src/main/java/datadog/trace/api/ConfigOrigin.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ public enum ConfigOrigin {
77
REMOTE("remote_config"),
88
/** configurations that are set through JVM properties */
99
JVM_PROP("jvm_prop"),
10+
/** configuration read in the stable config file, managed by users */
11+
USER_STABLE_CONFIG("user_stable_config"),
12+
/** configuration read in the stable config file, managed by fleet */
13+
MANAGED_STABLE_CONFIG("managed_stable_config"),
1014
/** set when the user has not set any configuration for the key (defaults to a value) */
1115
DEFAULT("default");
1216

‎internal-api/src/main/java/datadog/trace/bootstrap/config/provider/ConfigProvider.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,15 +356,19 @@ public static ConfigProvider createDefault() {
356356
if (configProperties.isEmpty()) {
357357
return new ConfigProvider(
358358
new SystemPropertiesConfigSource(),
359+
StableConfigSource.MANAGED,
359360
new EnvironmentConfigSource(),
360361
new OtelEnvironmentConfigSource(),
362+
StableConfigSource.USER,
361363
new CapturedEnvironmentConfigSource());
362364
} else {
363365
return new ConfigProvider(
364366
new SystemPropertiesConfigSource(),
367+
StableConfigSource.MANAGED,
365368
new EnvironmentConfigSource(),
366369
new PropertiesConfigSource(configProperties, true),
367370
new OtelEnvironmentConfigSource(configProperties),
371+
StableConfigSource.USER,
368372
new CapturedEnvironmentConfigSource());
369373
}
370374
}
@@ -378,16 +382,20 @@ public static ConfigProvider withoutCollector() {
378382
return new ConfigProvider(
379383
false,
380384
new SystemPropertiesConfigSource(),
385+
StableConfigSource.MANAGED,
381386
new EnvironmentConfigSource(),
382387
new OtelEnvironmentConfigSource(),
388+
StableConfigSource.USER,
383389
new CapturedEnvironmentConfigSource());
384390
} else {
385391
return new ConfigProvider(
386392
false,
387393
new SystemPropertiesConfigSource(),
394+
StableConfigSource.MANAGED,
388395
new EnvironmentConfigSource(),
389396
new PropertiesConfigSource(configProperties, true),
390397
new OtelEnvironmentConfigSource(configProperties),
398+
StableConfigSource.USER,
391399
new CapturedEnvironmentConfigSource());
392400
}
393401
}
@@ -403,17 +411,21 @@ public static ConfigProvider withPropertiesOverride(Properties properties) {
403411
if (configProperties.isEmpty()) {
404412
return new ConfigProvider(
405413
new SystemPropertiesConfigSource(),
414+
StableConfigSource.MANAGED,
406415
new EnvironmentConfigSource(),
407416
providedConfigSource,
408417
new OtelEnvironmentConfigSource(),
418+
StableConfigSource.USER,
409419
new CapturedEnvironmentConfigSource());
410420
} else {
411421
return new ConfigProvider(
412422
providedConfigSource,
413423
new SystemPropertiesConfigSource(),
424+
StableConfigSource.MANAGED,
414425
new EnvironmentConfigSource(),
415426
new PropertiesConfigSource(configProperties, true),
416427
new OtelEnvironmentConfigSource(configProperties),
428+
StableConfigSource.USER,
417429
new CapturedEnvironmentConfigSource());
418430
}
419431
}

‎internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigParser.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package datadog.trace.bootstrap.config.provider;
2+
3+
import java.io.File;
4+
import java.io.IOException;
5+
import java.nio.file.Files;
6+
import java.nio.file.Paths;
7+
import java.util.HashMap;
8+
import java.util.Map;
9+
import java.util.regex.Matcher;
10+
import java.util.regex.Pattern;
11+
import java.util.stream.Stream;
12+
import org.slf4j.Logger;
13+
import org.slf4j.LoggerFactory;
14+
15+
public class StableConfigParser {
16+
private static final Logger log = LoggerFactory.getLogger(StableConfigParser.class);
17+
// Match config_id:<value>
18+
private static final Pattern idPattern = Pattern.compile("^config_id\\s*:(.*)$");
19+
// Match 'apm_configuration_default:'
20+
private static final Pattern apmConfigPattern = Pattern.compile("^apm_configuration_default:$");
21+
// Match indented (2 spaces) key-value pairs, either with double quotes or without
22+
private static final Pattern keyValPattern =
23+
Pattern.compile("^\\s{2}([^:]+):\\s*(\"[^\"]*\"|[^\"\\n]*)$");;
24+
25+
public static StableConfigSource.StableConfig parse(String filePath) throws IOException {
26+
File file = new File(filePath);
27+
if (!file.exists()) {
28+
log.debug("Stable configuration file not available at specified path: {}", file);
29+
return StableConfigSource.StableConfig.EMPTY;
30+
}
31+
Map<String, String> configMap = new HashMap<>();
32+
String[] configId = new String[1];
33+
try (Stream<String> lines = Files.lines(Paths.get(filePath))) {
34+
int apmConfigNotFound = -1, apmConfigStarted = 0, apmConfigComplete = 1;
35+
int[] apmConfigFound = {apmConfigNotFound};
36+
lines.forEach(
37+
line -> {
38+
Matcher matcher = idPattern.matcher(line);
39+
if (matcher.find()) {
40+
// Do not allow duplicate config_id keys
41+
if (configId[0] != null) {
42+
throw new RuntimeException("Duplicate config_id keys found; file may be malformed");
43+
}
44+
configId[0] = trimQuotes(matcher.group(1).trim());
45+
return; // go to next line
46+
}
47+
// TODO: Do not allow duplicate apm_configuration_default keys; and/or return early once
48+
// apmConfigFound[0] == apmConfigComplete
49+
if (apmConfigFound[0] == apmConfigNotFound
50+
&& apmConfigPattern.matcher(line).matches()) {
51+
apmConfigFound[0] = apmConfigStarted;
52+
return; // go to next line
53+
}
54+
if (apmConfigFound[0] == apmConfigStarted) {
55+
Matcher keyValueMatcher = keyValPattern.matcher(line);
56+
if (keyValueMatcher.matches()) {
57+
configMap.put(
58+
keyValueMatcher.group(1).trim(),
59+
trimQuotes(keyValueMatcher.group(2).trim())); // Store key-value pair in map
60+
} else {
61+
// If we encounter a non-indented or non-key-value line, stop processing
62+
apmConfigFound[0] = apmConfigComplete;
63+
}
64+
}
65+
});
66+
return new StableConfigSource.StableConfig(configId[0], configMap);
67+
}
68+
}
69+
70+
private static String trimQuotes(String value) {
71+
if (value.length() > 1 && (value.startsWith("'") && value.endsWith("'"))
72+
|| (value.startsWith("\"") && value.endsWith("\""))) {
73+
return value.substring(1, value.length() - 1);
74+
}
75+
return value;
76+
}
77+
}

‎internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package datadog.trace.bootstrap.config.provider;
2+
3+
import static datadog.trace.util.Strings.propertyNameToEnvironmentVariableName;
4+
5+
import datadog.trace.api.ConfigOrigin;
6+
import java.util.Collections;
7+
import java.util.Map;
8+
import java.util.Set;
9+
import org.slf4j.Logger;
10+
import org.slf4j.LoggerFactory;
11+
12+
public final class StableConfigSource extends ConfigProvider.Source {
13+
private static final Logger log = LoggerFactory.getLogger(StableConfigSource.class);
14+
15+
public static final String USER_STABLE_CONFIG_PATH =
16+
"/etc/datadog-agent/application_monitoring.yaml";
17+
public static final String MANAGED_STABLE_CONFIG_PATH =
18+
"/etc/datadog-agent/managed/datadog-agent/stable/application_monitoring.yaml";
19+
public static final StableConfigSource USER =
20+
new StableConfigSource(USER_STABLE_CONFIG_PATH, ConfigOrigin.USER_STABLE_CONFIG);
21+
public static final StableConfigSource MANAGED =
22+
new StableConfigSource(
23+
StableConfigSource.MANAGED_STABLE_CONFIG_PATH, ConfigOrigin.MANAGED_STABLE_CONFIG);
24+
25+
private final ConfigOrigin fileOrigin;
26+
27+
private final StableConfig config;
28+
29+
StableConfigSource(String file, ConfigOrigin origin) {
30+
this.fileOrigin = origin;
31+
StableConfig cfg;
32+
try {
33+
cfg = StableConfigParser.parse(file);
34+
} catch (Throwable e) {
35+
log.debug("Stable configuration file not readable at specified path: {}", file);
36+
cfg = StableConfig.EMPTY;
37+
}
38+
this.config = cfg;
39+
}
40+
41+
@Override
42+
public String get(String key) {
43+
if (this.config == StableConfig.EMPTY) {
44+
return null;
45+
}
46+
return this.config.get(propertyNameToEnvironmentVariableName(key));
47+
}
48+
49+
@Override
50+
public ConfigOrigin origin() {
51+
return fileOrigin;
52+
}
53+
54+
public Set<String> getKeys() {
55+
return this.config.getKeys();
56+
}
57+
58+
public String getConfigId() {
59+
return this.config.getConfigId();
60+
}
61+
62+
public static class StableConfig {
63+
public static final StableConfig EMPTY = new StableConfig(null, Collections.emptyMap());
64+
private final Map<String, String> apmConfiguration;
65+
private final String configId;
66+
67+
StableConfig(String configId, Map<String, String> configMap) {
68+
this.configId = configId;
69+
this.apmConfiguration = configMap;
70+
}
71+
72+
public String get(String key) {
73+
return this.apmConfiguration.get(key);
74+
}
75+
76+
public Set<String> getKeys() {
77+
return this.apmConfiguration.keySet();
78+
}
79+
80+
public String getConfigId() {
81+
return this.configId;
82+
}
83+
}
84+
}

0 commit comments

Comments
 (0)