Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add java TLS support #14620

Merged
merged 79 commits into from
Jan 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
79 commits
Select commit Hold shift + click to select a range
a23e37a
Attach an agent to a java process
nplanel Sep 19, 2022
8e07f3f
add Java TLS config support
nplanel Dec 5, 2022
19a85bc
compile java with java7 jdk 1.7.0
nplanel Dec 6, 2022
9420250
ebpf java tls, subscribe process monitor
nplanel Dec 13, 2022
7cdc82a
USMAgent.jar
nplanel Dec 7, 2022
8019623
agent-usm-7.43.0.jar
nplanel Dec 12, 2022
f1769a1
loadCommand string assign
nplanel Dec 12, 2022
edf5d41
hotspot close unix conn on error
nplanel Dec 13, 2022
ed91d79
typo Exists()
nplanel Dec 13, 2022
a5eec89
addrandomIDdoc
nplanel Dec 13, 2022
95ba881
add doc jattach_test PID namespace
nplanel Dec 13, 2022
904555b
attachChownAttachFile
nplanel Dec 13, 2022
fb7c775
add _ in gotls and javatls => go_tls java_tls
nplanel Dec 21, 2022
f9a9880
hotspot-doc
nplanel Dec 21, 2022
3d424b3
simplyfysocketExists
nplanel Dec 21, 2022
650c478
ftypo
nplanel Dec 21, 2022
70ed58e
addcheckconfigdisabledbydefault
nplanel Dec 21, 2022
f7deef4
reusable-private-socketPath
nplanel Dec 21, 2022
361936e
freturnnilinsteademptyfunc
nplanel Dec 21, 2022
0856d80
dstCloseifcopyfailed
nplanel Dec 21, 2022
ad05c40
attachPIDPath
nplanel Dec 21, 2022
8c0caf0
fixupmissingerrocheckontouchattachpid
nplanel Dec 21, 2022
4da0889
simplyfyAttachCommand
nplanel Dec 21, 2022
2c2411d
fcleanupjavaUSMAgentJarPath
nplanel Dec 21, 2022
a547a19
renameto-agent-usm.jar
nplanel Dec 22, 2022
d8d8f61
https-usm-doc
nplanel Jan 5, 2023
8fbe051
fhotspotdoc
nplanel Jan 5, 2023
bcba297
fnonakedreturn
nplanel Jan 5, 2023
43749ba
fdocnspid
nplanel Jan 5, 2023
c11b1ba
trapStat_tcast
nplanel Jan 5, 2023
bd4147a
frenamelocalsrcAgent
nplanel Jan 5, 2023
4d06d11
fnsDstPath
nplanel Jan 5, 2023
62a37b9
docdstClose
nplanel Jan 5, 2023
2796ec4
fremovedstpathonerror
nplanel Jan 5, 2023
c297362
parseresponseerror
nplanel Jan 5, 2023
4c0fdca
fixupdoc
nplanel Jan 5, 2023
ea06234
attachJVMusealternativepathattachpath
nplanel Jan 5, 2023
8ef5b2c
fixupattachdoc
nplanel Jan 5, 2023
c5eace4
fdoc
nplanel Jan 5, 2023
c0880b2
renameAttachagentPath
nplanel Jan 5, 2023
d0518d3
fsimplify
nplanel Jan 5, 2023
a05bdc2
docfsuidgid
nplanel Jan 5, 2023
8420c82
docTestInject
nplanel Jan 5, 2023
758bcc4
fsimplifytestreadability
nplanel Jan 5, 2023
83aeabe
fdocsubprog
nplanel Jan 5, 2023
1172600
agentusmjarconst
nplanel Jan 5, 2023
9df6a47
javasubprogramlog
nplanel Jan 5, 2023
e723449
fixupcleanupexec
nplanel Jan 5, 2023
e428839
removetodorandom
nplanel Jan 5, 2023
e98ae08
renameauthID
nplanel Jan 5, 2023
df62eda
move enable_go_tls_support and java_tls to service_monitoring_config
nplanel Jan 5, 2023
cc6ae46
Update pkg/network/java/hotspot.go
nplanel Jan 5, 2023
2c4efa4
Update pkg/network/java/hotspot.go
nplanel Jan 5, 2023
6de2acf
revert-usm-gotls-config
nplanel Jan 5, 2023
e0f504e
addinternaldoc
nplanel Jan 5, 2023
e0baf98
remove p==nil check
nplanel Jan 6, 2023
136dac0
faster scan response
nplanel Jan 6, 2023
135b4cd
check if the socket exist before doing the hotspot handshake protocol
nplanel Jan 10, 2023
9486a86
check if agent-usm.jar is accessible
nplanel Jan 10, 2023
a16e66b
dont change gotls feature name
nplanel Jan 10, 2023
2a05ddc
removemanifest and TestAgent code
nplanel Jan 10, 2023
67f64de
javaslowstart
nplanel Jan 11, 2023
edabb5f
wait java process to start the JVM to avoid crashing the java process…
nplanel Jan 12, 2023
d02f5ad
test log java -version
nplanel Jan 12, 2023
4b46c32
5sec
nplanel Jan 12, 2023
103557f
testdumpinfoonerror
nplanel Jan 12, 2023
fadff58
18threadsmin
nplanel Jan 12, 2023
dabacc2
6threadon1core
nplanel Jan 13, 2023
b46232d
15mintests
nplanel Jan 13, 2023
67ae957
seqtests
nplanel Jan 13, 2023
351ebf7
testlimtedby3min
nplanel Jan 16, 2023
e32366c
checkjavaage5s
nplanel Jan 16, 2023
6cedc01
24mult5
nplanel Jan 16, 2023
137e601
wait for an older java : age > 10s minimum
nplanel Jan 16, 2023
f8c3952
testwaitafterattach
nplanel Jan 17, 2023
31a8e7e
f12tests
nplanel Jan 17, 2023
cfd7429
onetestiteration
nplanel Jan 18, 2023
cc4fefb
givetimejavatospawn
nplanel Jan 18, 2023
1225dd4
gofmt-s
nplanel Jan 19, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@
/pkg/network/encoding/*usm* @DataDog/universal-service-monitoring
/pkg/network/etw/ @DataDog/windows-kernel-integrations
/pkg/network/go/ @DataDog/universal-service-monitoring
/pkg/network/java/ @DataDog/universal-service-monitoring
/pkg/network/protocols/ @DataDog/universal-service-monitoring
/pkg/network/protocols/http/driver_*.go @DataDog/windows-kernel-integrations
/pkg/network/protocols/http/etw_*.go @DataDog/windows-kernel-integrations
Expand Down
3 changes: 3 additions & 0 deletions omnibus/config/software/system-probe.rb
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,12 @@
mkdir "#{install_dir}/embedded/share/system-probe/ebpf/runtime"
mkdir "#{install_dir}/embedded/share/system-probe/ebpf/co-re"
mkdir "#{install_dir}/embedded/share/system-probe/ebpf/co-re/btf"
mkdir "#{install_dir}/embedded/share/system-probe/java"
mkdir "#{install_dir}/embedded/nikos/embedded/bin"
mkdir "#{install_dir}/embedded/nikos/embedded/lib"

copy 'pkg/network/java/agent-usm.jar', "#{install_dir}/embedded/share/system-probe/java/"

if ENV.has_key?('SYSTEM_PROBE_BIN') and not ENV['SYSTEM_PROBE_BIN'].empty?
copy "#{ENV['SYSTEM_PROBE_BIN']}/system-probe", "#{install_dir}/embedded/bin/system-probe"
copy "#{ENV['SYSTEM_PROBE_BIN']}/http.o", "#{install_dir}/embedded/share/system-probe/ebpf/"
Expand Down
1 change: 1 addition & 0 deletions omnibus/package-scripts/agent-deb/postinst
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ fi
chown root:root ${INSTALL_DIR}/embedded/bin/system-probe
chown root:root ${INSTALL_DIR}/embedded/bin/security-agent
chown -R root:root ${INSTALL_DIR}/embedded/share/system-probe/ebpf
chown -R root:root ${INSTALL_DIR}/embedded/share/system-probe/java

# Enable and restart the agent service here on Debian platforms
# On RHEL, this is done in the posttrans script
Expand Down
1 change: 1 addition & 0 deletions omnibus/package-scripts/agent-rpm/postinst
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,6 @@ fi
chown root:root ${INSTALL_DIR}/embedded/bin/system-probe
chown root:root ${INSTALL_DIR}/embedded/bin/security-agent
chown -R root:root ${INSTALL_DIR}/embedded/share/system-probe/ebpf
chown -R root:root ${INSTALL_DIR}/embedded/share/system-probe/java

exit 0
6 changes: 6 additions & 0 deletions pkg/config/system_probe.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ const (
// defaultSystemProbeBPFDir is the default path for eBPF programs
defaultSystemProbeBPFDir = "/opt/datadog-agent/embedded/share/system-probe/ebpf"

// defaultSystemProbeJavaDir is the default path for java agent program
defaultSystemProbeJavaDir = "/opt/datadog-agent/embedded/share/system-probe/java"

// defaultRuntimeCompilerOutputDir is the default path for output from the system-probe runtime compiler
defaultRuntimeCompilerOutputDir = "/var/tmp/datadog-agent/system-probe/build"

Expand Down Expand Up @@ -92,6 +95,7 @@ func InitSystemProbeConfig(cfg Config) {
// ebpf general settings
cfg.BindEnvAndSetDefault(join(spNS, "bpf_debug"), false)
cfg.BindEnvAndSetDefault(join(spNS, "bpf_dir"), defaultSystemProbeBPFDir, "DD_SYSTEM_PROBE_BPF_DIR")
cfg.BindEnvAndSetDefault(join(spNS, "java_dir"), defaultSystemProbeJavaDir, "DD_SYSTEM_PROBE_JAVA_DIR")
cfg.BindEnvAndSetDefault(join(spNS, "excluded_linux_versions"), []string{})
cfg.BindEnvAndSetDefault(join(spNS, "enable_tracepoints"), false)
cfg.BindEnvAndSetDefault(join(spNS, "enable_co_re"), true, "DD_ENABLE_CO_RE")
Expand Down Expand Up @@ -147,6 +151,8 @@ func InitSystemProbeConfig(cfg Config) {

cfg.BindEnvAndSetDefault(join(spNS, "enable_go_tls_support"), false)

cfg.BindEnvAndSetDefault(join(smNS, "enable_java_tls_support"), false)

cfg.BindEnvAndSetDefault(join(netNS, "enable_gateway_lookup"), true, "DD_SYSTEM_PROBE_NETWORK_ENABLE_GATEWAY_LOOKUP")
cfg.BindEnvAndSetDefault(join(netNS, "max_http_stats_buffered"), 100000, "DD_SYSTEM_PROBE_NETWORK_MAX_HTTP_STATS_BUFFERED")
httpRules := join(netNS, "http_replace_rules")
Expand Down
4 changes: 4 additions & 0 deletions pkg/ebpf/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ type Config struct {
// BPFDir is the directory to load the eBPF program from
BPFDir string

// JavaDir is the directory to load the java agent program from
JavaDir string

// ExcludedBPFLinuxVersions lists Linux kernel versions that should not use BPF features
ExcludedBPFLinuxVersions []string

Expand Down Expand Up @@ -85,6 +88,7 @@ func NewConfig() *Config {
return &Config{
BPFDebug: cfg.GetBool(key(spNS, "bpf_debug")),
BPFDir: cfg.GetString(key(spNS, "bpf_dir")),
JavaDir: cfg.GetString(key(spNS, "java_dir")),
ExcludedBPFLinuxVersions: cfg.GetStringSlice(key(spNS, "excluded_linux_versions")),
EnableTracepoints: cfg.GetBool(key(spNS, "enable_tracepoints")),
ProcRoot: util.GetProcRoot(),
Expand Down
1 change: 1 addition & 0 deletions pkg/metadata/inventories/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ The payload is a JSON dict with the following fields
- `feature_networks_http_enabled` - **bool**: True if HTTP monitoring is enabled for Network Performance Monitoring (see: `network_config.enable_http_monitoring` config option in `system-proble.yaml`).
- `feature_networks_https_enabled` - **bool**: True if HTTPS monitoring is enabled for Network Performance Monitoring (see: `network_config.enable_https_monitoring` config option in `system-proble.yaml`).
- `feature_networks_gotls_enabled` - **bool**: True if HTTPS monitoring through GoTLS is enabled for Network Performance Monitoring (see: `system_probe_config.enable_go_tls_support` config option in `system-proble.yaml`).
- `feature_usm_java_tls_enabled` - **bool**: True if HTTPS monitoring through java TLS is enabled for Universal Service Monitoring (see: `service_monitoring_config.enable_java_tls_support` config option in `system-proble.yaml`).
- `feature_logs_enabled` - **bool**: True if the logs collection is enabled (see: `logs_enabled` config option).
- `feature_cspm_enabled` - **bool**: True if the Cloud Security Posture Management is enabled (see:
`compliance_config.enabled` config option).
Expand Down
2 changes: 2 additions & 0 deletions pkg/metadata/inventories/inventories.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ const (
AgentNetworksHTTPEnabled AgentMetadataName = "feature_networks_http_enabled"
AgentNetworksHTTPSEnabled AgentMetadataName = "feature_networks_https_enabled"
AgentNetworksGoTLSEnabled AgentMetadataName = "feature_networks_gotls_enabled"
AgentUSMJavaTLSEnabled AgentMetadataName = "feature_usm_java_tls_enabled"
AgentLogsEnabled AgentMetadataName = "feature_logs_enabled"
AgentCSPMEnabled AgentMetadataName = "feature_cspm_enabled"
AgentAPMEnabled AgentMetadataName = "feature_apm_enabled"
Expand Down Expand Up @@ -414,6 +415,7 @@ func initializeConfig(cfg config.Config) {
SetAgentMetadata(AgentNetworksHTTPEnabled, config.Datadog.GetBool("network_config.enable_http_monitoring"))
SetAgentMetadata(AgentNetworksHTTPSEnabled, config.Datadog.GetBool("network_config.enable_https_monitoring"))
SetAgentMetadata(AgentNetworksGoTLSEnabled, config.Datadog.GetBool("system_probe_config.enable_go_tls_support"))
SetAgentMetadata(AgentUSMJavaTLSEnabled, config.Datadog.GetBool("service_monitoring_config.enable_java_tls_support"))
SetAgentMetadata(AgentLogsEnabled, config.Datadog.GetBool("logs_enabled"))
SetAgentMetadata(AgentCSPMEnabled, config.Datadog.GetBool("compliance_config.enabled"))
SetAgentMetadata(AgentAPMEnabled, config.Datadog.GetBool("apm_config.enabled"))
Expand Down
7 changes: 7 additions & 0 deletions pkg/network/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ type Config struct {
// traffic done through Go's standard library's TLS implementation
EnableGoTLSSupport bool

// EnableJavaTLSSupport specifies whether the tracer should monitor HTTPS
// traffic done through Java's TLS implementation
EnableJavaTLSSupport bool

// MaxTrackedHTTPConnections max number of http(s) flows that will be concurrently tracked.
// value is currently Windows only
MaxTrackedHTTPConnections int64
Expand Down Expand Up @@ -267,6 +271,9 @@ func New() *Config {

HTTPMapCleanerInterval: time.Duration(cfg.GetInt(join(spNS, "http_map_cleaner_interval_in_s"))) * time.Second,
HTTPIdleConnectionTTL: time.Duration(cfg.GetInt(join(spNS, "http_idle_connection_ttl_in_s"))) * time.Second,

// Service Monitoring
EnableJavaTLSSupport: cfg.GetBool(join(smNS, "enable_java_tls_support")),
}

if !cfg.IsSet(join(spNS, "max_closed_connections_buffered")) {
Expand Down
36 changes: 36 additions & 0 deletions pkg/network/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,42 @@ func TestEnableHTTPMonitoring(t *testing.T) {
})
}

func TestEnableJavaTLSSupport(t *testing.T) {
t.Run("via YAML", func(t *testing.T) {
newConfig()
defer restoreGlobalConfig()

_, err := sysconfig.New("./testdata/TestDDAgentConfigYamlAndSystemProbeConfig-EnableJavaTLS.yaml")
require.NoError(t, err)
cfg := New()

assert.True(t, cfg.EnableJavaTLSSupport)
})

t.Run("via ENV variable", func(t *testing.T) {
newConfig()
defer restoreGlobalConfig()

t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_JAVA_TLS_SUPPORT", "true")
_, err := sysconfig.New("")
require.NoError(t, err)
cfg := New()

assert.True(t, cfg.EnableJavaTLSSupport)
})
}

func TestDefaultDisabledJavaTLSSupport(t *testing.T) {
newConfig()
defer restoreGlobalConfig()

_, err := sysconfig.New("")
require.NoError(t, err)
cfg := New()

assert.False(t, cfg.EnableJavaTLSSupport)
}

func TestDisableGatewayLookup(t *testing.T) {
t.Run("via YAML", func(t *testing.T) {
newConfig()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
service_monitoring_config:
enable_java_tls_support: true
1 change: 1 addition & 0 deletions pkg/network/java/agent-usm.jar
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fake agent-usm.jar will be overwritten in next PR
Loading