Skip to content

Commit

Permalink
[jmxfetch] Add agent5-like JMXFetch helper commands (#1208)
Browse files Browse the repository at this point in the history
  • Loading branch information
arbll authored Apr 19, 2018
1 parent e529886 commit 7c492b6
Show file tree
Hide file tree
Showing 11 changed files with 548 additions and 287 deletions.
5 changes: 5 additions & 0 deletions cmd/agent/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,8 @@ func StopServer() {
listener.Close()
}
}

// ServerAddress retruns the server address.
func ServerAddress() *net.TCPAddr {
return listener.Addr().(*net.TCPAddr)
}
189 changes: 189 additions & 0 deletions cmd/agent/app/jmx.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2018 Datadog, Inc.

// +build jmx

package app

import (
"fmt"
"strings"

"github.com/DataDog/datadog-agent/cmd/agent/api"
"github.com/DataDog/datadog-agent/cmd/agent/common"
"github.com/DataDog/datadog-agent/pkg/collector/check"
"github.com/DataDog/datadog-agent/pkg/collector/corechecks/embed"
"github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/jmxfetch"
"github.com/spf13/cobra"
)

var (
jmxCmd = &cobra.Command{
Use: "jmx",
Short: "",
Long: ``,
}

jmxListCmd = &cobra.Command{
Use: "list",
Short: "List attributes matched by JMXFetch.",
Long: ``,
}

jmxCollectCmd = &cobra.Command{
Use: "collect",
Short: "Start the collection of metrics based on your current configuration and display them in the console.",
Long: ``,
RunE: doJmxCollect,
}

jmxListEverythingCmd = &cobra.Command{
Use: "everything",
Short: "List every attributes available that has a type supported by JMXFetch.",
Long: ``,
RunE: doJmxListEverything,
}

jmxListMatchingCmd = &cobra.Command{
Use: "matching",
Short: "List attributes that match at least one of your instances configuration.",
Long: ``,
RunE: doJmxListMatching,
}

jmxListLimitedCmd = &cobra.Command{
Use: "limited",
Short: "List attributes that do match one of your instances configuration but that are not being collected because it would exceed the number of metrics that can be collected.",
Long: ``,
RunE: doJmxListLimited,
}

jmxListCollectedCmd = &cobra.Command{
Use: "collected",
Short: "List attributes that will actually be collected by your current instances configuration.",
Long: ``,
RunE: doJmxListCollected,
}

jmxListNotMatchingCmd = &cobra.Command{
Use: "not-matching",
Short: "List attributes that don’t match any of your instances configuration.",
Long: ``,
RunE: doJmxListNotCollected,
}

checks = []string{}
)

func init() {
// attach list and collect commands to jmx command
jmxCmd.AddCommand(jmxListCmd)
jmxCmd.AddCommand(jmxCollectCmd)

//attach list commands to list root
jmxListCmd.AddCommand(jmxListEverythingCmd, jmxListMatchingCmd, jmxListLimitedCmd, jmxListCollectedCmd, jmxListNotMatchingCmd)

jmxListCmd.PersistentFlags().StringSliceVar(&checks, "checks", []string{}, "JMX checks (ex: jmx,tomcat)")
jmxCollectCmd.PersistentFlags().StringSliceVar(&checks, "checks", []string{}, "JMX checks (ex: jmx,tomcat)")

// attach the command to the root
AgentCmd.AddCommand(jmxCmd)
}

func doJmxCollect(cmd *cobra.Command, args []string) error {
return runJmxCommand("collect")
}

func doJmxListEverything(cmd *cobra.Command, args []string) error {
return runJmxCommand("list_everything")
}

func doJmxListMatching(cmd *cobra.Command, args []string) error {
return runJmxCommand("list_matching_attributes")
}

func doJmxListLimited(cmd *cobra.Command, args []string) error {
return runJmxCommand("list_limited_attributes")
}

func doJmxListCollected(cmd *cobra.Command, args []string) error {
return runJmxCommand("list_collected_attributes")
}

func doJmxListNotCollected(cmd *cobra.Command, args []string) error {
return runJmxCommand("list_not_matching_attributes")
}

func setupAgent() error {

err := common.SetupConfig(confFilePath)
if err != nil {
return fmt.Errorf("unable to set up global agent configuration: %v", err)
}

common.SetupAutoConfig(config.Datadog.GetString("confd_path"))

// let the os assign an available port
config.Datadog.Set("cmd_port", 0)

// start the cmd HTTP server
if err := api.StartServer(); err != nil {
return fmt.Errorf("Error while starting api server, exiting: %v", err)
}

return nil
}

func runJmxCommand(command string) error {
err := setupAgent()
if err != nil {
return err
}

runner := jmxfetch.New()

runner.ReportOnConsole = true
runner.Command = command
runner.IPCPort = api.ServerAddress().Port

loadConfigs()

err = runner.Start()
if err != nil {
return err
}

err = runner.Wait()
if err != nil {
return err
}

fmt.Println("JMXFetch exited successfully. If nothing was displayed please check your configuration, flags and the JMXFetch log file.")
return nil
}

func loadConfigs() {
fmt.Println("Loading configs :")

configs := common.AC.GetAllConfigs()
includeEverything := len(checks) == 0

for _, c := range configs {
if c.IsJMX() && (includeEverything || configIncluded(c)) {
fmt.Println("Config ", c.Name, " was loaded.")
embed.AddJMXCachedConfig(c)
}
}
}

func configIncluded(config check.Config) bool {
for _, c := range checks {
if strings.EqualFold(config.Name, c) {
return true
}
}
return false
}
46 changes: 15 additions & 31 deletions docs/agent/changes.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ The new command line interface for the Agent is sub-command based:
| start-service | starts the agent within the service control manager |
| status | Print the current status |
| stopservice | stops the agent within the service control manager |
| jmx | JMX troubleshooting |
| version | Print the version info |

To see the list of available sub-commands on your platform, run:
Expand Down Expand Up @@ -472,44 +473,27 @@ You will have to install the missing dependencies manually as described above.

## JMX

The Agent 6 ships JMXFetch and supports all of its features, except those listed below.
The Agent 6 ships JMXFetch and include a few changes :

The Agent 6 does not ship the `jmxterm` JAR. If you wish to download and use `jmxterm`, please refer to the [upstream project](https://github.com/jiaqi/jmxterm).

We still don't have a full featured interface to JMXFetch, so for now you may
have to run some commands manually to debug the list of beans collected, JVMs,
etc. A typical manual call will take the following form:
Troubleshooting commands syntax have changed :

```shell
/usr/bin/java -Xmx200m -Xms50m -classpath /usr/lib/jvm/java-8-oracle/lib/tools.jar:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch-0.18.2-jar-with-dependencies.jar org.datadog.jmxfetch.App --check <check list> --conf_directory /etc/datadog-agent/conf.d --log_level INFO --log_location /var/log/datadog/jmxfetch.log --reporter console <command>
```
`sudo -u dd-agent datadog-agent jmx list matching`: List attributes that match at least one of your instances configuration.

where `<command>` can be any of:
- `list_everything`
- `list_collected_attributes`
- `list_matching_attributes`
- `list_not_matching_attributes`
- `list_limited_attributes`
- `list_jvms`

and `<check list>` corresponds to a list of valid `yaml` configurations in
`/etc/datadog-agent/conf.d/`. For instance:
- `cassandra.d/conf.yaml`
- `kafka.d/conf.yaml`
- `jmx.d/conf.yaml`
- ...

Example:
```
/usr/bin/java -Xmx200m -Xms50m -classpath /usr/lib/jvm/java-8-oracle/lib/tools.jar:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch-0.18.2-jar-with-dependencies.jar org.datadog.jmxfetch.App --check cassandra.d/conf.yaml jmx.d/conf.yaml --conf_directory /etc/datadog-agent/conf.d --log_level INFO --log_location /var/log/datadog/jmxfetch.log --reporter console list_everything
```
`sudo -u dd-agent datadog-agent jmx list limited`: List attributes that do match one of your instances configuration but that are not being collected because it would exceed the number of metrics that can be collected.

`sudo -u dd-agent datadog-agent jmx list collected`: List attributes that will actually be collected by your current instances configuration.

`sudo -u dd-agent datadog-agent jmx list not-matching`: List attributes that don’t match any of your instances configuration.

`sudo -u dd-agent datadog-agent jmx list everything`: List every attributes available that has a type supported by JMXFetch.

Note: the location to the JRE tools.jar (`/usr/lib/jvm/java-8-oracle/lib/tools.jar`
in the example) might reside elsewhere in your system. You should be able to easily
find it with `sudo find / -type f -name 'tools.jar'`.
`sudo -u dd-agent datadog-agent jmx collect`: Start the collection of metrics based on your current configuration and display them in the console.

Note: you may wish to specify alternative JVM heap parameters `-Xmx`, `-Xms`, the
values used in the example correspond to the JMXFetch defaults.
By default theses command will run on all the configured jmx checks. If you want to
use them for specific checks, you can specify them using the `--checks` flag :
`sudo datadog-agent jmx list collected --checks tomcat`

### GCE hostname

Expand Down
8 changes: 4 additions & 4 deletions pkg/collector/autodiscovery/autoconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ func (ac *AutoConfig) AddProvider(provider providers.ConfigProvider, shouldPoll
// Check instances. Should always be run once so providers that don't need
// polling will be queried at least once
func (ac *AutoConfig) LoadAndRun() {
resolvedConfigs := ac.getAllConfigs()
resolvedConfigs := ac.GetAllConfigs()
checks := ac.getChecksFromConfigs(resolvedConfigs, true)
ac.schedule(checks)
}
Expand All @@ -182,7 +182,7 @@ func (ac *AutoConfig) GetChecksByName(checkName string) []check.Check {
titleCheck := fmt.Sprintf("%s%s", strings.Title(checkName), "Check")
checks := []check.Check{}

for _, check := range ac.getChecksFromConfigs(ac.getAllConfigs(), false) {
for _, check := range ac.getChecksFromConfigs(ac.GetAllConfigs(), false) {
if checkName == check.String() || titleCheck == check.String() {
checks = append(checks, check)
}
Expand All @@ -191,9 +191,9 @@ func (ac *AutoConfig) GetChecksByName(checkName string) []check.Check {
return checks
}

// getAllConfigs queries all the providers and returns all the check
// GetAllConfigs queries all the providers and returns all the check
// configurations found, resolving the ones it can
func (ac *AutoConfig) getAllConfigs() []check.Config {
func (ac *AutoConfig) GetAllConfigs() []check.Config {
resolvedConfigs := []check.Config{}

for _, pd := range ac.providers {
Expand Down
49 changes: 22 additions & 27 deletions pkg/collector/check/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
package check

import (
"bytes"
"fmt"
"hash/fnv"
"log"
"regexp"
"strconv"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -163,35 +162,26 @@ func (c *Config) Equal(config *Config) bool {

// String YAML representation of the config
func (c *Config) String() string {
var yamlBuff bytes.Buffer

yamlBuff.Write([]byte("init_config:\n"))
if c.InitConfig != nil {
yamlBuff.Write([]byte("- "))
strInit := strings.Split(string(c.InitConfig[:]), "\n")
for i, line := range strInit {
if i > 0 {
yamlBuff.Write([]byte(" "))
}
yamlBuff.Write([]byte(line))
yamlBuff.Write([]byte("\n"))
}
rawConfig := make(map[interface{}]interface{})
var initConfig interface{}
var instances []interface{}

yaml.Unmarshal(c.InitConfig, &initConfig)
rawConfig["init_config"] = initConfig

for _, i := range c.Instances {
var instance interface{}
yaml.Unmarshal(i, &instance)
instances = append(instances, instance)
}
rawConfig["instances"] = instances

yamlBuff.Write([]byte("instances:\n"))
for _, instance := range c.Instances {
strInst := strings.Split(string(instance[:]), "\n")
yamlBuff.Write([]byte("- "))
for i, line := range strInst {
if i > 0 {
yamlBuff.Write([]byte(" "))
}
yamlBuff.Write([]byte(line))
yamlBuff.Write([]byte("\n"))
}
buffer, err := yaml.Marshal(&rawConfig)
if err != nil {
log.Fatal(err)
}

return yamlBuff.String()
return string(buffer)
}

// IsTemplate returns if the config has AD identifiers
Expand Down Expand Up @@ -334,6 +324,11 @@ func (c *Config) Digest() string {
return strconv.FormatUint(h.Sum64(), 16)
}

// IsJMX checks if the config is a JMX config
func (c *Config) IsJMX() bool {
return IsConfigJMX(c.Name, c.InitConfig)
}

// IsConfigJMX checks if a certain YAML config is a JMX config
func IsConfigJMX(name string, initConf ConfigData) bool {

Expand Down
4 changes: 2 additions & 2 deletions pkg/collector/check/check_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ func TestString(t *testing.T) {
assert.False(t, config.Equal(nil))

config.Name = "foo"
config.InitConfig = ConfigData("fooBarBaz")
config.InitConfig = ConfigData("fooBarBaz: test")
config.Instances = []ConfigData{ConfigData("justFoo")}

expected := `init_config:
- fooBarBaz
fooBarBaz: test
instances:
- justFoo
`
Expand Down
Loading

0 comments on commit 7c492b6

Please sign in to comment.