Enhance GET _node/pipelines/:id API for Metricbeat monitoring

[ycombinator]

Background

Today, when the xpack.monitoring.* settings are configured in logstash.yml, Logstash performs internal collection of it's own monitoring metrics and ships them to .monitoring-logstash-*. The Monitoring UI in Kibana then reads documents from these indices to create visualizations of a user's Logstash instances and pipelines.

In the future, we're aiming to have Metricbeat externally collect Logstash metrics and ship them to .monitoring-logstash-* indices. This will mean that Metricbeat has to poll various Logstash HTTP APIs to collect said metrics.

The Request

Currently the GET _node/pipelines/:id API returns much of the information that Metricbeat will need in order to build the documents to be indexed into .monitoring-logstash-*. However, there are a few additional pieces of information that this API would need to return to complete the picture:

• ephemeral_id: Ephemeral ID of pipeline
• hash: Hash of pipeline representation
• representation: Shape of pipeline (LIR) + plugins available to pipeline

Here is an example JSON document showing just these additional fields:

{
  "ephemeral_id":"6d69db08-f2a5-411f-81cf-0f395d5de358",
  "hash":"4a5913a37e548b5921a3a7aec761cc0e59f0ce988c44bca8bcf14163a0cfc550",
  "representation":{
    "type":"lir",
    "hash":"4a5913a37e548b5921a3a7aec761cc0e59f0ce988c44bca8bcf14163a0cfc550",
    "version":"0.0.0",
    "plugins":[
      {
        "name":"logstash-codec-cef",
        "version":"5.0.6"
      },
      {
        "name":"logstash-codec-collectd",
        "version":"3.0.8"
      },
      {
        "name":"logstash-codec-dots",
        "version":"3.0.6"
      },
      {
        "name":"logstash-codec-edn",
        "version":"3.0.6"
      },
      {
        "name":"logstash-codec-line",
        "version":"3.0.8"
      },
      {
        "name":"logstash-codec-edn_lines",
        "version":"3.0.6"
      },
      {
        "name":"logstash-codec-es_bulk",
        "version":"3.0.6"
      },
      {
        "name":"logstash-codec-fluent",
        "version":"3.2.0"
      },
      {
        "name":"logstash-codec-graphite",
        "version":"3.0.5"
      },
      {
        "name":"logstash-codec-json",
        "version":"3.0.5"
      },
      {
        "name":"logstash-codec-json_lines",
        "version":"3.0.6"
      },
      {
        "name":"logstash-codec-msgpack",
        "version":"3.0.7"
      },
      {
        "name":"logstash-patterns-core",
        "version":"4.1.2"
      },
      {
        "name":"logstash-codec-multiline",
        "version":"3.0.10"
      },
      {
        "name":"logstash-codec-netflow",
        "version":"4.1.2"
      },
      {
        "name":"logstash-codec-plain",
        "version":"3.0.6"
      },
      {
        "name":"logstash-codec-rubydebug",
        "version":"3.0.6"
      },
      {
        "name":"logstash-filter-aggregate",
        "version":"2.8.0"
      },
      {
        "name":"logstash-filter-anonymize",
        "version":"3.0.6"
      },
      {
        "name":"logstash-filter-cidr",
        "version":"3.1.2"
      },
      {
        "name":"logstash-filter-clone",
        "version":"3.0.6"
      },
      {
        "name":"logstash-filter-csv",
        "version":"3.0.8"
      },
      {
        "name":"logstash-filter-date",
        "version":"3.1.9"
      },
      {
        "name":"logstash-filter-de_dot",
        "version":"1.0.3"
      },
      {
        "name":"logstash-filter-dissect",
        "version":"1.2.0"
      },
      {
        "name":"logstash-filter-dns",
        "version":"3.0.11"
      },
      {
        "name":"logstash-filter-drop",
        "version":"3.0.5"
      },
      {
        "name":"logstash-filter-elasticsearch",
        "version":"3.4.0"
      },
      {
        "name":"logstash-filter-fingerprint",
        "version":"3.2.0"
      },
      {
        "name":"logstash-filter-geoip",
        "version":"5.0.3"
      },
      {
        "name":"logstash-filter-grok",
        "version":"4.0.3"
      },
      {
        "name":"logstash-filter-jdbc_static",
        "version":"1.0.5"
      },
      {
        "name":"logstash-filter-jdbc_streaming",
        "version":"1.0.4"
      },
      {
        "name":"logstash-filter-json",
        "version":"3.0.5"
      },
      {
        "name":"logstash-filter-kv",
        "version":"4.2.1"
      },
      {
        "name":"logstash-filter-metrics",
        "version":"4.0.5"
      },
      {
        "name":"logstash-filter-multiline",
        "version":"3.0.4"
      },
      {
        "name":"logstash-filter-mutate",
        "version":"3.3.3"
      },
      {
        "name":"logstash-filter-ruby",
        "version":"3.1.5"
      },
      {
        "name":"logstash-filter-sleep",
        "version":"3.0.6"
      },
      {
        "name":"logstash-filter-split",
        "version":"3.1.6"
      },
      {
        "name":"logstash-filter-syslog_pri",
        "version":"3.0.5"
      },
      {
        "name":"logstash-filter-throttle",
        "version":"4.0.4"
      },
      {
        "name":"logstash-filter-translate",
        "version":"3.2.3"
      },
      {
        "name":"logstash-filter-truncate",
        "version":"1.0.4"
      },
      {
        "name":"logstash-filter-urldecode",
        "version":"3.0.6"
      },
      {
        "name":"logstash-filter-useragent",
        "version":"3.2.3"
      },
      {
        "name":"logstash-filter-xml",
        "version":"4.0.5"
      },
      {
        "name":"logstash-input-azure_event_hubs",
        "version":"1.0.4"
      },
      {
        "name":"logstash-input-beats",
        "version":"5.1.6"
      },
      {
        "name":"logstash-input-dead_letter_queue",
        "version":"1.1.4"
      },
      {
        "name":"logstash-input-elasticsearch",
        "version":"4.2.1"
      },
      {
        "name":"logstash-input-exec",
        "version":"3.3.2"
      },
      {
        "name":"logstash-input-file",
        "version":"4.1.6"
      },
      {
        "name":"logstash-input-ganglia",
        "version":"3.1.4"
      },
      {
        "name":"logstash-input-gelf",
        "version":"3.1.1"
      },
      {
        "name":"logstash-input-generator",
        "version":"3.0.6"
      },
      {
        "name":"logstash-input-tcp",
        "version":"5.1.0"
      },
      {
        "name":"logstash-input-graphite",
        "version":"3.0.6"
      },
      {
        "name":"logstash-input-heartbeat",
        "version":"3.0.7"
      },
      {
        "name":"logstash-input-http",
        "version":"3.2.2"
      },
      {
        "name":"logstash-input-http_poller",
        "version":"4.0.5"
      },
      {
        "name":"logstash-input-imap",
        "version":"3.0.6"
      },
      {
        "name":"logstash-input-jdbc",
        "version":"4.3.13"
      },
      {
        "name":"logstash-input-kafka",
        "version":"8.2.0"
      },
      {
        "name":"logstash-input-pipe",
        "version":"3.0.7"
      },
      {
        "name":"logstash-input-rabbitmq",
        "version":"6.0.3"
      },
      {
        "name":"logstash-input-redis",
        "version":"3.4.0"
      },
      {
        "name":"logstash-input-s3",
        "version":"3.4.1"
      },
      {
        "name":"logstash-input-snmp",
        "version":"1.0.0"
      },
      {
        "name":"logstash-input-snmptrap",
        "version":"3.0.6"
      },
      {
        "name":"logstash-input-sqs",
        "version":"3.1.1"
      },
      {
        "name":"logstash-input-stdin",
        "version":"3.2.6"
      },
      {
        "name":"logstash-input-syslog",
        "version":"3.4.1"
      },
      {
        "name":"logstash-input-twitter",
        "version":"3.0.8"
      },
      {
        "name":"logstash-input-udp",
        "version":"3.3.4"
      },
      {
        "name":"logstash-input-unix",
        "version":"3.0.7"
      },
      {
        "name":"logstash-output-appsearch",
        "version":"1.0.0.beta1"
      },
      {
        "name":"logstash-output-cloudwatch",
        "version":"3.0.8"
      },
      {
        "name":"logstash-output-file",
        "version":"4.2.5"
      },
      {
        "name":"logstash-output-csv",
        "version":"3.0.7"
      },
      {
        "name":"logstash-output-elasticsearch",
        "version":"9.2.1"
      },
      {
        "name":"logstash-output-email",
        "version":"4.1.1"
      },
      {
        "name":"logstash-output-graphite",
        "version":"3.1.6"
      },
      {
        "name":"logstash-output-http",
        "version":"5.2.2"
      },
      {
        "name":"logstash-output-kafka",
        "version":"7.2.0"
      },
      {
        "name":"logstash-output-lumberjack",
        "version":"3.1.7"
      },
      {
        "name":"logstash-output-nagios",
        "version":"3.0.6"
      },
      {
        "name":"logstash-output-null",
        "version":"3.0.5"
      },
      {
        "name":"logstash-output-pagerduty",
        "version":"3.0.7"
      },
      {
        "name":"logstash-output-pipe",
        "version":"3.0.6"
      },
      {
        "name":"logstash-output-rabbitmq",
        "version":"5.1.1"
      },
      {
        "name":"logstash-output-redis",
        "version":"4.0.4"
      },
      {
        "name":"logstash-output-s3",
        "version":"4.1.6"
      },
      {
        "name":"logstash-output-sns",
        "version":"4.0.7"
      },
      {
        "name":"logstash-output-sqs",
        "version":"5.1.2"
      },
      {
        "name":"logstash-output-stdout",
        "version":"3.1.4"
      },
      {
        "name":"logstash-output-tcp",
        "version":"5.0.3"
      },
      {
        "name":"logstash-output-udp",
        "version":"3.0.6"
      },
      {
        "name":"logstash-output-webhdfs",
        "version":"3.0.6"
      }
    ],
    "graph":{
      "vertices":[
        {
          "explicit_id":false,
          "plugin_type":"input",
          "type":"plugin",
          "config_name":"stdin",
          "meta":{
            "source":{
              "protocol":"str",
              "line":1,
              "column":9,
              "id":"pipeline"
            }
          },
          "id":"a6288a968df1bdcfe6433498f8bbabd15e6be53134f3594128330fd37b9ca837"
        },
        {
          "type":"queue",
          "meta":null,
          "explicit_id":false,
          "id":"__QUEUE__"
        },
        {
          "explicit_id":false,
          "plugin_type":"filter",
          "type":"plugin",
          "config_name":"drop",
          "meta":{
            "source":{
              "protocol":"str",
              "line":1,
              "column":29,
              "id":"pipeline"
            }
          },
          "id":"991a78728a853623523a25457cf2932267f11f34b0c8af6310c48e44a6d69c43"
        },
        {
          "explicit_id":false,
          "plugin_type":"output",
          "type":"plugin",
          "config_name":"stdout",
          "meta":{
            "source":{
              "protocol":"str",
              "line":1,
              "column":48,
              "id":"pipeline"
            }
          },
          "id":"42811067c43d0beafc02fa83221107895efc12eb37f9c4d92b94758bdf1dd3e8"
        }
      ],
      "edges":[
        {
          "from":"a6288a968df1bdcfe6433498f8bbabd15e6be53134f3594128330fd37b9ca837",
          "type":"plain",
          "to":"__QUEUE__",
          "id":"ceb29a7e399e41ca3014e3e7a68e9507bc5c516968eff66c4a4b4f6ba9070a2c"
        },
        {
          "from":"__QUEUE__",
          "type":"plain",
          "to":"991a78728a853623523a25457cf2932267f11f34b0c8af6310c48e44a6d69c43",
          "id":"1044d8f9a483ce5914048d3e337e321bc5ee94033abe534bc7de53bb398e199a"
        },
        {
          "from":"991a78728a853623523a25457cf2932267f11f34b0c8af6310c48e44a6d69c43",
          "type":"plain",
          "to":"42811067c43d0beafc02fa83221107895efc12eb37f9c4d92b94758bdf1dd3e8",
          "id":"b09c74922aeb08b5ebe2f2b54d07e07b4b08a4816e320ac3570b2bccff215bf6"
        }
      ]
    }
  }
}

[jsvd]

Looking at the overall structure of the document:

{
  "ephemeral_id":"6d69db08-f2a5-411f-81cf-0f395d5de358",
  "hash":"4a5913a37e548b5921a3a7aec761cc0e59f0ce988c44bca8bcf14163a0cfc550",
  "representation":{
    "type":"lir",
    "hash":"4a5913a37e548b5921a3a7aec761cc0e59f0ce988c44bca8bcf14163a0cfc550",
    "version":"0.0.0",
    "plugins":[  ... ]
    "graph":{
      "vertices":[ .. ],
      "edges":[ .. ]
    }
  }
}

I suggest a couple of things:

• the list of plugins seems to show all installed plugins, so I think we can skip this since the information is available in another api call
• the representation is only returned if the query parameter ?representation=true is passed

[ycombinator]

I'm good with both suggestions.