diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 8dd0b70d11538..2220f046e9bb0 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -18,8 +18,6 @@ include::setup.asciidoc[] include::setup/setup-xes.asciidoc[] -include::monitoring/configuring-monitoring.asciidoc[] - include::setup/setup-xclient.asciidoc[] include::setup/bootstrap-checks-xes.asciidoc[] diff --git a/docs/reference/monitoring/collecting-monitoring-data.asciidoc b/docs/reference/monitoring/collecting-monitoring-data.asciidoc index 8ab0443edb167..904595ae510d8 100644 --- a/docs/reference/monitoring/collecting-monitoring-data.asciidoc +++ b/docs/reference/monitoring/collecting-monitoring-data.asciidoc @@ -1,10 +1,7 @@ [role="xpack"] [testenv="gold"] [[collecting-monitoring-data]] -=== Collecting monitoring data -++++ -Collecting monitoring data -++++ +== Collecting monitoring data If you enable the Elastic {monitor-features} in your cluster, you can optionally collect metrics about {es}. By default, monitoring is enabled but diff --git a/docs/reference/monitoring/collectors.asciidoc b/docs/reference/monitoring/collectors.asciidoc index 64d56e81cdc87..568d21e835476 100644 --- a/docs/reference/monitoring/collectors.asciidoc +++ b/docs/reference/monitoring/collectors.asciidoc @@ -110,7 +110,7 @@ For more information about the configuration options for the collectors, see [float] [[es-monitoring-stack]] -=== Collecting data from across the Elastic Stack +==== Collecting data from across the Elastic Stack {monitoring} in {es} also receives monitoring data from other parts of the Elastic Stack. In this way, it serves as an unscheduled monitoring data diff --git a/docs/reference/monitoring/configuring-metricbeat.asciidoc b/docs/reference/monitoring/configuring-metricbeat.asciidoc index c765a47cb8d25..c8f41b18bfa91 100644 --- a/docs/reference/monitoring/configuring-metricbeat.asciidoc +++ b/docs/reference/monitoring/configuring-metricbeat.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="gold"] [[configuring-metricbeat]] -=== Collecting {es} monitoring data with {metricbeat} +== Collecting {es} monitoring data with {metricbeat} [subs="attributes"] ++++ diff --git a/docs/reference/monitoring/configuring-monitoring.asciidoc b/docs/reference/monitoring/configuring-monitoring.asciidoc deleted file mode 100644 index 9fe5b71d7a18c..0000000000000 --- a/docs/reference/monitoring/configuring-monitoring.asciidoc +++ /dev/null @@ -1,20 +0,0 @@ -[role="xpack"] -[testenv="gold"] -[[configuring-monitoring]] -== Configuring monitoring in {es} -++++ -Configuring monitoring -++++ - -If you enable the Elastic {monitor-features} in your cluster, there are two -methods to collect metrics about {es}: - -* <> -* <> - -To learn about monitoring in general, see -{stack-ov}/xpack-monitoring.html[Monitoring the {stack}]. - -include::collecting-monitoring-data.asciidoc[] -include::configuring-metricbeat.asciidoc[] -include::indices.asciidoc[] \ No newline at end of file diff --git a/docs/reference/monitoring/exporters.asciidoc b/docs/reference/monitoring/exporters.asciidoc index fee09015dbbd9..742b24608f250 100644 --- a/docs/reference/monitoring/exporters.asciidoc +++ b/docs/reference/monitoring/exporters.asciidoc @@ -158,6 +158,3 @@ which is used to determine whether the resource should be replaced. The `version field value represents the latest version of {monitoring} that changed the resource. If a resource is edited by someone or something external to {monitoring}, those changes are lost the next time an automatic update occurs. - -include::local-export.asciidoc[] -include::http-export.asciidoc[] diff --git a/docs/reference/monitoring/how-monitoring-works.asciidoc b/docs/reference/monitoring/how-monitoring-works.asciidoc new file mode 100644 index 0000000000000..e80c04eceacb6 --- /dev/null +++ b/docs/reference/monitoring/how-monitoring-works.asciidoc @@ -0,0 +1,40 @@ +[role="xpack"] +[testenv="basic"] +[[how-monitoring-works]] +== How monitoring works +++++ +How it works +++++ + +Each {es} node, {ls} node, {kib} instance, and Beat is considered unique in the +cluster based on its persistent UUID, which is written to the +<> directory when the node or instance starts. + +Monitoring documents are just ordinary JSON documents built by monitoring each +{stack} component at some collection interval. If you want to alter the +templates for these indices, see <>. + +Each component in the stack is responsible for monitoring itself and then +forwarding those documents to the {es} production cluster for both routing and +indexing (storage). The routing and indexing processes in {es} are handled by +what are called <> and +<>. + +Alternatively, in 6.4 and later, you can use {metricbeat} to collect +monitoring data about {kib} and ship it directly to the monitoring cluster, +rather than routing it through the production cluster. In 6.5 and later, you +can also use {metricbeat} to collect and ship data about {es}. + +To learn how to collect monitoring data, see: + +* <> +* <> +* {kibana-ref}/xpack-monitoring.html[Monitoring {kib}] +* {logstash-ref}/monitoring-logstash.html[Monitoring Logstash] +* Monitoring Beats: +** {auditbeat-ref}/monitoring.html[Auditbeat] +** {filebeat-ref}/monitoring.html[Filebeat] +** {heartbeat-ref}/monitoring.html[Heartbeat] +** {metricbeat-ref}/monitoring.html[Metricbeat] +** {packetbeat-ref}/monitoring.html[Packetbeat] +** {winlogbeat-ref}/monitoring.html[Winlogbeat] diff --git a/docs/reference/monitoring/http-export.asciidoc b/docs/reference/monitoring/http-export.asciidoc index a875e5a0169ee..eaca9904d04d1 100644 --- a/docs/reference/monitoring/http-export.asciidoc +++ b/docs/reference/monitoring/http-export.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[http-exporter]] -=== HTTP Exporters +=== HTTP exporters The `http` exporter is the preferred exporter in {monitoring} because it enables the use of a separate monitoring cluster. As a secondary benefit, it avoids diff --git a/docs/reference/monitoring/images/architecture10.png b/docs/reference/monitoring/images/architecture10.png new file mode 100644 index 0000000000000..dfcb417d812a5 Binary files /dev/null and b/docs/reference/monitoring/images/architecture10.png differ diff --git a/docs/reference/monitoring/images/architecture20.png b/docs/reference/monitoring/images/architecture20.png new file mode 100644 index 0000000000000..9a45ce0423e1c Binary files /dev/null and b/docs/reference/monitoring/images/architecture20.png differ diff --git a/docs/reference/monitoring/index.asciidoc b/docs/reference/monitoring/index.asciidoc index fbda72e0f979a..561070141ccf4 100644 --- a/docs/reference/monitoring/index.asciidoc +++ b/docs/reference/monitoring/index.asciidoc @@ -1,54 +1,30 @@ [role="xpack"] [testenv="basic"] -[[es-monitoring]] -= Monitoring {es} +[[monitor-elasticsearch-cluster]] += Monitor a cluster [partintro] -- -The Elastic {monitor-features} enable you to easily monitor the health of -your {es} cluster. The monitoring metrics are collected from each node and -stored in {es} indices. - -TIP: In production environments, it is recommended to store the monitoring data -in a separate _monitoring cluster_. See -{stack-ov}/monitoring-production.html[Monitoring in a production environment]. - -Each {es} node is considered unique based on its persistent UUID, which is -written on first start to its <> directory, which -defaults to `./data`. - -All settings associated with monitoring in {es} must be set in either the -`elasticsearch.yml` file for each node or, where possible, in the dynamic -cluster settings. For more information, see <>. - -[[es-monitoring-overview]] -{es} is also at the core of monitoring across the {stack}. In all cases, -monitoring documents are just ordinary JSON documents built by monitoring each -{stack} component at some collection interval, then indexing those -documents into the monitoring cluster. - -Each component in the stack is responsible for monitoring itself and then -forwarding those documents to the {es} production cluster for both routing and -indexing (storage). The routing and indexing processes in {es} are handled by -what are called <> and -<>. - -Alternatively, in 6.4 and later, you can use {metricbeat} to collect -monitoring data about {kib} and ship it directly to the monitoring cluster, -rather than routing it through the production cluster. In 6.5 and later, you -can also use {metricbeat} to collect and ship data about {es}. - -You can view monitoring data from {kib} where it’s easy to spot issues at a -glance or delve into the system behavior over time to diagnose operational -issues. In addition to the built-in status warnings, you can also set up custom -alerts based on the data in the monitoring indices. - -For an introduction to monitoring your {stack}, including Beats, {ls}, and {kib}, -see {stack-ov}/xpack-monitoring.html[Monitoring the {stack}]. - +The {stack} {monitor-features} provide a way to keep a pulse on the health and +performance of your {es} cluster. + +* <> +* <> +* <> +* <> +* <> +* <> +* <> -- +include::overview.asciidoc[] +include::how-monitoring-works.asciidoc[] +include::collecting-monitoring-data.asciidoc[] +include::pause-export.asciidoc[] +include::configuring-metricbeat.asciidoc[] +include::indices.asciidoc[] include::collectors.asciidoc[] include::exporters.asciidoc[] -include::pause-export.asciidoc[] +include::local-export.asciidoc[] +include::http-export.asciidoc[] diff --git a/docs/reference/monitoring/indices.asciidoc b/docs/reference/monitoring/indices.asciidoc index 34cbced1c4332..a63c215428f73 100644 --- a/docs/reference/monitoring/indices.asciidoc +++ b/docs/reference/monitoring/indices.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[config-monitoring-indices]] -=== Configuring indices for monitoring +== Configuring indices for monitoring <> are used to configure the indices that store the monitoring data collected from a cluster. diff --git a/docs/reference/monitoring/local-export.asciidoc b/docs/reference/monitoring/local-export.asciidoc index 821a6b1fc0e13..8723b226ca766 100644 --- a/docs/reference/monitoring/local-export.asciidoc +++ b/docs/reference/monitoring/local-export.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[local-exporter]] -=== Local Exporters +=== Local exporters The `local` exporter is the default exporter in {monitoring}. It routes data back into the same (local) cluster. In other words, it uses the production @@ -56,7 +56,7 @@ For more information about the configuration options for the `local` exporter, see <>. [[local-exporter-cleaner]] -==== Cleaner Service +==== Cleaner service One feature of the `local` exporter, which is not present in the `http` exporter, is a cleaner service. The cleaner service runs once per day at 01:00 AM UTC on diff --git a/docs/reference/monitoring/overview.asciidoc b/docs/reference/monitoring/overview.asciidoc new file mode 100644 index 0000000000000..7fd6899ab38e1 --- /dev/null +++ b/docs/reference/monitoring/overview.asciidoc @@ -0,0 +1,40 @@ +[role="xpack"] +[testenv="basic"] +[[monitoring-overview]] +== Monitoring overview +++++ +Overview +++++ + +When you monitor a cluster, you collect data from the {es} nodes, {ls} nodes, +and {kib} instances in your cluster. + +All of the monitoring metrics are stored in {es}, which enables you to easily +visualize the data from {kib}. By default, the monitoring metrics are stored in +local indices. + +TIP: In production, we strongly recommend using a separate monitoring cluster. Using a separate monitoring cluster prevents production cluster outages from impacting your ability to access your monitoring data. It also prevents +monitoring activities from impacting the performance of your production cluster. For the same reason, we also recommend using a separate {kib} instance for +viewing the monitoring data. + +The following diagram illustrates a typical monitoring architecture with separate +production and monitoring clusters: + +image::monitoring/images/architecture10.png["A typical monitoring environment"] + +In 6.4 and later, you can use {metricbeat} to collect and ship data about +{kib}, rather than routing it through {es}. In 6.5 and later, you can also use +{metricbeat} to collect and ship data about {es}. For example: + +image::monitoring/images/architecture20.png[A typical monitoring environment that includes {metricbeat}] + +If you have the appropriate license, you can route data from multiple production +clusters to a single monitoring cluster. For more information about the +differences between various subscription levels, see: +https://www.elastic.co/subscriptions + +IMPORTANT: In general, the monitoring cluster and the clusters being monitored +should be running the same version of the stack. A monitoring cluster cannot +monitor production clusters running newer versions of the stack. If necessary, +the monitoring cluster can monitor production clusters running the latest +release of the previous major version. \ No newline at end of file diff --git a/docs/reference/monitoring/pause-export.asciidoc b/docs/reference/monitoring/pause-export.asciidoc index 7a8bc664ffc38..6cf02a1f2401d 100644 --- a/docs/reference/monitoring/pause-export.asciidoc +++ b/docs/reference/monitoring/pause-export.asciidoc @@ -1,7 +1,7 @@ [role="xpack"] [testenv="basic"] [[pause-export]] -== Pausing Data Collection +=== Pausing data collection To stop generating {monitoring} data in {es}, disable data collection: diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 6575b65d2a6cf..cc1f9174799c6 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -649,4 +649,14 @@ See <>. [role="exclude",id="ccs-reduction"] === {ccs-cap} reduction -See <>. \ No newline at end of file +See <>. + +[role="exclude",id="es-monitoring"] +=== Monitoring {es} + +See <>. + +[role="exclude",id="configuring-monitoring"] +=== Configuring monitoring in {es} + +See <>.